Add OCI Referrers API and image index support

.gitignore

@@ -33,4 +33,5 @@ TODO.md
 
 # Local things
 .elixir_ls
+.claude/settings.local.json
 priv/plts

.tool-versions

@@ -0,0 +1,2 @@
+elixir 1.18.4-otp-27
+erlang 27.2

lib/oci/plug.ex

@@ -24,16 +24,13 @@ defmodule OCI.Plug do
   @impl true
   def call(%{script_name: [@api_version]} = conn, %{registry: registry}) do
     conn
+    |> maybe_enable_inspection()
     |> OCI.Plug.Context.call()
     |> put_private(:oci_registry, registry)
     |> authenticate()
     |> fetch_query_params()
     |> authorize()
-    # |> OCI.Inspector.log_info(nil, "before:handle/1")
-    |> OCI.Inspector.inspect("before:handle/1")
     |> OCI.Plug.Handler.handle()
-
-    # |> OCI.Inspector.log_info(nil, "after:handle/1")
   end
 
   def call(conn, _opts) do
@@ -98,4 +95,14 @@ defmodule OCI.Plug do
     |> send_resp(error.http_status, body)
     |> halt()
   end
+
+  defp maybe_enable_inspection(conn) do
+    if System.get_env("OCI_TEST_ENABLE_HTTP_PRY") == "true" do
+      # You can set this inspect call anywhere on the conn and it will trigger
+      # Inspector.pry calls wherever they are placed based on the HTTP request.
+      conn |> OCI.Inspector.inspect("OCI_TEST_ENABLE_HTTP_PRY enabled!")
+    else
+      conn
+    end
+  end
 end

lib/oci/plug/context.ex

@@ -12,7 +12,7 @@
 
   def init(opts), do: opts
 
   def call(conn, _opts \\ []) do
     segments = conn.path_info |> Enum.reverse()
 
     {rest, endpoint, id} =
@@ -23,6 +23,7 @@
         [uuid, "uploads", "blobs" | rest] -> {rest, :blobs_uploads, uuid}
         [digest, "blobs" | rest] -> {rest, :blobs, digest}
         [reference, "manifests" | rest] -> {rest, :manifests, reference}
+        [digest, "referrers" | rest] -> {rest, :referrers, digest}
       end
 
     # Reverse the path info, and the last parts after the known API path portions is the repo name.

lib/oci/plug/handler.ex

@@ -57,7 +57,21 @@ defmodule OCI.Plug.Handler do
           ctx :: OCI.Context.t()
         ) :: Plug.Conn.t() | {:error, atom()} | {:error, atom(), String.t()}
   defp dispatch(%{method: "GET"} = conn, :tags_list, registry, repo, _id, ctx) do
-    with {:ok, tags} <- Registry.list_tags(registry, repo, pagination(conn.query_params), ctx) do
+    pag = pagination(conn.query_params)
+
+    with {:ok, tags} <- Registry.list_tags(registry, repo, pag, ctx) do
+      conn =
+        if pag.n != nil and length(tags) == pag.n do
+          last = List.last(tags)
+
+          link =
+            "</#{Registry.api_version()}/#{repo}/tags/list?n=#{pag.n}&last=#{last}>; rel=\"next\""
+
+          put_resp_header(conn, "link", link)
+        else
+          conn
+        end
+
       conn
       |> put_resp_content_type("application/json")
       |> send_resp(200, Jason.encode!(%{name: repo, tags: tags}))
@@ -113,7 +127,7 @@ defmodule OCI.Plug.Handler do
           {:ok, _, _} ->
             :ok
 
-          # TODO: we are swallowing this error!!! Yikes.
+          # Swallowed error: https://github.com/massdriver-cloud/oci/issues/12
           err ->
             err
         end
@@ -234,12 +248,50 @@ defmodule OCI.Plug.Handler do
     end
   end
 
+  defp dispatch(%{method: "GET"} = conn, :referrers, registry, repo, digest, ctx) do
+    filters =
+      case conn.query_params["artifactType"] do
+        nil -> %{}
+        artifact_type -> %{"artifactType" => artifact_type}
+      end
+
+    with {:ok, referrers} <- Registry.list_referrers(registry, repo, digest, filters, ctx) do
+      index = %{
+        "schemaVersion" => 2,
+        "mediaType" => "application/vnd.oci.image.index.v1+json",
+        "manifests" => referrers
+      }
+
+      conn =
+        if filters["artifactType"] do
+          put_resp_header(conn, "oci-filters-applied", "artifactType")
+        else
+          conn
+        end
+
+      conn
+      |> put_resp_header("content-type", "application/vnd.oci.image.index.v1+json")
+      |> send_resp(200, Jason.encode!(index))
+    end
+  end
+
   defp dispatch(%{method: "PUT"} = conn, :manifests, registry, repo, reference, ctx) do
     manifest = conn.params
     manifest_digest = conn.assigns[:oci_digest]
 
     with :ok <- Registry.store_manifest(registry, repo, reference, manifest, manifest_digest, ctx) do
+      maybe_set_oci_subject = fn conn ->
+        case get_in(conn.params, ["subject", "digest"]) do
+          nil ->
+            conn
+
+          subject_digest ->
+            put_resp_header(conn, "oci-subject", subject_digest)
+        end
+      end
+
       conn
+      |> maybe_set_oci_subject.()
       |> put_resp_header("location", Registry.manifests_reference_path(repo, reference))
       |> send_resp(201, "")
     end

lib/oci/plug/parser.ex

@@ -65,14 +65,21 @@ defmodule OCI.Plug.Parser do
     end
   end
 
-  def parse(conn, "application", "vnd.oci.image.manifest.v1+json", _headers, opts) do
+  def parse(conn, "application", "vnd.oci.image.manifest.v1+json", headers, opts) do
+    parse_oci_manifest(conn, headers, opts)
+  end
+
+  def parse(conn, "application", "vnd.oci.image.index.v1+json", headers, opts) do
+    parse_oci_manifest(conn, headers, opts)
+  end
+
+  def parse(conn, _type, _subtype, _headers, _opts), do: {:next, conn}
+
+  defp parse_oci_manifest(conn, _headers, opts) do
     read_full_body(conn, opts, "")
     |> case do
       {:ok, full_body, conn} ->
         digest = :crypto.hash(:sha256, full_body) |> Base.encode16(case: :lower)
-
-        # Note: this is not the 'digest' as is in the query string, but the byte-for-byte digest of the body.
-        # before it is ready by the json decoder.
         conn = Plug.Conn.assign(conn, :oci_digest, "sha256:#{digest}")
 
         decoder = Keyword.fetch!(opts, :json_decoder)
@@ -90,8 +97,6 @@ defmodule OCI.Plug.Parser do
     end
   end
 
-  def parse(conn, _type, _subtype, _headers, _opts), do: {:next, conn}
-
   @spec read_full_body(Plug.Conn.t(), opts(), String.t()) ::
           {:ok, String.t(), Plug.Conn.t()} | {:error, term()}
   defp read_full_body(conn, opts, acc) do

lib/oci/registry.ex

@@ -93,7 +93,7 @@ defmodule OCI.Registry do
     {:ok, reg}
   end
 
-  def repo_exists?(%{storage: storage}, repo, ctx) do
+  def repo_exists?(storage, repo, ctx) do
     adapter(storage).repo_exists?(storage, repo, ctx)
   end
 
@@ -207,16 +207,81 @@ defmodule OCI.Registry do
   end
 
   def store_manifest(%{storage: storage}, repo, reference, manifest, manifest_digest, ctx) do
-    adapter(storage).store_manifest(
-      storage,
-      repo,
-      reference,
-      manifest,
-      manifest_digest,
-      ctx
-    )
+    required_blobs = referenced_blobs(manifest)
+
+    missing =
+      Enum.filter(required_blobs, fn digest ->
+        !adapter(storage).blob_exists?(storage, repo, digest, ctx)
+      end)
+
+    if missing != [] do
+      {:error, :MANIFEST_BLOB_UNKNOWN, %{missing: missing}}
+    else
+      with :ok <-
+             adapter(storage).store_manifest(
+               storage,
+               repo,
+               reference,
+               manifest,
+               manifest_digest,
+               ctx
+             ) do
+        maybe_index_referrer(storage, repo, manifest, manifest_digest, ctx)
+        :ok
+      end
+    end
+  end
+
+  @doc """
+  Extracts the list of blob digests referenced by a manifest.
+
+  Image manifests reference a config blob and layer blobs.
+  Image indexes reference other manifests (not blobs), so they return an empty list.
+  """
+  def list_referrers(%{storage: storage}, repo, digest, filters, ctx) do
+    adapter(storage).list_referrers(storage, repo, digest, filters, ctx)
+  end
+
+  defp maybe_index_referrer(storage, repo, manifest, manifest_digest, ctx) do
+    case manifest["subject"] do
+      %{"digest" => subject_digest} when is_binary(subject_digest) ->
+        descriptor = build_referrer_descriptor(manifest, manifest_digest)
+        adapter(storage).put_referrer(storage, repo, subject_digest, descriptor, ctx)
+
+      _ ->
+        :ok
+    end
   end
 
+  defp build_referrer_descriptor(manifest, manifest_digest) do
+    manifest_json = Jason.encode!(manifest)
+
+    artifact_type =
+      manifest["artifactType"] || get_in(manifest, ["config", "mediaType"])
+
+    descriptor = %{
+      "mediaType" => manifest["mediaType"],
+      "digest" => manifest_digest,
+      "size" => byte_size(manifest_json),
+      "artifactType" => artifact_type
+    }
+
+    case manifest["annotations"] do
+      annotations when is_map(annotations) and map_size(annotations) > 0 ->
+        Map.put(descriptor, "annotations", annotations)
+
+      _ ->
+        descriptor
+    end
+  end
+
+  def referenced_blobs(%{"layers" => layers, "config" => config}) when is_list(layers) do
+    [config["digest"] | Enum.map(layers, & &1["digest"])]
+    |> Enum.reject(&is_nil/1)
+  end
+
+  def referenced_blobs(_manifest), do: []
+
   def get_manifest(%{storage: storage}, repo, reference, ctx) do
     adapter(storage).get_manifest(storage, repo, reference, ctx)
   end
@@ -226,7 +291,11 @@ defmodule OCI.Registry do
   end
 
   def list_tags(%{storage: storage}, repo, pagination, ctx) do
-    adapter(storage).list_tags(storage, repo, pagination, ctx)
+    if repo_exists?(storage, repo, ctx) do
+      adapter(storage).list_tags(storage, repo, pagination, ctx)
+    else
+      {:error, :NAME_UNKNOWN, %{repo: repo}}
+    end
   end
 
   @doc """
@@ -282,7 +351,7 @@ defmodule OCI.Registry do
   Returns {:ok, location} on success, {:error, :BLOB_UNKNOWN} if the source blob doesn't exist.
   """
   def mount_blob(%__MODULE__{storage: storage} = registry, repo, digest, from_repo, ctx) do
-    if repo_exists?(registry, from_repo, ctx) do
+    if repo_exists?(storage, from_repo, ctx) do
       if blob_exists?(registry, from_repo, digest, ctx) do
         # credo:disable-for-next-line
         with :ok <- adapter(storage).mount_blob(storage, repo, digest, from_repo, ctx) do

lib/oci/storage/adapter.ex

@@ -231,6 +231,33 @@ defmodule OCI.Storage.Adapter do
               | {:error, atom()}
               | {:error, atom(), error_details_t}
 
+  @doc """
+  Lists manifest descriptors that reference the given subject digest.
+
+  The `filters` map may contain:
+    - `"artifactType"` - filter referrers by artifact type
+  """
+  @callback list_referrers(
+              storage :: t(),
+              repo :: String.t(),
+              subject_digest :: String.t(),
+              filters :: map(),
+              ctx :: OCI.Context.t()
+            ) ::
+              {:ok, [map()]}
+
+  @doc """
+  Stores a referrer descriptor for a given subject digest.
+  """
+  @callback put_referrer(
+              storage :: t(),
+              repo :: String.t(),
+              subject_digest :: String.t(),
+              descriptor :: map(),
+              ctx :: OCI.Context.t()
+            ) ::
+              :ok
+
   @doc """
   Checks if an upload exists.
   """