diff --git a/.gitignore b/.gitignore index b16a14c..c64e3bc 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ TODO.md # Local things .elixir_ls +.claude/settings.local.json priv/plts diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..def9100 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,2 @@ +elixir 1.18.4-otp-27 +erlang 27.2 diff --git a/lib/oci/plug.ex b/lib/oci/plug.ex index 73828cc..1e0827f 100644 --- a/lib/oci/plug.ex +++ b/lib/oci/plug.ex @@ -24,16 +24,13 @@ defmodule OCI.Plug do @impl true def call(%{script_name: [@api_version]} = conn, %{registry: registry}) do conn + |> maybe_enable_inspection() |> OCI.Plug.Context.call() |> put_private(:oci_registry, registry) |> authenticate() |> fetch_query_params() |> authorize() - # |> OCI.Inspector.log_info(nil, "before:handle/1") - |> OCI.Inspector.inspect("before:handle/1") |> OCI.Plug.Handler.handle() - - # |> OCI.Inspector.log_info(nil, "after:handle/1") end def call(conn, _opts) do @@ -98,4 +95,14 @@ defmodule OCI.Plug do |> send_resp(error.http_status, body) |> halt() end + + defp maybe_enable_inspection(conn) do + if System.get_env("OCI_TEST_ENABLE_HTTP_PRY") == "true" do + # You can set this inspect call anywhere on the conn and it will trigger + # Inspector.pry calls wherever they are placed based on the HTTP request. + conn |> OCI.Inspector.inspect("OCI_TEST_ENABLE_HTTP_PRY enabled!") + else + conn + end + end end diff --git a/lib/oci/plug/context.ex b/lib/oci/plug/context.ex index 85fd943..21ca1cb 100644 --- a/lib/oci/plug/context.ex +++ b/lib/oci/plug/context.ex @@ -23,6 +23,7 @@ defmodule OCI.Plug.Context do [uuid, "uploads", "blobs" | rest] -> {rest, :blobs_uploads, uuid} [digest, "blobs" | rest] -> {rest, :blobs, digest} [reference, "manifests" | rest] -> {rest, :manifests, reference} + [digest, "referrers" | rest] -> {rest, :referrers, digest} end # Reverse the path info, and the last parts after the known API path portions is the repo name. diff --git a/lib/oci/plug/handler.ex b/lib/oci/plug/handler.ex index e90066f..a33b8af 100644 --- a/lib/oci/plug/handler.ex +++ b/lib/oci/plug/handler.ex @@ -57,7 +57,21 @@ defmodule OCI.Plug.Handler do ctx :: OCI.Context.t() ) :: Plug.Conn.t() | {:error, atom()} | {:error, atom(), String.t()} defp dispatch(%{method: "GET"} = conn, :tags_list, registry, repo, _id, ctx) do - with {:ok, tags} <- Registry.list_tags(registry, repo, pagination(conn.query_params), ctx) do + pag = pagination(conn.query_params) + + with {:ok, tags} <- Registry.list_tags(registry, repo, pag, ctx) do + conn = + if pag.n != nil and length(tags) == pag.n do + last = List.last(tags) + + link = + "; rel=\"next\"" + + put_resp_header(conn, "link", link) + else + conn + end + conn |> put_resp_content_type("application/json") |> send_resp(200, Jason.encode!(%{name: repo, tags: tags})) @@ -113,7 +127,7 @@ defmodule OCI.Plug.Handler do {:ok, _, _} -> :ok - # TODO: we are swallowing this error!!! Yikes. + # Swallowed error: https://github.com/massdriver-cloud/oci/issues/12 err -> err end @@ -234,12 +248,50 @@ defmodule OCI.Plug.Handler do end end + defp dispatch(%{method: "GET"} = conn, :referrers, registry, repo, digest, ctx) do + filters = + case conn.query_params["artifactType"] do + nil -> %{} + artifact_type -> %{"artifactType" => artifact_type} + end + + with {:ok, referrers} <- Registry.list_referrers(registry, repo, digest, filters, ctx) do + index = %{ + "schemaVersion" => 2, + "mediaType" => "application/vnd.oci.image.index.v1+json", + "manifests" => referrers + } + + conn = + if filters["artifactType"] do + put_resp_header(conn, "oci-filters-applied", "artifactType") + else + conn + end + + conn + |> put_resp_header("content-type", "application/vnd.oci.image.index.v1+json") + |> send_resp(200, Jason.encode!(index)) + end + end + defp dispatch(%{method: "PUT"} = conn, :manifests, registry, repo, reference, ctx) do manifest = conn.params manifest_digest = conn.assigns[:oci_digest] with :ok <- Registry.store_manifest(registry, repo, reference, manifest, manifest_digest, ctx) do + maybe_set_oci_subject = fn conn -> + case get_in(conn.params, ["subject", "digest"]) do + nil -> + conn + + subject_digest -> + put_resp_header(conn, "oci-subject", subject_digest) + end + end + conn + |> maybe_set_oci_subject.() |> put_resp_header("location", Registry.manifests_reference_path(repo, reference)) |> send_resp(201, "") end diff --git a/lib/oci/plug/parser.ex b/lib/oci/plug/parser.ex index a6417f5..a460f2f 100644 --- a/lib/oci/plug/parser.ex +++ b/lib/oci/plug/parser.ex @@ -65,14 +65,21 @@ defmodule OCI.Plug.Parser do end end - def parse(conn, "application", "vnd.oci.image.manifest.v1+json", _headers, opts) do + def parse(conn, "application", "vnd.oci.image.manifest.v1+json", headers, opts) do + parse_oci_manifest(conn, headers, opts) + end + + def parse(conn, "application", "vnd.oci.image.index.v1+json", headers, opts) do + parse_oci_manifest(conn, headers, opts) + end + + def parse(conn, _type, _subtype, _headers, _opts), do: {:next, conn} + + defp parse_oci_manifest(conn, _headers, opts) do read_full_body(conn, opts, "") |> case do {:ok, full_body, conn} -> digest = :crypto.hash(:sha256, full_body) |> Base.encode16(case: :lower) - - # Note: this is not the 'digest' as is in the query string, but the byte-for-byte digest of the body. - # before it is ready by the json decoder. conn = Plug.Conn.assign(conn, :oci_digest, "sha256:#{digest}") decoder = Keyword.fetch!(opts, :json_decoder) @@ -90,8 +97,6 @@ defmodule OCI.Plug.Parser do end end - def parse(conn, _type, _subtype, _headers, _opts), do: {:next, conn} - @spec read_full_body(Plug.Conn.t(), opts(), String.t()) :: {:ok, String.t(), Plug.Conn.t()} | {:error, term()} defp read_full_body(conn, opts, acc) do diff --git a/lib/oci/registry.ex b/lib/oci/registry.ex index fbb6c34..2d26f49 100644 --- a/lib/oci/registry.ex +++ b/lib/oci/registry.ex @@ -93,7 +93,7 @@ defmodule OCI.Registry do {:ok, reg} end - def repo_exists?(%{storage: storage}, repo, ctx) do + def repo_exists?(storage, repo, ctx) do adapter(storage).repo_exists?(storage, repo, ctx) end @@ -207,16 +207,81 @@ defmodule OCI.Registry do end def store_manifest(%{storage: storage}, repo, reference, manifest, manifest_digest, ctx) do - adapter(storage).store_manifest( - storage, - repo, - reference, - manifest, - manifest_digest, - ctx - ) + required_blobs = referenced_blobs(manifest) + + missing = + Enum.filter(required_blobs, fn digest -> + !adapter(storage).blob_exists?(storage, repo, digest, ctx) + end) + + if missing != [] do + {:error, :MANIFEST_BLOB_UNKNOWN, %{missing: missing}} + else + with :ok <- + adapter(storage).store_manifest( + storage, + repo, + reference, + manifest, + manifest_digest, + ctx + ) do + maybe_index_referrer(storage, repo, manifest, manifest_digest, ctx) + :ok + end + end + end + + @doc """ + Extracts the list of blob digests referenced by a manifest. + + Image manifests reference a config blob and layer blobs. + Image indexes reference other manifests (not blobs), so they return an empty list. + """ + def list_referrers(%{storage: storage}, repo, digest, filters, ctx) do + adapter(storage).list_referrers(storage, repo, digest, filters, ctx) + end + + defp maybe_index_referrer(storage, repo, manifest, manifest_digest, ctx) do + case manifest["subject"] do + %{"digest" => subject_digest} when is_binary(subject_digest) -> + descriptor = build_referrer_descriptor(manifest, manifest_digest) + adapter(storage).put_referrer(storage, repo, subject_digest, descriptor, ctx) + + _ -> + :ok + end end + defp build_referrer_descriptor(manifest, manifest_digest) do + manifest_json = Jason.encode!(manifest) + + artifact_type = + manifest["artifactType"] || get_in(manifest, ["config", "mediaType"]) + + descriptor = %{ + "mediaType" => manifest["mediaType"], + "digest" => manifest_digest, + "size" => byte_size(manifest_json), + "artifactType" => artifact_type + } + + case manifest["annotations"] do + annotations when is_map(annotations) and map_size(annotations) > 0 -> + Map.put(descriptor, "annotations", annotations) + + _ -> + descriptor + end + end + + def referenced_blobs(%{"layers" => layers, "config" => config}) when is_list(layers) do + [config["digest"] | Enum.map(layers, & &1["digest"])] + |> Enum.reject(&is_nil/1) + end + + def referenced_blobs(_manifest), do: [] + def get_manifest(%{storage: storage}, repo, reference, ctx) do adapter(storage).get_manifest(storage, repo, reference, ctx) end @@ -226,7 +291,11 @@ defmodule OCI.Registry do end def list_tags(%{storage: storage}, repo, pagination, ctx) do - adapter(storage).list_tags(storage, repo, pagination, ctx) + if repo_exists?(storage, repo, ctx) do + adapter(storage).list_tags(storage, repo, pagination, ctx) + else + {:error, :NAME_UNKNOWN, %{repo: repo}} + end end @doc """ @@ -282,7 +351,7 @@ defmodule OCI.Registry do Returns {:ok, location} on success, {:error, :BLOB_UNKNOWN} if the source blob doesn't exist. """ def mount_blob(%__MODULE__{storage: storage} = registry, repo, digest, from_repo, ctx) do - if repo_exists?(registry, from_repo, ctx) do + if repo_exists?(storage, from_repo, ctx) do if blob_exists?(registry, from_repo, digest, ctx) do # credo:disable-for-next-line with :ok <- adapter(storage).mount_blob(storage, repo, digest, from_repo, ctx) do diff --git a/lib/oci/storage/adapter.ex b/lib/oci/storage/adapter.ex index 2a8c094..8c14dec 100644 --- a/lib/oci/storage/adapter.ex +++ b/lib/oci/storage/adapter.ex @@ -231,6 +231,33 @@ defmodule OCI.Storage.Adapter do | {:error, atom()} | {:error, atom(), error_details_t} + @doc """ + Lists manifest descriptors that reference the given subject digest. + + The `filters` map may contain: + - `"artifactType"` - filter referrers by artifact type + """ + @callback list_referrers( + storage :: t(), + repo :: String.t(), + subject_digest :: String.t(), + filters :: map(), + ctx :: OCI.Context.t() + ) :: + {:ok, [map()]} + + @doc """ + Stores a referrer descriptor for a given subject digest. + """ + @callback put_referrer( + storage :: t(), + repo :: String.t(), + subject_digest :: String.t(), + descriptor :: map(), + ctx :: OCI.Context.t() + ) :: + :ok + @doc """ Checks if an upload exists. """ diff --git a/lib/oci/storage/local.ex b/lib/oci/storage/local.ex index dec86c8..d8709de 100644 --- a/lib/oci/storage/local.ex +++ b/lib/oci/storage/local.ex @@ -194,22 +194,56 @@ defmodule OCI.Storage.Local do end @impl true - def list_tags(storage, repo, pagination, _ctx) do - if File.dir?(tags_dir(storage, repo)) do - tags = - tags_dir(storage, repo) - |> File.ls!() - |> Enum.sort() - - paginated_tags = - tags - |> cursor(pagination.last) - |> limit(pagination.n) - - {:ok, paginated_tags} - else - {:error, :NAME_UNKNOWN} + def list_referrers(storage, repo, subject_digest, filters, _ctx) do + path = referrer_path(storage, repo, subject_digest) + + referrers = + case File.read(path) do + {:ok, content} -> Jason.decode!(content) + {:error, :enoent} -> [] + end + + filtered = + case Map.get(filters, "artifactType") do + nil -> referrers + artifact_type -> Enum.filter(referrers, &(&1["artifactType"] == artifact_type)) + end + + {:ok, filtered} + end + + @impl true + def put_referrer(storage, repo, subject_digest, descriptor, _ctx) do + dir = referrers_dir(storage, repo) + path = referrer_path(storage, repo, subject_digest) + + :ok = File.mkdir_p!(dir) + + existing = + case File.read(path) do + {:ok, content} -> Jason.decode!(content) + {:error, :enoent} -> [] + end + + # Avoid duplicates by digest + unless Enum.any?(existing, &(&1["digest"] == descriptor["digest"])) do + File.write!(path, Jason.encode!(existing ++ [descriptor])) end + + :ok + end + + @impl true + def list_tags(storage, repo, pagination, _ctx) do + paginated_tags = + storage + |> tags_dir(repo) + |> File.ls!() + |> Enum.sort() + |> cursor(pagination.last) + |> limit(pagination.n) + + {:ok, paginated_tags} end @impl true @@ -229,30 +263,19 @@ defmodule OCI.Storage.Local do end @impl true - def store_manifest(storage, repo, reference, manifest, manifest_digest, ctx) do - blobs = [manifest["config"]["digest"]] ++ Enum.map(manifest["layers"], & &1["digest"]) - - if Enum.any?(blobs, fn digest -> - !blob_exists?(storage, repo, digest, ctx) - end) do - # TODO; return which blobs are missing. - # TODO: is this the right error or MANIFEST_INVALID? - {:error, :MANIFEST_BLOB_UNKNOWN, ""} - else - # Store manifest by digest - manifest_json = Jason.encode!(manifest) - - :ok = File.mkdir_p!(manifests_dir(storage, repo)) - File.write!(digest_path(storage, repo, manifest_digest), manifest_json) + def store_manifest(storage, repo, reference, manifest, manifest_digest, _ctx) do + manifest_json = Jason.encode!(manifest) - # If reference is a tag, create a tag reference - if !String.starts_with?(reference, "sha256:") do - :ok = File.mkdir_p!(tags_dir(storage, repo)) - File.write!(tag_path(storage, repo, reference), manifest_digest) - end + :ok = File.mkdir_p!(manifests_dir(storage, repo)) + File.write!(digest_path(storage, repo, manifest_digest), manifest_json) - :ok + # If reference is a tag, create a tag reference + if !String.starts_with?(reference, "sha256:") do + :ok = File.mkdir_p!(tags_dir(storage, repo)) + File.write!(tag_path(storage, repo, reference), manifest_digest) end + + :ok end @impl true @@ -318,6 +341,14 @@ defmodule OCI.Storage.Local do Path.join([repo_dir(storage, repo), "manifests"]) end + defp referrers_dir(storage, repo) do + Path.join([repo_dir(storage, repo), "referrers"]) + end + + defp referrer_path(storage, repo, subject_digest) do + Path.join([referrers_dir(storage, repo), subject_digest]) + end + defp repo_dir(storage, repo) do Path.join([storage.path, repo]) end diff --git a/mix.exs b/mix.exs index 750fe6b..778b423 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule OCI.MixProject do def project do [ app: :oci, - version: "0.0.6", + version: "0.1.0", elixir: "~> 1.15", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, diff --git a/test/conformance_test.exs b/test/conformance_test.exs index 51f8b14..54beddc 100644 --- a/test/conformance_test.exs +++ b/test/conformance_test.exs @@ -42,6 +42,6 @@ defmodule OCI.ConformanceTest do end end - Module.eval_quoted(__MODULE__, quote) + Code.eval_quoted(quote, [], __ENV__) end) end diff --git a/test/support/conformance_suite.ex b/test/support/conformance_suite.ex index 5b082fe..292d11c 100644 --- a/test/support/conformance_suite.ex +++ b/test/support/conformance_suite.ex @@ -72,7 +72,7 @@ defmodule ConformanceSuite do {"OCI_PASSWORD", "mypass"}, {"OCI_TEST_PULL", "1"}, {"OCI_TEST_PUSH", "1"}, - {"OCI_TEST_CONTENT_DISCOVERY", "0"}, + {"OCI_TEST_CONTENT_DISCOVERY", "1"}, {"OCI_TEST_CONTENT_MANAGEMENT", "0"}, {"OCI_AUTOMATIC_CROSSMOUNT", "0"}, {"OCI_DELETE_MANIFEST_BEFORE_BLOBS", "0"},