diff --git a/lib/mix/tasks/hex.registry.ex b/lib/mix/tasks/hex.registry.ex index ce61ab56..13d7331c 100644 --- a/lib/mix/tasks/hex.registry.ex +++ b/lib/mix/tasks/hex.registry.ex @@ -3,6 +3,7 @@ defmodule Mix.Tasks.Hex.Registry do @behaviour Hex.Mix.TaskDescription @switches [ + incremental: :boolean, name: :string, private_key: :string ] @@ -53,11 +54,36 @@ defmodule Mix.Tasks.Hex.Registry do {:decimal, "~> 2.0", repo: "acme"} end + ### Incremental Builds + + By default, `mix hex.registry build` will create a registry that includes packages and versions + present in the tarball directory. This means that missing tarballs will remove the corresponding + versions or packages from the registry. + + You can optionally perform an incremental build of the registry using the `--incremental` + command line option. This will add artifacts in the tarball directory without removing any of + the other versions or packages. This may be useful, for example, in a CI environment in which + you would like to publish to a local registry without downloading tarballs. + + To successfully run an incremental build, the following files are still required: + + * PUBLIC_DIR/names + * PUBLIC_DIR/versions + + as well as + + * PUBLIC_DIR/packages/PACKAGE_NAME + + for any existing packages to which you intend to add an additional version. + ### Command line options * `--name` - The name of the registry * `--private-key` - Path to the private key + + * `--incremental` - Use incremental registry building (see Incremental Builds) + """ @impl true def run(args) do @@ -88,11 +114,34 @@ defmodule Mix.Tasks.Hex.Registry do repo_name = opts[:name] || raise "missing --name" private_key_path = opts[:private_key] || raise "missing --private-key" private_key = private_key_path |> File.read!() |> decode_private_key() - build(repo_name, public_dir, private_key) + incremental? = opts[:incremental] == true + + build(repo_name, public_dir, private_key, incremental?) end - defp build(repo_name, public_dir, private_key) do - ensure_public_key(private_key, public_dir) + defp build(repo_name, public_dir, private_key, incremental?) do + public_key = ensure_public_key(private_key, public_dir) + + existing_names = + if incremental? do + read_names!(repo_name, public_dir, public_key) + |> Enum.map(fn %{name: name, updated_at: updated_at} -> {name, updated_at} end) + |> Enum.into(%{}) + else + %{} + end + + existing_versions = + if incremental? do + read_versions!(repo_name, public_dir, public_key) + |> Enum.map(fn %{name: name, versions: versions} -> + {name, %{updated_at: existing_names[name], versions: versions}} + end) + |> Enum.into(%{}) + else + %{} + end + create_directory(Path.join(public_dir, "tarballs")) paths_per_name = @@ -103,10 +152,19 @@ defmodule Mix.Tasks.Hex.Registry do versions = Enum.map(paths_per_name, fn {name, paths} -> + existing_releases = + if incremental? do + read_package(repo_name, public_dir, public_key, name) + else + [] + end + releases = paths |> Enum.map(&build_release(repo_name, &1)) + |> Enum.concat(existing_releases) |> Enum.sort(&(Hex.Version.compare(&1.version, &2.version) == :lt)) + |> Enum.uniq_by(& &1.version) updated_at = paths @@ -114,7 +172,8 @@ defmodule Mix.Tasks.Hex.Registry do |> Enum.sort() |> Enum.at(-1) - updated_at = updated_at && %{seconds: to_unix(updated_at), nanos: 0} + previous_updated_at = get_in(existing_names, [name, :updated_at, :seconds]) + updated_at = %{seconds: max_updated_at(previous_updated_at, updated_at), nanos: 0} package = :mix_hex_registry.build_package( @@ -126,10 +185,15 @@ defmodule Mix.Tasks.Hex.Registry do versions = Enum.map(releases, & &1.version) {name, %{updated_at: updated_at, versions: versions}} end) + |> Enum.into(%{}) + + versions = Map.merge(existing_versions, versions) - for path <- Path.wildcard("#{public_dir}/packages/*"), - not Enum.member?(Map.keys(paths_per_name), Path.basename(path)) do - remove_file(path) + if not incremental? do + for path <- Path.wildcard("#{public_dir}/packages/*"), + not Enum.member?(Map.keys(paths_per_name), Path.basename(path)) do + remove_file(path) + end end names = @@ -151,6 +215,8 @@ defmodule Mix.Tasks.Hex.Registry do write_file("#{public_dir}/versions", versions) end + ## Build utilities + @unix_epoch :calendar.datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}) @doc false @@ -158,6 +224,13 @@ defmodule Mix.Tasks.Hex.Registry do :calendar.datetime_to_gregorian_seconds(erl_datetime) - @unix_epoch end + defp max_updated_at(previous_as_unix_or_nil, nil), do: previous_as_unix_or_nil + defp max_updated_at(nil, current_as_datetime), do: to_unix(current_as_datetime) + + defp max_updated_at(previous_as_unix, current_as_datetime) do + max(previous_as_unix, to_unix(current_as_datetime)) + end + defp build_release(repo_name, tarball_path) do tarball = File.read!(tarball_path) {:ok, result} = :mix_hex_tarball.unpack(tarball, :memory) @@ -206,8 +279,60 @@ defmodule Mix.Tasks.Hex.Registry do {:error, :enoent} -> write_file(path, encoded_public_key) end + + encoded_public_key + end + + ## Incremental build utilities + + defp read_names!(repo_name, public_dir, public_key) do + path = Path.join(public_dir, "names") + payload = read_file!(path) + + case :mix_hex_registry.unpack_names(payload, repo_name, public_key) do + {:ok, names} -> + names + + _ -> + Mix.raise(""" + Invalid package name manifest at #{path} + + Is the repository name #{repo_name} correct? + """) + end + end + + defp read_versions!(repo_name, public_dir, public_key) do + path = Path.join(public_dir, "versions") + payload = read_file!(path) + + case :mix_hex_registry.unpack_versions(payload, repo_name, public_key) do + {:ok, versions} -> + versions + + _ -> + Mix.raise(""" + Invalid package version manifest at #{path} + + Is the repository name #{repo_name} correct? + """) + end end + defp read_package(repo_name, public_dir, public_key, package_name) do + path = Path.join([public_dir, "packages", package_name]) + + with {:ok, payload} <- read_file(path), + {:ok, package} <- + :mix_hex_registry.unpack_package(payload, repo_name, package_name, public_key) do + package + else + _ -> [] + end + end + + ## File utilities + defp create_directory(path) do unless File.dir?(path) do Hex.Shell.info(["* creating ", path]) @@ -215,6 +340,30 @@ defmodule Mix.Tasks.Hex.Registry do end end + defp read_file!(path) do + if File.exists?(path) do + Hex.Shell.info(["* reading ", path]) + else + Mix.raise(""" + Error reading file #{path} + + Using --incremental requires an existing registry + """) + end + + File.read!(path) + end + + defp read_file(path) do + if File.exists?(path) do + Hex.Shell.info(["* reading ", path]) + else + Hex.Shell.info(["* skipping ", path]) + end + + File.read(path) + end + defp write_file(path, data) do if File.exists?(path) do Hex.Shell.info(["* updating ", path]) diff --git a/test/mix/tasks/hex.registry_test.exs b/test/mix/tasks/hex.registry_test.exs index c2e1d61c..bd3a19d1 100644 --- a/test/mix/tasks/hex.registry_test.exs +++ b/test/mix/tasks/hex.registry_test.exs @@ -198,6 +198,164 @@ defmodule Mix.Tasks.Hex.RegistryTest do end) end + test "build --incremental" do + in_tmp(fn -> + bypass = setup_bypass() + + 0 = Mix.shell().cmd("openssl genrsa -out private_key.pem") + flush() + + # Initial run (--incremental cannot be used to create a new registry) + + Mix.Task.run( + "hex.registry", + ~w(build public --name acme --private-key private_key.pem) + ) + + assert_received {:mix_shell, :info, ["* creating public/public_key"]} + assert_received {:mix_shell, :info, ["* creating public/tarballs"]} + assert_received {:mix_shell, :info, ["* creating public/names"]} + assert_received {:mix_shell, :info, ["* creating public/versions"]} + refute_received _ + + config = %{ + :mix_hex_core.default_config() + | repo_url: "http://localhost:#{bypass.port}", + repo_verify: false, + repo_verify_origin: false + } + + assert {:ok, {200, _, []}} = :mix_hex_repo.get_names(config) + assert {:ok, {200, _, []}} = :mix_hex_repo.get_versions(config) + + # Add package to empty registry + + {:ok, %{tarball: tarball}} = :mix_hex_tarball.create(%{name: "foo", version: "0.10.0"}, []) + File.write!("public/tarballs/foo-0.10.0.tar", tarball) + + Mix.Task.reenable("hex.registry") + + Mix.Task.run( + "hex.registry", + ~w(build public --name acme --private-key private_key.pem --incremental) + ) + + assert_received {:mix_shell, :info, ["* reading public/names"]} + assert_received {:mix_shell, :info, ["* reading public/versions"]} + assert_received {:mix_shell, :info, ["* skipping public/packages/foo"]} + assert_received {:mix_shell, :info, ["* creating public/packages/foo"]} + assert_received {:mix_shell, :info, ["* updating public/names"]} + assert_received {:mix_shell, :info, ["* updating public/versions"]} + refute_received _ + + assert {:ok, {200, _, names}} = :mix_hex_repo.get_names(config) + assert [%{name: "foo", updated_at: %{seconds: updated_at}}] = names + + assert updated_at == + "public/tarballs/foo-0.10.0.tar" + |> File.stat!() + |> Map.fetch!(:mtime) + |> Mix.Tasks.Hex.Registry.to_unix() + + assert {:ok, {200, _, versions}} = :mix_hex_repo.get_versions(config) + assert versions == [%{name: "foo", retired: [], versions: ["0.10.0"]}] + + # Add new package version without tarballs for other versions + + File.rm!("public/tarballs/foo-0.10.0.tar") + {:ok, %{tarball: tarball}} = :mix_hex_tarball.create(%{name: "foo", version: "0.9.0"}, []) + File.write!("public/tarballs/foo-0.9.0.tar", tarball) + + Mix.Task.reenable("hex.registry") + + Mix.Task.run( + "hex.registry", + ~w(build public --name acme --private-key private_key.pem --incremental) + ) + + assert_received {:mix_shell, :info, ["* reading public/names"]} + assert_received {:mix_shell, :info, ["* reading public/versions"]} + assert_received {:mix_shell, :info, ["* reading public/packages/foo"]} + assert_received {:mix_shell, :info, ["* updating public/packages/foo"]} + assert_received {:mix_shell, :info, ["* updating public/names"]} + assert_received {:mix_shell, :info, ["* updating public/versions"]} + refute_received _ + + assert {:ok, {200, _, names}} = :mix_hex_repo.get_names(config) + assert [%{name: "foo", updated_at: %{seconds: updated_at}}] = names + + assert updated_at == + "public/tarballs/foo-0.9.0.tar" + |> File.stat!() + |> Map.fetch!(:mtime) + |> Mix.Tasks.Hex.Registry.to_unix() + + assert {:ok, {200, _, versions}} = :mix_hex_repo.get_versions(config) + assert versions == [%{name: "foo", retired: [], versions: ["0.9.0", "0.10.0"]}] + + # Add new package without tarballs for other packages + + File.rm!("public/tarballs/foo-0.9.0.tar") + + metadata = %{ + name: "bar", + version: "0.1.0", + requirements: %{ + "foo" => %{ + "app" => "foo", + "optional" => false, + "repository" => "acme", + "requirement" => "~> 0.1.0" + }, + "baz" => %{ + "app" => "baz", + "optional" => false, + "repository" => "external", + "requirement" => "~> 0.1.0" + } + } + } + + {:ok, %{tarball: tarball}} = :mix_hex_tarball.create(metadata, []) + File.write!("public/tarballs/bar-0.1.0.tar", tarball) + + Mix.Task.reenable("hex.registry") + + Mix.Task.run( + "hex.registry", + ~w(build public --name acme --private-key private_key.pem --incremental) + ) + + assert_received {:mix_shell, :info, ["* reading public/names"]} + assert_received {:mix_shell, :info, ["* reading public/versions"]} + assert_received {:mix_shell, :info, ["* skipping public/packages/bar"]} + assert_received {:mix_shell, :info, ["* creating public/packages/bar"]} + assert_received {:mix_shell, :info, ["* updating public/names"]} + assert_received {:mix_shell, :info, ["* updating public/versions"]} + refute_received _ + + assert {:ok, {200, _, names}} = :mix_hex_repo.get_names(config) + assert [%{name: "bar", updated_at: _}, %{name: "foo", updated_at: _}] = names + assert {:ok, {200, _, [package]}} = :mix_hex_repo.get_package(config, "bar") + + assert package.dependencies == [ + %{ + app: "baz", + optional: false, + package: "baz", + requirement: "~> 0.1.0", + repository: "external" + }, + %{ + app: "foo", + optional: false, + package: "foo", + requirement: "~> 0.1.0" + } + ] + end) + end + defp setup_bypass() do bypass = Bypass.open()