Add --incremental option to hex.registry build

lib/mix/tasks/hex.registry.ex

@@ -3,6 +3,7 @@ defmodule Mix.Tasks.Hex.Registry do
   @behaviour Hex.Mix.TaskDescription
 
   @switches [
+    incremental: :boolean,
     name: :string,
     private_key: :string
   ]
@@ -53,11 +54,36 @@ defmodule Mix.Tasks.Hex.Registry do
         {:decimal, "~> 2.0", repo: "acme"}
       end
 
+  ### Incremental Builds
+
+  By default, `mix hex.registry build` will create a registry that includes packages and versions
+  present in the tarball directory. This means that missing tarballs will remove the corresponding
+  versions or packages from the registry.
+
+  You can optionally perform an incremental build of the registry using the `--incremental`
+  command line option. This will add artifacts in the tarball directory without removing any of
+  the other versions or packages. This may be useful, for example, in a CI environment in which
+  you would like to publish to a local registry without downloading tarballs.
+
+  To successfully run an incremental build, the following files are still required:
+
+      * PUBLIC_DIR/names
+      * PUBLIC_DIR/versions
+
+  as well as
+
+      * PUBLIC_DIR/packages/PACKAGE_NAME
+
+  for any existing packages to which you intend to add an additional version.
+
   ### Command line options
 
     * `--name` - The name of the registry
 
     * `--private-key` - Path to the private key
+
+    * `--incremental` - Use incremental registry building (see Incremental Builds)
+
   """
   @impl true
   def run(args) do
@@ -88,11 +114,34 @@ defmodule Mix.Tasks.Hex.Registry do
     repo_name = opts[:name] || raise "missing --name"
     private_key_path = opts[:private_key] || raise "missing --private-key"
     private_key = private_key_path |> File.read!() |> decode_private_key()
-    build(repo_name, public_dir, private_key)
+    incremental? = opts[:incremental] == true
+
+    build(repo_name, public_dir, private_key, incremental?)
   end
 
-  defp build(repo_name, public_dir, private_key) do
-    ensure_public_key(private_key, public_dir)
+  defp build(repo_name, public_dir, private_key, incremental?) do
+    public_key = ensure_public_key(private_key, public_dir)
+
+    existing_names =
+      if incremental? do
+        read_names!(repo_name, public_dir, public_key)
+        |> Enum.map(fn %{name: name, updated_at: updated_at} -> {name, updated_at} end)
+        |> Enum.into(%{})
+      else
+        %{}
+      end
+
+    existing_versions =
+      if incremental? do
+        read_versions!(repo_name, public_dir, public_key)
+        |> Enum.map(fn %{name: name, versions: versions} ->
+          {name, %{updated_at: existing_names[name], versions: versions}}
+        end)
+        |> Enum.into(%{})
+      else
+        %{}
+      end
+
     create_directory(Path.join(public_dir, "tarballs"))
 
     paths_per_name =
@@ -103,18 +152,28 @@ defmodule Mix.Tasks.Hex.Registry do
 
     versions =
       Enum.map(paths_per_name, fn {name, paths} ->
+        existing_releases =
+          if incremental? do
+            read_package(repo_name, public_dir, public_key, name)
+          else
+            []
+          end
+
         releases =
           paths
           |> Enum.map(&build_release(repo_name, &1))
+          |> Enum.concat(existing_releases)
           |> Enum.sort(&(Hex.Version.compare(&1.version, &2.version) == :lt))
+          |> Enum.uniq_by(& &1.version)
 
         updated_at =
           paths
           |> Enum.map(&File.stat!(&1).mtime)
           |> Enum.sort()
           |> Enum.at(-1)
 
-        updated_at = updated_at && %{seconds: to_unix(updated_at), nanos: 0}
+        previous_updated_at = get_in(existing_names, [name, :updated_at, :seconds])
+        updated_at = %{seconds: max_updated_at(previous_updated_at, updated_at), nanos: 0}
 
         package =
           :mix_hex_registry.build_package(
@@ -126,10 +185,15 @@ defmodule Mix.Tasks.Hex.Registry do
         versions = Enum.map(releases, & &1.version)
         {name, %{updated_at: updated_at, versions: versions}}
       end)
+      |> Enum.into(%{})
+
+    versions = Map.merge(existing_versions, versions)
 
-    for path <- Path.wildcard("#{public_dir}/packages/*"),
-        not Enum.member?(Map.keys(paths_per_name), Path.basename(path)) do
-      remove_file(path)
+    if not incremental? do
+      for path <- Path.wildcard("#{public_dir}/packages/*"),
+          not Enum.member?(Map.keys(paths_per_name), Path.basename(path)) do
+        remove_file(path)
+      end
     end
 
     names =
@@ -151,13 +215,22 @@ defmodule Mix.Tasks.Hex.Registry do
     write_file("#{public_dir}/versions", versions)
   end
 
+  ## Build utilities
+
   @unix_epoch :calendar.datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}})
 
   @doc false
   def to_unix(erl_datetime) do
     :calendar.datetime_to_gregorian_seconds(erl_datetime) - @unix_epoch
   end
 
+  defp max_updated_at(previous_as_unix_or_nil, nil), do: previous_as_unix_or_nil
+  defp max_updated_at(nil, current_as_datetime), do: to_unix(current_as_datetime)
+
+  defp max_updated_at(previous_as_unix, current_as_datetime) do
+    max(previous_as_unix, to_unix(current_as_datetime))
+  end
+
   defp build_release(repo_name, tarball_path) do
     tarball = File.read!(tarball_path)
     {:ok, result} = :mix_hex_tarball.unpack(tarball, :memory)
@@ -206,15 +279,91 @@ defmodule Mix.Tasks.Hex.Registry do
       {:error, :enoent} ->
         write_file(path, encoded_public_key)
     end
+
+    encoded_public_key
+  end
+
+  ## Incremental build utilities
+
+  defp read_names!(repo_name, public_dir, public_key) do
+    path = Path.join(public_dir, "names")
+    payload = read_file!(path)
+
+    case :mix_hex_registry.unpack_names(payload, repo_name, public_key) do
+      {:ok, names} ->
+        names
+
+      _ ->
+        Mix.raise("""
+        Invalid package name manifest at #{path}
+
+        Is the repository name #{repo_name} correct?
+        """)
+    end
+  end
+
+  defp read_versions!(repo_name, public_dir, public_key) do
+    path = Path.join(public_dir, "versions")
+    payload = read_file!(path)
+
+    case :mix_hex_registry.unpack_versions(payload, repo_name, public_key) do
+      {:ok, versions} ->
+        versions
+
+      _ ->
+        Mix.raise("""
+        Invalid package version manifest at #{path}
+
+        Is the repository name #{repo_name} correct?
+        """)
+    end
   end
 
+  defp read_package(repo_name, public_dir, public_key, package_name) do
+    path = Path.join([public_dir, "packages", package_name])
+
+    with {:ok, payload} <- read_file(path),
+         {:ok, package} <-
+           :mix_hex_registry.unpack_package(payload, repo_name, package_name, public_key) do
+      package
+    else
+      _ -> []
+    end
+  end
+
+  ## File utilities
+
   defp create_directory(path) do
     unless File.dir?(path) do
       Hex.Shell.info(["* creating ", path])
       File.mkdir_p!(path)
     end
   end
 
+  defp read_file!(path) do
+    if File.exists?(path) do
+      Hex.Shell.info(["* reading ", path])
+    else
+      Mix.raise("""
+      Error reading file #{path}
+
+      Using --incremental requires an existing registry
+      """)
+    end
+
+    File.read!(path)
+  end
+
+  defp read_file(path) do
+    if File.exists?(path) do
+      Hex.Shell.info(["* reading ", path])
+    else
+      Hex.Shell.info(["* skipping ", path])
+    end
+
+    File.read(path)
+  end
+
   defp write_file(path, data) do
     if File.exists?(path) do
       Hex.Shell.info(["* updating ", path])