From bd858bdfbabee1ae4d29e3471b182187fe09ca03 Mon Sep 17 00:00:00 2001 From: Philip Sampaio Date: Wed, 28 Aug 2024 14:26:52 -0300 Subject: [PATCH] Revert "Add support for headers and dynamic base URL (#65) (#77)" This reverts commit ae78223763005efdf98c10b82d189c1dfd5d6497. This change is temporary, since we need to release a patch first. --- lib/rustler_precompiled.ex | 99 +++----- lib/rustler_precompiled/config.ex | 22 +- test/rustler_precompiled_test.exs | 368 ++++++++++-------------------- 3 files changed, 148 insertions(+), 341 deletions(-) diff --git a/lib/rustler_precompiled.ex b/lib/rustler_precompiled.ex index c781e04..7b3ea27 100644 --- a/lib/rustler_precompiled.ex +++ b/lib/rustler_precompiled.ex @@ -24,20 +24,7 @@ defmodule RustlerPrecompiled do * `:crate` - The name of Rust crate if different from the `:otp_app`. This is optional. - * `:base_url` - Location where to find the NIFs from. This should be one of the following: - - * A URL to a directory containing the NIFs. The name of the NIF will be appended to it - and a GET request will be made. Works well with public GitHub releases. - - * A tuple of `{URL, headers}`. The headers should be a list of key-value pairs. - This is useful when the NIFs are hosted in a private server. - - * A tuple of `{module, function}` where the `function` is an atom representing the function - name in that module. It's expected a function of arity 1, where the NIF file name is given, - and it should return a URL or a tuple of `{URL, headers}`. - This should be used for all cases not covered by the above. - For example when multiple requests have to be made, like when using a private GitHub release - through the GitHub API, or when the URLs don't resemble a simple directory. + * `:base_url` - A valid URL that is used as base path for the NIF file. * `:version` - The version of precompiled assets (it is part of the NIF filename). @@ -275,19 +262,13 @@ defmodule RustlerPrecompiled do @native_dir "priv/native" - @doc deprecated: "Use available_nifs/1 instead" - def available_nif_urls(nif_module) when is_atom(nif_module) do - available_nifs(nif_module) - |> Enum.map(fn {_lib_name, {url, _headers}} -> url end) - end - @doc """ - Returns URLs for NIFs based on its module name as a list of tuples: `[{lib_name, {url, headers}}]`. + Returns URLs for NIFs based on its module name. The module name is the one that defined the NIF and this information is stored in a metadata file. """ - def available_nifs(nif_module) when is_atom(nif_module) do + def available_nif_urls(nif_module) when is_atom(nif_module) do nif_module |> metadata_file() |> read_map_from_file() @@ -305,13 +286,6 @@ defmodule RustlerPrecompiled do @doc false def nif_urls_from_metadata(metadata) when is_map(metadata) do - with {:ok, nifs} <- nifs_from_metadata(metadata) do - {:ok, Enum.map(nifs, fn {_lib_name, {url, _headers}} -> url end)} - end - end - - @doc false - def nifs_from_metadata(metadata) when is_map(metadata) do case metadata do %{ targets: targets, @@ -346,27 +320,22 @@ defmodule RustlerPrecompiled do variants = Map.fetch!(variants, target_triple) for variant <- variants do - lib_name = lib_name_with_ext(target_triple, lib_name <> "--" <> Atom.to_string(variant)) - {lib_name, tar_gz_file_url(base_url, lib_name)} + tar_gz_file_url( + base_url, + lib_name_with_ext(target_triple, lib_name <> "--" <> Atom.to_string(variant)) + ) end end defp maybe_variants_tar_gz_urls(_, _, _, _), do: [] - @doc deprecated: "Use current_target_nifs/1 instead" - def current_target_nif_urls(nif_module) when is_atom(nif_module) do - nif_module - |> current_target_nifs() - |> Enum.map(fn {_lib_name, {url, _headers}} -> url end) - end - @doc """ - Returns the file URLs to be downloaded for current target as a list of tuples: `[{lib_name, {url, headers}}]`. + Returns the file URLs to be downloaded for current target. It is in the plural because a target may have some variants for it. It receives the NIF module. """ - def current_target_nifs(nif_module) when is_atom(nif_module) do + def current_target_nif_urls(nif_module) when is_atom(nif_module) do metadata = nif_module |> metadata_file() @@ -393,10 +362,9 @@ defmodule RustlerPrecompiled do defp tar_gz_urls(base_url, basename, version, nif_version, target_triple, variants) do lib_name = lib_name(basename, version, nif_version, target_triple) - lib_name_with_ext = lib_name_with_ext(target_triple, lib_name) [ - {lib_name_with_ext, tar_gz_file_url(base_url, lib_name_with_ext(target_triple, lib_name))} + tar_gz_file_url(base_url, lib_name_with_ext(target_triple, lib_name)) | maybe_variants_tar_gz_urls(variants, base_url, target_triple, lib_name) ] end @@ -648,7 +616,7 @@ defmodule RustlerPrecompiled do # `cache_base_dir` is a "private" option used only in tests. cache_dir = cache_dir(config.base_cache_dir, "precompiled_nifs") - cached_tar_gz = Path.join(cache_dir, file_name) + cached_tar_gz = Path.join(cache_dir, "#{file_name}.tar.gz") {:ok, Map.merge(basic_metadata, %{ @@ -873,34 +841,21 @@ defmodule RustlerPrecompiled do "so" end - "#{lib_name}.#{ext}.tar.gz" - end - - defp tar_gz_file_url({module, function_name}, file_name) - when is_atom(module) and is_atom(function_name) do - apply(module, function_name, [file_name]) + "#{lib_name}.#{ext}" end - defp tar_gz_file_url({base_url, request_headers}, file_name) do + defp tar_gz_file_url(base_url, file_name) do uri = URI.parse(base_url) uri = Map.update!(uri, :path, fn path -> - Path.join(path || "", file_name) + Path.join(path || "", "#{file_name}.tar.gz") end) - {to_string(uri), request_headers} - end - - defp tar_gz_file_url(base_url, file_name) do - tar_gz_file_url({base_url, []}, file_name) - end - - defp download_nif_artifact(url) when is_binary(url) do - download_nif_artifact({url, []}) + to_string(uri) end - defp download_nif_artifact({url, request_headers}) do + defp download_nif_artifact(url) do url = String.to_charlist(url) Logger.debug("Downloading NIF from #{url}") @@ -941,10 +896,7 @@ defmodule RustlerPrecompiled do options = [body_format: :binary] - request_headers = - Enum.map(request_headers, fn {k, v} when is_binary(k) -> {String.to_charlist(k), v} end) - - case :httpc.request(:get, {url, request_headers}, http_options, options) do + case :httpc.request(:get, {url, []}, http_options, options) do {:ok, {{_, 200, _}, _headers, body}} -> {:ok, body} @@ -961,17 +913,16 @@ defmodule RustlerPrecompiled do attempts = max_retries(options) download_results = - for {lib_name, url} <- urls, - do: {lib_name, with_retry(fn -> download_nif_artifact(url) end, attempts)} + for url <- urls, do: {url, with_retry(fn -> download_nif_artifact(url) end, attempts)} cache_dir = cache_dir("precompiled_nifs") :ok = File.mkdir_p(cache_dir) Enum.flat_map(download_results, fn result -> - with {:download, {lib_name, download_result}} <- {:download, result}, + with {:download, {url, download_result}} <- {:download, result}, {:download_result, {:ok, body}} <- {:download_result, download_result}, hash <- :crypto.hash(@checksum_algo, body), - path <- Path.join(cache_dir, lib_name), + path <- Path.join(cache_dir, basename_from_url(url)), {:file, :ok} <- {:file, File.write(path, body)} do checksum = Base.encode16(hash, case: :lower) @@ -981,7 +932,7 @@ defmodule RustlerPrecompiled do [ %{ - lib_name: lib_name, + url: url, path: path, checksum: checksum, checksum_algo: @checksum_algo @@ -1035,6 +986,14 @@ defmodule RustlerPrecompiled do end) end + defp basename_from_url(url) do + uri = URI.parse(url) + + uri.path + |> String.split("/") + |> List.last() + end + defp read_map_from_file(file) do with {:ok, contents} <- File.read(file), {%{} = contents, _} <- Code.eval_string(contents) do diff --git a/lib/rustler_precompiled/config.ex b/lib/rustler_precompiled/config.ex index ce608e2..3515c41 100644 --- a/lib/rustler_precompiled/config.ex +++ b/lib/rustler_precompiled/config.ex @@ -83,34 +83,16 @@ defmodule RustlerPrecompiled.Config do defp validate_base_url!(nil), do: raise_for_nil_field_value(:base_url) - defp validate_base_url!(base_url) when is_binary(base_url) do - validate_base_url!({base_url, []}) - end - - defp validate_base_url!({base_url, headers}) when is_binary(base_url) and is_list(headers) do + defp validate_base_url!(base_url) do case :uri_string.parse(base_url) do %{} -> - if Enum.all?(headers, &match?({key, value} when is_binary(key) and is_binary(value), &1)) do - {base_url, headers} - else - raise "`:base_url` headers for `RustlerPrecompiled` must be a list of `{binary(),binary()}`" - end + base_url {:error, :invalid_uri, error} -> raise "`:base_url` for `RustlerPrecompiled` is invalid: #{inspect(to_string(error))}" end end - defp validate_base_url!({module, function}) when is_atom(module) and is_atom(function) do - Code.ensure_compiled!(module) - - if Kernel.function_exported?(module, function, 1) do - {module, function} - else - raise "`:base_url` for `RustlerPrecompiled` is a function that does not exist: `#{inspect(module)}.#{function}/1`" - end - end - defp validate_list!(nil, option, _valid_values), do: raise_for_nil_field_value(option) defp validate_list!([_ | _] = values, option, valid_values) do diff --git a/test/rustler_precompiled_test.exs b/test/rustler_precompiled_test.exs index e627443..6e432ac 100644 --- a/test/rustler_precompiled_test.exs +++ b/test/rustler_precompiled_test.exs @@ -436,19 +436,17 @@ defmodule RustlerPrecompiledTest do result = capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: nif_fixtures_dir, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_cache_dir: nif_fixtures_dir, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: @available_targets, + nif_versions: @default_nif_versions + } {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -487,127 +485,16 @@ defmodule RustlerPrecompiledTest do result = capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: "http://localhost:#{bypass.port}/download", - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) - - {:ok, metadata} = RustlerPrecompiled.build_metadata(config) - - assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config, metadata) - - assert result.load? - assert {:rustler_precompiled, path} = result.load_from - - assert path =~ "priv/native" - assert path =~ "example-v0.2.0-nif" - end) - - assert result =~ "Downloading" - assert result =~ "http://localhost:#{bypass.port}/download" - assert result =~ "NIF cached at" - end) - end - - @tag :tmp_dir - test "a project downloading precompiled NIFs with custom header", %{ - tmp_dir: tmp_dir, - checksum_sample: checksum_sample, - nif_fixtures_dir: nif_fixtures_dir - } do - bypass = Bypass.open() - - in_tmp(tmp_dir, fn -> - File.write!("checksum-Elixir.RustlerPrecompilationExample.Native.exs", checksum_sample) - - Bypass.expect_once(bypass, fn conn -> - file_name = List.last(conn.path_info) - file = File.read!(Path.join([nif_fixtures_dir, "precompiled_nifs", file_name])) - - if Plug.Conn.get_req_header(conn, "authorization") == ["Token 123"] do - Plug.Conn.resp(conn, 200, file) - else - Plug.Conn.resp(conn, 401, "Unauthorized") - end - end) - - result = - capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: - {"http://localhost:#{bypass.port}/download", [{"authorization", "Token 123"}]}, - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) - - {:ok, metadata} = RustlerPrecompiled.build_metadata(config) - - assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config, metadata) - - assert result.load? - assert {:rustler_precompiled, path} = result.load_from - - assert path =~ "priv/native" - assert path =~ "example-v0.2.0-nif" - end) - - assert result =~ "Downloading" - assert result =~ "http://localhost:#{bypass.port}/download" - assert result =~ "NIF cached at" - end) - end - - @tag :tmp_dir - test "a project downloading precompiled NIFs with custom handler", %{ - tmp_dir: tmp_dir, - checksum_sample: checksum_sample, - nif_fixtures_dir: nif_fixtures_dir - } do - bypass = Bypass.open(port: 1234) - - in_tmp(tmp_dir, fn -> - File.write!("checksum-Elixir.RustlerPrecompilationExample.Native.exs", checksum_sample) - - Bypass.expect_once(bypass, fn conn -> - %{"file_name" => file_name} = URI.decode_query(conn.query_string) - file = File.read!(Path.join([nif_fixtures_dir, "precompiled_nifs", file_name])) - - if Plug.Conn.get_req_header(conn, "authorization") == ["Token 123"] do - Plug.Conn.resp(conn, 200, file) - else - Plug.Conn.resp(conn, 401, "Unauthorized") - end - end) - - result = - capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: {__MODULE__, :url_with_headers}, - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_cache_dir: tmp_dir, + base_url: "http://localhost:#{bypass.port}/download", + version: "0.2.0", + crate: "example", + targets: @available_targets, + nif_versions: @default_nif_versions + } {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -655,18 +542,16 @@ defmodule RustlerPrecompiledTest do result = capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: "http://localhost:#{bypass.port}/download", - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_cache_dir: tmp_dir, + base_url: "http://localhost:#{bypass.port}/download", + version: "0.2.0", + crate: "example", + targets: @available_targets, + nif_versions: @default_nif_versions + } {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -708,19 +593,17 @@ defmodule RustlerPrecompiledTest do end) capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: "http://localhost:#{bypass.port}/download", - version: "0.2.0", - crate: "example", - max_retries: 0, - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_cache_dir: tmp_dir, + base_url: "http://localhost:#{bypass.port}/download", + version: "0.2.0", + crate: "example", + max_retries: 0, + targets: @available_targets, + nif_versions: @default_nif_versions + } {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -749,18 +632,16 @@ defmodule RustlerPrecompiledTest do end) capture_log(fn -> - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_cache_dir: tmp_dir, - base_url: "http://localhost:#{bypass.port}/download", - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: @default_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_cache_dir: tmp_dir, + base_url: "http://localhost:#{bypass.port}/download", + version: "0.2.0", + crate: "example", + targets: @available_targets, + nif_versions: @default_nif_versions + } {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -777,19 +658,17 @@ defmodule RustlerPrecompiledTest do describe "build_metadata/1" do test "builds a valid metadata" do - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: @available_targets, - variants: %{}, - nif_versions: @available_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: @available_targets, + variants: %{}, + nif_versions: @available_nif_versions + } assert {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -806,18 +685,16 @@ defmodule RustlerPrecompiledTest do end test "returns error when current target is not available" do - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: ["hexagon-unknown-linux-musl"], - nif_versions: @available_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: ["hexagon-unknown-linux-musl"], + nif_versions: @available_nif_versions + } assert {:error, error} = RustlerPrecompiled.build_metadata(config) assert error =~ "precompiled NIF is not available for this target: " @@ -888,18 +765,16 @@ defmodule RustlerPrecompiledTest do end test "builds a valid metadata with a restrict NIF versions list" do - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: @available_targets, - nif_versions: ["2.15"], - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: @available_targets, + nif_versions: ["2.15"] + } assert {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -907,24 +782,22 @@ defmodule RustlerPrecompiledTest do end test "builds a valid metadata with specified variants" do - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: @available_targets, - variants: %{ - "x86_64-unknown-linux-gnu" => [ - old_glibc: fn _config -> true end, - legacy_cpus: fn _config -> true end - ] - }, - nif_versions: @available_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: @available_targets, + variants: %{ + "x86_64-unknown-linux-gnu" => [ + old_glibc: fn _config -> true end, + legacy_cpus: fn _config -> true end + ] + }, + nif_versions: @available_nif_versions + } assert {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -933,29 +806,27 @@ defmodule RustlerPrecompiledTest do # We need this guard because not every one is running the tests in the same OS/Arch. if metadata.lib_name =~ "x86_64-unknown-linux-gnu" do assert String.ends_with?(metadata.lib_name, "--old_glibc") - assert String.ends_with?(metadata.file_name, "--old_glibc.so.tar.gz") + assert String.ends_with?(metadata.file_name, "--old_glibc.so") end end test "builds a valid metadata saving the current variant as legacy CPU" do - config = - RustlerPrecompiled.Config.new( - otp_app: :rustler_precompiled, - module: RustlerPrecompilationExample.Native, - base_url: - "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", - version: "0.2.0", - crate: "example", - targets: @available_targets, - variants: %{ - "x86_64-unknown-linux-gnu" => [ - old_glibc: fn _config -> false end, - legacy_cpus: fn _config -> true end - ] - }, - nif_versions: @available_nif_versions, - force_build: false - ) + config = %RustlerPrecompiled.Config{ + otp_app: :rustler_precompiled, + module: RustlerPrecompilationExample.Native, + base_url: + "https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0", + version: "0.2.0", + crate: "example", + targets: @available_targets, + variants: %{ + "x86_64-unknown-linux-gnu" => [ + old_glibc: fn _config -> false end, + legacy_cpus: fn _config -> true end + ] + }, + nif_versions: @available_nif_versions + } assert {:ok, metadata} = RustlerPrecompiled.build_metadata(config) @@ -963,7 +834,7 @@ defmodule RustlerPrecompiledTest do if metadata.lib_name =~ "x86_64-unknown-linux-gnu" do assert String.ends_with?(metadata.lib_name, "--legacy_cpus") - assert String.ends_with?(metadata.file_name, "--legacy_cpus.so.tar.gz") + assert String.ends_with?(metadata.file_name, "--legacy_cpus.so") end end end @@ -1097,9 +968,4 @@ defmodule RustlerPrecompiledTest do |> Base.encode64() |> binary_part(0, len) end - - def url_with_headers(file_name) do - {"http://localhost:1234/download?file_name=#{file_name}&foo=bar", - [{"authorization", "Token 123"}]} - end end