Skip to content

Commit

Permalink
Always write the metadata file in compilation (#37)
Browse files Browse the repository at this point in the history 
This makes the metadata file available for mix tasks, making the process
of releasing a new version easier.
So even if "force compilation" is used, we always have the metadata
file available.

To clarify: metadata is written when the Elixir file is compiled.

Closes #27
  • Loading branch information
@philss
philss authored Oct 19, 2022
1 parent 8ed0016 commit 56254db
Show file tree
Hide file tree
Showing 2 changed files with 145 additions and 77 deletions.
169 changes: 95 additions & 74 deletions lib/rustler_precompiled.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,26 +116,36 @@ defmodule RustlerPrecompiled do
|> Keyword.put_new(:module, module)
|> RustlerPrecompiled.Config.new()

if config.force_build? do
rustler_opts = Keyword.drop(opts, [:base_url, :version, :force_build, :targets])
case build_metadata(config) do
{:ok, metadata} ->
# We need to write metadata in order to run Mix tasks.
_ = write_metadata(module, metadata)

{:force_build, rustler_opts}
else
with {:error, precomp_error} <- RustlerPrecompiled.download_or_reuse_nif_file(config) do
message = """
Error while downloading precompiled NIF: #{precomp_error}.
if config.force_build? do
rustler_opts = Keyword.drop(opts, [:base_url, :version, :force_build, :targets])

{:force_build, rustler_opts}
else
with {:error, precomp_error} <-
RustlerPrecompiled.download_or_reuse_nif_file(config, metadata) do
message = """
Error while downloading precompiled NIF: #{precomp_error}.
You can force the project to build from scratch with:
You can force the project to build from scratch with:
config :rustler_precompiled, :force_build, #{config.otp_app}: true
config :rustler_precompiled, :force_build, #{config.otp_app}: true
In order to force the build, you also need to add Rustler as a dependency in your `mix.exs`:
In order to force the build, you also need to add Rustler as a dependency in your `mix.exs`:
{:rustler, ">= 0.0.0", optional: true}
"""
{:rustler, ">= 0.0.0", optional: true}
"""

{:error, message}
end
{:error, message}
end
end

{:error, _} = error ->
error
end
end

Expand Down Expand Up @@ -399,81 +409,92 @@ defmodule RustlerPrecompiled do
Enum.join(values, "-")
end

# Calculates metadata based in the TARGET and options
# from `config`.
@doc false
def build_metadata(%Config{} = config) do
with {:ok, target} <- target(target_config(), config.targets) do
basename = config.crate || config.otp_app
lib_name = "#{lib_prefix(target)}#{basename}-v#{config.version}-#{target}"

file_name = lib_name_with_ext(target, lib_name)

# `cache_base_dir` is a "private" option used only in tests.
cache_dir = cache_dir(config.base_cache_dir, "precompiled_nifs")
cached_tar_gz = Path.join(cache_dir, "#{file_name}.tar.gz")

base_url = config.base_url

{:ok,
%{
otp_app: config.otp_app,
crate: config.crate,
cached_tar_gz: cached_tar_gz,
base_url: base_url,
basename: basename,
lib_name: lib_name,
file_name: file_name,
target: target,
targets: config.targets,
version: config.version
}}
end
end

# Perform the download or load of the precompiled NIF
# It will look in the "priv/native/otp_app" first, and if
# that file doesn't exist, it will try to fetch from cache.
# In case there is no valid cached file, then it will try
# to download the NIF from the provided base URL.
#
# The `metadata` is a map built by `build_metadata/1` and
# has details about what is the current target and where
# to save the downloaded tar.gz.
@doc false
def download_or_reuse_nif_file(%Config{} = config) do
def download_or_reuse_nif_file(%Config{} = config, metadata) when is_map(metadata) do
name = config.otp_app
version = config.version

native_dir = Application.app_dir(name, @native_dir)

# NOTE: this `cache_base_dir` is a "private" option used only in tests.
cache_dir = cache_dir(config.base_cache_dir, "precompiled_nifs")
lib_name = Map.fetch!(metadata, :lib_name)
cached_tar_gz = Map.fetch!(metadata, :cached_tar_gz)
cache_dir = Path.dirname(cached_tar_gz)

with {:ok, target} <- target(target_config(), config.targets) do
basename = config.crate || name
lib_name = "#{lib_prefix(target)}#{basename}-v#{version}-#{target}"

file_name = lib_name_with_ext(target, lib_name)
cached_tar_gz = Path.join(cache_dir, "#{file_name}.tar.gz")
file_name = Map.fetch!(metadata, :file_name)
lib_file = Path.join(native_dir, file_name)

lib_file = Path.join(native_dir, file_name)
base_url = config.base_url
nif_module = config.module

base_url = config.base_url
nif_module = config.module

metadata = %{
otp_app: name,
crate: config.crate,
cached_tar_gz: cached_tar_gz,
base_url: base_url,
basename: basename,
lib_name: lib_name,
file_name: file_name,
target: target,
targets: config.targets,
version: version
}

write_metadata(nif_module, metadata)

result = %{
load?: true,
load_from: {name, Path.join("priv/native", lib_name)},
load_data: config.load_data
}

# TODO: add option to only write metadata
cond do
File.exists?(cached_tar_gz) ->
# Remove existing NIF file so we don't have processes using it.
# See: https://github.com/rusterlium/rustler/blob/46494d261cbedd3c798f584459e42ab7ee6ea1f4/rustler_mix/lib/rustler/compiler.ex#L134
File.rm(lib_file)

with :ok <- check_file_integrity(cached_tar_gz, nif_module),
:ok <- :erl_tar.extract(cached_tar_gz, [:compressed, cwd: Path.dirname(lib_file)]) do
Logger.debug("Copying NIF from cache and extracting to #{lib_file}")
{:ok, result}
end

true ->
dirname = Path.dirname(lib_file)
result = %{
load?: true,
load_from: {name, Path.join("priv/native", lib_name)},
load_data: config.load_data
}

with :ok <- File.mkdir_p(cache_dir),
:ok <- File.mkdir_p(dirname),
{:ok, tar_gz} <- download_tar_gz(base_url, lib_name, cached_tar_gz),
:ok <- File.write(cached_tar_gz, tar_gz),
:ok <- check_file_integrity(cached_tar_gz, nif_module),
:ok <-
:erl_tar.extract({:binary, tar_gz}, [:compressed, cwd: Path.dirname(lib_file)]) do
Logger.debug("NIF cached at #{cached_tar_gz} and extracted to #{lib_file}")
if File.exists?(cached_tar_gz) do
# Remove existing NIF file so we don't have processes using it.
# See: https://github.com/rusterlium/rustler/blob/46494d261cbedd3c798f584459e42ab7ee6ea1f4/rustler_mix/lib/rustler/compiler.ex#L134
File.rm(lib_file)

{:ok, result}
end
with :ok <- check_file_integrity(cached_tar_gz, nif_module),
:ok <- :erl_tar.extract(cached_tar_gz, [:compressed, cwd: Path.dirname(lib_file)]) do
Logger.debug("Copying NIF from cache and extracting to #{lib_file}")
{:ok, result}
end
else
dirname = Path.dirname(lib_file)

with :ok <- File.mkdir_p(cache_dir),
:ok <- File.mkdir_p(dirname),
{:ok, tar_gz} <- download_tar_gz(base_url, lib_name, cached_tar_gz),
:ok <- File.write(cached_tar_gz, tar_gz),
:ok <- check_file_integrity(cached_tar_gz, nif_module),
:ok <-
:erl_tar.extract({:binary, tar_gz}, [:compressed, cwd: Path.dirname(lib_file)]) do
Logger.debug("NIF cached at #{cached_tar_gz} and extracted to #{lib_file}")

{:ok, result}
end
end
end
Expand Down
53 changes: 50 additions & 3 deletions test/rustler_precompiled_test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,9 @@ defmodule RustlerPrecompiledTest do
targets: @available_targets
}

assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config)
{:ok, metadata} = RustlerPrecompiled.build_metadata(config)

assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config, metadata)

assert result.load?
assert {:rustler_precompiled, path} = result.load_from
Expand Down Expand Up @@ -317,7 +319,9 @@ defmodule RustlerPrecompiledTest do
targets: @available_targets
}

assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config)
{:ok, metadata} = RustlerPrecompiled.build_metadata(config)

assert {:ok, result} = RustlerPrecompiled.download_or_reuse_nif_file(config, metadata)

assert result.load?
assert {:rustler_precompiled, path} = result.load_from
Expand Down Expand Up @@ -358,7 +362,9 @@ defmodule RustlerPrecompiledTest do
targets: @available_targets
}

assert {:error, error} = RustlerPrecompiled.download_or_reuse_nif_file(config)
{:ok, metadata} = RustlerPrecompiled.build_metadata(config)

assert {:error, error} = RustlerPrecompiled.download_or_reuse_nif_file(config, metadata)

assert error =~
"the precompiled NIF file does not exist in the checksum file. " <>
Expand All @@ -369,6 +375,47 @@ defmodule RustlerPrecompiledTest do
end
end

describe "build_metadata/1" do
test "builds a valid metadata" do
config = %RustlerPrecompiled.Config{
otp_app: :rustler_precompiled,
module: RustlerPrecompilationExample.Native,
base_url:
"https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0",
version: "0.2.0",
crate: "example",
targets: @available_targets
}

assert {:ok, metadata} = RustlerPrecompiled.build_metadata(config)

assert metadata.otp_app == :rustler_precompiled
assert metadata.basename == "example"
assert metadata.crate == "example"

assert String.ends_with?(metadata.cached_tar_gz, "tar.gz")
assert [_ | _] = metadata.targets
assert metadata.version == "0.2.0"
assert metadata.base_url == config.base_url
end

test "returns error when current target is not available" do
config = %RustlerPrecompiled.Config{
otp_app: :rustler_precompiled,
module: RustlerPrecompilationExample.Native,
base_url:
"https://github.com/philss/rustler_precompilation_example/releases/download/v0.2.0",
version: "0.2.0",
crate: "example",
targets: ["hexagon-unknown-linux-musl"]
}

assert {:error, error} = RustlerPrecompiled.build_metadata(config)
assert error =~ "precompiled NIF is not available for this target: "
assert error =~ ".\nThe available targets are:\n - hexagon-unknown-linux-musl"
end
end

def in_tmp(tmp_path, function) do
path = Path.join([tmp_path, random_string(10)])

Expand Down

0 comments on commit 56254db

Please sign in to comment.