Skip to content

Commit

Permalink
Prebuild URLs (#37)
Browse files Browse the repository at this point in the history
Closes #36.
  • Loading branch information
josevalim authored Oct 20, 2023
1 parent 6bd37be commit 53c2667
Show file tree
Hide file tree
Showing 2 changed files with 212 additions and 29 deletions.
143 changes: 114 additions & 29 deletions lib/adbc_driver.ex
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,121 @@ defmodule Adbc.Driver do
alias Adbc.Helper
require Logger

@official_drivers ~w(sqlite postgresql flightsql snowflake)a
@official_driver_base_url "https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-"
# == CONSTANTS ==

# Generated by update.exs at 2023-10-20T06:45:32. Do not change manually.
@version "0.7.0"
@driver_names [:sqlite, :postgresql, :flightsql, :snowflake]
@driver_data %{
flightsql: %{
"aarch64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_flightsql-0.7.0-py3-none-macosx_11_0_arm64.whl"
},
"aarch64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_flightsql-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
},
"x86_64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_flightsql-0.7.0-py3-none-macosx_10_9_x86_64.whl"
},
"x86_64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_flightsql-0.7.0-py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
},
"x86_64-windows-msvc" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_flightsql-0.7.0-py3-none-win_amd64.whl"
}
},
postgresql: %{
"aarch64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_postgresql-0.7.0-py3-none-macosx_11_0_arm64.whl"
},
"aarch64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_postgresql-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
},
"x86_64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_postgresql-0.7.0-py3-none-macosx_10_9_x86_64.whl"
},
"x86_64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_postgresql-0.7.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
},
"x86_64-windows-msvc" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_postgresql-0.7.0-py3-none-win_amd64.whl"
}
},
snowflake: %{
"aarch64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_snowflake-0.7.0-py3-none-macosx_11_0_arm64.whl"
},
"aarch64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_snowflake-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
},
"x86_64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_snowflake-0.7.0-py3-none-macosx_10_9_x86_64.whl"
},
"x86_64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_snowflake-0.7.0-py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
},
"x86_64-windows-msvc" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_snowflake-0.7.0-py3-none-win_amd64.whl"
}
},
sqlite: %{
"aarch64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_sqlite-0.7.0-py3-none-macosx_11_0_arm64.whl"
},
"aarch64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_sqlite-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
},
"x86_64-apple-darwin" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_sqlite-0.7.0-py3-none-macosx_10_9_x86_64.whl"
},
"x86_64-linux-gnu" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_sqlite-0.7.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
},
"x86_64-windows-msvc" => %{
url:
"https://github.com/apache/arrow-adbc/releases/download/apache-arrow-adbc-0.7.0/adbc_driver_sqlite-0.7.0-py3-none-win_amd64.whl"
}
}
}

# == CONSTANTS ==

def download(driver_name, opts \\ [])

def download(driver_name, opts) when driver_name in @official_drivers and is_list(opts) do
base_url = opts[:base_url] || @official_driver_base_url
def download(driver_name, opts) when driver_name in @driver_names and is_list(opts) do
url = opts[:url]
version = opts[:version] || @version
ignore_proxy = opts[:ignore_proxy] || false

with {:ok, triplet} <- current_triplet(),
:missing <- driver_status(driver_name, version, triplet),
{:ok, wheel} <- driver_wheel(driver_name, version, triplet),
url = base_url <> version <> "/" <> wheel,
{:ok, url} <- driver_wheel(url, driver_name, version, triplet),
{:ok, cache_path} <- cached_download(url, ignore_proxy, driver_name, version, triplet) do
extract!(cache_path, driver_name, version, triplet)
end
end

def download(driver_name, opts) when is_list(opts) do
known = Enum.map_join(@official_drivers, ", ", &inspect/1)
known = Enum.map_join(@driver_names, ", ", &inspect/1)
{:error, "unknown driver #{inspect(driver_name)}, expected one of #{known}"}
end

Expand Down Expand Up @@ -82,27 +175,19 @@ defmodule Adbc.Driver do
end
end

defp driver_wheel(driver_name, version, triplet) do
case triplet do
"aarch64-apple-darwin" ->
{:ok, "adbc_driver_#{driver_name}-#{version}-py3-none-macosx_11_0_arm64.whl"}

"x86_64-apple-darwin" ->
{:ok, "adbc_driver_#{driver_name}-#{version}-py3-none-macosx_10_9_x86_64.whl"}

"aarch64-linux-gnu" ->
{:ok,
"adbc_driver_#{driver_name}-#{version}-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"}

"x86_64-linux-gnu" ->
{:ok,
"adbc_driver_#{driver_name}-#{version}-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl"}
defp driver_wheel(url, _driver_name, _version, _triplet) when is_binary(url) do
{:ok, url}
end

"x86_64-windows-msvc" ->
{:ok, "adbc_driver_#{driver_name}-#{version}-py3-none-win_amd64.whl"}
defp driver_wheel(_url, _driver_name, version, triplet) when version != @version do
{:error,
"when passing a custom version for a driver, you must also specify the :url to find the precompiled version for `#{triplet}`"}
end

_ ->
{:error, "official driver does not have a precompiled version for `#{triplet}`"}
defp driver_wheel(_url, driver_name, _version, triplet) do
case @driver_data do
%{^driver_name => %{^triplet => %{url: url}}} -> {:ok, url}
%{} -> {:error, "official driver does not have a precompiled version for `#{triplet}`"}
end
end

Expand Down Expand Up @@ -159,7 +244,7 @@ defmodule Adbc.Driver do

def so_path(driver_name, opts \\ [])

def so_path(driver_name, opts) when driver_name in @official_drivers do
def so_path(driver_name, opts) when driver_name in @driver_names do
version = opts[:version] || @version

case current_triplet() do
Expand All @@ -183,7 +268,7 @@ defmodule Adbc.Driver do
end

def so_path(driver_name, _opts) do
known = Enum.map_join(@official_drivers, ", ", &inspect/1)
known = Enum.map_join(@driver_names, ", ", &inspect/1)

{:error,
"unknown driver #{inspect(driver_name)}, expected one of #{known} or a string representing the full path to a driver"}
Expand All @@ -202,4 +287,4 @@ defmodule Adbc.Driver do
_ -> Application.app_dir(:adbc, "priv/lib")
end
end
end
end
98 changes: 98 additions & 0 deletions update.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
# To support a new version of ADBC, you must:
#
# 1. Update the contents in 3rd_party with VERSION (only root files and c/)
# 2. Invoke `elixir update.exs VERSION`
# 3. ...
# 4. Profit!
#
Mix.install([{:req, "~> 0.4"}])

drivers = ~w(sqlite postgresql flightsql snowflake)a
file = "lib/adbc_driver.ex"

version =
case System.argv() do
[version] -> version
_ -> raise "expected VERSION as argument"
end

opts =
if token = System.get_env("GITHUB_TOKEN") do
[auth: {:bearer, token}]
else
[]
end

release =
Req.get!(
"https://api.github.com/repos/apache/arrow-adbc/releases/tags/apache-arrow-adbc-#{version}",
opts
)

if release.status != 200 do
raise "unknown GitHub release for version #{version}\n\n#{inspect(release)}"
end

assets = release.body["assets"]

mapping =
for driver <- drivers, into: %{} do
prefix = "adbc_driver_#{driver}-#{version}"
suffix = ".whl"

wheels =
Enum.filter(assets, fn %{"name" => name} ->
String.starts_with?(name, prefix) and String.ends_with?(name, suffix)
end)

data_for = fn wheels, parts ->
case Enum.split_with(wheels, fn %{"name" => name} -> Enum.all?(parts, &(name =~ &1)) end) do
{[%{"browser_download_url" => url}], rest} -> {%{url: url}, rest}
{[], _rest} -> raise "no wheel matching #{inspect(parts)}\n\n#{inspect(wheels)}"
{_, _rest} -> raise "many wheels matching #{inspect(parts)}\n\n#{inspect(wheels)}"
end
end

{aarch64_apple_darwin, wheels} = data_for.(wheels, ["macosx", "arm64"])
{x86_64_apple_darwin, wheels} = data_for.(wheels, ["macosx", "x86_64"])
{aarch64_linux_gnu, wheels} = data_for.(wheels, ["manylinux", "aarch64"])
{x86_64_linux_gnu, wheels} = data_for.(wheels, ["manylinux", "x86_64"])
{x86_64_windows_msvc, wheels} = data_for.(wheels, ["win_amd64"])

if wheels != [] do
IO.puts("The following wheels for #{driver} are not being used:\n\n#{inspect(wheels)}")
end

data =
%{
"aarch64-apple-darwin" => aarch64_apple_darwin,
"x86_64-apple-darwin" => x86_64_apple_darwin,
"aarch64-linux-gnu" => aarch64_linux_gnu,
"x86_64-linux-gnu" => x86_64_linux_gnu,
"x86_64-windows-msvc" => x86_64_windows_msvc
}

{driver, data}
end

case String.split(File.read!(file), "# == CONSTANTS ==") do
[pre, _mid, post] ->
time =
NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) |> NaiveDateTime.to_iso8601()

mid = """
# == CONSTANTS ==
# Generated by update.exs at #{time}. Do not change manually.
@version #{inspect(version)}
@driver_names #{inspect(drivers)}
@driver_data #{inspect(mapping)}
# == CONSTANTS ==
"""

File.write!(file, Code.format_string!(pre <> mid <> post) <> "\n")

_ ->
raise "could not find # == CONSTANTS == chunks in #{file}"
end

0 comments on commit 53c2667

Please sign in to comment.