diff --git a/.env.example b/.env.example index 7ca2b3c96a..777160890d 100644 --- a/.env.example +++ b/.env.example @@ -280,3 +280,9 @@ # exist, Lightning will attempt to fetch the file and write it to the same location. # For this reason, you have to make sure that the directory exists and it is writable # ADAPTORS_REGISTRY_JSON_PATH=/path/to/adaptor_registry_cache.json +# +# These 2 envs are used to enable local adaptors mode. OPENFN_ADAPTORS_REPO points +# to the repo directory which must have a `packages` subdir. LOCAL_ADAPTORS env is +# the flag used to enable/disable this mode +# LOCAL_ADAPTORS=true +# OPENFN_ADAPTORS_REPO=/path/to/repo/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a13f9c0e6d..3991221c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ and this project adheres to ### Added +- Add support for local adaptors. This can be enabled via `LOCAL_ADAPTORS=true` + and path specified via `OPENFN_ADAPTORS_REPO=./path/to/repo/` + [#905](https://github.com/OpenFn/lightning/issues/905) +- Add component injection for AI responses feedback + [#2495](https://github.com/OpenFn/lightning/issues/2495) - Audit the provisioning of projects via the API [#2718](https://github.com/OpenFn/lightning/issues/2718) diff --git a/RUNNINGLOCAL.md b/RUNNINGLOCAL.md index 37fd9335de..bf914c1b7a 100644 --- a/RUNNINGLOCAL.md +++ b/RUNNINGLOCAL.md @@ -142,6 +142,41 @@ you. [Learn more about configuring workers](WORKERS.md) +### Using Local Adaptors + +You can force lightning to use adaptor builds from your local +[adaptors](https://github.com/openfn/adaptors) repo. + +Note that this is a global toggle: ALL runs will use local adaptor versions, and +the adaptor picklist in the Workflow Editor will only suggest adaptors present +in the monorepo. + +Remember to re-build your adaptors after making changes (use +`pnpm build --watch` in the monorepo). + +To start, set up the following environment variables: + +- `LOCAL_ADAPTORS`: Used to enable or disable the local adaptors mode. Set it to + `true` to enable. +- `OPENFN_ADAPTORS_REPO`: This should point to the adaptors monorepo. This is + the same variable used when you pass `-m` to the CLI. + +Example configuration: + +```sh +export LOCAL_ADAPTORS=true +export OPENFN_ADAPTORS_REPO=/path/to/repo/ +``` + +You can also run the server directly in local mode with: + +```sh +LOCAL_ADAPTORS=true mix phx.server +``` + +Ensure that the `OPENFN_ADAPTORS_REPO` directory is correctly set up with the +necessary `packages` subdirectory, otherwise the app wont start + ### Problems with Apple Silicon You might run into some errors when running the docker containers on Apple diff --git a/assets/package-lock.json b/assets/package-lock.json index cb2d2a4ca0..1dade308ef 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -32,7 +32,7 @@ "zustand": "^4.3.7" }, "devDependencies": { - "@openfn/ws-worker": "^1.8.6", + "@openfn/ws-worker": "^1.9.1", "@types/marked": "^4.0.8", "@types/react": "^18.0.15", "@types/react-dom": "^18.0.6", @@ -566,13 +566,14 @@ } }, "node_modules/@openfn/compiler": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-0.4.2.tgz", - "integrity": "sha512-S0Ojh5VMe+r27Y/GdThZ1czVursj+niFiIq2sEY0RXYwuONjedg6CQ78CkNifjxVeoH/Zo7+P20fmVfymOuMfw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@openfn/compiler/-/compiler-1.0.0.tgz", + "integrity": "sha512-ljfotbzGl8f6QcXd6ILk+QGz5xCwKVx/OBoQuI9BYPMxc2YK/Vc8JykksprOEiHAA7pV7UzQs9fS1OngH/eehw==", "dev": true, "dependencies": { - "@openfn/describe-package": "0.1.3", - "@openfn/logger": "1.0.2", + "@openfn/describe-package": "0.1.4", + "@openfn/lexicon": "^1.1.0", + "@openfn/logger": "1.0.4", "acorn": "^8.8.0", "ast-types": "^0.14.2", "recast": "^0.21.5" @@ -583,9 +584,9 @@ } }, "node_modules/@openfn/describe-package": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@openfn/describe-package/-/describe-package-0.1.3.tgz", - "integrity": "sha512-/nXTBoxERM3tCIaG1myq+iFE49z49XjVHjtDt9GrAyAnzSJaLtJuLXHe61dHAItHTXvHkgT4gEgsif+CmK18lg==", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@openfn/describe-package/-/describe-package-0.1.4.tgz", + "integrity": "sha512-QYY/PSlwoRc2gUHeHP5tbRnckpDjflIuY8j4HCb0tXtAmSy6flWaQ8W1tR5zz+rbeeBcYOahmkkEOmNSNyahSA==", "dependencies": { "@typescript/vfs": "^1.3.5", "cross-fetch": "^3.1.5", @@ -598,16 +599,16 @@ } }, "node_modules/@openfn/engine-multi": { - "version": "1.4.5", - "resolved": "https://registry.npmjs.org/@openfn/engine-multi/-/engine-multi-1.4.5.tgz", - "integrity": "sha512-JKD3v6TdZn8WJPGdZLInsI4UedsS7PvrjnjycBLKY4Jqb2p08lzpm+h0yIq2KbT1D25BPu4YNeJP4Bqi/P/b0Q==", + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/@openfn/engine-multi/-/engine-multi-1.4.8.tgz", + "integrity": "sha512-u6G3lXkE5INad7hVnjD+yCPjrPNG+SDhAN9QBxSpIZFpe5XNO9PkNmmE5DTDmhO2KkapStUiEsvYa8/v6VhZMg==", "dev": true, "dependencies": { - "@openfn/compiler": "0.4.2", + "@openfn/compiler": "1.0.0", "@openfn/language-common": "2.0.0-rc3", "@openfn/lexicon": "^1.1.0", - "@openfn/logger": "1.0.2", - "@openfn/runtime": "1.5.3", + "@openfn/logger": "1.0.4", + "@openfn/runtime": "1.6.1", "fast-safe-stringify": "^2.1.1" } }, @@ -624,9 +625,9 @@ "dev": true }, "node_modules/@openfn/logger": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@openfn/logger/-/logger-1.0.2.tgz", - "integrity": "sha512-zFRfCqAUZ35d5lujgl2s8MOSAQUmZ9NKvj/JsHvf4eA6fIfez5fxRQbgxPEqgf4IguYUH3s2fmvVyxI/AfZtKA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@openfn/logger/-/logger-1.0.4.tgz", + "integrity": "sha512-XJ11PPHHHIDpIpP0XSQkonnbvU6EzQiDWTkVHdM8XgkNLnfb74wSkyn0c1Jjgtdr1zC1689xMCQe6kbpFl9iGQ==", "dev": true, "dependencies": { "@inquirer/confirm": "2.0.6", @@ -639,27 +640,37 @@ } }, "node_modules/@openfn/runtime": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.5.3.tgz", - "integrity": "sha512-UDeMPDcylLvsrg/q9fLot0VSpSbrnnYWM+hJ8GrNQw531x04LtGB/6t6b7Pj1kRODcGgPpBFDKygDDEadyTJmg==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@openfn/runtime/-/runtime-1.6.1.tgz", + "integrity": "sha512-dzKDaz/B71ZmeML4KUhzsxgRPCiXprvhqPeH7k+VQO2iZm+zLPXgV9Sdt+dm/v2KZX3Alhrx2yx+n1aAe5d4Ww==", "dev": true, "dependencies": { - "@openfn/logger": "1.0.2", + "@openfn/logger": "1.0.4", "fast-safe-stringify": "^2.1.1", - "semver": "^7.5.4" + "semver": "^7.5.4", + "source-map": "^0.7.4" + } + }, + "node_modules/@openfn/runtime/node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "dev": true, + "engines": { + "node": ">= 8" } }, "node_modules/@openfn/ws-worker": { - "version": "1.8.6", - "resolved": "https://registry.npmjs.org/@openfn/ws-worker/-/ws-worker-1.8.6.tgz", - "integrity": "sha512-KjVoj6+fWqbVN/tibRA76uWkSABWu1peGqZiDjD8JN8ATeqsxHez7PxJYo+vr8mbB9iMkpy7vpJJg5aQQFmhCw==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@openfn/ws-worker/-/ws-worker-1.9.1.tgz", + "integrity": "sha512-9/H3YMBxSUHliJJ8d8EXpxYwUANQeQjf9g0xLaOQQkVq1tyJyDP55g3+6NnrRa78hxuGkR5QMK4be15TBrj7dw==", "dev": true, "dependencies": { "@koa/router": "^12.0.0", - "@openfn/engine-multi": "1.4.5", + "@openfn/engine-multi": "1.4.8", "@openfn/lexicon": "^1.1.0", - "@openfn/logger": "1.0.2", - "@openfn/runtime": "1.5.3", + "@openfn/logger": "1.0.4", + "@openfn/runtime": "1.6.1", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", diff --git a/assets/package.json b/assets/package.json index db295db449..baa706acdf 100644 --- a/assets/package.json +++ b/assets/package.json @@ -34,7 +34,7 @@ "zustand": "^4.3.7" }, "devDependencies": { - "@openfn/ws-worker": "^1.8.6", + "@openfn/ws-worker": "^1.9.1", "@types/marked": "^4.0.8", "@types/react": "^18.0.15", "@types/react-dom": "^18.0.6", diff --git a/lib/lightning/adaptor_registry.ex b/lib/lightning/adaptor_registry.ex index ec9fe3f48e..ba1fa6ac20 100644 --- a/lib/lightning/adaptor_registry.ex +++ b/lib/lightning/adaptor_registry.ex @@ -89,34 +89,31 @@ defmodule Lightning.AdaptorRegistry do @impl GenServer def handle_continue(opts, _state) do - cache_path = - case opts[:use_cache] do - true -> - Path.join([ - System.tmp_dir!(), - "lightning", - "adaptor_registry_cache.json" - ]) - - path when is_binary(path) -> - path - - _ -> - nil + adaptors = + case Enum.into(opts, %{}) do + %{local_adaptors_repo: repo_path} when is_binary(repo_path) -> + read_adaptors_from_local_repo(repo_path) + + %{use_cache: use_cache} + when use_cache === true or is_binary(use_cache) -> + cache_path = + if is_binary(use_cache) do + use_cache + else + Path.join([ + System.tmp_dir!(), + "lightning", + "adaptor_registry_cache.json" + ]) + end + + read_from_cache(cache_path) || write_to_cache(cache_path, fetch()) + + _other -> + fetch() end - if cache_path do - read_from_cache(cache_path) - |> case do - nil -> - {:noreply, write_to_cache(cache_path, fetch())} - - adaptors -> - {:noreply, adaptors} - end - else - {:noreply, fetch()} - end + {:noreply, adaptors} end # false positive, it's a file from init @@ -273,6 +270,22 @@ defmodule Lightning.AdaptorRegistry do } end + defp read_adaptors_from_local_repo(repo_path) do + Logger.debug("Using local adaptors repo at #{repo_path}") + + repo_path + |> Path.join("packages") + |> File.ls!() + |> Enum.map(fn package -> + %{ + name: "@openfn/language-" <> package, + repo: "file://" <> Path.join([repo_path, "packages", package]), + latest: "local", + versions: [] + } + end) + end + @doc """ Destructures an NPM style package name into module name and version. @@ -303,6 +316,17 @@ defmodule Lightning.AdaptorRegistry do _ -> {nil, nil} end + |> then(fn + {name, version} when is_binary(name) -> + if local_adaptors_enabled?() do + {name, "local"} + else + {name, version} + end + + other -> + other + end) end @doc """ @@ -326,6 +350,9 @@ defmodule Lightning.AdaptorRegistry do {nil, nil} -> "" + {adaptor_name, "local"} -> + "#{adaptor_name}@local" + {adaptor_name, "latest"} -> "#{adaptor_name}@#{latest_for(adaptor_name)}" @@ -333,4 +360,10 @@ defmodule Lightning.AdaptorRegistry do adaptor end end + + def local_adaptors_enabled? do + config = Lightning.Config.adaptor_registry() + + if config[:local_adaptors_repo], do: true, else: false + end end diff --git a/lib/lightning/config.ex b/lib/lightning/config.ex index 888155f416..e551806147 100644 --- a/lib/lightning/config.ex +++ b/lib/lightning/config.ex @@ -7,6 +7,11 @@ defmodule Lightning.Config do @behaviour Lightning.Config alias Lightning.Services.AdapterHelper + @impl true + def adaptor_registry do + Application.get_env(:lightning, Lightning.AdaptorRegistry, []) + end + @impl true def token_signer do :persistent_term.get({__MODULE__, "token_signer"}, nil) @@ -284,6 +289,14 @@ defmodule Lightning.Config do @callback usage_tracking_run_chunk_size() :: integer() @callback worker_secret() :: binary() | nil @callback worker_token_signer() :: Joken.Signer.t() + @callback adaptor_registry() :: Keyword.t() + + @doc """ + Returns the configuration for the `Lightning.AdaptorRegistry` service + """ + def adaptor_registry do + impl().adaptor_registry() + end @doc """ Returns the Apollo server configuration. diff --git a/lib/lightning/config/bootstrap.ex b/lib/lightning/config/bootstrap.ex index b2b7b1f400..029a9e8baa 100644 --- a/lib/lightning/config/bootstrap.ex +++ b/lib/lightning/config/bootstrap.ex @@ -154,13 +154,36 @@ defmodule Lightning.Config.Bootstrap do config :lightning, :adaptor_service, adaptors_path: env!("ADAPTORS_PATH", :string, "./priv/openfn") + local_adaptors_repo = + env!( + "OPENFN_ADAPTORS_REPO", + :string, + Utils.get_env([ + :lightning, + Lightning.AdaptorRegistry, + :local_adaptors_repo + ]) + ) + + use_local_adaptors_repo? = + env!("LOCAL_ADAPTORS", &Utils.ensure_boolean/1, false) + |> tap(fn v -> + if v && !is_binary(local_adaptors_repo) do + raise """ + LOCAL_ADAPTORS is set to true, but OPENFN_ADAPTORS_REPO is not set. + """ + end + end) + config :lightning, Lightning.AdaptorRegistry, use_cache: env!( "ADAPTORS_REGISTRY_JSON_PATH", :string, Utils.get_env([:lightning, Lightning.AdaptorRegistry, :use_cache]) - ) + ), + local_adaptors_repo: + use_local_adaptors_repo? && Path.expand(local_adaptors_repo) config :lightning, :oauth_clients, google: [ diff --git a/lib/lightning_web/live/job_live/adaptor_picker.ex b/lib/lightning_web/live/job_live/adaptor_picker.ex index 5c4d665dd9..acd4ac3714 100644 --- a/lib/lightning_web/live/job_live/adaptor_picker.ex +++ b/lib/lightning_web/live/job_live/adaptor_picker.ex @@ -19,7 +19,7 @@ defmodule LightningWeb.JobLive.AdaptorPicker do
+ (<%= @adaptor_name |> display_name_for_adaptor() |> elem(0) %>)
+
+ is not available <%= if @local_adaptors_enabled?,
+ do: "locally",
+ else: "in NPM" %>
+
+