Support local adaptors (#2834)

* Support local adaptors

* add tests

* fix failing worker test

* update changelog

* make credo happy

* add test for config

* fix versions not getting sent as @Local

* update instructions

* tweal readme

---------

Co-authored-by: Joe Clark <[email protected]>
Co-authored-by: Stuart Corbishley <[email protected]>

.env.example

@@ -280,3 +280,9 @@
 # exist, Lightning will attempt to fetch the file and write it to the same location.
 # For this reason, you have to make sure that the directory exists and it is writable
 # ADAPTORS_REGISTRY_JSON_PATH=/path/to/adaptor_registry_cache.json
+#
+# These 2 envs are used to enable local adaptors mode. OPENFN_ADAPTORS_REPO points
+# to the repo directory which must have a `packages` subdir. LOCAL_ADAPTORS env is
+# the flag used to enable/disable this mode
+# LOCAL_ADAPTORS=true
+# OPENFN_ADAPTORS_REPO=/path/to/repo/

CHANGELOG.md

@@ -17,6 +17,11 @@ and this project adheres to
 
 ### Added
 
+- Add support for local adaptors. This can be enabled via `LOCAL_ADAPTORS=true`
+  and path specified via `OPENFN_ADAPTORS_REPO=./path/to/repo/`
+  [#905](https://github.com/OpenFn/lightning/issues/905)
+- Add component injection for AI responses feedback
+  [#2495](https://github.com/OpenFn/lightning/issues/2495)
 - Audit the provisioning of projects via the API
   [#2718](https://github.com/OpenFn/lightning/issues/2718)
 

RUNNINGLOCAL.md

@@ -142,6 +142,41 @@ you.
 
 [Learn more about configuring workers](WORKERS.md)
 
+### Using Local Adaptors
+
+You can force lightning to use adaptor builds from your local
+[adaptors](https://github.com/openfn/adaptors) repo.
+
+Note that this is a global toggle: ALL runs will use local adaptor versions, and
+the adaptor picklist in the Workflow Editor will only suggest adaptors present
+in the monorepo.
+
+Remember to re-build your adaptors after making changes (use
+`pnpm build --watch` in the monorepo).
+
+To start, set up the following environment variables:
+
+- `LOCAL_ADAPTORS`: Used to enable or disable the local adaptors mode. Set it to
+  `true` to enable.
+- `OPENFN_ADAPTORS_REPO`: This should point to the adaptors monorepo. This is
+  the same variable used when you pass `-m` to the CLI.
+
+Example configuration:
+
+```sh
+export LOCAL_ADAPTORS=true
+export OPENFN_ADAPTORS_REPO=/path/to/repo/
+```
+
+You can also run the server directly in local mode with:
+
+```sh
+LOCAL_ADAPTORS=true mix phx.server
+```
+
+Ensure that the `OPENFN_ADAPTORS_REPO` directory is correctly set up with the
+necessary `packages` subdirectory, otherwise the app wont start
+
 ### Problems with Apple Silicon
 
 You might run into some errors when running the docker containers on Apple

assets/package.json

@@ -34,7 +34,7 @@
     "zustand": "^4.3.7"
   },
   "devDependencies": {
-    "@openfn/ws-worker": "^1.8.6",
+    "@openfn/ws-worker": "^1.9.1",
     "@types/marked": "^4.0.8",
     "@types/react": "^18.0.15",
     "@types/react-dom": "^18.0.6",

lib/lightning/adaptor_registry.ex

@@ -89,34 +89,31 @@ defmodule Lightning.AdaptorRegistry do
 
   @impl GenServer
   def handle_continue(opts, _state) do
-    cache_path =
-      case opts[:use_cache] do
-        true ->
-          Path.join([
-            System.tmp_dir!(),
-            "lightning",
-            "adaptor_registry_cache.json"
-          ])
-
-        path when is_binary(path) ->
-          path
-
-        _ ->
-          nil
+    adaptors =
+      case Enum.into(opts, %{}) do
+        %{local_adaptors_repo: repo_path} when is_binary(repo_path) ->
+          read_adaptors_from_local_repo(repo_path)
+
+        %{use_cache: use_cache}
+        when use_cache === true or is_binary(use_cache) ->
+          cache_path =
+            if is_binary(use_cache) do
+              use_cache
+            else
+              Path.join([
+                System.tmp_dir!(),
+                "lightning",
+                "adaptor_registry_cache.json"
+              ])
+            end
+
+          read_from_cache(cache_path) || write_to_cache(cache_path, fetch())
+
+        _other ->
+          fetch()
       end
 
-    if cache_path do
-      read_from_cache(cache_path)
-      |> case do
-        nil ->
-          {:noreply, write_to_cache(cache_path, fetch())}
-
-        adaptors ->
-          {:noreply, adaptors}
-      end
-    else
-      {:noreply, fetch()}
-    end
+    {:noreply, adaptors}
   end
 
   # false positive, it's a file from init
@@ -273,6 +270,22 @@ defmodule Lightning.AdaptorRegistry do
     }
   end
 
+  defp read_adaptors_from_local_repo(repo_path) do
+    Logger.debug("Using local adaptors repo at #{repo_path}")
+
+    repo_path
+    |> Path.join("packages")
+    |> File.ls!()
+    |> Enum.map(fn package ->
+      %{
+        name: "@openfn/language-" <> package,
+        repo: "file://" <> Path.join([repo_path, "packages", package]),
+        latest: "local",
+        versions: []
+      }
+    end)
+  end
+
   @doc """
   Destructures an NPM style package name into module name and version.
 
@@ -303,6 +316,17 @@ defmodule Lightning.AdaptorRegistry do
       _ ->
         {nil, nil}
     end
+    |> then(fn
+      {name, version} when is_binary(name) ->
+        if local_adaptors_enabled?() do
+          {name, "local"}
+        else
+          {name, version}
+        end
+
+      other ->
+        other
+    end)
   end
 
   @doc """
@@ -326,11 +350,20 @@ defmodule Lightning.AdaptorRegistry do
       {nil, nil} ->
         ""
 
+      {adaptor_name, "local"} ->
+        "#{adaptor_name}@local"
+
       {adaptor_name, "latest"} ->
         "#{adaptor_name}@#{latest_for(adaptor_name)}"
 
       _ ->
         adaptor
     end
   end
+
+  def local_adaptors_enabled? do
+    config = Lightning.Config.adaptor_registry()
+
+    if config[:local_adaptors_repo], do: true, else: false
+  end
 end

lib/lightning/config.ex

@@ -7,6 +7,11 @@ defmodule Lightning.Config do
     @behaviour Lightning.Config
     alias Lightning.Services.AdapterHelper
 
+    @impl true
+    def adaptor_registry do
+      Application.get_env(:lightning, Lightning.AdaptorRegistry, [])
+    end
+
     @impl true
     def token_signer do
       :persistent_term.get({__MODULE__, "token_signer"}, nil)
@@ -284,6 +289,14 @@ defmodule Lightning.Config do
   @callback usage_tracking_run_chunk_size() :: integer()
   @callback worker_secret() :: binary() | nil
   @callback worker_token_signer() :: Joken.Signer.t()
+  @callback adaptor_registry() :: Keyword.t()
+
+  @doc """
+  Returns the configuration for the `Lightning.AdaptorRegistry` service
+  """
+  def adaptor_registry do
+    impl().adaptor_registry()
+  end
 
   @doc """
   Returns the Apollo server configuration.

lib/lightning/config/bootstrap.ex

@@ -154,13 +154,36 @@ defmodule Lightning.Config.Bootstrap do
     config :lightning, :adaptor_service,
       adaptors_path: env!("ADAPTORS_PATH", :string, "./priv/openfn")
 
+    local_adaptors_repo =
+      env!(
+        "OPENFN_ADAPTORS_REPO",
+        :string,
+        Utils.get_env([
+          :lightning,
+          Lightning.AdaptorRegistry,
+          :local_adaptors_repo
+        ])
+      )
+
+    use_local_adaptors_repo? =
+      env!("LOCAL_ADAPTORS", &Utils.ensure_boolean/1, false)
+      |> tap(fn v ->
+        if v && !is_binary(local_adaptors_repo) do
+          raise """
+          LOCAL_ADAPTORS is set to true, but OPENFN_ADAPTORS_REPO is not set.
+          """
+        end
+      end)
+
     config :lightning, Lightning.AdaptorRegistry,
       use_cache:
         env!(
           "ADAPTORS_REGISTRY_JSON_PATH",
           :string,
           Utils.get_env([:lightning, Lightning.AdaptorRegistry, :use_cache])
-        )
+        ),
+      local_adaptors_repo:
+        use_local_adaptors_repo? && Path.expand(local_adaptors_repo)
 
     config :lightning, :oauth_clients,
       google: [