support collections in provisioner

CHANGELOG.md

@@ -17,6 +17,9 @@ and this project adheres to
 
 ### Added
 
+- Extend provisioner to support collections
+  [#2830](https://github.com/OpenFn/lightning/issues/2830)
+
 ### Changed
 
 ### Fixed

lib/lightning/collections/collection.ex

@@ -18,6 +18,7 @@ defmodule Lightning.Collections.Collection do
   schema "collections" do
     field :name, :string
     field :byte_size_sum, :integer
+    field :delete, :boolean, virtual: true
     belongs_to :project, Lightning.Projects.Project
     has_many :items, Lightning.Collections.Item
 
@@ -29,6 +30,11 @@ defmodule Lightning.Collections.Collection do
     entry
     |> cast(attrs, [:project_id, :name])
     |> validate_required([:project_id, :name])
+    |> validate()
+  end
+
+  def validate(changeset) do
+    changeset
     |> validate_format(:name, ~r/^[a-z0-9]+([\-_.][a-z0-9]+)*$/,
       message: "Collection name must be URL safe"
     )

lib/lightning/export_utils.ex

@@ -16,7 +16,15 @@ defmodule Lightning.ExportUtils do
   ]
 
   @ordering_map %{
-    project: [:name, :description, :credentials, :globals, :workflows],
+    project: [
+      :name,
+      :description,
+      :collections,
+      :credentials,
+      :globals,
+      :workflows
+    ],
+    collection: [:name],
     credential: [:name, :owner],
     workflow: [:name, :jobs, :triggers, :edges],
     job: [:name, :adaptor, :credential, :globals, :body],
@@ -321,12 +329,22 @@ defmodule Lightning.ExportUtils do
         )
       end)
 
+    collections_map =
+      project.collections
+      |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
+      |> Enum.reduce(%{}, fn collection, acc ->
+        ytree = build_collection_yaml_tree(collection)
+
+        Map.put(acc, hyphenate(collection.name), ytree)
+      end)
+
     %{
       name: project.name,
       description: project.description,
       node_type: :project,
       workflows: workflows_map,
-      credentials: credentials_map
+      credentials: credentials_map,
+      collections: collections_map
     }
   end
 
@@ -344,6 +362,13 @@ defmodule Lightning.ExportUtils do
     }
   end
 
+  defp build_collection_yaml_tree(collection) do
+    %{
+      name: collection.name,
+      node_type: :collection
+    }
+  end
+
   defp build_workflow_yaml_tree(workflow, project_credentials) do
     jobs =
       workflow.jobs
@@ -374,7 +399,10 @@ defmodule Lightning.ExportUtils do
 
   def generate_new_yaml(project, nil) do
     project =
-      Lightning.Repo.preload(project, project_credentials: [credential: :user])
+      Lightning.Repo.preload(project,
+        project_credentials: [credential: :user],
+        collections: []
+      )
 
     yaml =
       project
@@ -387,7 +415,10 @@ defmodule Lightning.ExportUtils do
 
   def generate_new_yaml(project, snapshots) when is_list(snapshots) do
     project =
-      Lightning.Repo.preload(project, project_credentials: [credential: :user])
+      Lightning.Repo.preload(project,
+        project_credentials: [credential: :user],
+        collections: []
+      )
 
     yaml =
       snapshots

lib/lightning/projects/project.ex

@@ -40,6 +40,8 @@ defmodule Lightning.Projects.Project do
 
     has_many :project_credentials, ProjectCredential
     has_many :credentials, through: [:project_credentials, :credential]
+    has_many :collections, Lightning.Collections.Collection
+
     timestamps()
   end
 

lib/lightning/projects/provisioner.ex

@@ -13,6 +13,7 @@ defmodule Lightning.Projects.Provisioner do
 
   alias Ecto.Multi
   alias Lightning.Accounts.User
+  alias Lightning.Collections.Collection
   alias Lightning.Projects.Project
   alias Lightning.Projects.ProjectCredential
   alias Lightning.Projects.ProjectUser
@@ -180,6 +181,7 @@ defmodule Lightning.Projects.Provisioner do
   def parse_document(%Project{} = project, data) when is_map(data) do
     project
     |> project_changeset(data)
+    |> cast_assoc(:collections, with: &collection_changeset/2)
     |> cast_assoc(:workflows, with: &workflow_changeset/2)
     |> then(fn changeset ->
       case WorkflowUsageLimiter.limit_workflows_activation(
@@ -234,6 +236,7 @@ defmodule Lightning.Projects.Provisioner do
       project,
       [
         :project_users,
+        :collections,
         project_credentials: [credential: [:user]],
         workflows: {w, [:jobs, :triggers, :edges]}
       ],
@@ -255,6 +258,15 @@ defmodule Lightning.Projects.Provisioner do
     |> Project.validate()
   end
 
+  defp collection_changeset(collection, attrs) do
+    collection
+    |> cast(attrs, [:id, :name, :delete])
+    |> validate_required([:id])
+    |> maybe_mark_for_deletion()
+    |> validate_extraneous_params()
+    |> Collection.validate()
+  end
+
   defp workflow_changeset(workflow, attrs) do
     workflow
     |> cast(attrs, [:id, :name, :delete])

lib/lightning_web/controllers/api/provisioning_json.ex

@@ -4,6 +4,7 @@ defmodule LightningWeb.API.ProvisioningJSON do
   import LightningWeb.CoreComponents, only: [translate_error: 1]
   import Ecto.Changeset
 
+  alias Lightning.Collections.Collection
   alias Lightning.Projects.Project
   alias Lightning.Projects.ProjectCredential
   alias Lightning.Workflows.Edge
@@ -34,6 +35,12 @@ defmodule LightningWeb.API.ProvisioningJSON do
       |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
       |> Enum.map(&as_json/1)
     )
+    |> Map.put(
+      :collections,
+      project.collections
+      |> Enum.sort_by(& &1.inserted_at, NaiveDateTime)
+      |> Enum.map(&as_json/1)
+    )
   end
 
   def as_json(%module{} = workflow_or_snapshot)
@@ -109,6 +116,10 @@ defmodule LightningWeb.API.ProvisioningJSON do
     }
   end
 
+  def as_json(%Collection{} = collection) do
+    %{id: collection.id, name: collection.name}
+  end
+
   defp drop_keys_with_nil_value(map) do
     Map.reject(map, fn {_, v} -> is_nil(v) end)
   end

lib/mix/tasks/install_runtime.ex

@@ -43,7 +43,7 @@ defmodule Mix.Tasks.Lightning.InstallRuntime do
 
   def packages do
     ~W(
-      @openfn/[email protected].2
+      @openfn/[email protected].3
       @openfn/language-common@latest
     )
   end

test/fixtures/canonical_project.yaml

@@ -1,6 +1,9 @@
 name: a-test-project
 description: |
   This is only a test
+collections:
+  cannonical-collection:
+    name: cannonical-collection
 credentials:
   [email protected]:
     name: new credential

test/integration/cli_deploy_test.exs

@@ -286,7 +286,20 @@ defmodule Lightning.CliDeployTest do
          expected_project_credential_state(pc)}
       end)
 
-    Map.merge(state, %{workflows: workflows, project_credentials: credentials})
+    collections =
+      Map.new(project.collections, fn collection ->
+        {hyphenize(collection.name), expected_collection_state(collection)}
+      end)
+
+    Map.merge(state, %{
+      workflows: workflows,
+      project_credentials: credentials,
+      collections: collections
+    })
+  end
+
+  defp expected_collection_state(collection) do
+    Map.take(collection, [:id, :name])
   end
 
   defp expected_project_credential_state(project_credential) do

test/lightning/projects/provisioner_test.exs

@@ -319,6 +319,42 @@ defmodule Lightning.Projects.ProvisionerTest do
 
       assert workflow_ids == expected_workflow_ids
     end
+
+    test "creates collections" do
+      Mox.verify_on_exit!()
+      user = insert(:user)
+
+      %{body: body, project_id: project_id} = valid_document()
+
+      collection_id = Ecto.UUID.generate()
+      collection_name = "test-collection"
+
+      body_with_collections =
+        Map.put(body, "collections", [
+          %{id: collection_id, name: collection_name}
+        ])
+
+      Mox.stub(
+        Lightning.Extensions.MockUsageLimiter,
+        :limit_action,
+        fn _action, _context -> :ok end
+      )
+
+      {:ok, project} =
+        Provisioner.import_document(
+          %Lightning.Projects.Project{},
+          user,
+          body_with_collections
+        )
+
+      assert %{id: ^project_id, collections: [collection]} = project
+
+      assert %{
+               id: ^collection_id,
+               name: ^collection_name,
+               project_id: ^project_id
+             } = collection
+    end
   end
 
   describe "import_document/2 with an existing project" do
@@ -872,6 +908,108 @@ defmodule Lightning.Projects.ProvisionerTest do
 
       assert Repo.all(Audit) == []
     end
+
+    test "creating collection", %{
+      project: %{id: project_id} = project,
+      user: user
+    } do
+      Mox.stub(
+        Lightning.Extensions.MockUsageLimiter,
+        :limit_action,
+        fn _action, %{project_id: ^project_id} ->
+          :ok
+        end
+      )
+
+      %{body: body} = valid_document(project.id)
+
+      {:ok, project} = Provisioner.import_document(project, user, body)
+      collection_id = Ecto.UUID.generate()
+      collection_name = "test-collection"
+
+      body_with_collections =
+        Map.put(body, "collections", [
+          %{id: collection_id, name: collection_name}
+        ])
+
+      {:ok, project} =
+        Provisioner.import_document(project, user, body_with_collections)
+
+      assert %{id: ^project_id, collections: [collection]} = project
+
+      assert %{
+               id: ^collection_id,
+               name: ^collection_name,
+               project_id: ^project_id
+             } = collection
+    end
+
+    test "updating a collection", %{
+      project: %{id: project_id} = project,
+      user: user
+    } do
+      Mox.stub(
+        Lightning.Extensions.MockUsageLimiter,
+        :limit_action,
+        fn _action, %{project_id: ^project_id} ->
+          :ok
+        end
+      )
+
+      collection = insert(:collection, project: project)
+      collection_id = collection.id
+      new_collection_name = "new-collection-name"
+
+      assert collection.name != new_collection_name
+
+      body = %{
+        "id" => project_id,
+        "name" => "test-project",
+        "collections" => [
+          %{"id" => collection_id, "name" => new_collection_name}
+        ]
+      }
+
+      assert {:ok, %{id: ^project_id, collections: [collection]}} =
+               Provisioner.import_document(project, user, body)
+
+      assert %{
+               id: ^collection_id,
+               name: ^new_collection_name,
+               project_id: ^project_id
+             } = collection
+    end
+
+    test "deleting a collection", %{
+      project: %{id: project_id} = project,
+      user: user
+    } do
+      Mox.stub(
+        Lightning.Extensions.MockUsageLimiter,
+        :limit_action,
+        fn _action, %{project_id: ^project_id} ->
+          :ok
+        end
+      )
+
+      collection = insert(:collection, project: project)
+      collection_to_delete = insert(:collection, project: project)
+
+      body = %{
+        "id" => project_id,
+        "name" => "test-project",
+        "collections" => [
+          %{"id" => collection.id},
+          %{"id" => collection_to_delete.id, "delete" => true}
+        ]
+      }
+
+      assert {:ok, %{id: ^project_id, collections: [remaining_collection]}} =
+               Provisioner.import_document(project, user, body)
+
+      assert Repo.reload(collection_to_delete) |> is_nil()
+      assert remaining_collection.id == collection.id
+    end
   end
 
   defp valid_document(project_id \\ nil, number_of_workflows \\ 1) do