1254 remove run fks

lib/lightning/projects.ex

@@ -19,7 +19,7 @@ defmodule Lightning.Projects do
   alias Lightning.Accounts.User
   alias Lightning.ExportUtils
   alias Lightning.Workflows.Workflow
-  alias Lightning.Invocation.{Run, Dataclip}
+  alias Lightning.Invocation.{Run, Dataclip, LogLine}
   alias Lightning.WorkOrder
 
   require Logger
@@ -210,16 +210,20 @@ defmodule Lightning.Projects do
     end)
 
     Repo.transaction(fn ->
-      project_attempts_query(project) |> Repo.delete_all()
+      project_log_lines_query(project) |> Repo.delete_all()
 
       project_attempt_run_query(project) |> Repo.delete_all()
 
+      project_attempts_query(project) |> Repo.delete_all()
+
       project_workorders_query(project) |> Repo.delete_all()
 
       project_runs_query(project) |> Repo.delete_all()
 
       project_jobs_query(project) |> Repo.delete_all()
 
+      # No explicit test for triggers? Confirm when the noise from 
+      # attempts has been silenced
       project_triggers_query(project) |> Repo.delete_all()
 
       project_workflows_query(project) |> Repo.delete_all()
@@ -259,6 +263,15 @@ defmodule Lightning.Projects do
     )
   end
 
+  def project_log_lines_query(project) do
+    from(ll in LogLine,
+      join: att in assoc(ll, :attempt),
+      join: wo in assoc(att, :work_order),
+      join: w in assoc(wo, :workflow),
+      where: w.project_id == ^project.id
+    )
+  end
+
   def project_workorders_query(project) do
     from(wo in WorkOrder,
       join: w in assoc(wo, :workflow),

lib/lightning/setup_utils.ex

@@ -893,6 +893,7 @@ defmodule Lightning.SetupUtils do
       Lightning.Projects.ProjectCredential,
       Lightning.WorkOrder,
       Lightning.Invocation.Run,
+      Lightning.Invocation.LogLine,
       Lightning.Credentials.Credential,
       Lightning.Workflows.Job,
       Lightning.Workflows.Trigger,

priv/repo/migrations/20231127112907_remove_run_fk_on_attempt_runs.exs

@@ -0,0 +1,19 @@
+defmodule Lightning.Repo.Migrations.RemoveRunFkOnAttemptRuns do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    ALTER TABLE attempt_runs
+    DROP CONSTRAINT attempt_runs_run_id_fkey
+    """)
+  end
+
+  def down do
+    execute("""
+      ALTER TABLE attempt_runs
+      ADD CONSTRAINT attempt_runs_run_id_fkey
+      FOREIGN KEY (run_id)
+      REFERENCES runs(id) ON DELETE CASCADE
+    """)
+  end
+end

priv/repo/migrations/20231127112928_remove_run_fk_on_log_lines.exs

@@ -0,0 +1,19 @@
+defmodule Lightning.Repo.Migrations.RemoveRunFkOnLogLines do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    ALTER TABLE log_lines
+    DROP CONSTRAINT log_lines_run_id_fkey
+    """)
+  end
+
+  def down do
+    execute("""
+    ALTER TABLE log_lines
+    ADD CONSTRAINT log_lines_run_id_fkey
+    FOREIGN KEY (run_id)
+    REFERENCES runs(id) ON DELETE CASCADE
+    """)
+  end
+end

priv/repo/migrations/20231127112943_remove_run_fk_on_log_lines_monolith.exs

@@ -0,0 +1,19 @@
+defmodule Lightning.Repo.Migrations.RemoveRunFkOnLogLinesMonolith do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    ALTER TABLE log_lines_monolith
+    DROP CONSTRAINT log_lines_monolith_run_id_fkey
+    """)
+  end
+
+  def down do
+    execute("""
+    ALTER TABLE log_lines_monolith
+    ADD CONSTRAINT "log_lines_monolith_run_id_fkey"
+    FOREIGN KEY (run_id)
+    REFERENCES runs(id) ON DELETE CASCADE
+    """)
+  end
+end

priv/repo/migrations/20231127114133_remove_run_fk_on_runs.exs

@@ -0,0 +1,19 @@
+defmodule Lightning.Repo.Migrations.RemoveRunFkOnRuns do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    ALTER TABLE runs
+    DROP CONSTRAINT runs_previous_id_fkey
+    """)
+  end
+
+  def down do
+    execute("""
+    ALTER TABLE runs
+    ADD CONSTRAINT runs_previous_id_fkey
+    FOREIGN KEY (previous_id)
+    REFERENCES runs(id) ON DELETE CASCADE
+    """)
+  end
+end

priv/repo/migrations/20231127115754_shim_migration_not_for_merging.exs

@@ -0,0 +1,33 @@
+# NBNB For WIP branch only - this is just a shim branch until
+# the branch that removes attempts FKs is merged in.
+defmodule Lightning.Repo.Migrations.ShimMigrationNotForMerging do
+  use Ecto.Migration
+
+  def up do
+    execute("""
+    ALTER TABLE attempt_runs
+    DROP CONSTRAINT attempt_runs_attempt_id_fkey
+    """)
+    execute("""
+    ALTER TABLE log_lines
+    DROP CONSTRAINT log_lines_attempt_id_fkey
+    """)
+  end
+
+  def change do
+    execute("""
+    ALTER TABLE attempt_runs
+    ADD CONSTRAINT attempt_runs_attempt_id_fkey
+    FOREIGN KEY (attempt_id)
+    REFERENCES attempts(id) ON DELETE CASCADE
+    """)
+
+    execute("""
+    ALTER TABLE log_lines
+    ADD CONSTRAINT log_lines_attempt_id_fkey
+    FOREIGN KEY (attempt_id)
+    REFERENCES attempts(id)
+    ON DELETE CASCADE
+    """)
+  end
+end

test/lightning/projects_test.exs

@@ -261,13 +261,14 @@ defmodule Lightning.ProjectsTest do
         ]
       )
 
+
       p2_dataclip = insert(:dataclip, body: %{foo: "bar"}, project: p2)
 
       p2_run = insert(:run, input_dataclip: p2_dataclip, job: e2.target_job)
 
       p2_log_line = build(:log_line, run: p2_run)
 
-      insert(:workorder,
+      p2_workorder = insert(:workorder,
         trigger: t2,
         workflow: w2,
         dataclip: p2_dataclip,
@@ -280,6 +281,17 @@ defmodule Lightning.ProjectsTest do
           )
       )
 
+      {:ok, p2_pu} = p2.project_users |> Enum.fetch(0)
+
+      {:ok, p2_pc} = Repo.all(Ecto.assoc(p2, :project_credentials)) |> Enum.fetch(0)
+
+      p2_dataclip = Repo.get_by(
+        Lightning.Invocation.Dataclip,
+        project_id: p2.id
+      )
+
+      p2_attempt_run = Repo.get_by(Lightning.AttemptRun, run_id: p2_run.id)
+
       runs_query = Lightning.Projects.project_runs_query(p1)
 
       work_order_query = Lightning.Projects.project_workorders_query(p1)
@@ -319,24 +331,28 @@ defmodule Lightning.ProjectsTest do
 
       assert {:ok, %Project{}} = Projects.delete_project(p1)
 
-      assert runs_query |> Repo.aggregate(:count, :id) == 0
+      assert only_record_for_type?(p2_run)
+
+      assert only_record_for_type?(p2_workorder)
 
       assert work_order_query |> Repo.aggregate(:count, :id) == 0
 
       assert attempt_query |> Repo.aggregate(:count, :id) == 0
 
-      assert attempt_run_query |> Repo.aggregate(:count, :id) == 0
+      assert only_record_for_type?(p2_attempt_run)
 
-      assert pu_query |> Repo.aggregate(:count, :id) == 0
+      assert only_record_for_type?(p2_pu)
 
-      assert pc_query |> Repo.aggregate(:count, :id) == 0
+      assert only_record_for_type?(p2_pc)
 
       assert workflows_query |> Repo.aggregate(:count, :id) == 0
 
       assert jobs_query |> Repo.aggregate(:count, :id) == 0
 
       assert only_record_for_type?(p2_log_line)
 
+      assert only_record_for_type?(p2_dataclip)
+
       assert_raise Ecto.NoResultsError, fn ->
         Projects.get_project!(p1.id)
       end

test/support/model_helpers.ex

@@ -83,7 +83,7 @@ defmodule Lightning.ModelHelpers do
       where: r.id == ^expected_instance.id,
       left_join: others in ^model,
       on: others.id != ^expected_instance.id,
-      select: [count(r.id), count(others.id)]
+      select: [count(r.id, :distinct), count(others.id)]
     )
     |> Lightning.Repo.one!()
     |> case do