diff --git a/lib/lightning/projects.ex b/lib/lightning/projects.ex index 1d747443be..abd06a1a95 100644 --- a/lib/lightning/projects.ex +++ b/lib/lightning/projects.ex @@ -19,7 +19,7 @@ defmodule Lightning.Projects do alias Lightning.Accounts.User alias Lightning.ExportUtils alias Lightning.Workflows.Workflow - alias Lightning.Invocation.{Run, Dataclip} + alias Lightning.Invocation.{Run, Dataclip, LogLine} alias Lightning.WorkOrder require Logger @@ -210,16 +210,20 @@ defmodule Lightning.Projects do end) Repo.transaction(fn -> - project_attempts_query(project) |> Repo.delete_all() + project_log_lines_query(project) |> Repo.delete_all() project_attempt_run_query(project) |> Repo.delete_all() + project_attempts_query(project) |> Repo.delete_all() + project_workorders_query(project) |> Repo.delete_all() project_runs_query(project) |> Repo.delete_all() project_jobs_query(project) |> Repo.delete_all() + # No explicit test for triggers? Confirm when the noise from + # attempts has been silenced project_triggers_query(project) |> Repo.delete_all() project_workflows_query(project) |> Repo.delete_all() @@ -259,6 +263,15 @@ defmodule Lightning.Projects do ) end + def project_log_lines_query(project) do + from(ll in LogLine, + join: att in assoc(ll, :attempt), + join: wo in assoc(att, :work_order), + join: w in assoc(wo, :workflow), + where: w.project_id == ^project.id + ) + end + def project_workorders_query(project) do from(wo in WorkOrder, join: w in assoc(wo, :workflow), diff --git a/lib/lightning/setup_utils.ex b/lib/lightning/setup_utils.ex index f85448ef6b..d84ac3f81f 100644 --- a/lib/lightning/setup_utils.ex +++ b/lib/lightning/setup_utils.ex @@ -893,6 +893,7 @@ defmodule Lightning.SetupUtils do Lightning.Projects.ProjectCredential, Lightning.WorkOrder, Lightning.Invocation.Run, + Lightning.Invocation.LogLine, Lightning.Credentials.Credential, Lightning.Workflows.Job, Lightning.Workflows.Trigger, diff --git a/priv/repo/migrations/20231127112907_remove_run_fk_on_attempt_runs.exs b/priv/repo/migrations/20231127112907_remove_run_fk_on_attempt_runs.exs new file mode 100644 index 0000000000..a241a8ecac --- /dev/null +++ b/priv/repo/migrations/20231127112907_remove_run_fk_on_attempt_runs.exs @@ -0,0 +1,19 @@ +defmodule Lightning.Repo.Migrations.RemoveRunFkOnAttemptRuns do + use Ecto.Migration + + def up do + execute(""" + ALTER TABLE attempt_runs + DROP CONSTRAINT attempt_runs_run_id_fkey + """) + end + + def down do + execute(""" + ALTER TABLE attempt_runs + ADD CONSTRAINT attempt_runs_run_id_fkey + FOREIGN KEY (run_id) + REFERENCES runs(id) ON DELETE CASCADE + """) + end +end diff --git a/priv/repo/migrations/20231127112928_remove_run_fk_on_log_lines.exs b/priv/repo/migrations/20231127112928_remove_run_fk_on_log_lines.exs new file mode 100644 index 0000000000..ce0d53340c --- /dev/null +++ b/priv/repo/migrations/20231127112928_remove_run_fk_on_log_lines.exs @@ -0,0 +1,19 @@ +defmodule Lightning.Repo.Migrations.RemoveRunFkOnLogLines do + use Ecto.Migration + + def up do + execute(""" + ALTER TABLE log_lines + DROP CONSTRAINT log_lines_run_id_fkey + """) + end + + def down do + execute(""" + ALTER TABLE log_lines + ADD CONSTRAINT log_lines_run_id_fkey + FOREIGN KEY (run_id) + REFERENCES runs(id) ON DELETE CASCADE + """) + end +end diff --git a/priv/repo/migrations/20231127112943_remove_run_fk_on_log_lines_monolith.exs b/priv/repo/migrations/20231127112943_remove_run_fk_on_log_lines_monolith.exs new file mode 100644 index 0000000000..9c93ebe94d --- /dev/null +++ b/priv/repo/migrations/20231127112943_remove_run_fk_on_log_lines_monolith.exs @@ -0,0 +1,19 @@ +defmodule Lightning.Repo.Migrations.RemoveRunFkOnLogLinesMonolith do + use Ecto.Migration + + def up do + execute(""" + ALTER TABLE log_lines_monolith + DROP CONSTRAINT log_lines_monolith_run_id_fkey + """) + end + + def down do + execute(""" + ALTER TABLE log_lines_monolith + ADD CONSTRAINT "log_lines_monolith_run_id_fkey" + FOREIGN KEY (run_id) + REFERENCES runs(id) ON DELETE CASCADE + """) + end +end diff --git a/priv/repo/migrations/20231127114133_remove_run_fk_on_runs.exs b/priv/repo/migrations/20231127114133_remove_run_fk_on_runs.exs new file mode 100644 index 0000000000..2a75c83973 --- /dev/null +++ b/priv/repo/migrations/20231127114133_remove_run_fk_on_runs.exs @@ -0,0 +1,19 @@ +defmodule Lightning.Repo.Migrations.RemoveRunFkOnRuns do + use Ecto.Migration + + def up do + execute(""" + ALTER TABLE runs + DROP CONSTRAINT runs_previous_id_fkey + """) + end + + def down do + execute(""" + ALTER TABLE runs + ADD CONSTRAINT runs_previous_id_fkey + FOREIGN KEY (previous_id) + REFERENCES runs(id) ON DELETE CASCADE + """) + end +end diff --git a/priv/repo/migrations/20231127115754_shim_migration_not_for_merging.exs b/priv/repo/migrations/20231127115754_shim_migration_not_for_merging.exs new file mode 100644 index 0000000000..e361278e36 --- /dev/null +++ b/priv/repo/migrations/20231127115754_shim_migration_not_for_merging.exs @@ -0,0 +1,33 @@ +# NBNB For WIP branch only - this is just a shim branch until +# the branch that removes attempts FKs is merged in. +defmodule Lightning.Repo.Migrations.ShimMigrationNotForMerging do + use Ecto.Migration + + def up do + execute(""" + ALTER TABLE attempt_runs + DROP CONSTRAINT attempt_runs_attempt_id_fkey + """) + execute(""" + ALTER TABLE log_lines + DROP CONSTRAINT log_lines_attempt_id_fkey + """) + end + + def change do + execute(""" + ALTER TABLE attempt_runs + ADD CONSTRAINT attempt_runs_attempt_id_fkey + FOREIGN KEY (attempt_id) + REFERENCES attempts(id) ON DELETE CASCADE + """) + + execute(""" + ALTER TABLE log_lines + ADD CONSTRAINT log_lines_attempt_id_fkey + FOREIGN KEY (attempt_id) + REFERENCES attempts(id) + ON DELETE CASCADE + """) + end +end diff --git a/test/lightning/projects_test.exs b/test/lightning/projects_test.exs index c1c47d9acf..d379a32eee 100644 --- a/test/lightning/projects_test.exs +++ b/test/lightning/projects_test.exs @@ -261,13 +261,14 @@ defmodule Lightning.ProjectsTest do ] ) + p2_dataclip = insert(:dataclip, body: %{foo: "bar"}, project: p2) p2_run = insert(:run, input_dataclip: p2_dataclip, job: e2.target_job) p2_log_line = build(:log_line, run: p2_run) - insert(:workorder, + p2_workorder = insert(:workorder, trigger: t2, workflow: w2, dataclip: p2_dataclip, @@ -280,6 +281,17 @@ defmodule Lightning.ProjectsTest do ) ) + {:ok, p2_pu} = p2.project_users |> Enum.fetch(0) + + {:ok, p2_pc} = Repo.all(Ecto.assoc(p2, :project_credentials)) |> Enum.fetch(0) + + p2_dataclip = Repo.get_by( + Lightning.Invocation.Dataclip, + project_id: p2.id + ) + + p2_attempt_run = Repo.get_by(Lightning.AttemptRun, run_id: p2_run.id) + runs_query = Lightning.Projects.project_runs_query(p1) work_order_query = Lightning.Projects.project_workorders_query(p1) @@ -319,17 +331,19 @@ defmodule Lightning.ProjectsTest do assert {:ok, %Project{}} = Projects.delete_project(p1) - assert runs_query |> Repo.aggregate(:count, :id) == 0 + assert only_record_for_type?(p2_run) + + assert only_record_for_type?(p2_workorder) assert work_order_query |> Repo.aggregate(:count, :id) == 0 assert attempt_query |> Repo.aggregate(:count, :id) == 0 - assert attempt_run_query |> Repo.aggregate(:count, :id) == 0 + assert only_record_for_type?(p2_attempt_run) - assert pu_query |> Repo.aggregate(:count, :id) == 0 + assert only_record_for_type?(p2_pu) - assert pc_query |> Repo.aggregate(:count, :id) == 0 + assert only_record_for_type?(p2_pc) assert workflows_query |> Repo.aggregate(:count, :id) == 0 @@ -337,6 +351,8 @@ defmodule Lightning.ProjectsTest do assert only_record_for_type?(p2_log_line) + assert only_record_for_type?(p2_dataclip) + assert_raise Ecto.NoResultsError, fn -> Projects.get_project!(p1.id) end diff --git a/test/support/model_helpers.ex b/test/support/model_helpers.ex index f2fc9b46df..480d201476 100644 --- a/test/support/model_helpers.ex +++ b/test/support/model_helpers.ex @@ -83,7 +83,7 @@ defmodule Lightning.ModelHelpers do where: r.id == ^expected_instance.id, left_join: others in ^model, on: others.id != ^expected_instance.id, - select: [count(r.id), count(others.id)] + select: [count(r.id, :distinct), count(others.id)] ) |> Lightning.Repo.one!() |> case do