Replace DB package tables by sip and aip tables

Stop using package across the persistence layers. Use SIP on ingest
and AIP on storage renaming DB tables, types and enums. The use of
package still happens on the API design/spec and the dashboard, these
will be modified in following PRs.

- Allow generating migrations for both databases using ariga.io/atlas.
- Add `atlas-hash` Make rule to re-generate sum file in migration dirs.
- Update ingest database, ent schema and client:
  - Rename `package` table to `sip`.
  - Adapt `preservation_action` table relation.
  - Generate `use_atlas` migration to match schema from ent/Atlas.
  - Rename package related functions.
- Update storage database, ent schema and client:
  - Rename `package` table to `aip`.
  - Rename package related functions.
- Rename datatype SIP and PreservationAction.SIPID.
- Rename SIP statuses and types.
- Rename AIP statuses.

Makefile

@@ -15,6 +15,7 @@ else
 endif
 
 include hack/make/bootstrap.mk
+include hack/make/dep_atlas.mk
 include hack/make/dep_ent.mk
 include hack/make/dep_go_enums.mk
 include hack/make/dep_goa.mk
@@ -30,7 +31,8 @@ include hack/make/dep_tparse.mk
 include hack/make/dep_workflowcheck.mk
 
 # Lazy-evaluated list of tools.
-TOOLS = $(ENT) \
+TOOLS = $(ATLAS) \
+	$(ENT) \
 	$(GO_ENUM) \
 	$(GOA) \
 	$(GOLANGCI_LINT) \
@@ -68,6 +70,14 @@ TEST_IGNORED_PACKAGES = $(filter $(IGNORED_PACKAGES),$(PACKAGES))
 
 export PATH:=$(GOBIN):$(PATH)
 
+atlas-hash: $(ATLAS) # @HELP Recalculate the migration hashes.
+	atlas migrate hash \
+		--dir="file://internal/db/migrations" \
+		--dir-format="atlas"
+	atlas migrate hash \
+		--dir="file://internal/storage/persistence/migrations" \
+		--dir-format="atlas"
+
 db: # @HELP Opens the MySQL shell connected to the enduro development database.
 db:
 	mysql -h127.0.0.1 -P3306 -uroot -proot123 enduro
@@ -106,8 +116,8 @@ gen-enums: # @HELP Generate go-enum assets.
 gen-enums: ENUM_FLAGS = --names --template=$(CURDIR)/hack/make/enums.tmpl
 gen-enums: $(GO_ENUM)
 	go-enum $(ENUM_FLAGS) \
-		-f internal/enums/package_type.go \
-		-f internal/enums/pkg_status.go \
+		-f internal/enums/sip_type.go \
+		-f internal/enums/sip_status.go \
 		-f internal/enums/preprocessing_task_outcome.go \
 		-f internal/enums/pres_action_status.go \
 		-f internal/enums/pres_action_type.go \

cmd/migrate/main.go

@@ -1,22 +1,33 @@
-// Example:
+// Example (ingest):
+//
+//  1. Make changes to schema files (internal/persistence/ent/schema),
+//  2. Re-generate (make gen-ent),
+//  3. Use an empty MySQL database,
+//  4. Run:
+//     $ go run ./cmd/migrate/ \
+//     --db="ingest" \
+//     --dsn="mysql://root:root123@tcp(localhost:3306)/enduro_migrate" \
+//     --path="./internal/db/migrations" \
+//     --name="changes"
+//
+// Example (storage):
 //
 //  1. Make changes to schema files (internal/storage/persistence/ent/schema),
 //  2. Re-generate (make gen-ent),
-//  3. Drop any existing database tables or delete and re-create the database,
+//  3. Use an empty MySQL database,
 //  4. Run:
 //     $ go run ./cmd/migrate/ \
-//     --config="./enduro.toml" \
-//     --dsn="mysql://enduro:enduro123@tcp(localhost:3306)/enduro_storage" \
-//     --name="init" \
-//     --path="./internal/storage/persistence/migrations"
+//     --db="storage" \
+//     --dsn="mysql://root:root123@tcp(localhost:3306)/enduro_migrate" \
+//     --path="./internal/storage/persistence/migrations" \
+//     --name="changes"
 package main
 
 import (
 	"context"
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"ariga.io/atlas/sql/sqltool"
@@ -25,40 +36,38 @@ import (
 	"github.com/go-sql-driver/mysql"
 	"github.com/spf13/pflag"
 
-	"github.com/artefactual-sdps/enduro/internal/config"
-	"github.com/artefactual-sdps/enduro/internal/storage/persistence/ent/db/migrate"
+	ingest_migrate "github.com/artefactual-sdps/enduro/internal/persistence/ent/db/migrate"
+	storage_migrate "github.com/artefactual-sdps/enduro/internal/storage/persistence/ent/db/migrate"
 )
 
 func main() {
 	p := pflag.NewFlagSet("migrate", pflag.ExitOnError)
-	p.String("config", "", "Configuration file")
+	p.String("db", "", "Enduro database ('ingest' or 'storage')")
 	p.String("dsn", "", "MySQL DSN")
 	p.String("path", "", "Migration directory")
 	p.String("name", "changes", "Migration name")
 	_ = p.Parse(os.Args[1:])
 
-	path, _ := p.GetString("path")
-	if path == "" {
-		wd, err := os.Getwd()
-		if err != nil {
-			os.Exit(1)
-		}
-		// Guessing that running it from the root folder.
-		path = filepath.Join(wd, "internal/storage/persistence/migrations")
+	db, _ := p.GetString("db")
+	if db == "" {
+		fmt.Printf("--db flag is missing")
+		os.Exit(1)
+	}
+	if db != "ingest" && db != "storage" {
+		fmt.Printf("--db flag has an unexpected value (use 'ingest' or 'storage')")
+		os.Exit(1)
 	}
 
-	var cfg config.Configuration
-	configFile, _ := p.GetString("config")
-	_, _, err := config.Read(&cfg, configFile)
-	if err != nil {
-		fmt.Printf("Failed to read configuration: %v\n", err)
+	DSN, _ := p.GetString("dsn")
+	if DSN == "" {
+		fmt.Printf("--dsn flag is missing")
 		os.Exit(1)
 	}
 
-	DSN := cfg.Storage.Database.DSN
-	flagDSN, _ := p.GetString("dsn")
-	if flagDSN != "" {
-		DSN = flagDSN
+	path, _ := p.GetString("path")
+	if path == "" {
+		fmt.Printf("--path flag is missing")
+		os.Exit(1)
 	}
 
 	// MySQL's DSN format is not accepted by Ent, convert as needed (remove Net).
@@ -89,11 +98,17 @@ func main() {
 		schema.WithDir(dir),                         // provide migration directory
 		schema.WithMigrationMode(schema.ModeReplay), // provide migration mode
 		schema.WithDialect(dialect.MySQL),           // Ent dialect to use
+		schema.WithDropIndex(true),
+		schema.WithDropColumn(true),
 	}
 
 	// Generate migrations using Atlas support for TiDB (note the Ent dialect option passed above).
 	name, _ := p.GetString("name")
-	err = migrate.NamedDiff(ctx, entDSN, name, opts...)
+	if db == "ingest" {
+		err = ingest_migrate.NamedDiff(ctx, entDSN, name, opts...)
+	} else {
+		err = storage_migrate.NamedDiff(ctx, entDSN, name, opts...)
+	}
 	if err != nil {
 		log.Fatalf("failed generating migration file: %v", err)
 	}

go.mod

@@ -3,7 +3,7 @@ module github.com/artefactual-sdps/enduro
 go 1.23.6
 
 require (
-	ariga.io/atlas v0.19.2
+	ariga.io/atlas v0.31.0
 	ariga.io/sqlcomment v0.1.0
 	buf.build/gen/go/artefactual/a3m/grpc/go v1.3.0-20230508184533-2e9432075630.2
 	buf.build/gen/go/artefactual/a3m/protocolbuffers/go v1.31.0-20230508184533-2e9432075630.2
@@ -95,6 +95,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
 	github.com/aws/smithy-go v1.20.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bmatcuk/doublestar v1.3.4 // indirect
 	github.com/bodgit/plumbing v1.2.0 // indirect
 	github.com/bodgit/sevenzip v1.3.0 // indirect
 	github.com/bodgit/windows v1.0.0 // indirect
@@ -158,7 +159,8 @@ require (
 	github.com/therootcompany/xz v1.0.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	github.com/zclconf/go-cty v1.14.1 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
+	github.com/zclconf/go-cty-yaml v1.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0 // indirect
 	go.opentelemetry.io/otel/metric v1.29.0 // indirect

go.sum

@@ -1,5 +1,5 @@
-ariga.io/atlas v0.19.2 h1:ulK06d4joEaMP06HNNPxdpD8dFgZGzjzjk+Mb5VfF08=
-ariga.io/atlas v0.19.2/go.mod h1:VPlcXdd4w2KqKnH54yEZcry79UAhpaWaxEsmn5JRNoE=
+ariga.io/atlas v0.31.0 h1:Nw6/Jdc7OpZfiy6oh/dJAYPp5XxGYvMTWLOUutwWjeY=
+ariga.io/atlas v0.31.0/go.mod h1:J3chwsQAgjDF6Ostz7JmJJRTCbtqIupUbVR/gqZrMiA=
 ariga.io/sqlcomment v0.1.0 h1:8kQPlVe3sXpTloEFlpX5dhFAXB28i6rwq9ktqqnPx70=
 ariga.io/sqlcomment v0.1.0/go.mod h1:NT1IZMfBTQl1MUU5wgVONmnDqFRqtZrdDRgAXfc1g5k=
 buf.build/gen/go/artefactual/a3m/grpc/go v1.3.0-20230508184533-2e9432075630.2 h1:+AADIVDD4GabjtfMHO/M9WIQBc/0IXN7ddYuN8VU8tc=
@@ -487,6 +487,8 @@ github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
+github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bodgit/plumbing v1.2.0 h1:gg4haxoKphLjml+tgnecR4yLBV5zo4HAZGCtAh3xCzM=
 github.com/bodgit/plumbing v1.2.0/go.mod h1:b9TeRi7Hvc6Y05rjm8VML3+47n4XTZPtQ/5ghqic2n8=
 github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY=
@@ -942,8 +944,10 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA=
-github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8=
+github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty-yaml v1.1.0 h1:nP+jp0qPHv2IhUVqmQSzjvqAWcObN0KBkUl2rWBdig0=
+github.com/zclconf/go-cty-yaml v1.1.0/go.mod h1:9YLUH4g7lOhVWqUbctnVlZ5KLpg7JAprQNgxSZ1Gyxs=
 go.artefactual.dev/amclient v0.4.1-0.20240705155055-0c5abef5207c h1:kMgKuV9yx0LTQpkWdPxPC4GHAKMS9pDQ0x3CD8SmLfI=
 go.artefactual.dev/amclient v0.4.1-0.20240705155055-0c5abef5207c/go.mod h1:oAOlZHC78IWjWDCqRM1/8K1x/WYmRUL3peujKCdoXaA=
 go.artefactual.dev/tools v0.14.0 h1:ESLbemsnkdIPmYXtz0uZTcPqVnTUXIEZd9DSTRyTZqY=

hack/make/dep_atlas.mk

@@ -0,0 +1,22 @@
+$(call _assert_var,MAKEDIR)
+$(call _conditional_include,$(MAKEDIR)/base.mk)
+$(call _assert_var,UNAME_OS)
+$(call _assert_var,UNAME_ARCH2)
+$(call _assert_var,CACHE_VERSIONS)
+$(call _assert_var,CACHE_BIN)
+
+ATLAS_VERSION ?= 0.31.0
+
+ATLAS := $(CACHE_VERSIONS)/atlas/$(ATLAS_VERSION)
+$(ATLAS):
+	rm -f $(CACHE_BIN)/atlas
+	mkdir -p $(CACHE_BIN)
+	$(eval TMP := $(shell mktemp -d))
+	$(eval OS := $(shell echo $(UNAME_OS) | tr A-Z a-z))
+	curl -sSL \
+		https://release.ariga.io/atlas/atlas-$(OS)-$(UNAME_ARCH2)-v$(ATLAS_VERSION) \
+		> $(CACHE_BIN)/atlas
+	chmod +x $(CACHE_BIN)/atlas
+	rm -rf $(dir $(ATLAS))
+	mkdir -p $(dir $(ATLAS))
+	touch $(ATLAS)

internal/am/job_tracker_test.go

@@ -211,7 +211,7 @@ func TestConvertJobToPreservationTask(t *testing.T) {
 			want: datatypes.PreservationTask{
 				TaskID: "f60018ac-da79-4769-9509-c6c41d5efe7e",
 				Name:   "Move to processing directory",
-				Status: enums.PreservationTaskStatus(enums.PackageStatusDone),
+				Status: enums.PreservationTaskStatusDone,
 				StartedAt: sql.NullTime{
 					Time:  time.Date(2024, time.January, 18, 1, 27, 49, 0, time.UTC),
 					Valid: true,
@@ -251,7 +251,7 @@ func TestConvertJobToPreservationTask(t *testing.T) {
 			want: datatypes.PreservationTask{
 				TaskID: "c2128d39-2ace-47c5-8cac-39ded8d9c9ef",
 				Name:   "Verify SIP compliance",
-				Status: enums.PreservationTaskStatus(enums.PackageStatusInProgress),
+				Status: enums.PreservationTaskStatusInProgress,
 				StartedAt: sql.NullTime{
 					Time:  time.Date(2024, time.January, 18, 1, 27, 49, 0, time.UTC),
 					Valid: true,

internal/api/design/package_.go

@@ -266,7 +266,7 @@ var _ = Service("package", func() {
 })
 
 var EnumPackageStatus = func() {
-	Enum(enums.PackageStatusInterfaces()...)
+	Enum(enums.SIPStatusInterfaces()...)
 }
 
 var Package_ = Type("Package", func() {
@@ -275,7 +275,7 @@ var Package_ = Type("Package", func() {
 	TypedAttributeUUID("location_id", "Identifier of storage location")
 	Attribute("status", String, "Status of the package", func() {
 		EnumPackageStatus()
-		Default(enums.PackageStatusNew.String())
+		Default(enums.SIPStatusNew.String())
 	})
 	AttributeUUID("workflow_id", "Identifier of processing workflow")
 	AttributeUUID("run_id", "Identifier of latest processing workflow run")

internal/datatypes/preservation_action.go

@@ -14,5 +14,5 @@ type PreservationAction struct {
 	Status      enums.PreservationActionStatus `db:"status"`
 	StartedAt   sql.NullTime                   `db:"started_at"`
 	CompletedAt sql.NullTime                   `db:"completed_at"`
-	PackageID   int                            `db:"package_id"`
+	SIPID       int                            `db:"sip_id"`
 }