From a26fc081935a1fbd2100d67cfcee58430b9c08a2 Mon Sep 17 00:00:00 2001 From: "Giau. Tran Minh" Date: Mon, 27 Nov 2023 15:45:58 +0700 Subject: [PATCH] atlas/schema: added concurrent_index support --- api/v1alpha1/atlasschema_types.go | 23 +++++++-- api/v1alpha1/zz_generated.deepcopy.go | 50 +++++++++++++++++-- charts/atlas-operator/templates/crds/crd.yaml | 16 ++++++ .../crd/bases/db.atlasgo.io_atlasschemas.yaml | 16 ++++++ controllers/atlasschema_controller.go | 17 +++++-- controllers/atlasschema_controller_test.go | 30 +++++++---- controllers/templates/atlas_schema.tmpl | 16 +++++- 7 files changed, 143 insertions(+), 25 deletions(-) diff --git a/api/v1alpha1/atlasschema_types.go b/api/v1alpha1/atlasschema_types.go index 17b40b58..9442479b 100644 --- a/api/v1alpha1/atlasschema_types.go +++ b/api/v1alpha1/atlasschema_types.go @@ -74,8 +74,11 @@ type ( DevURLFrom Secret `json:"devURLFrom,omitempty"` // Exclude a list of glob patterns used to filter existing resources being taken into account. Exclude []string `json:"exclude,omitempty"` + // TxMode defines the transaction mode to use when applying the schema. + // +kubebuilder:default=file + TxMode TransactionMode `json:"txMode,omitempty"` // Policy defines the policies to apply when managing the schema change lifecycle. - Policy Policy `json:"policy,omitempty"` + Policy *Policy `json:"policy,omitempty"` // The names of the schemas (named databases) on the target database to be managed. Schemas []string `json:"schemas,omitempty"` } @@ -87,12 +90,12 @@ type ( } // Policy defines the policies to apply when managing the schema change lifecycle. Policy struct { - Lint Lint `json:"lint,omitempty"` - Diff Diff `json:"diff,omitempty"` + Lint *Lint `json:"lint,omitempty"` + Diff *Diff `json:"diff,omitempty"` } // Lint defines the linting policies to apply before applying the schema. Lint struct { - Destructive CheckConfig `json:"destructive,omitempty"` + Destructive *CheckConfig `json:"destructive,omitempty"` } // CheckConfig defines the configuration of a linting check. CheckConfig struct { @@ -100,7 +103,8 @@ type ( } // Diff defines the diff policies to apply when planning schema changes. Diff struct { - Skip SkipChanges `json:"skip,omitempty"` + ConcurrentIndex *ConcurrentIndex `json:"concurrent_index,omitempty"` + Skip *SkipChanges `json:"skip,omitempty"` } // SkipChanges represents the skip changes policy. SkipChanges struct { @@ -135,6 +139,15 @@ type ( // +optional ModifyForeignKey bool `json:"modify_foreign_key,omitempty"` } + ConcurrentIndex struct { + // +optional + Create bool `json:"create,omitempty"` + // +optional + Drop bool `json:"drop,omitempty"` + } + // TransactionMode + // +kubebuilder:validation:Enum=file;all;none + TransactionMode string ) func init() { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 01cc9a01..7b613176 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -193,7 +193,11 @@ func (in *AtlasSchemaSpec) DeepCopyInto(out *AtlasSchemaSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - out.Policy = in.Policy + if in.Policy != nil { + in, out := &in.Policy, &out.Policy + *out = new(Policy) + (*in).DeepCopyInto(*out) + } if in.Schemas != nil { in, out := &in.Schemas, &out.Schemas *out = make([]string, len(*in)) @@ -264,6 +268,21 @@ func (in *Cloud) DeepCopy() *Cloud { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConcurrentIndex) DeepCopyInto(out *ConcurrentIndex) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConcurrentIndex. +func (in *ConcurrentIndex) DeepCopy() *ConcurrentIndex { + if in == nil { + return nil + } + out := new(ConcurrentIndex) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Credentials) DeepCopyInto(out *Credentials) { *out = *in @@ -292,7 +311,16 @@ func (in *Credentials) DeepCopy() *Credentials { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Diff) DeepCopyInto(out *Diff) { *out = *in - out.Skip = in.Skip + if in.ConcurrentIndex != nil { + in, out := &in.ConcurrentIndex, &out.ConcurrentIndex + *out = new(ConcurrentIndex) + **out = **in + } + if in.Skip != nil { + in, out := &in.Skip, &out.Skip + *out = new(SkipChanges) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Diff. @@ -336,7 +364,11 @@ func (in *Dir) DeepCopy() *Dir { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Lint) DeepCopyInto(out *Lint) { *out = *in - out.Destructive = in.Destructive + if in.Destructive != nil { + in, out := &in.Destructive, &out.Destructive + *out = new(CheckConfig) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lint. @@ -352,8 +384,16 @@ func (in *Lint) DeepCopy() *Lint { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Policy) DeepCopyInto(out *Policy) { *out = *in - out.Lint = in.Lint - out.Diff = in.Diff + if in.Lint != nil { + in, out := &in.Lint, &out.Lint + *out = new(Lint) + (*in).DeepCopyInto(*out) + } + if in.Diff != nil { + in, out := &in.Diff, &out.Diff + *out = new(Diff) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Policy. diff --git a/charts/atlas-operator/templates/crds/crd.yaml b/charts/atlas-operator/templates/crds/crd.yaml index 021dc62d..dfb00870 100644 --- a/charts/atlas-operator/templates/crds/crd.yaml +++ b/charts/atlas-operator/templates/crds/crd.yaml @@ -523,6 +523,13 @@ spec: description: Diff defines the diff policies to apply when planning schema changes. properties: + concurrent_index: + properties: + create: + type: boolean + drop: + type: boolean + type: object skip: description: SkipChanges represents the skip changes policy. properties: @@ -603,6 +610,15 @@ spec: items: type: string type: array + txMode: + default: file + description: TxMode defines the transaction mode to use when applying + the schema. + enum: + - file + - all + - none + type: string url: description: URL of the target database schema. type: string diff --git a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml index 3bd328e4..96c13faa 100644 --- a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml +++ b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml @@ -192,6 +192,13 @@ spec: description: Diff defines the diff policies to apply when planning schema changes. properties: + concurrent_index: + properties: + create: + type: boolean + drop: + type: boolean + type: object skip: description: SkipChanges represents the skip changes policy. properties: @@ -272,6 +279,15 @@ spec: items: type: string type: array + txMode: + default: file + description: TxMode defines the transaction mode to use when applying + the schema. + enum: + - file + - all + - none + type: string url: description: URL of the target database schema. type: string diff --git a/controllers/atlasschema_controller.go b/controllers/atlasschema_controller.go index afe79a23..e987d7c8 100644 --- a/controllers/atlasschema_controller.go +++ b/controllers/atlasschema_controller.go @@ -67,7 +67,8 @@ type ( DevURL string Schemas []string Exclude []string - Policy dbv1alpha1.Policy + Policy *dbv1alpha1.Policy + TxMode dbv1alpha1.TransactionMode desired []byte ext string @@ -199,7 +200,7 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result(err) } } - report, err := r.apply(ctx, wd.Path(), data.EnvName) + report, err := r.apply(ctx, wd.Path(), data.EnvName, string(data.TxMode)) if err != nil { res.SetNotReady("ApplyingSchema", err.Error()) r.recorder.Event(res, corev1.EventTypeWarning, "ApplyingSchema", err.Error()) @@ -255,13 +256,14 @@ func (r *AtlasSchemaReconciler) watchRefs(res *dbv1alpha1.AtlasSchema) { } } -func (r *AtlasSchemaReconciler) apply(ctx context.Context, dir, envName string) (*atlas.SchemaApply, error) { +func (r *AtlasSchemaReconciler) apply(ctx context.Context, dir, envName, txMode string) (*atlas.SchemaApply, error) { cli, err := atlas.NewClient(dir, r.execPath) if err != nil { return nil, err } return cli.SchemaApply(ctx, &atlas.SchemaApplyParams{ - Env: envName, + Env: envName, + TxMode: txMode, }) } @@ -275,6 +277,7 @@ func (r *AtlasSchemaReconciler) extractData(ctx context.Context, res *dbv1alpha1 Schemas: s.Schemas, Exclude: s.Exclude, Policy: s.Policy, + TxMode: s.TxMode, } ) data.URL, err = s.DatabaseURL(ctx, r, res.Namespace) @@ -311,7 +314,11 @@ func (d *managedData) Source() string { // ShouldLint returns true if the linting policy is set to error. func (d *managedData) shouldLint() bool { - return d.Policy.Lint.Destructive.Error + p := d.Policy + if p == nil || p.Lint == nil || p.Lint.Destructive == nil { + return false + } + return p.Lint.Destructive.Error } // hash returns the sha256 hash of the desired. diff --git a/controllers/atlasschema_controller_test.go b/controllers/atlasschema_controller_test.go index d5770842..03a3cd96 100644 --- a/controllers/atlasschema_controller_test.go +++ b/controllers/atlasschema_controller_test.go @@ -312,7 +312,11 @@ func TestReconcile_Lint(t *testing.T) { tt := cliTest(t) sc := conditionReconciling() sc.Spec.URL = tt.dburl - sc.Spec.Policy.Lint.Destructive.Error = true + sc.Spec.Policy = &dbv1alpha1.Policy{ + Lint: &dbv1alpha1.Lint{ + Destructive: &dbv1alpha1.CheckConfig{Error: true}, + }, + } sc.Status.LastApplied = 1 tt.k8s.put(sc) tt.initDB("create table x (c int);") @@ -360,7 +364,11 @@ func TestBadSQL(t *testing.T) { sc := conditionReconciling() sc.Spec.Schema.SQL = "bad sql;" sc.Spec.URL = tt.dburl - sc.Spec.Policy.Lint.Destructive.Error = true + sc.Spec.Policy = &dbv1alpha1.Policy{ + Lint: &dbv1alpha1.Lint{ + Destructive: &dbv1alpha1.CheckConfig{Error: true}, + }, + } sc.Status.LastApplied = 1 tt.k8s.put(sc) resp, err := tt.r.Reconcile(context.Background(), req()) @@ -378,8 +386,12 @@ func TestDiffPolicy(t *testing.T) { sc := conditionReconciling() sc.Spec.URL = tt.dburl sc.Spec.Schema.SQL = "create table y (c int);" - sc.Spec.Policy.Diff.Skip = dbv1alpha1.SkipChanges{ - DropTable: true, + sc.Spec.Policy = &dbv1alpha1.Policy{ + Diff: &dbv1alpha1.Diff{ + Skip: &dbv1alpha1.SkipChanges{ + DropTable: true, + }, + }, } sc.Status.LastApplied = 1 tt.k8s.put(sc) @@ -402,12 +414,12 @@ func TestConfigTemplate(t *testing.T) { EnvName: defaultEnvName, URL: must(url.Parse("mysql://root:password@localhost:3306/test")), DevURL: "mysql://root:password@localhost:3306/dev", - Policy: dbv1alpha1.Policy{ - Lint: dbv1alpha1.Lint{ - Destructive: dbv1alpha1.CheckConfig{Error: true}, + Policy: &dbv1alpha1.Policy{ + Lint: &dbv1alpha1.Lint{ + Destructive: &dbv1alpha1.CheckConfig{Error: true}, }, - Diff: dbv1alpha1.Diff{ - Skip: dbv1alpha1.SkipChanges{ + Diff: &dbv1alpha1.Diff{ + Skip: &dbv1alpha1.SkipChanges{ DropSchema: true, DropTable: true, }, diff --git a/controllers/templates/atlas_schema.tmpl b/controllers/templates/atlas_schema.tmpl index 66213a0c..290092b0 100644 --- a/controllers/templates/atlas_schema.tmpl +++ b/controllers/templates/atlas_schema.tmpl @@ -7,8 +7,20 @@ variable "lint_destructive" { {{- end }} } {{- with .Policy }} - {{- with .Diff.Skip }} + {{- with .Diff }} + {{- if or .ConcurrentIndex .Skip }} diff { + {{- with .ConcurrentIndex }} + concurrent_index { + {{- if .Create }} + create = true + {{- end }} + {{- if .Drop }} + drop = true + {{- end }} + } + {{- end }} + {{- with .Skip }} skip { {{- if .AddSchema }} add_schema = true @@ -56,7 +68,9 @@ diff { modify_foreign_key = true {{- end }} } + {{- end }} } + {{- end }} {{- end }} {{- with .Lint }} lint {