diff --git a/go.mod b/go.mod index 33235ddf..65e10da5 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/open-policy-agent/gatekeeper/v3 v3.17.1 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/pkg/errors v0.9.1 - github.com/pluralsh/console/go/client v1.17.0 + github.com/pluralsh/console/go/client v1.19.0 github.com/pluralsh/controller-reconcile-helper v0.1.0 github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 github.com/pluralsh/polly v0.1.10 diff --git a/go.sum b/go.sum index 5ddf5363..ccc3b58e 100644 --- a/go.sum +++ b/go.sum @@ -666,8 +666,8 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pluralsh/console/go/client v1.17.0 h1:ExP+HUWa94e8cIbFY+6ARBq33oC87xpxTVw3+eZS7+I= -github.com/pluralsh/console/go/client v1.17.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= +github.com/pluralsh/console/go/client v1.19.0 h1:x5JQWYtxr5S/PVUJS4Jiiir2IYAqMruONXydu9NSliM= +github.com/pluralsh/console/go/client v1.19.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k= github.com/pluralsh/controller-reconcile-helper v0.1.0 h1:BV3dYZFH5rn8ZvZjtpkACSv/GmLEtRftNQj/Y4ddHEo= github.com/pluralsh/controller-reconcile-helper v0.1.0/go.mod h1:RxAbvSB4/jkvx616krCdNQXPbpGJXW3J1L3rASxeFOA= github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 h1:ab2PN+6if/Aq3/sJM0AVdy1SYuMAnq4g20VaKhTm/Bw= diff --git a/pkg/controller/stacks/job.go b/pkg/controller/stacks/job.go index cd4437e1..32725825 100644 --- a/pkg/controller/stacks/job.go +++ b/pkg/controller/stacks/job.go @@ -6,6 +6,8 @@ import ( "os" "strings" + "k8s.io/apimachinery/pkg/api/resource" + console "github.com/pluralsh/console/go/client" "github.com/pluralsh/deployment-operator/internal/metrics" "github.com/pluralsh/deployment-operator/internal/utils" @@ -94,7 +96,10 @@ func (r *StackReconciler) reconcileRunJob(ctx context.Context, run *console.Stac return nil, err } - job := r.GenerateRunJob(run, jobSpec, name, namespace) + job, err := r.GenerateRunJob(run, jobSpec, name, namespace) + if err != nil { + return nil, err + } logger.V(2).Info("creating job for stack run", "id", run.ID, "namespace", job.Namespace, "name", job.Name) if err := r.k8sClient.Create(ctx, job); err != nil { logger.Error(err, "unable to create job") @@ -141,7 +146,8 @@ func (r *StackReconciler) GetRunResourceNamespace(jobSpec *batchv1.JobSpec) (nam return } -func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec *batchv1.JobSpec, name, namespace string) *batchv1.Job { +func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec *batchv1.JobSpec, name, namespace string) (*batchv1.Job, error) { + var err error // If user-defined job spec was not available initialize it here. if jobSpec == nil { jobSpec = &batchv1.JobSpec{} @@ -163,6 +169,11 @@ func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec jobSpec.Template.Spec.Containers = r.ensureDefaultContainer(jobSpec.Template.Spec.Containers, run) + jobSpec.Template.Spec.Containers, err = r.ensureDefaultContainerResourcesRequests(jobSpec.Template.Spec.Containers, run) + if err != nil { + return nil, err + } + jobSpec.Template.Spec.Volumes = ensureDefaultVolumes(jobSpec.Template.Spec.Volumes) jobSpec.Template.Spec.SecurityContext = ensureDefaultPodSecurityContext(jobSpec.Template.Spec.SecurityContext) @@ -175,7 +186,7 @@ func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec Labels: map[string]string{jobSelector: name}, }, Spec: *jobSpec, - } + }, nil } func getRunJobSpec(name string, jobSpecFragment *console.JobSpecFragment) *batchv1.JobSpec { @@ -378,3 +389,55 @@ func ensureDefaultContainerSecurityContext(sc *corev1.SecurityContext) *corev1.S RunAsGroup: lo.ToPtr(nonRootGID), } } + +func (r *StackReconciler) ensureDefaultContainerResourcesRequests(containers []corev1.Container, run *console.StackRunFragment) ([]corev1.Container, error) { + if run.JobSpec == nil || run.JobSpec.Requests == nil { + return containers, nil + } + if run.JobSpec.Requests.Requests == nil && run.JobSpec.Requests.Limits == nil { + return containers, nil + } + + for i, container := range containers { + if run.JobSpec.Requests.Requests != nil { + if len(container.Resources.Requests) == 0 { + containers[i].Resources.Requests = map[corev1.ResourceName]resource.Quantity{} + } + if run.JobSpec.Requests.Requests.CPU != nil { + cpu, err := resource.ParseQuantity(*run.JobSpec.Requests.Requests.CPU) + if err != nil { + return nil, err + } + containers[i].Resources.Requests[corev1.ResourceCPU] = cpu + } + if run.JobSpec.Requests.Requests.Memory != nil { + memory, err := resource.ParseQuantity(*run.JobSpec.Requests.Requests.Memory) + if err != nil { + return nil, err + } + containers[i].Resources.Requests[corev1.ResourceMemory] = memory + } + } + if run.JobSpec.Requests.Limits != nil { + if len(container.Resources.Limits) == 0 { + containers[i].Resources.Limits = map[corev1.ResourceName]resource.Quantity{} + } + if run.JobSpec.Requests.Limits.CPU != nil { + cpu, err := resource.ParseQuantity(*run.JobSpec.Requests.Limits.CPU) + if err != nil { + return nil, err + } + containers[i].Resources.Limits[corev1.ResourceCPU] = cpu + } + if run.JobSpec.Requests.Limits.Memory != nil { + memory, err := resource.ParseQuantity(*run.JobSpec.Requests.Limits.Memory) + if err != nil { + return nil, err + } + containers[i].Resources.Limits[corev1.ResourceMemory] = memory + } + } + } + + return containers, nil +} diff --git a/pkg/controller/stacks/job_test.go b/pkg/controller/stacks/job_test.go index d181e234..02625c3b 100644 --- a/pkg/controller/stacks/job_test.go +++ b/pkg/controller/stacks/job_test.go @@ -1,9 +1,15 @@ package stacks import ( + "fmt" "testing" "time" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + console "github.com/pluralsh/console/go/client" "github.com/samber/lo" "github.com/stretchr/testify/assert" @@ -13,6 +19,8 @@ import ( "github.com/pluralsh/deployment-operator/pkg/test/mocks" ) +const defaultName = "default" + func TestGetDefaultContainerImage(t *testing.T) { var kClient client.Client fakeConsoleClient := mocks.NewClientMock(t) @@ -103,3 +111,169 @@ func TestGetDefaultContainerImage(t *testing.T) { }) } } + +func TestGenerateRunJob(t *testing.T) { + var kClient client.Client + fakeConsoleClient := mocks.NewClientMock(t) + namespace := defaultName + runID := "1" + reconciler := NewStackReconciler(fakeConsoleClient, kClient, scheme.Scheme, time.Minute, 0, namespace, "", "") + cases := []struct { + name string + run *console.StackRunFragment + expectedJobSpec batchv1.JobSpec + }{ + { + name: "use_empty_job_spec", + run: &console.StackRunFragment{ + ID: runID, + Type: console.StackTypeTerraform, + Configuration: console.StackConfigurationFragment{}, + }, + expectedJobSpec: func() batchv1.JobSpec { + js := genDefaultJobSpec(namespace, "use_empty_job_spec", runID) + js.Template.Labels = nil + return js + }(), + }, + { + name: "use_defaults", + run: &console.StackRunFragment{ + ID: runID, + Type: console.StackTypeTerraform, + Configuration: console.StackConfigurationFragment{}, + JobSpec: &console.JobSpecFragment{ + Namespace: namespace, + }, + }, + expectedJobSpec: genDefaultJobSpec(namespace, "use_defaults", runID), + }, + { + name: "add_labels", + run: &console.StackRunFragment{ + ID: runID, + Type: console.StackTypeTerraform, + Configuration: console.StackConfigurationFragment{}, + JobSpec: &console.JobSpecFragment{ + Namespace: namespace, + Labels: map[string]interface{}{ + "test": "test", + }, + }, + }, + expectedJobSpec: func() batchv1.JobSpec { + js := genDefaultJobSpec(namespace, "add_labels", runID) + js.Template.Labels = map[string]string{ + "test": "test", + } + return js + }(), + }, + { + name: "add_sa", + run: &console.StackRunFragment{ + ID: runID, + Type: console.StackTypeTerraform, + Configuration: console.StackConfigurationFragment{}, + JobSpec: &console.JobSpecFragment{ + Namespace: namespace, + ServiceAccount: lo.ToPtr(defaultName), + }, + }, + expectedJobSpec: func() batchv1.JobSpec { + js := genDefaultJobSpec(namespace, "add_sa", runID) + js.Template.Spec.ServiceAccountName = defaultName + return js + }(), + }, + { + name: "add_resources", + run: &console.StackRunFragment{ + ID: runID, + Type: console.StackTypeTerraform, + Configuration: console.StackConfigurationFragment{}, + JobSpec: &console.JobSpecFragment{ + Namespace: namespace, + Requests: &console.ContainerResourcesFragment{ + Requests: &console.ResourceRequestFragment{ + CPU: lo.ToPtr("2Mi"), + }, + Limits: &console.ResourceRequestFragment{ + Memory: lo.ToPtr("2M"), + }, + }, + }, + }, + expectedJobSpec: func() batchv1.JobSpec { + js := genDefaultJobSpec(namespace, "add_resources", runID) + js.Template.Spec.Containers[0].Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2M"), + }, + } + return js + }(), + }, + } + + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + name := GetRunResourceName(test.run) + jobSpec := getRunJobSpec(name, test.run.JobSpec) + job, err := reconciler.GenerateRunJob(test.run, jobSpec, test.name, namespace) + assert.Nil(t, err) + assert.NotNil(t, job) + assert.Equal(t, test.expectedJobSpec, job.Spec) + }) + } +} + +func genDefaultJobSpec(namespace, name, runID string) batchv1.JobSpec { + return batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{podDefaultContainerAnnotation: DefaultJobContainer}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: defaultName, + Image: "ghcr.io/pluralsh/harness:0.4.29-terraform-1.8.2", + WorkingDir: "", + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("stack-%s", runID), + }, + }, + }, + }, + Env: make([]corev1.EnvVar, 0), + Resources: corev1.ResourceRequirements{}, + VolumeMounts: ensureDefaultVolumeMounts(nil), + TerminationMessagePath: "", + TerminationMessagePolicy: "", + ImagePullPolicy: "", + SecurityContext: ensureDefaultContainerSecurityContext(nil), + Stdin: false, + StdinOnce: false, + TTY: false, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + Volumes: ensureDefaultVolumes(nil), + SecurityContext: ensureDefaultPodSecurityContext(nil), + }, + }, + TTLSecondsAfterFinished: lo.ToPtr(int32(60 * 60)), + BackoffLimit: lo.ToPtr(int32(0)), + } + +}