Skip to content

Commit

Permalink
feat: add resource requests into stack run jobs (#293)
Browse files Browse the repository at this point in the history 
* add resource requests into stack run jobs

* fix linter
  • Loading branch information
@zreigz
zreigz authored Oct 2, 2024
1 parent b4e2e9a commit 6d29f7d
Show file tree
Hide file tree
Showing 4 changed files with 243 additions and 6 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ require (
github.com/open-policy-agent/gatekeeper/v3 v3.17.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/pkg/errors v0.9.1
github.com/pluralsh/console/go/client v1.17.0
github.com/pluralsh/console/go/client v1.19.0
github.com/pluralsh/controller-reconcile-helper v0.1.0
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34
github.com/pluralsh/polly v0.1.10
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,8 +666,8 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pluralsh/console/go/client v1.17.0 h1:ExP+HUWa94e8cIbFY+6ARBq33oC87xpxTVw3+eZS7+I=
github.com/pluralsh/console/go/client v1.17.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
github.com/pluralsh/console/go/client v1.19.0 h1:x5JQWYtxr5S/PVUJS4Jiiir2IYAqMruONXydu9NSliM=
github.com/pluralsh/console/go/client v1.19.0/go.mod h1:lpoWASYsM9keNePS3dpFiEisUHEfObIVlSL3tzpKn8k=
github.com/pluralsh/controller-reconcile-helper v0.1.0 h1:BV3dYZFH5rn8ZvZjtpkACSv/GmLEtRftNQj/Y4ddHEo=
github.com/pluralsh/controller-reconcile-helper v0.1.0/go.mod h1:RxAbvSB4/jkvx616krCdNQXPbpGJXW3J1L3rASxeFOA=
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 h1:ab2PN+6if/Aq3/sJM0AVdy1SYuMAnq4g20VaKhTm/Bw=
Expand Down
69 changes: 66 additions & 3 deletions pkg/controller/stacks/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"os"
"strings"

"k8s.io/apimachinery/pkg/api/resource"

console "github.com/pluralsh/console/go/client"
"github.com/pluralsh/deployment-operator/internal/metrics"
"github.com/pluralsh/deployment-operator/internal/utils"
Expand Down Expand Up @@ -94,7 +96,10 @@ func (r *StackReconciler) reconcileRunJob(ctx context.Context, run *console.Stac
return nil, err
}

job := r.GenerateRunJob(run, jobSpec, name, namespace)
job, err := r.GenerateRunJob(run, jobSpec, name, namespace)
if err != nil {
return nil, err
}
logger.V(2).Info("creating job for stack run", "id", run.ID, "namespace", job.Namespace, "name", job.Name)
if err := r.k8sClient.Create(ctx, job); err != nil {
logger.Error(err, "unable to create job")
Expand Down Expand Up @@ -141,7 +146,8 @@ func (r *StackReconciler) GetRunResourceNamespace(jobSpec *batchv1.JobSpec) (nam
return
}

func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec *batchv1.JobSpec, name, namespace string) *batchv1.Job {
func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec *batchv1.JobSpec, name, namespace string) (*batchv1.Job, error) {
var err error
// If user-defined job spec was not available initialize it here.
if jobSpec == nil {
jobSpec = &batchv1.JobSpec{}
Expand All @@ -163,6 +169,11 @@ func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec

jobSpec.Template.Spec.Containers = r.ensureDefaultContainer(jobSpec.Template.Spec.Containers, run)

jobSpec.Template.Spec.Containers, err = r.ensureDefaultContainerResourcesRequests(jobSpec.Template.Spec.Containers, run)
if err != nil {
return nil, err
}

jobSpec.Template.Spec.Volumes = ensureDefaultVolumes(jobSpec.Template.Spec.Volumes)

jobSpec.Template.Spec.SecurityContext = ensureDefaultPodSecurityContext(jobSpec.Template.Spec.SecurityContext)
Expand All @@ -175,7 +186,7 @@ func (r *StackReconciler) GenerateRunJob(run *console.StackRunFragment, jobSpec
Labels: map[string]string{jobSelector: name},
},
Spec: *jobSpec,
}
}, nil
}

func getRunJobSpec(name string, jobSpecFragment *console.JobSpecFragment) *batchv1.JobSpec {
Expand Down Expand Up @@ -378,3 +389,55 @@ func ensureDefaultContainerSecurityContext(sc *corev1.SecurityContext) *corev1.S
RunAsGroup: lo.ToPtr(nonRootGID),
}
}

func (r *StackReconciler) ensureDefaultContainerResourcesRequests(containers []corev1.Container, run *console.StackRunFragment) ([]corev1.Container, error) {
if run.JobSpec == nil || run.JobSpec.Requests == nil {
return containers, nil
}
if run.JobSpec.Requests.Requests == nil && run.JobSpec.Requests.Limits == nil {
return containers, nil
}

for i, container := range containers {
if run.JobSpec.Requests.Requests != nil {
if len(container.Resources.Requests) == 0 {
containers[i].Resources.Requests = map[corev1.ResourceName]resource.Quantity{}
}
if run.JobSpec.Requests.Requests.CPU != nil {
cpu, err := resource.ParseQuantity(*run.JobSpec.Requests.Requests.CPU)
if err != nil {
return nil, err
}
containers[i].Resources.Requests[corev1.ResourceCPU] = cpu
}
if run.JobSpec.Requests.Requests.Memory != nil {
memory, err := resource.ParseQuantity(*run.JobSpec.Requests.Requests.Memory)
if err != nil {
return nil, err
}
containers[i].Resources.Requests[corev1.ResourceMemory] = memory
}
}
if run.JobSpec.Requests.Limits != nil {
if len(container.Resources.Limits) == 0 {
containers[i].Resources.Limits = map[corev1.ResourceName]resource.Quantity{}
}
if run.JobSpec.Requests.Limits.CPU != nil {
cpu, err := resource.ParseQuantity(*run.JobSpec.Requests.Limits.CPU)
if err != nil {
return nil, err
}
containers[i].Resources.Limits[corev1.ResourceCPU] = cpu
}
if run.JobSpec.Requests.Limits.Memory != nil {
memory, err := resource.ParseQuantity(*run.JobSpec.Requests.Limits.Memory)
if err != nil {
return nil, err
}
containers[i].Resources.Limits[corev1.ResourceMemory] = memory
}
}
}

return containers, nil
}
174 changes: 174 additions & 0 deletions pkg/controller/stacks/job_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
package stacks

import (
"fmt"
"testing"
"time"

batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

console "github.com/pluralsh/console/go/client"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
Expand All @@ -13,6 +19,8 @@ import (
"github.com/pluralsh/deployment-operator/pkg/test/mocks"
)

const defaultName = "default"

func TestGetDefaultContainerImage(t *testing.T) {
var kClient client.Client
fakeConsoleClient := mocks.NewClientMock(t)
Expand Down Expand Up @@ -103,3 +111,169 @@ func TestGetDefaultContainerImage(t *testing.T) {
})
}
}

func TestGenerateRunJob(t *testing.T) {
var kClient client.Client
fakeConsoleClient := mocks.NewClientMock(t)
namespace := defaultName
runID := "1"
reconciler := NewStackReconciler(fakeConsoleClient, kClient, scheme.Scheme, time.Minute, 0, namespace, "", "")
cases := []struct {
name string
run *console.StackRunFragment
expectedJobSpec batchv1.JobSpec
}{
{
name: "use_empty_job_spec",
run: &console.StackRunFragment{
ID: runID,
Type: console.StackTypeTerraform,
Configuration: console.StackConfigurationFragment{},
},
expectedJobSpec: func() batchv1.JobSpec {
js := genDefaultJobSpec(namespace, "use_empty_job_spec", runID)
js.Template.Labels = nil
return js
}(),
},
{
name: "use_defaults",
run: &console.StackRunFragment{
ID: runID,
Type: console.StackTypeTerraform,
Configuration: console.StackConfigurationFragment{},
JobSpec: &console.JobSpecFragment{
Namespace: namespace,
},
},
expectedJobSpec: genDefaultJobSpec(namespace, "use_defaults", runID),
},
{
name: "add_labels",
run: &console.StackRunFragment{
ID: runID,
Type: console.StackTypeTerraform,
Configuration: console.StackConfigurationFragment{},
JobSpec: &console.JobSpecFragment{
Namespace: namespace,
Labels: map[string]interface{}{
"test": "test",
},
},
},
expectedJobSpec: func() batchv1.JobSpec {
js := genDefaultJobSpec(namespace, "add_labels", runID)
js.Template.Labels = map[string]string{
"test": "test",
}
return js
}(),
},
{
name: "add_sa",
run: &console.StackRunFragment{
ID: runID,
Type: console.StackTypeTerraform,
Configuration: console.StackConfigurationFragment{},
JobSpec: &console.JobSpecFragment{
Namespace: namespace,
ServiceAccount: lo.ToPtr(defaultName),
},
},
expectedJobSpec: func() batchv1.JobSpec {
js := genDefaultJobSpec(namespace, "add_sa", runID)
js.Template.Spec.ServiceAccountName = defaultName
return js
}(),
},
{
name: "add_resources",
run: &console.StackRunFragment{
ID: runID,
Type: console.StackTypeTerraform,
Configuration: console.StackConfigurationFragment{},
JobSpec: &console.JobSpecFragment{
Namespace: namespace,
Requests: &console.ContainerResourcesFragment{
Requests: &console.ResourceRequestFragment{
CPU: lo.ToPtr("2Mi"),
},
Limits: &console.ResourceRequestFragment{
Memory: lo.ToPtr("2M"),
},
},
},
},
expectedJobSpec: func() batchv1.JobSpec {
js := genDefaultJobSpec(namespace, "add_resources", runID)
js.Template.Spec.Containers[0].Resources = corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("2M"),
},
}
return js
}(),
},
}

for _, test := range cases {
t.Run(test.name, func(t *testing.T) {
name := GetRunResourceName(test.run)
jobSpec := getRunJobSpec(name, test.run.JobSpec)
job, err := reconciler.GenerateRunJob(test.run, jobSpec, test.name, namespace)
assert.Nil(t, err)
assert.NotNil(t, job)
assert.Equal(t, test.expectedJobSpec, job.Spec)
})
}
}

func genDefaultJobSpec(namespace, name, runID string) batchv1.JobSpec {
return batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{},
Annotations: map[string]string{podDefaultContainerAnnotation: DefaultJobContainer},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: defaultName,
Image: "ghcr.io/pluralsh/harness:0.4.29-terraform-1.8.2",
WorkingDir: "",
EnvFrom: []corev1.EnvFromSource{
{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: fmt.Sprintf("stack-%s", runID),
},
},
},
},
Env: make([]corev1.EnvVar, 0),
Resources: corev1.ResourceRequirements{},
VolumeMounts: ensureDefaultVolumeMounts(nil),
TerminationMessagePath: "",
TerminationMessagePolicy: "",
ImagePullPolicy: "",
SecurityContext: ensureDefaultContainerSecurityContext(nil),
Stdin: false,
StdinOnce: false,
TTY: false,
},
},
RestartPolicy: corev1.RestartPolicyNever,
Volumes: ensureDefaultVolumes(nil),
SecurityContext: ensureDefaultPodSecurityContext(nil),
},
},
TTLSecondsAfterFinished: lo.ToPtr(int32(60 * 60)),
BackoffLimit: lo.ToPtr(int32(0)),
}

}

0 comments on commit 6d29f7d

Please sign in to comment.