Skip to content

Commit

Permalink
test(e2e): end-to-end test for updating instrumentation at startup
Browse files Browse the repository at this point in the history
  • Loading branch information
@basti1302
basti1302 committed Jul 5, 2024
1 parent a8baac8 commit b533608
Show file tree
Hide file tree
Showing 6 changed files with 219 additions and 47 deletions.
2 changes: 1 addition & 1 deletion cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ func startOperatorManager(
}
setupLog.Info("Dash0 reconciler has been set up.")

if os.Getenv("ENABLE_WEBHOOKS") != "false" {
if os.Getenv("ENABLE_WEBHOOK") != "false" {
if err = (&dash0webhook.Handler{
Client: mgr.GetClient(),
Recorder: mgr.GetEventRecorderFor("dash0-webhook"),
Expand Down
4 changes: 4 additions & 0 deletions helm-chart/dash0-operator/templates/operator/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ spec:
- name: DASH0_INIT_CONTAINER_IMAGE_PULL_POLICY
value: {{ .Values.operator.initContainerImage.pullPolicy }}
{{- end }}
{{- if not .Values.operator.enableWebhook }}
- name: ENABLE_WEBHOOK
value: "false"
{{- end }}
{{- if .Values.operator.developmentMode }}
- name: DASH0_DEVELOPMENT_MODE
value: {{ .Values.operator.developmentMode | toString | quote }}
Expand Down
14 changes: 14 additions & 0 deletions helm-chart/dash0-operator/tests/operator/deployment_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,15 @@ tests:
imagePullSecrets:
- name: regcred
- name: anotherSecret
enableWebhook: false
managerPodResources:
limits:
cpu: 123m
memory: 456Mi
requests:
cpu: 5m
memory: 32Mi
developmentMode: true
asserts:
- equal:
path: metadata.labels['label1']
Expand Down Expand Up @@ -125,6 +127,18 @@ tests:
- equal:
path: spec.template.spec.containers[0].env[3].value
value: Always
- equal:
path: spec.template.spec.containers[0].env[4].name
value: ENABLE_WEBHOOK
- equal:
path: spec.template.spec.containers[0].env[4].value
value: "false"
- equal:
path: spec.template.spec.containers[0].env[5].name
value: DASH0_DEVELOPMENT_MODE
- equal:
path: spec.template.spec.containers[0].env[5].value
value: "true"
- equal:
path: spec.template.spec.containers[0].resources.limits.cpu
value: 123m
Expand Down
3 changes: 3 additions & 0 deletions helm-chart/dash0-operator/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ operator:
# label2: "value 2"
podLabels: {}

# Set this to "false" to disable the admission webhook to instrument new workloads at deploy time.
enableWebhook: true

# resources for the controller manager pod(s)
managerPodResources:
limits:
Expand Down
176 changes: 130 additions & 46 deletions test/e2e/e2e_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
verifyTelemetryTimeout = 90 * time.Second
verifyTelemetryPollingInterval = 500 * time.Millisecond
dash0CustomResourceName = "dash0-sample"
additionalImageTag = "e2e-test"
)

var (
Expand Down Expand Up @@ -305,6 +306,19 @@ func RebuildOperatorControllerImage(operatorImage ImageSpec, buildImageLocally b
fmt.Sprintf("IMG_REPOSITORY=%s", operatorImage.repository),
fmt.Sprintf("IMG_TAG=%s", operatorImage.tag),
))).To(Succeed())

additionalTag := ImageSpec{
repository: operatorImage.repository,
tag: additionalImageTag,
}
Expect(
RunAndIgnoreOutput(
exec.Command(
"docker",
"tag",
renderFullyQualifiedImageName(operatorImage),
renderFullyQualifiedImageName(additionalTag),
))).To(Succeed())
}

func RebuildDash0InstrumentationImage(instrumentationImage ImageSpec, buildImageLocally bool) {
Expand All @@ -328,13 +342,27 @@ func RebuildDash0InstrumentationImage(instrumentationImage ImageSpec, buildImage
instrumentationImage.repository,
instrumentationImage.tag,
))).To(Succeed())

additionalTag := ImageSpec{
repository: instrumentationImage.repository,
tag: additionalImageTag,
}
Expect(
RunAndIgnoreOutput(
exec.Command(
"docker",
"tag",
renderFullyQualifiedImageName(instrumentationImage),
renderFullyQualifiedImageName(additionalTag),
))).To(Succeed())
}

func DeployOperatorWithCollectorAndClearExportedTelemetry(
operatorNamespace string,
operatorHelmChart string,
operatorHelmChartUrl string,
images Images,
enableWebhook bool,
) {
By("removing old captured telemetry files")
_ = os.Remove("test-resources/e2e-test-volumes/collector-received-data/traces.jsonl")
Expand All @@ -344,7 +372,7 @@ func DeployOperatorWithCollectorAndClearExportedTelemetry(
ensureOtelCollectorHelmRepoIsInstalled()
ensureDash0OperatorHelmRepoIsInstalled(operatorHelmChart, operatorHelmChartUrl)

By("deploying the controller-manager")
By("deploying the operator controller")
arguments := []string{
"install",
"--namespace",
Expand All @@ -355,56 +383,15 @@ func DeployOperatorWithCollectorAndClearExportedTelemetry(
"--set", "operator.developmentMode=true",
"--set", "operator.disableSecretCheck=true",
"--set", "operator.disableOtlpEndpointCheck=true",
"--set", fmt.Sprintf("operator.enableWebhook=%t", enableWebhook),
}
arguments = setIfNotEmpty(arguments, "operator.image.repository", images.operator.repository)
arguments = setIfNotEmpty(arguments, "operator.image.tag", images.operator.tag)
arguments = setIfNotEmpty(arguments, "operator.image.digest", images.operator.digest)
arguments = setIfNotEmpty(arguments, "operator.image.pullPolicy", images.operator.pullPolicy)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.repository", images.instrumentation.repository)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.tag", images.instrumentation.tag)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.digest", images.instrumentation.digest)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.pullPolicy", images.instrumentation.pullPolicy)
arguments = append(arguments, operatorHelmReleaseName)
arguments = append(arguments, operatorHelmChart)
arguments = addOptionalHelmParameters(arguments, operatorHelmChart, images)

output, err := Run(exec.Command("helm", arguments...))
Expect(err).NotTo(HaveOccurred())
fmt.Fprintf(GinkgoWriter, "output of helm install:\n%s", output)

var controllerPodName string
By("validating that the controller-manager pod is running as expected")
verifyControllerUp := func() error {
cmd := exec.Command("kubectl", "get",
"pods", "-l", "control-plane=controller-manager",
"-o", "go-template={{ range .items }}"+
"{{ if not .metadata.deletionTimestamp }}"+
"{{ .metadata.name }}"+
"{{ \"\\n\" }}{{ end }}{{ end }}",
"-n", operatorNamespace,
)

podOutput, err := Run(cmd, false)
Expect(err).NotTo(HaveOccurred())
podNames := GetNonEmptyLines(podOutput)
if len(podNames) != 1 {
return fmt.Errorf("expect 1 controller pods running, but got %d -- %s", len(podNames), podOutput)
}
controllerPodName = podNames[0]
Expect(controllerPodName).To(ContainSubstring("controller-manager"))

cmd = exec.Command("kubectl", "get",
"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
"-n", operatorNamespace,
)
status, err := Run(cmd)
Expect(err).NotTo(HaveOccurred())
if status != "Running" {
return fmt.Errorf("controller pod in %s status", status)
}
return nil
}

Eventually(verifyControllerUp, 120*time.Second, time.Second).Should(Succeed())
verifyThatControllerPodIsRunning(operatorNamespace)

// verify that the OTel collector is also up and running
Expect(RunAndIgnoreOutput(
Expand All @@ -420,6 +407,20 @@ func DeployOperatorWithCollectorAndClearExportedTelemetry(
))).To(Succeed())
}

func addOptionalHelmParameters(arguments []string, operatorHelmChart string, images Images) []string {
arguments = setIfNotEmpty(arguments, "operator.image.repository", images.operator.repository)
arguments = setIfNotEmpty(arguments, "operator.image.tag", images.operator.tag)
arguments = setIfNotEmpty(arguments, "operator.image.digest", images.operator.digest)
arguments = setIfNotEmpty(arguments, "operator.image.pullPolicy", images.operator.pullPolicy)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.repository", images.instrumentation.repository)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.tag", images.instrumentation.tag)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.digest", images.instrumentation.digest)
arguments = setIfNotEmpty(arguments, "operator.initContainerImage.pullPolicy", images.instrumentation.pullPolicy)
arguments = append(arguments, operatorHelmReleaseName)
arguments = append(arguments, operatorHelmChart)
return arguments
}

func setIfNotEmpty(arguments []string, key string, value string) []string {
if value != "" {
arguments = append(arguments, "--set")
Expand Down Expand Up @@ -496,8 +497,45 @@ func ensureDash0OperatorHelmRepoIsInstalled(operatorHelmChart string, operatorHe
}
}

func verifyThatControllerPodIsRunning(operatorNamespace string) {
var controllerPodName string
By("validating that the controller-manager pod is running as expected")
verifyControllerUp := func() error {
cmd := exec.Command("kubectl", "get",
"pods", "-l", "control-plane=controller-manager",
"-o", "go-template={{ range .items }}"+
"{{ if not .metadata.deletionTimestamp }}"+
"{{ .metadata.name }}"+
"{{ \"\\n\" }}{{ end }}{{ end }}",
"-n", operatorNamespace,
)

podOutput, err := Run(cmd, false)
Expect(err).NotTo(HaveOccurred())
podNames := GetNonEmptyLines(podOutput)
if len(podNames) != 1 {
return fmt.Errorf("expect 1 controller pods running, but got %d -- %s", len(podNames), podOutput)
}
controllerPodName = podNames[0]
Expect(controllerPodName).To(ContainSubstring("controller-manager"))

cmd = exec.Command("kubectl", "get",
"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
"-n", operatorNamespace,
)
status, err := Run(cmd)
Expect(err).NotTo(HaveOccurred())
if status != "Running" {
return fmt.Errorf("controller pod in %s status", status)
}
return nil
}

Eventually(verifyControllerUp, 120*time.Second, time.Second).Should(Succeed())
}

func UndeployOperatorAndCollector(operatorNamespace string) {
By("undeploying the controller-manager")
By("undeploying the operator controller")
Expect(
RunAndIgnoreOutput(
exec.Command(
Expand Down Expand Up @@ -535,6 +573,52 @@ func VerifyDash0OperatorReleaseIsNotInstalled(g Gomega, operatorNamespace string
))).To(Succeed())
}

func UpgradeOperator(
operatorNamespace string,
operatorHelmChart string,
operatorHelmChartUrl string,
images Images,
enableWebhook bool,
) {
ensureDash0OperatorHelmRepoIsInstalled(operatorHelmChart, operatorHelmChartUrl)

By("upgrading the operator controller")
arguments := []string{
"upgrade",
"--namespace",
operatorNamespace,
"--values",
"test-resources/helm/e2e.values.yaml",
"--set", "operator.developmentMode=true",
"--set", "operator.disableSecretCheck=true",
"--set", "operator.disableOtlpEndpointCheck=true",
"--set", fmt.Sprintf("operator.enableWebhook=%t", enableWebhook),
}
arguments = addOptionalHelmParameters(arguments, operatorHelmChart, images)

output, err := Run(exec.Command("helm", arguments...))
Expect(err).NotTo(HaveOccurred())
fmt.Fprintf(GinkgoWriter, "output of helm upgrade:\n%s", output)

By("waiting shortly, to give the operator time to restart after helm upgrade")
time.Sleep(5 * time.Second)

verifyThatControllerPodIsRunning(operatorNamespace)

// verify that the OTel collector is also up and running
Expect(RunAndIgnoreOutput(
exec.Command("kubectl",
"rollout",
"status",
"daemonset",
fmt.Sprintf("%s-opentelemetry-collector-agent", operatorHelmReleaseName),
"--namespace",
operatorNamespace,
"--timeout",
"60s",
))).To(Succeed())
}

func DeployDash0CustomResource(namespace string) {
Expect(
RunAndIgnoreOutput(exec.Command(
Expand Down
Loading

0 comments on commit b533608

Please sign in to comment.