From 154a5e05db88af2838b0db6beb864409586ad7dd Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Tue, 22 Apr 2025 14:33:10 +0200 Subject: [PATCH 1/2] Pass opentelemetry environment variables to v1alpha ETOS The Suite Runner job will get its environment variables automatically from the deployed ETOS cluster whereas for the environment requests we check if we have a cluster to fetch and get it from there when possible. I also make sure that we pass in the traceparents that can be set in the annotations field of our testruns and environmentrequests so that the opentelemetry trace graph is complete. This change requires an update of the ETOS API as well as the environment provider to be considered complete. --- internal/controller/environment_controller.go | 48 +++++++++++++------ .../environmentrequest_controller.go | 46 ++++++++++++++---- internal/controller/testrun_controller.go | 17 ++++++- 3 files changed, 86 insertions(+), 25 deletions(-) diff --git a/internal/controller/environment_controller.go b/internal/controller/environment_controller.go index 17759df1..983683d2 100644 --- a/internal/controller/environment_controller.go +++ b/internal/controller/environment_controller.go @@ -261,6 +261,39 @@ func (r EnvironmentReconciler) releaseJob(environment *etosv1alpha1.Environment, } clusterName = cluster.Name } + traceparent, ok := environmentRequest.Annotations["etos.eiffel-community.github.io/traceparent"] + if !ok { + traceparent = "" + } + + envList := []corev1.EnvVar{ + { + Name: "REQUEST", + Value: environmentRequest.Name, + }, + { + Name: "ENVIRONMENT", + Value: environment.Name, + }, + { + Name: "ETOS_ETCD_HOST", + Value: databaseHost, + }, + { + Name: "OTEL_CONTEXT", + Value: traceparent, + }, + } + if cluster != nil && cluster.Spec.OpenTelemetry.Enabled { + envList = append(envList, corev1.EnvVar{ + Name: "OTEL_EXPORTER_OTLP_ENDPOINT", + Value: cluster.Spec.OpenTelemetry.Endpoint, + }) + envList = append(envList, corev1.EnvVar{ + Name: "OTEL_EXPORTER_OTLP_INSECURE", + Value: cluster.Spec.OpenTelemetry.Insecure, + }) + } return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -303,20 +336,7 @@ func (r EnvironmentReconciler) releaseJob(environment *etosv1alpha1.Environment, }, Command: []string{"python", "-u", "-m", "environment_provider.environment"}, Args: []string{environment.Name}, - Env: []corev1.EnvVar{ - { - Name: "REQUEST", - Value: environmentRequest.Name, - }, - { - Name: "ENVIRONMENT", - Value: environment.Name, - }, - { - Name: "ETOS_ETCD_HOST", - Value: databaseHost, - }, - }, + Env: envList, }, }, }, diff --git a/internal/controller/environmentrequest_controller.go b/internal/controller/environmentrequest_controller.go index 743954a8..c82c7322 100644 --- a/internal/controller/environmentrequest_controller.go +++ b/internal/controller/environmentrequest_controller.go @@ -264,7 +264,7 @@ func (r *EnvironmentRequestReconciler) reconcileEnvironmentProvider(ctx context. } // envVarListFrom creates a list of EnvVar key-value pairs from an EnvironmentRequest instance -func (r EnvironmentRequestReconciler) envVarListFrom(ctx context.Context, environmentrequest *etosv1alpha1.EnvironmentRequest) ([]corev1.EnvVar, error) { +func (r EnvironmentRequestReconciler) envVarListFrom(ctx context.Context, environmentrequest *etosv1alpha1.EnvironmentRequest, cluster *etosv1alpha1.Cluster) ([]corev1.EnvVar, error) { etosEncryptionKey, err := environmentrequest.Spec.Config.EncryptionKey.Get(ctx, r.Client, environmentrequest.Namespace) if err != nil { return nil, err @@ -277,6 +277,10 @@ func (r EnvironmentRequestReconciler) envVarListFrom(ctx context.Context, enviro if err != nil { return nil, err } + traceparent, ok := environmentrequest.Annotations["etos.eiffel-community.github.io/traceparent"] + if !ok { + traceparent = "" + } envList := []corev1.EnvVar{ { Name: "REQUEST", @@ -410,6 +414,20 @@ func (r EnvironmentRequestReconciler) envVarListFrom(ctx context.Context, enviro Name: "ETOS_ROUTING_KEY_TAG", Value: environmentrequest.Spec.Config.RoutingKeyTag, }, + { + Name: "OTEL_CONTEXT", + Value: traceparent, + }, + } + if cluster != nil && cluster.Spec.OpenTelemetry.Enabled { + envList = append(envList, corev1.EnvVar{ + Name: "OTEL_EXPORTER_OTLP_ENDPOINT", + Value: cluster.Spec.OpenTelemetry.Endpoint, + }) + envList = append(envList, corev1.EnvVar{ + Name: "OTEL_EXPORTER_OTLP_INSECURE", + Value: cluster.Spec.OpenTelemetry.Insecure, + }) } return envList, nil } @@ -473,19 +491,29 @@ func (r EnvironmentRequestReconciler) environmentProviderJob(ctx context.Context grace := int64(30) backoff := int32(0) - envVarList, err := r.envVarListFrom(ctx, environmentrequest) - if err != nil { - logger.Error(err, "Failed to create environment variable list for environment provider") - return nil, err - } - labels := map[string]string{ "etos.eiffel-community.github.io/id": environmentrequest.Spec.Identifier, // TODO: omitempty "app.kubernetes.io/name": "environment-provider", "app.kubernetes.io/part-of": "etos", } - if cluster := environmentrequest.Labels["etos.eiffel-community.github.io/cluster"]; cluster != "" { - labels["etos.eiffel-community.github.io/cluster"] = cluster + + var cluster *etosv1alpha1.Cluster + if clusterName := environmentrequest.Labels["etos.eiffel-community.github.io/cluster"]; clusterName != "" { + labels["etos.eiffel-community.github.io/cluster"] = clusterName + clusterNamespacedName := types.NamespacedName{ + Name: clusterName, + Namespace: environmentrequest.Namespace, + } + if err := r.Get(ctx, clusterNamespacedName, cluster); err != nil { + logger.Info("Failed to get cluster resource!") + return nil, err + } + } + + envVarList, err := r.envVarListFrom(ctx, environmentrequest, cluster) + if err != nil { + logger.Error(err, "Failed to create environment variable list for environment provider") + return nil, err } return &batchv1.Job{ diff --git a/internal/controller/testrun_controller.go b/internal/controller/testrun_controller.go index d9946b0a..e917a19a 100644 --- a/internal/controller/testrun_controller.go +++ b/internal/controller/testrun_controller.go @@ -59,7 +59,6 @@ type TestRunReconciler struct { client.Client Scheme *runtime.Scheme Clock - Cluster *etosv1alpha1.Cluster } /* @@ -455,6 +454,12 @@ func (r TestRunReconciler) environmentRequest(cluster *etosv1alpha1.Cluster, tes databasePort = "2379" } + annotations := make(map[string]string) + traceparent, ok := testrun.Annotations["etos.eiffel-community.github.io/traceparent"] + if ok { + annotations["etos.eiffel-community.github.io/traceparent"] = traceparent + } + return &etosv1alpha1.EnvironmentRequest{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -463,7 +468,7 @@ func (r TestRunReconciler) environmentRequest(cluster *etosv1alpha1.Cluster, tes "app.kubernetes.io/name": "suite-runner", "app.kubernetes.io/part-of": "etos", }, - Annotations: make(map[string]string), + Annotations: annotations, GenerateName: fmt.Sprintf("%s-", testrun.Name), Namespace: testrun.Namespace, }, @@ -516,6 +521,10 @@ func (r TestRunReconciler) environmentRequest(cluster *etosv1alpha1.Cluster, tes // suiteRunnerJob is the job definition for an etos suite runner. func (r TestRunReconciler) suiteRunnerJob(testrun *etosv1alpha1.TestRun) *batchv1.Job { + traceparent, ok := testrun.Annotations["etos.eiffel-community.github.io/traceparent"] + if !ok { + traceparent = "" + } grace := int64(30) backoff := int32(0) return &batchv1.Job{ @@ -654,6 +663,10 @@ func (r TestRunReconciler) suiteRunnerJob(testrun *etosv1alpha1.TestRun) *batchv Name: "IDENTIFIER", Value: testrun.Spec.ID, }, + { + Name: "OTEL_CONTEXT", + Value: traceparent, + }, { Name: "KUBEXIT_NAME", Value: "esr", From 42d77285687122777b277dd9ef1b4d5e6c693569 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Tue, 22 Apr 2025 14:52:55 +0200 Subject: [PATCH 2/2] Fix a bug with a nil cluster --- internal/controller/environmentrequest_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/environmentrequest_controller.go b/internal/controller/environmentrequest_controller.go index c82c7322..97c01952 100644 --- a/internal/controller/environmentrequest_controller.go +++ b/internal/controller/environmentrequest_controller.go @@ -504,6 +504,7 @@ func (r EnvironmentRequestReconciler) environmentProviderJob(ctx context.Context Name: clusterName, Namespace: environmentrequest.Namespace, } + cluster = &etosv1alpha1.Cluster{} if err := r.Get(ctx, clusterNamespacedName, cluster); err != nil { logger.Info("Failed to get cluster resource!") return nil, err