Chart: add OpenSearch remote logging options (apache#45082)

* add OpenSearch remote logging options
LefterisXefteris · Jan 5, 2025 · f96e0c7 · f96e0c7
1 parent 154318e
commit f96e0c7
Show file tree

Hide file tree

Showing 10 changed files with 659 additions and 51 deletions.
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
@@ -142,6 +142,15 @@ If release name contains chart name it will be used as a full name.
         key: connection
     {{- end }}
   {{- end }}
+  {{- if .Values.opensearch.enabled }}
+    {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__OPENSEARCH__HOST }}
+  - name: AIRFLOW__OPENSEARCH__HOST
+    valueFrom:
+      secretKeyRef:
+        name: {{ template "opensearch_secret" . }}
+        key: connection
+    {{- end }}
+  {{- end }}
 {{- end }}
 
 {{/* User defined Airflow environment variables */}}
@@ -427,6 +436,10 @@ If release name contains chart name it will be used as a full name.
   {{- default (printf "%s-elasticsearch" (include "airflow.fullname" .)) .Values.elasticsearch.secretName }}
 {{- end }}
 
+{{- define "opensearch_secret" -}}
+  {{- default (printf "%s-opensearch" (include "airflow.fullname" .)) .Values.opensearch.secretName }}
+{{- end }}
+
 {{- define "flower_secret" -}}
   {{- default (printf "%s-flower" (include "airflow.fullname" .)) .Values.flower.secretName }}
 {{- end }}

diff --git a/chart/templates/check-values.yaml b/chart/templates/check-values.yaml
@@ -62,6 +62,10 @@ The sole purpose of this yaml file is it to check the values file is consistent
 
   {{- end }}
 
+  {{- if and .Values.elasticsearch.enabled .Values.opensearch.enabled  }}
+    {{ required "You must not set both values elasticsearch.enabled and opensearch.enabled" nil }}
+  {{- end }}
+
   {{- if .Values.elasticsearch.enabled  }}
     {{- if and .Values.elasticsearch.secretName .Values.elasticsearch.connection }}
       {{ required "You must not set both values elasticsearch.secretName and elasticsearch.connection" nil }}
@@ -72,3 +76,14 @@ The sole purpose of this yaml file is it to check the values file is consistent
     {{- end }}
 
   {{- end }}
+
+  {{- if .Values.opensearch.enabled  }}
+    {{- if and .Values.opensearch.secretName .Values.opensearch.connection }}
+      {{ required "You must not set both values opensearch.secretName and opensearch.connection" nil }}
+    {{- end }}
+
+    {{- if not (or .Values.opensearch.secretName .Values.opensearch.connection) }}
+      {{ required "You must set one of the values opensearch.secretName or opensearch.connection when using OpenSearch" nil }}
+    {{- end }}
+
+  {{- end }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -30,8 +30,8 @@
 {{- $stateful := and $local $persistence }}
 # We can skip DAGs mounts on scheduler if dagProcessor is enabled, except with $local mode
 {{- $localOrDagProcessorDisabled := or (not .Values.dagProcessor.enabled) $local }}
-# If we're using elasticsearch logging
-{{- $elasticsearch := .Values.elasticsearch.enabled }}
+# If we're using elasticsearch or opensearch logging
+{{- $remoteLogging := or .Values.elasticsearch.enabled .Values.opensearch.enabled }}
 {{- $nodeSelector := or .Values.scheduler.nodeSelector .Values.nodeSelector }}
 {{- $affinity := or .Values.scheduler.affinity .Values.affinity }}
 {{- $tolerations := or .Values.scheduler.tolerations .Values.tolerations }}
@@ -217,8 +217,8 @@ spec:
                 {{- else }}
                   {{- include "scheduler_startup_check_command" . | indent 14 }}
                 {{- end }}
-          {{- if and $local (not $elasticsearch) }}
-          # Serve logs if we're in local mode and we don't have elasticsearch enabled.
+          {{- if and $local (not $remoteLogging) }}
+          # Serve logs if we're in local mode and we have neither elasticsearch nor opensearch enabled.
           ports:
             - name: worker-logs
               containerPort: {{ .Values.ports.workerLogs }}

diff --git a/chart/templates/secrets/opensearch-secret.yaml b/chart/templates/secrets/opensearch-secret.yaml
@@ -0,0 +1,44 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## OpenSearch Secret
+#################################
+{{- if (and .Values.opensearch.enabled (not .Values.opensearch.secretName)) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "airflow.fullname" . }}-opensearch
+  labels:
+    release: {{ .Release.Name }}
+    chart: {{ .Chart.Name }}
+    heritage: {{ .Release.Service }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+type: Opaque
+data:
+  {{- with .Values.opensearch.connection }}
+    {{- if and .user .pass }}
+      connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery)) "host" (printf "%s:%s" .host ((default 9200 .port) | toString) ) ) | b64enc | quote }}
+    {{- else }}
+    connection: {{ urlJoin (dict "scheme" (default "http" .scheme) "host" (printf "%s:%s" .host ((default 9200 .port) | toString))) | b64enc | quote }}
+    {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
@@ -1093,6 +1093,11 @@
                     "description": "Enable ``AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST`` variable to be read from the Elasticsearch Host Secret - Airflow <1.10.4 variant",
                     "type": "boolean",
                     "default": true
+                },
+                "AIRFLOW__OPENSEARCH__HOST": {
+                    "description": "Enable ``AIRFLOW__OPENSEARCH__HOST`` variable to be read from the OpenSearch Host Secret",
+                    "type": "boolean",
+                    "default": true
                 }
             }
         },
@@ -8044,6 +8049,69 @@
                 }
             }
         },
+        "opensearch": {
+            "description": "OpenSearch logging configuration.",
+            "type": "object",
+            "x-docsSection": "Airflow",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "description": "Enable OpenSearch task logging.",
+                    "type": "boolean",
+                    "default": false
+                },
+                "secretName": {
+                    "description": "A secret containing the connection string.",
+                    "type": [
+                        "string",
+                        "null"
+                    ],
+                    "default": null
+                },
+                "connection": {
+                    "description": "OpenSearch connection configuration.",
+                    "type": "object",
+                    "default": {},
+                    "additionalProperties": false,
+                    "properties": {
+                        "scheme": {
+                            "description": "Scheme",
+                            "type": "string",
+                            "default": "http"
+                        },
+                        "user": {
+                            "description": "Username",
+                            "type": "string",
+                            "default": ""
+                        },
+                        "pass": {
+                            "description": "Password",
+                            "type": "string",
+                            "default": ""
+                        },
+                        "host": {
+                            "description": "Host",
+                            "type": "string",
+                            "default": ""
+                        },
+                        "port": {
+                            "description": "Port",
+                            "type": "number",
+                            "default": 80
+                        }
+                    },
+                    "examples": [
+                        {
+                            "scheme": "https",
+                            "user": "...",
+                            "pass": "...",
+                            "host": "...",
+                            "port": "..."
+                        }
+                    ]
+                }
+            }
+        },
         "ports": {
             "description": "All ports used by chart.",
             "type": "object",

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -366,6 +366,7 @@ enableBuiltInSecretEnvVars:
   AIRFLOW__CELERY__BROKER_URL: true
   AIRFLOW__ELASTICSEARCH__HOST: true
   AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST: true
+  AIRFLOW__OPENSEARCH__HOST: true
 
 # Priority Classes that will be installed by charts.
 # Ideally, there should be an entry for dagProcessor, flower,
@@ -2473,6 +2474,22 @@ elasticsearch:
   #   port: ~
   connection: {}
 
+# OpenSearch logging configuration
+opensearch:
+  # Enable opensearch task logging
+  enabled: false
+  # A secret containing the connection
+  secretName: ~
+  # Or an object representing the connection
+  # Example:
+  # connection:
+  #   scheme: ~
+  #   user: ~
+  #   pass: ~
+  #   host: ~
+  #   port: ~
+  connection: {}
+
 # All ports used by chart
 ports:
   flowerUI: 5555
@@ -2591,9 +2608,9 @@ config:
     executor: '{{ .Values.executor }}'
     # For Airflow 1.10, backward compatibility; moved to [logging] in 2.0
     colored_console_log: 'False'
-    remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
+    remote_logging: '{{- ternary "True" "False" (or .Values.elasticsearch.enabled .Values.opensearch.enabled) }}'
   logging:
-    remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}'
+    remote_logging: '{{- ternary "True" "False" (or .Values.elasticsearch.enabled .Values.opensearch.enabled) }}'
     colored_console_log: 'False'
   metrics:
     statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'

diff --git a/helm_tests/airflow_aux/test_airflow_common.py b/helm_tests/airflow_aux/test_airflow_common.py
@@ -321,7 +321,10 @@ def test_should_disable_some_variables(self):
                     "AIRFLOW__CORE__SQL_ALCHEMY_CONN": False,
                     "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN": False,
                     "AIRFLOW__WEBSERVER__SECRET_KEY": False,
+                    # the following vars only appear if remote logging is set, so disabling them in this test is kind of a no-op
                     "AIRFLOW__ELASTICSEARCH__HOST": False,
+                    "AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST": False,
+                    "AIRFLOW__OPENSEARCH__HOST": False,
                 },
             },
             show_only=[