feat: add embedded serving mode in frontend (#84)

Signed-off-by: arkbriar <[email protected]>
risingwavelabs · May 13, 2024 · 974728f · 974728f
1 parent 7e3dc1a
commit 974728f
Show file tree

Hide file tree

Showing 6 changed files with 153 additions and 4 deletions.
diff --git a/charts/risingwave/templates/compute-sts.yaml b/charts/risingwave/templates/compute-sts.yaml
@@ -123,6 +123,10 @@ spec:
         command:
         - /risingwave/bin/risingwave
         - compute-node
+        {{- if .Values.frontendComponent.embeddedServing }}
+        args:
+        - --role=streaming
+        {{- end }}
         {{- if .Values.computeComponent.autoDeregistration.enabled }}
         lifecycle:
           preStop:

diff --git a/charts/risingwave/templates/frontend-deploy.yaml b/charts/risingwave/templates/frontend-deploy.yaml
@@ -107,17 +107,47 @@ spec:
         command: {{ toYaml .Values.diagnosticMode.command | nindent 8 }}
         args: {{ toYaml .Values.diagnosticMode.args | nindent 8 }}
         {{- else }}
+        {{- if .Values.frontendComponent.embeddedServing }}
+        command:
+        - /risingwave/bin/risingwave
+        - standalone
+        args:
+        - >-
+          --compute-opts=--config-path /risingwave/config/risingwave.toml
+            --listen-addr 0.0.0.0:{{ .Values.ports.compute.svc }}
+            --advertise-addr $(POD_IP):{{ .Values.ports.compute.svc }}
+            --role serving
+        - >-
+          --frontend-opts=--config-path /risingwave/config/risingwave.toml
+            --listen-addr 0.0.0.0:{{ .Values.ports.frontend.svc }}
+            --advertise-addr $(POD_IP):{{ .Values.ports.frontend.svc }}
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - bash
+              - -c
+              - >-
+                /risingwave/bin/risingwave ctl meta unregister-workers --yes \
+                  --workers ${POD_IP}:{{ .Values.ports.compute.svc }}
+        {{- else }}
         command:
         - /risingwave/bin/risingwave
         - frontend-node
         {{- end }}
+        {{- end }}
         ports:
         - containerPort: {{ .Values.ports.frontend.svc }}
           name: svc
           protocol: TCP
         - containerPort: {{ .Values.ports.frontend.metrics }}
           name: metrics
           protocol: TCP
+        {{- if .Values.frontendComponent.embeddedServing }}
+        - containerPort: {{ .Values.ports.compute.svc }}
+          name: c-svc
+          protocol: TCP
+        {{- end }}
         envFrom:
         {{- if .Values.frontendComponent.extraEnvVarsConfigMap }}
         - configMapRef:
@@ -127,6 +157,13 @@ spec:
         - secretRef:
             name: {{ .Values.frontendComponent.extraEnvVarsSecret }}
         {{- end }}
+        {{- if .Values.frontendComponent.embeddedServing }}
+        {{- $credentialsSecret := (include "risingwave.credentialsSecretName" . ) -}}
+        {{- if $credentialsSecret }}
+        - secretRef:
+            name: {{ $credentialsSecret }}
+        {{- end }}
+        {{- end }}
         env:
         # Disable auto region loading. Refer to the original source for more information.
         # https://github.com/awslabs/aws-sdk-rust/blob/main/sdk/aws-config/src/imds/region.rs
@@ -172,6 +209,65 @@ spec:
         - name: RW_SSL_KEY
           value: /risingwave/certs/tls.key
         {{- end }}
+        {{- if .Values.frontendComponent.embeddedServing }}
+        {{- if .Values.stateStore.s3.enabled }}
+        - name: AWS_REGION
+          value: {{ .Values.stateStore.s3.region }}
+        {{- end }}
+        {{- if and .Values.stateStore.s3.enabled .Values.stateStore.s3.forcePathStyle }}
+        - name: RW_IS_FORCE_PATH_STYLE
+          value: 'true'
+        {{- end }}
+        {{- if .Values.stateStore.oss.enabled }}
+        - name: OSS_REGION
+          value: {{ .Values.stateStore.oss.region }}
+        {{- end }}
+        {{- if .Values.stateStore.obs.enabled }}
+        - name: OBS_REGION
+          value: {{ .Values.stateStore.obs.region }}
+        {{- end }}
+        {{- if (include "risingwave.bundle.minio.enabled" .) }}
+        - name: MINIO_USERNAME
+          valueFrom:
+            secretKeyRef:
+              key: root-user
+              name: {{ default (include "common.names.fullname" .Subcharts.minio) .Values.minio.auth.existingSecret }}
+        - name: MINIO_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              key: root-password
+              name: {{ default (include "common.names.fullname" .Subcharts.minio) .Values.minio.auth.existingSecret }}
+        {{- end }}
+        {{- if .Values.stateStore.s3.enabled }}
+        {{- if .Values.stateStore.s3.endpoint }}
+        - name: RW_S3_ENDPOINT
+          value: {{ .Values.stateStore.s3.endpoint }}
+        {{- end }}
+        {{- else if .Values.stateStore.oss.enabled }}
+        - name: OSS_ENDPOINT
+          value: {{ include "risingwave.oss.endpoint" . }}
+        {{- else if .Values.stateStore.obs.enabled }}
+        - name: OBS_ENDPOINT
+          value: {{ include "risingwave.obs.endpoint" . }}
+        {{- else if .Values.stateStore.azblob.enabled }}
+        - name: AZBLOB_ENDPOINT
+          value: {{ .Values.stateStore.azblob.endpoint }}
+        {{- end }}
+        - name: RW_STATE_STORE
+          value: {{ include "risingwave.hummockConnectionString" . }}
+        - name: RW_DATA_DIRECTORY
+          value: {{ include "risingwave.stateStoreDataDirectory" . }}
+        {{- if .Values.frontendComponent.resources.limits }}
+        {{- if .Values.frontendComponent.resources.limits.cpu }}
+        - name: RW_PARALLELISM
+          value: $(CONTAINER_CPU_LIMIT)
+        {{- end }}
+        {{- if .Values.frontendComponent.resources.limits.memory }}
+        - name: RW_TOTAL_MEMORY_BYTES
+          value: $(CONTAINER_MEMORY_LIMIT)
+        {{- end }}
+        {{- end }}
+        {{- end }}
         {{- if .Values.frontendComponent.extraEnvVars }}
         {{- .Values.frontendComponent.extraEnvVars | toYaml | nindent 8 }}
         {{- end }}

diff --git a/charts/risingwave/tests/compute_auto_deregistration_test.yaml b/charts/risingwave/tests/compute_auto_deregistration_test.yaml
@@ -28,4 +28,4 @@ tests:
       - -c
       - >-
         /risingwave/bin/risingwave ctl meta unregister-workers --yes \
-          --workers ${POD_NAME}.RELEASE-NAME-risingwave-compute-headless.NAMESPACE.svc:5688
+          --workers ${POD_NAME}.RELEASE-NAME-risingwave-compute-headless.NAMESPACE.svc:5688
diff --git a/charts/risingwave/tests/frontend_embedded_serving_test.yaml b/charts/risingwave/tests/frontend_embedded_serving_test.yaml
@@ -0,0 +1,39 @@
+suite: Test Embedded Frontend Serving
+templates:
+- frontend-deploy.yaml
+- compute-sts.yaml
+chart:
+  appVersion: 1.0.0
+  version: 0.0.1
+set:
+  frontendComponent:
+    embeddedServing: true
+tests:
+- it: command should be standalone
+  template: frontend-deploy.yaml
+  asserts:
+  - equal:
+      path: spec.template.spec.containers[0].command
+      value:
+      - /risingwave/bin/risingwave
+      - standalone
+- it: should have pre-stop lifecycle hook
+  template: frontend-deploy.yaml
+  asserts:
+  - exists:
+      path: spec.template.spec.containers[0].lifecycle.preStop
+  - equal:
+      path: spec.template.spec.containers[0].lifecycle.preStop.exec.command
+      value:
+      - bash
+      - -c
+      - >-
+        /risingwave/bin/risingwave ctl meta unregister-workers --yes \
+          --workers ${POD_IP}:5688
+- it: compute node should set role to streaming
+  template: compute-sts.yaml
+  asserts:
+  - contains:
+      path: spec.template.spec.containers[0].args
+      content:
+        --role=streaming
diff --git a/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml b/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml
@@ -1,15 +1,19 @@
-suite: Test state store env vars (meta/compute/compactor)
+suite: Test state store env vars (meta/frontend/compute/compactor)
 templates:
 - templates/meta-sts.yaml
 - templates/compute-sts.yaml
 - templates/compactor-deploy.yaml
+- templates/frontend-deploy.yaml
 set:
   metaComponent:
     extraEnvVarsSecret: dummy
   computeComponent:
     extraEnvVarsSecret: dummy
   compactorComponent:
     extraEnvVarsSecret: dummy
+  frontendComponent:
+    embeddedServing: true
+    extraEnvVarsSecret: dummy
 chart:
   appVersion: 1.0.0
   version: 0.0.1

diff --git a/charts/risingwave/values.yaml b/charts/risingwave/values.yaml
@@ -904,9 +904,9 @@ frontendComponent:
   affinity: { }
 
   ## @param frontendComponent.terminationGracePeriodSeconds Termination grace period in seconds.
-  ## Defaults to 1 seconds.
+  ## Defaults to 10 seconds.
   ##
-  terminationGracePeriodSeconds: 1
+  terminationGracePeriodSeconds: 10
 
   ## @param frontendComponent.schedulerName Name of an alternate scheduler.
   ## Ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
@@ -940,6 +940,12 @@ frontendComponent:
   ## Defaults to full.
   rustBacktrace: full
 
+  ## @param frontendComponent.embeddedServing Embed serving node in frontend.
+  ## If enabled, the compute nodes will be used for streaming only.
+  ## Defaults to false.
+  ##
+  embeddedServing: false
+
 computeComponent:
   ## @param computeComponent.podLabels Labels to add to the component pods.
   ##