From 974728fa5e9047ed046a9a297d35a49b28a09fb1 Mon Sep 17 00:00:00 2001 From: ArkBriar Date: Mon, 13 May 2024 17:30:19 +0800 Subject: [PATCH] feat: add embedded serving mode in frontend (#84) Signed-off-by: arkbriar --- charts/risingwave/templates/compute-sts.yaml | 4 + .../risingwave/templates/frontend-deploy.yaml | 96 +++++++++++++++++++ .../compute_auto_deregistration_test.yaml | 2 +- .../tests/frontend_embedded_serving_test.yaml | 39 ++++++++ .../cluster_workloads_env_test.yaml | 6 +- charts/risingwave/values.yaml | 10 +- 6 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 charts/risingwave/tests/frontend_embedded_serving_test.yaml diff --git a/charts/risingwave/templates/compute-sts.yaml b/charts/risingwave/templates/compute-sts.yaml index cd7ca7c..d4dc1a6 100644 --- a/charts/risingwave/templates/compute-sts.yaml +++ b/charts/risingwave/templates/compute-sts.yaml @@ -123,6 +123,10 @@ spec: command: - /risingwave/bin/risingwave - compute-node + {{- if .Values.frontendComponent.embeddedServing }} + args: + - --role=streaming + {{- end }} {{- if .Values.computeComponent.autoDeregistration.enabled }} lifecycle: preStop: diff --git a/charts/risingwave/templates/frontend-deploy.yaml b/charts/risingwave/templates/frontend-deploy.yaml index a9bf727..7f54247 100644 --- a/charts/risingwave/templates/frontend-deploy.yaml +++ b/charts/risingwave/templates/frontend-deploy.yaml @@ -107,10 +107,35 @@ spec: command: {{ toYaml .Values.diagnosticMode.command | nindent 8 }} args: {{ toYaml .Values.diagnosticMode.args | nindent 8 }} {{- else }} + {{- if .Values.frontendComponent.embeddedServing }} + command: + - /risingwave/bin/risingwave + - standalone + args: + - >- + --compute-opts=--config-path /risingwave/config/risingwave.toml + --listen-addr 0.0.0.0:{{ .Values.ports.compute.svc }} + --advertise-addr $(POD_IP):{{ .Values.ports.compute.svc }} + --role serving + - >- + --frontend-opts=--config-path /risingwave/config/risingwave.toml + --listen-addr 0.0.0.0:{{ .Values.ports.frontend.svc }} + --advertise-addr $(POD_IP):{{ .Values.ports.frontend.svc }} + lifecycle: + preStop: + exec: + command: + - bash + - -c + - >- + /risingwave/bin/risingwave ctl meta unregister-workers --yes \ + --workers ${POD_IP}:{{ .Values.ports.compute.svc }} + {{- else }} command: - /risingwave/bin/risingwave - frontend-node {{- end }} + {{- end }} ports: - containerPort: {{ .Values.ports.frontend.svc }} name: svc @@ -118,6 +143,11 @@ spec: - containerPort: {{ .Values.ports.frontend.metrics }} name: metrics protocol: TCP + {{- if .Values.frontendComponent.embeddedServing }} + - containerPort: {{ .Values.ports.compute.svc }} + name: c-svc + protocol: TCP + {{- end }} envFrom: {{- if .Values.frontendComponent.extraEnvVarsConfigMap }} - configMapRef: @@ -127,6 +157,13 @@ spec: - secretRef: name: {{ .Values.frontendComponent.extraEnvVarsSecret }} {{- end }} + {{- if .Values.frontendComponent.embeddedServing }} + {{- $credentialsSecret := (include "risingwave.credentialsSecretName" . ) -}} + {{- if $credentialsSecret }} + - secretRef: + name: {{ $credentialsSecret }} + {{- end }} + {{- end }} env: # Disable auto region loading. Refer to the original source for more information. # https://github.com/awslabs/aws-sdk-rust/blob/main/sdk/aws-config/src/imds/region.rs @@ -172,6 +209,65 @@ spec: - name: RW_SSL_KEY value: /risingwave/certs/tls.key {{- end }} + {{- if .Values.frontendComponent.embeddedServing }} + {{- if .Values.stateStore.s3.enabled }} + - name: AWS_REGION + value: {{ .Values.stateStore.s3.region }} + {{- end }} + {{- if and .Values.stateStore.s3.enabled .Values.stateStore.s3.forcePathStyle }} + - name: RW_IS_FORCE_PATH_STYLE + value: 'true' + {{- end }} + {{- if .Values.stateStore.oss.enabled }} + - name: OSS_REGION + value: {{ .Values.stateStore.oss.region }} + {{- end }} + {{- if .Values.stateStore.obs.enabled }} + - name: OBS_REGION + value: {{ .Values.stateStore.obs.region }} + {{- end }} + {{- if (include "risingwave.bundle.minio.enabled" .) }} + - name: MINIO_USERNAME + valueFrom: + secretKeyRef: + key: root-user + name: {{ default (include "common.names.fullname" .Subcharts.minio) .Values.minio.auth.existingSecret }} + - name: MINIO_PASSWORD + valueFrom: + secretKeyRef: + key: root-password + name: {{ default (include "common.names.fullname" .Subcharts.minio) .Values.minio.auth.existingSecret }} + {{- end }} + {{- if .Values.stateStore.s3.enabled }} + {{- if .Values.stateStore.s3.endpoint }} + - name: RW_S3_ENDPOINT + value: {{ .Values.stateStore.s3.endpoint }} + {{- end }} + {{- else if .Values.stateStore.oss.enabled }} + - name: OSS_ENDPOINT + value: {{ include "risingwave.oss.endpoint" . }} + {{- else if .Values.stateStore.obs.enabled }} + - name: OBS_ENDPOINT + value: {{ include "risingwave.obs.endpoint" . }} + {{- else if .Values.stateStore.azblob.enabled }} + - name: AZBLOB_ENDPOINT + value: {{ .Values.stateStore.azblob.endpoint }} + {{- end }} + - name: RW_STATE_STORE + value: {{ include "risingwave.hummockConnectionString" . }} + - name: RW_DATA_DIRECTORY + value: {{ include "risingwave.stateStoreDataDirectory" . }} + {{- if .Values.frontendComponent.resources.limits }} + {{- if .Values.frontendComponent.resources.limits.cpu }} + - name: RW_PARALLELISM + value: $(CONTAINER_CPU_LIMIT) + {{- end }} + {{- if .Values.frontendComponent.resources.limits.memory }} + - name: RW_TOTAL_MEMORY_BYTES + value: $(CONTAINER_MEMORY_LIMIT) + {{- end }} + {{- end }} + {{- end }} {{- if .Values.frontendComponent.extraEnvVars }} {{- .Values.frontendComponent.extraEnvVars | toYaml | nindent 8 }} {{- end }} diff --git a/charts/risingwave/tests/compute_auto_deregistration_test.yaml b/charts/risingwave/tests/compute_auto_deregistration_test.yaml index c3c0b80..ba421c0 100644 --- a/charts/risingwave/tests/compute_auto_deregistration_test.yaml +++ b/charts/risingwave/tests/compute_auto_deregistration_test.yaml @@ -28,4 +28,4 @@ tests: - -c - >- /risingwave/bin/risingwave ctl meta unregister-workers --yes \ - --workers ${POD_NAME}.RELEASE-NAME-risingwave-compute-headless.NAMESPACE.svc:5688 \ No newline at end of file + --workers ${POD_NAME}.RELEASE-NAME-risingwave-compute-headless.NAMESPACE.svc:5688 diff --git a/charts/risingwave/tests/frontend_embedded_serving_test.yaml b/charts/risingwave/tests/frontend_embedded_serving_test.yaml new file mode 100644 index 0000000..3c2e330 --- /dev/null +++ b/charts/risingwave/tests/frontend_embedded_serving_test.yaml @@ -0,0 +1,39 @@ +suite: Test Embedded Frontend Serving +templates: +- frontend-deploy.yaml +- compute-sts.yaml +chart: + appVersion: 1.0.0 + version: 0.0.1 +set: + frontendComponent: + embeddedServing: true +tests: +- it: command should be standalone + template: frontend-deploy.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - /risingwave/bin/risingwave + - standalone +- it: should have pre-stop lifecycle hook + template: frontend-deploy.yaml + asserts: + - exists: + path: spec.template.spec.containers[0].lifecycle.preStop + - equal: + path: spec.template.spec.containers[0].lifecycle.preStop.exec.command + value: + - bash + - -c + - >- + /risingwave/bin/risingwave ctl meta unregister-workers --yes \ + --workers ${POD_IP}:5688 +- it: compute node should set role to streaming + template: compute-sts.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: + --role=streaming diff --git a/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml b/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml index c123f22..4f07bb6 100644 --- a/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml +++ b/charts/risingwave/tests/statestore/cluster_workloads_env_test.yaml @@ -1,8 +1,9 @@ -suite: Test state store env vars (meta/compute/compactor) +suite: Test state store env vars (meta/frontend/compute/compactor) templates: - templates/meta-sts.yaml - templates/compute-sts.yaml - templates/compactor-deploy.yaml +- templates/frontend-deploy.yaml set: metaComponent: extraEnvVarsSecret: dummy @@ -10,6 +11,9 @@ set: extraEnvVarsSecret: dummy compactorComponent: extraEnvVarsSecret: dummy + frontendComponent: + embeddedServing: true + extraEnvVarsSecret: dummy chart: appVersion: 1.0.0 version: 0.0.1 diff --git a/charts/risingwave/values.yaml b/charts/risingwave/values.yaml index d9e9d5a..4094076 100644 --- a/charts/risingwave/values.yaml +++ b/charts/risingwave/values.yaml @@ -904,9 +904,9 @@ frontendComponent: affinity: { } ## @param frontendComponent.terminationGracePeriodSeconds Termination grace period in seconds. - ## Defaults to 1 seconds. + ## Defaults to 10 seconds. ## - terminationGracePeriodSeconds: 1 + terminationGracePeriodSeconds: 10 ## @param frontendComponent.schedulerName Name of an alternate scheduler. ## Ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ @@ -940,6 +940,12 @@ frontendComponent: ## Defaults to full. rustBacktrace: full + ## @param frontendComponent.embeddedServing Embed serving node in frontend. + ## If enabled, the compute nodes will be used for streaming only. + ## Defaults to false. + ## + embeddedServing: false + computeComponent: ## @param computeComponent.podLabels Labels to add to the component pods. ##