Add support for offload-s3 module

weaviate · Jun 21, 2024 · 12548e3 · 12548e3
1 parent 5f90ef3
commit 12548e3
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 3 deletions.
diff --git a/.cicd/test.sh b/.cicd/test.sh
@@ -159,6 +159,7 @@ function check_creates_template() {
   check_modules "--set modules.generative-ollama.enabled=true --set modules.text2vec-ollama.enabled=true" "value: generative-ollama,text2vec-ollama"
   check_modules "--set modules.text2vec-octoai.enabled=true" "value: text2vec-octoai"
   check_modules "--set modules.generative-octoai.enabled=true" "value: generative-octoai"
+  check_modules "--set offload.s3.enabled=true" "value: offload-s3"
 
   check_modules "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey" "value: text2vec-openai"
   check_modules "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "value: qna-openai"
@@ -190,6 +191,9 @@ function check_creates_template() {
   check_string_existence "--set modules.text2vec-octoai.enabled=true --set modules.text2vec-octoai.apiKey=octoaiApiKey" "name: OCTOAI_APIKEY"
   check_string_existence "--set modules.generative-octoai.enabled=true --set modules.generative-octoai.apiKey=octoaiApiKey" "name: OCTOAI_APIKEY"
   check_string_existence "--set modules.generative-octoai.enabled=true --set modules.generative-octoai.apiKey=octoaiApiKey --set modules.text2vec-octoai.enabled=true --set modules.text2vec-octoai.apiKey=octoaiApiKey" "name: OCTOAI_APIKEY"
+  check_string_existence "--set offload.s3.enabled=true --set modules.generative-aws.enabled=true --set modules.generative-aws.secrets.AWS_ACCESS_KEY_ID=key --set modules.generative-aws.secrets.AWS_SECRET_ACCESS_KEY=secret --set modules.text2vec-aws.enabled=true" "name: weaviate-aws"
+  check_string_existence "--set offload.s3.enabled=true --set modules.generative-aws.enabled=true --set offload.s3.secrets.AWS_ACCESS_KEY_ID=key --set offload.s3.secrets.AWS_SECRET_ACCESS_KEY=secret --set modules.text2vec-aws.enabled=true" "name: weaviate-aws"
+  check_string_existence "--set offload.s3.enabled=true --set modules.generative-aws.enabled=true --set offload.s3.envSecrets.AWS_ACCESS_KEY_ID=key-secret-name --set offload.s3.envSecrets.AWS_SECRET_ACCESS_KEY=secret --set modules.text2vec-aws.enabled=true" "name: key-secret-name"
 
   check_setting_has_value "--set replicas=3 --set env.RAFT_BOOTSTRAP_EXPECT=3" "name: RAFT_JOIN" "value: \"weaviate-0,weaviate-1,weaviate-2\""
   check_setting_has_value "--set replicas=3 --set env.RAFT_BOOTSTRAP_EXPECT=3" "name: RAFT_BOOTSTRAP_EXPECT" "value: \"3\""

diff --git a/weaviate/templates/_helpers.tpl b/weaviate/templates/_helpers.tpl
@@ -104,6 +104,9 @@
   {{- if (index .Values "modules" "reranker-voyageai" "enabled") -}}
     {{ $modules = append $modules "reranker-voyageai" }}
   {{- end -}}
+  {{- if (index .Values "offload" "s3" "enabled") -}}
+    {{ $modules = append $modules "offload-s3" }}
+  {{- end -}}
   {{- if (index .Values "backups" "filesystem" "enabled") -}}
     {{ $modules = append $modules "backup-filesystem" }}
   {{- end -}}

diff --git a/weaviate/templates/awsSecret.yaml b/weaviate/templates/awsSecret.yaml
@@ -1,7 +1,8 @@
 {{-  $t2vAWS := and (index .Values "modules" "text2vec-aws" "enabled") (index .Values "modules" "text2vec-aws" "secrets") -}}
 {{-  $generativeAWS := and (index .Values "modules" "generative-aws" "enabled") (index .Values "modules" "generative-aws" "secrets") }}
 {{-  $backupS3 := and (index .Values "backups" "s3" "enabled") (index .Values "backups" "s3" "secrets") -}}
-{{-  $awsSecrets := or ($t2vAWS) ($generativeAWS) ($backupS3) }}
+{{-  $offloadS3 := and (index .Values "offload" "s3" "enabled") (index .Values "offload" "s3" "secrets") -}}
+{{-  $awsSecrets := or ($t2vAWS) ($generativeAWS) ($backupS3) ($offloadS3) }}
 
 {{- $awsSecrets := list }}
 {{- if $t2vAWS -}}
@@ -13,6 +14,9 @@
 {{- if $backupS3 -}}
   {{ $awsSecrets = append $awsSecrets "backup-s3" }}
 {{- end }}
+{{- if $offloadS3 -}}
+  {{ $awsSecrets = append $awsSecrets "offload-s3" }}
+{{- end }}
 
 {{- if gt (len $awsSecrets) 1 -}}
   {{- fail "AWS modules activated, but with both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. Please choose one configuration to pass AWS credentials." }}
@@ -32,10 +36,18 @@ data:
   AWS_ACCESS_KEY_ID: {{ index .Values "modules" "text2vec-aws" "secrets" "AWS_ACCESS_KEY_ID" | b64enc }}
   {{- else if (index .Values "modules" "generative-aws" "secrets" "AWS_ACCESS_KEY_ID") }}
   AWS_ACCESS_KEY_ID: {{ index .Values "modules" "generative-aws" "secrets" "AWS_ACCESS_KEY_ID" | b64enc }}
+  {{- else if (index .Values "backups" "s3" "secrets" "AWS_ACCESS_KEY_ID") }}
+  AWS_ACCESS_KEY_ID: {{ index .Values "backups" "s3" "secrets" "AWS_ACCESS_KEY_ID" | b64enc }}
+  {{- else if (index .Values "offload" "s3" "secrets" "AWS_ACCESS_KEY_ID") }}
+  AWS_ACCESS_KEY_ID: {{ index .Values "offload" "s3" "secrets" "AWS_ACCESS_KEY_ID" | b64enc }}
   {{- end }}
   {{- if index .Values "modules" "text2vec-aws" "secrets" "AWS_SECRET_ACCESS_KEY" }}
   AWS_SECRET_ACCESS_KEY: {{ index .Values "modules" "text2vec-aws" "secrets" "AWS_SECRET_ACCESS_KEY" | b64enc }}
   {{- else if (index .Values "modules" "generative-aws" "secrets" "AWS_SECRET_ACCESS_KEY") }}
   AWS_SECRET_ACCESS_KEY: {{ index .Values "modules" "generative-aws" "secrets" "AWS_SECRET_ACCESS_KEY" | b64enc }}
+  {{- else if (index .Values "backups" "s3" "secrets" "AWS_SECRET_ACCESS_KEY") }}
+  AWS_SECRET_ACCESS_KEY: {{ index .Values "backups" "s3" "secrets" "AWS_SECRET_ACCESS_KEY" | b64enc }}
+  {{- else if (index .Values "offload" "s3" "secrets" "AWS_SECRET_ACCESS_KEY") }}
+  AWS_SECRET_ACCESS_KEY: {{ index .Values "offload" "s3" "secrets" "AWS_SECRET_ACCESS_KEY" | b64enc }}
   {{- end }}
 {{- end }}
diff --git a/weaviate/templates/weaviateStatefulset.yaml b/weaviate/templates/weaviateStatefulset.yaml
@@ -269,8 +269,16 @@ spec:
               {{- end }}
             {{- end }}
           {{- end  }}
-          {{- if or (index .Values "backups" "s3" "enabled") (index .Values "modules" "text2vec-aws" "enabled") (index .Values "modules" "generative-aws" "enabled") }}
-            {{- if index .Values "backups" "s3" "envSecrets" }}
+          {{- if or (index .Values "offload" "s3" "enabled") (index .Values "backups" "s3" "enabled") (index .Values "modules" "text2vec-aws" "enabled") (index .Values "modules" "generative-aws" "enabled") }}
+            {{- if index .Values "offload" "s3" "envSecrets" }}
+              {{- range $key, $secret := $.Values.offload.s3.envSecrets }}
+          - name: {{ $key }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret }}
+                key: {{ $key | quote }}
+              {{- end }}
+            {{- else if index .Values "backups" "s3" "envSecrets" }}
               {{- range $key, $secret := $.Values.backups.s3.envSecrets }}
           - name: {{ $key }}
             valueFrom:

diff --git a/weaviate/values.yaml b/weaviate/values.yaml
@@ -326,6 +326,57 @@ envSecrets:
   # create a Kubernetes secret with AUTHENTICATION_APIKEY_ALLOWED_KEYS key and its respective value
   # AUTHENTICATION_APIKEY_ALLOWED_KEYS: name-of-the-k8s-secret-containing-the-comma-separated-api-keys
 
+# Configure offload providers
+offload:
+  s3:
+    enabled: false
+    # If one is using AWS EKS and has already configured K8s Service Account
+    # that holds the AWS credentials one can pass a name of that service account
+    # here using this setting.
+    # NOTE: the root `serviceAccountName` config has priority over this one, and 
+    # if the root one is set this one will NOT overwrite it. This one is here for
+    # backwards compatibility.
+    serviceAccountName:
+
+    envconfig:
+      # Configure bucket where data should be saved, this setting is mandatory
+      OFFLOAD_S3_BUCKET: weaviate-offload
+
+      # Optional setting. Defaults to empty string.
+      # Set this option if you want to save data to a given location
+      # inside the bucket
+      # OFFLOAD_S3_PATH: path/inside/bucket
+
+      # Optional setting. Defaults to AWS S3 (s3.amazonaws.com). 
+      # Set this option if you have a MinIO storage configured in your environment
+      # and want to use it instead of the AWS S3.
+      # OFFLOAD_S3_ENDPOINT: custom.minio.endpoint.address
+
+      # Optional setting. Defaults to true.
+      # Set this option if you don't want to use SSL.
+      # OFFLOAD_S3_USE_SSL: true
+
+      # You can pass environment AWS settings here:
+      # Define the region
+      # AWS_REGION: eu-west-1
+
+    # For Weaviate to be able to create bucket objects it needs a user credentials to authenticate to AWS.
+    # The User must have permissions to read/create/delete bucket objects.
+    # You can pass the User credentials (access-key id and access-secret-key) in 2 ways:
+    # 1. by setting the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY plain values in the `secrets` section below
+    #     this chart will create a kubernetes secret for you with these key-values pairs
+    # 2. create Kubernetes secret/s with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY  keys and their respective values
+    #     Set the Key and the secret where it is set in `envSecrets` section below
+    secrets: {}
+    #   AWS_ACCESS_KEY_ID: access-key-id (plain text)
+    #   AWS_SECRET_ACCESS_KEY: secret-access-key (plain text)
+
+    # If one has already defined secrets with AWS credentials one can pass them using
+    # this setting:
+    envSecrets: {}
+    #   AWS_ACCESS_KEY_ID: name-of-the-k8s-secret-containing-the-key-id
+    #   AWS_SECRET_ACCESS_KEY: name-of-the-k8s-secret-containing-the-key
+
 # Configure backup providers
 backups:
   # The backup-filesystem module enables creation of the DB backups in