Merge pull request #156 from weaviate/add-support-for-azure-key

Add support for AZURE_APIKEY in OpenAI modules

.cicd/test.sh

@@ -143,6 +143,14 @@ function check_creates_template() {
   check_modules "--set modules.text2vec-contextionary.enabled=false --set modules.reranker-transformers.enabled=true" "value: reranker-transformers"
   check_modules "--set modules.text2vec-contextionary.enabled=true --set modules.text-spellcheck.enabled=true --set modules.reranker-transformers.enabled=true" "value: text2vec-contextionary,text-spellcheck,reranker-transformers"
 
+  check_modules "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey" "value: text2vec-openai"
+  check_modules "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "value: qna-openai"
+  check_modules "--set modules.generative-openai.enabled=true --set modules.generative-openai.azureApiKey=azureApiKey" "value: generative-openai"
+  check_modules "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey --set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey --set modules.generative-openai.enabled=true --set modules.generative-openai.azureApiKey=azureApiKey" "value: qna-openai,generative-openai,text2vec-openai"
+  check_string_existence "--set modules.text2vec-openai.enabled=true --set modules.text2vec-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY"
+  check_string_existence "--set modules.qna-openai.enabled=true --set modules.qna-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY"
+  check_string_existence "--set modules.generative-openai.enabled=true --set modules.generative-openai.azureApiKey=azureApiKey" "name: AZURE_APIKEY"
+
   _settingPassageQueryOn="--set modules.text2vec-contextionary.enabled=false --set modules.text2vec-transformers.passageQueryServices.passage.enabled=true --set modules.text2vec-transformers.passageQueryServices.query.enabled=true"
   check_setting_has_value "$_settingPassageQueryOn" "name: TRANSFORMERS_PASSAGE_INFERENCE_API" "value: http://transformers-inference-passage.default.svc.cluster.local:8080"
   check_setting_has_value "$_settingPassageQueryOn" "name: TRANSFORMERS_QUERY_INFERENCE_API" "value: http://transformers-inference-query.default.svc.cluster.local:8080"

weaviate/Chart.yaml

@@ -7,10 +7,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 16.3.1
+version: 16.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 1.19.8
+appVersion: 1.20.0
 icon: https://raw.githubusercontent.com/weaviate/weaviate/19de0956c69b66c5552447e84d016f4fe29d12c9/docs/assets/weaviate-logo.png

weaviate/templates/weaviateSecretOpenAI.yaml

@@ -1,8 +1,17 @@
 {{-  $t2vOpenAI := and (index .Values "modules" "text2vec-openai" "enabled") (index .Values "modules" "text2vec-openai" "apiKey") -}}
 {{-  $qnaOpenAI := and (index .Values "modules" "qna-openai" "enabled") (index .Values "modules" "qna-openai" "apiKey") }}
 {{-  $generativeOpenAI := and (index .Values "modules" "generative-openai" "enabled") (index .Values "modules" "generative-openai" "apiKey") }}
+{{-  $t2vAzureOpenAI := and (index .Values "modules" "text2vec-openai" "enabled") (index .Values "modules" "text2vec-openai" "azureApiKey") -}}
+{{-  $qnaAzureOpenAI := and (index .Values "modules" "qna-openai" "enabled") (index .Values "modules" "qna-openai" "azureApiKey") }}
+{{-  $generativeAzureOpenAI := and (index .Values "modules" "generative-openai" "enabled") (index .Values "modules" "generative-openai" "azureApiKey") }}
 {{-  $openAI := or ($t2vOpenAI) ($qnaOpenAI) ($generativeOpenAI) }}
-{{- if $openAI }}
+{{-  $azureOpenAI := or ($t2vAzureOpenAI) ($qnaAzureOpenAI) ($generativeAzureOpenAI) }}
+
+{{- if and ($openAI) ($azureOpenAI) -}}
+  {{- fail "OpenAI modules activated, but with both OpenAI and Azure OpenAI keys. Please choose one configuration to pass either OpenAI or Azure OpenAI api key." }}
+{{- end }}
+
+{{- if or ($openAI) ($azureOpenAI) }}
 
 {{- $apiKeys := list }}
 {{- if $t2vOpenAI }}
@@ -14,6 +23,15 @@
 {{- if $generativeOpenAI }}
   {{- $apiKeys = append $apiKeys (index .Values "modules" "generative-openai" "apiKey") -}}
 {{- end }}
+{{- if $t2vAzureOpenAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "text2vec-openai" "azureApiKey") -}}
+{{- end }}
+{{- if $qnaAzureOpenAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "qna-openai" "azureApiKey") -}}
+{{- end }}
+{{- if $generativeAzureOpenAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "generative-openai" "azureApiKey") -}}
+{{- end }}
 
 {{- if gt (len ($apiKeys | uniq)) 1 -}}
   {{- fail "OpenAI modules activated, but their keys differ. Use the same key on all activated OpenAI modules." }}
@@ -32,6 +50,12 @@ data:
   apiKey: {{ index .Values "modules" "qna-openai" "apiKey" | b64enc }}
   {{- else if (index .Values "modules" "generative-openai" "apiKey") }}
   apiKey: {{ index .Values "modules" "generative-openai" "apiKey" | b64enc }}
+  {{- else if (index .Values "modules" "qna-openai" "azureApiKey") }}
+  apiKey: {{ index .Values "modules" "qna-openai" "azureApiKey" | b64enc }}
+  {{- else if (index .Values "modules" "generative-openai" "azureApiKey") }}
+  apiKey: {{ index .Values "modules" "generative-openai" "azureApiKey" | b64enc }}
+  {{- else if (index .Values "modules" "text2vec-openai" "azureApiKey") }}
+  apiKey: {{ index .Values "modules" "text2vec-openai" "azureApiKey" | b64enc }}
   {{- else }}
   apiKey: {{ index .Values "modules" "text2vec-openai" "apiKey" | b64enc }}
   {{- end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -161,6 +161,13 @@ spec:
                 name: weaviate-openai
                 key: apiKey
           {{- end  }}
+          {{ if or (and (index .Values "modules" "text2vec-openai" "enabled") (index .Values "modules" "text2vec-openai" "azureApiKey")) (and (index .Values "modules" "qna-openai" "enabled") (index .Values "modules" "qna-openai" "azureApiKey")) (and (index .Values "modules" "generative-openai" "enabled") (index .Values "modules" "generative-openai" "azureApiKey")) }}
+          - name: AZURE_APIKEY
+            valueFrom:
+              secretKeyRef:
+                name: weaviate-openai
+                key: apiKey
+          {{- end  }}
           {{- if and (index .Values "modules" "text2vec-huggingface" "enabled") (index .Values "modules" "text2vec-huggingface" "apiKey") }}
           - name: HUGGINGFACE_APIKEY
             valueFrom:

weaviate/values.yaml

@@ -7,7 +7,7 @@ image:
   # of weaviate. In accordance with Infra-as-code, you should pin this value
   # down and only change it if you explicitly want to upgrade the Weaviate
   # version.
-  tag: 1.19.8
+  tag: 1.20.0
   repo: semitechnologies/weaviate
   # Image pull policy: https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
   pullPolicy: IfNotPresent
@@ -596,8 +596,10 @@ modules:
     enabled: false
 
     # Set your OpenAI API Key to be passed to Weaviate pod as
-    # an environment variable
+    # an environment variable. You can set either the OpenAI's Service API key
+    # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time.
     apiKey: ''
+    azureApiKey: ''
 
   # The text2vec-huggingface module uses HuggingFace API
   # to dynamically compute vector embeddings based on the
@@ -778,8 +780,10 @@ modules:
     enabled: false
 
     # Set your OpenAI API Key to be passed to Weaviate pod as
-    # an environment variable
+    # an environment variable. You can set either the OpenAI's Service API key
+    # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time.
     apiKey: ''
+    azureApiKey: ''
 
   # The generative-openai module uses OpenAI Completions API
   # along with text-davinci-003 model to behave as ChatGPT.
@@ -791,8 +795,10 @@ modules:
     enabled: false
 
     # Set your OpenAI API Key to be passed to Weaviate pod as
-    # an environment variable
+    # an environment variable. You can set either the OpenAI's Service API key
+    # OR the Azure OpenAI's Service Key. Note that you cannot set both of the at the same time.
     apiKey: ''
+    azureApiKey: ''
 
   # The generative-cohere module uses Cohere Generate API
   # More information about Cohere's Generate API can be found here: