Dev

build/ci/integration-values.yaml

@@ -57,7 +57,7 @@ localSettings:
   DEBUG: true
   DOMAIN_LANGUAGE: {}
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  SEARCH_ADMIN: "http://elasticsearch-data:9200"
+  SEARCH_URL: "http://elasticsearch-data:9200"
   TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://${NAME}.integration.sefaria.org"

build/ci/production-values.yaml

@@ -163,6 +163,7 @@ instrumentation:
   enabled: false
   otelEndpoint: "http://otel-collector-collector.monitoring:4317"
   jaegerEndpoint: "jaeger-agent-dev.monitoring.svc.cluster.local:6831"
+
 localSettings:
   DEBUG: false
   DOMAIN_LANGUAGE: {

build/ci/sandbox-values.yaml

@@ -53,7 +53,7 @@ localSettings:
   DEBUG: false
   DOMAIN_LANGUAGE: {}
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  SEARCH_ADMIN: "http://elasticsearch-data:9200"
+  SEARCH_URL: "http://elasticsearch-data:9200"
   TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://${NAME}.cauldron.sefaria.org" 

helm-chart/sefaria-project/templates/_helpers.tpl

@@ -54,6 +54,22 @@ elastic-certificate-{{ .Values.deployEnv }}
 {{- end }}
 {{- end }}
 
+{{- define "sefaria.secrets.elasticUser" }}
+{{- if .Values.secrets.elasticUser.ref -}}
+{{- .Values.secrets.elasticUser.ref }}
+{{- else -}}
+elastic-user-{{ .Values.deployEnv }}
+{{- end }}
+{{- end }}
+
+{{- define "sefaria.secrets.elasticAdmin" }}
+{{- if .Values.secrets.elasticAdmin.ref -}}
+{{- .Values.secrets.elasticAdmin.ref }}
+{{- else -}}
+elastic-admin-{{ .Values.deployEnv }}
+{{- end }}
+{{- end }}
+
 
 {{- define "sefaria.secrets.originTls" }}
 {{- if .Values.ingress.secrets.originTls.ref -}}

helm-chart/sefaria-project/templates/configmap/local-settings-file.yaml

@@ -136,16 +136,11 @@ data:
     }
     SERVER_EMAIL = os.getenv("SERVER_EMAIL")
 
-    SEARCH_HOST = "/api/search"
-    SEARCH_ADMIN = os.getenv("SEARCH_ADMIN")
-    SEARCH_ADMIN_USER = os.getenv("SEARCH_ADMIN_USER")
-    SEARCH_ADMIN_PW = os.getenv("SEARCH_ADMIN_PW")
-    SEARCH_ADMIN_K8S = os.getenv("SEARCH_ADMIN_K8S")
+    auth_str = f'{os.getenv("ELASTIC_USERNAME")}:{os.getenv("ELASTIC_PASSWORD")}@' if os.getenv("ELASTIC_USERNAME") else ''
+    SEARCH_URL = f'http://{auth_str}{os.getenv("SEARCH_HOST")}:9200'
     SEARCH_INDEX_ON_SAVE = True
-    SEARCH_INDEX_NAME = "sefaria"
     SEARCH_INDEX_NAME_TEXT = 'text'  # name of the ElasticSearch index to use
     SEARCH_INDEX_NAME_SHEET = 'sheet'
-    SEARCH_INDEX_NAME_MERGED = 'merged'
 
     TURN_SERVER = os.getenv("TURN_SERVER") #coturn.cauldron.sefaria.org
     TURN_SECRET= os.getenv("TURN_SECRET")

helm-chart/sefaria-project/templates/configmap/local-settings.yaml

@@ -9,7 +9,6 @@ data:
   DEBUG: "{{ .Values.localSettings.DEBUG }}"
   DOMAIN_LANGUAGE: {{ .Values.localSettings.DOMAIN_LANGUAGE | toJson | quote }}
   APSCHEDULER_NAME: {{ tpl .Values.localSettings.APSCHEDULER_NAME . | quote }}
-  SEARCH_ADMIN: "http://{{ .Values.nginx.SEARCH_HOST }}:9200"
   TURN_SERVER: {{ .Values.localSettings.TURN_SERVER | quote }}
   USE_CLOUDFLARE: "{{ .Values.localSettings.USE_CLOUDFLARE }}"
   FRONT_END_URL: {{ .Values.localSettings.FRONT_END_URL | quote }}
@@ -26,3 +25,4 @@ data:
   SENTRY_ENVIRONMENT: {{ .Values.deployEnv | quote }}
   SENTRY_CODE_VERSION: {{ .Values.web.containerImage.tag }}
   FAIL_GRACEFULLY: "{{ .Values.localSettings.FAIL_GRACEFULLY }}"
+  SEARCH_HOST: {{ .Values.nginx.SEARCH_HOST | quote }}

helm-chart/sefaria-project/templates/configmap/nginx.yaml

@@ -6,7 +6,43 @@ metadata:
     deployEnv: "{{ .Values.deployEnv }}"
     {{- include "sefaria.labels" . | nindent 4 }}
 data:
+  {{- if .Values.instrumentation.enabled }}
+  opentracing.json: |-
+    {
+      "service_name": "nginx-{{ .Values.deployEnv }}",
+      "propagation_format": "jaeger",
+      "sampler": {
+        "type": "const",
+        "param": 1,
+        "samplingServerURL": "http://127.0.0.1:5778/sampling"
+      },
+      "reporter": {
+        "endpoint": "",
+        "localAgentHostPort": "{{ .Values.instrumentation.jaegerEndpoint }}"
+      },
+      "headers": {
+        "TraceContextHeaderName": "",
+        "jaegerDebugHeader": "",
+        "jaegerBaggageHeader": "",
+        "traceBaggageHeaderPrefix": ""
+      }
+    }
+  {{- end }}
+  entrypoint.sh: |
+    #!/bin/bash
+
+    set -e
+
+    export ELASTIC_AUTH_HEADER=$(echo -n $ELASTIC_USERNAME:$ELASTIC_PASSWORD | base64)
+    envsubst '${ENV_NAME},${VARNISH_HOST},${SEARCH_HOST},${RELEASE_TAG},${STRAPI_LOCATION},${ELASTIC_AUTH_HEADER}{{- if .Values.linker.enabled }},${LINKER_HOST}{{- end }}{{- if .Values.instrumentation.enabled }},${NGINX_VERSION}{{- end }}' < /conf/nginx.template.conf > /nginx.conf 
+
+    nginx -c /nginx.conf -g 'daemon off;'
+
   nginx.template.conf: |-
+    {{- if .Values.instrumentation.enabled }}
+    load_module /etc/nginx/modules/ngx_http_opentracing_module.so;
+    {{- end }}
+
     user www-data;
     worker_processes 8;
     error_log  /var/log/nginx/error.log warn;
@@ -18,6 +54,10 @@ data:
     }
 
     http {
+      {{- if .Values.instrumentation.enabled }}
+      opentracing_load_tracer /usr/local/lib/libjaegertracing_plugin.so /etc/nginx/opentracing.json;
+      {{- end }}
+
       # https://nginx.org/en/docs/varindex.html
       log_format structured '{ "requestDuration": $request_time, "envName": "${ENV_NAME}", "stackComponent": "nginx", "host": "$hostname", "severity": "info", "httpRequest": { "requestMethod": "$request_method", "requestUrl": "$request_uri", "requestSize": $request_length, "status":  $status, "responseSize": $body_bytes_sent, "userAgent":  "$http_user_agent", "remoteIp": "$http_x_original_forwarded_for", "referer": "$http_referer", "latency": ${request_time}s, "protocol": "$server_protocol", "forwardedHTTP": "$http_x_forwarded_proto" }, "remoteUser": "$remote_user", "timeLocal": "$time_local" }';
       access_log /dev/stdout structured;
@@ -78,6 +118,7 @@ data:
         location /api/search/ {
           rewrite ^/(?:api/search)/(.*)$ /$1 break;
           proxy_set_header Content-Type application/json;  # es 6.0 requires this header
+          proxy_set_header Authorization "Basic ${ELASTIC_AUTH_HEADER}";
           add_header 'Access-Control-Allow-Origin' '';
           proxy_pass http://elasticsearch_upstream/;
         }
@@ -108,6 +149,10 @@ data:
         }
 
         location / {
+            {{- if .Values.instrumentation.enabled }}
+            opentracing on;
+            opentracing_propagate_context;
+            {{- end }}
             proxy_send_timeout  300;
             proxy_read_timeout  300;
             proxy_set_header Host $host;

helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch-es6.yaml

@@ -0,0 +1,77 @@
+{{- if .Values.cronJobs.reindexElasticSearch.enabled }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Values.deployEnv }}-reindex-elastic-search-es6
+  labels:
+    {{- include "sefaria.labels" . | nindent 4 }}
+spec:
+  schedule: "20 13 * * 0"
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        spec:
+          affinity:
+            podAntiAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+              - labelSelector:
+                  matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                    - mongo
+                topologyKey: kubernetes.io.hostname
+          containers:
+          - name: reindex-elastic-search-es6
+            image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}"
+            resources:
+              limits:
+                memory: 9Gi
+              requests:
+                memory: 7Gi
+            env:
+            - name: SEARCH_HOST
+              value: "{{ .Values.cronjobs.reindexElasticSearch.SEARCH_HOST_ES6 }}"
+            - name: REDIS_HOST
+              value: "redis-{{ .Values.deployEnv }}"
+            - name: NODEJS_HOST
+              value: "node-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
+            - name: VARNISH_HOST
+              value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
+            - name: SLACK_URL
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "sefaria.secrets.slackWebhook" . }}
+                  key: slack-webhook
+            envFrom:
+            - secretRef:
+                name: {{ .Values.secrets.localSettings.ref }}
+                optional: true
+            - configMapRef:
+                name: local-settings-{{ .Values.deployEnv }}
+            - secretRef:
+                name: local-settings-secrets-{{ .Values.deployEnv }}
+                optional: true
+            volumeMounts:
+              - mountPath: /app/sefaria/local_settings.py
+                name: local-settings
+                subPath: local_settings.py
+                readOnly: true
+            command: ["bash"]
+            args: [
+              "-c",
+              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy && /app/run /app/scripts/reindex_elasticsearch_cronjob_ES6.py"
+            ]
+          restartPolicy: Never
+          volumes:
+          - name: local-settings
+            configMap:
+              name: local-settings-file-{{ .Values.deployEnv }}
+              items:
+                - key: local_settings.py
+                  path: local_settings.py
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+{{- end }}

helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml

@@ -7,7 +7,7 @@ metadata:
   labels:
     {{- include "sefaria.labels" . | nindent 4 }}
 spec:
-  schedule: "20 13 * * 0"
+  schedule: "20 13 * * 2"
   jobTemplate:
     spec:
       backoffLimit: 1
@@ -32,6 +32,8 @@ spec:
               requests:
                 memory: 7Gi
             env:
+            - name: SEARCH_HOST
+              value: "{{ .Values.cronJobs.reindexElasticSearch.SEARCH_HOST_ES8 }}"
             - name: REDIS_HOST
               value: "redis-{{ .Values.deployEnv }}"
             - name: NODEJS_HOST
@@ -44,14 +46,16 @@ spec:
                   name: {{ template "sefaria.secrets.slackWebhook" . }}
                   key: slack-webhook
             envFrom:
+            - secretRef:
+                name: {{ template "sefaria.secrets.elasticAdmin" . }}  
             - secretRef:
                 name: {{ .Values.secrets.localSettings.ref }}
                 optional: true
+            - configMapRef:
+                name: local-settings-{{ .Values.deployEnv }}
             - secretRef:
                 name: local-settings-secrets-{{ .Values.deployEnv }}
                 optional: true
-            - configMapRef:
-                name: local-settings-{{ .Values.deployEnv }}
             volumeMounts:
               - mountPath: /app/sefaria/local_settings.py
                 name: local-settings
@@ -60,7 +64,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy && /app/run /app/scripts/reindex_elasticsearch_cronjob.py"
+              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy elasticsearch==8.8.2 git+https://github.com/Sefaria/[email protected]#egg=elasticsearch-dsl && /app/run /app/scripts/reindex_elasticsearch_cronjob.py"
             ]
           restartPolicy: Never
           volumes:

helm-chart/sefaria-project/templates/rollout/nginx.yaml

@@ -52,10 +52,7 @@ spec:
       - name: nginx
         image: "{{ .Values.nginx.containerImage.imageRegistry }}:{{ .Values.nginx.containerImage.tag }}"
         imagePullPolicy: Always
-        command: ["bash", "-c"]
-        # https://serverfault.com/questions/577370/how-can-i-use-environment-variables-in-nginx-conf
-        args: [ "envsubst '${ENV_NAME},${VARNISH_HOST},${SEARCH_HOST},${RELEASE_TAG}{{- if .Values.linker.enabled }},${LINKER_HOST}{{- end }}' < /conf/nginx.template.conf > /nginx.conf && exec nginx -c /nginx.conf -g 'daemon off;'" ]
-        args: [ "envsubst '${ENV_NAME},${VARNISH_HOST},${SEARCH_HOST},${RELEASE_TAG},${STRAPI_LOCATION}{{- if .Values.linker.enabled }},${LINKER_HOST}{{- end }}{{- if .Values.instrumentation.enabled }},${NGINX_VERSION}{{- end }}' < /conf/nginx.template.conf > /nginx.conf && exec nginx -c /nginx.conf -g 'daemon off;'" ]
+        command: ["bash", "-c", "/usr/src/entrypoint.sh"]
         ports:
         - containerPort: 80
         - containerPort: 443
@@ -73,9 +70,18 @@ spec:
           periodSeconds: 30
         resources: {{ toYaml .Values.nginx.resources | nindent 10 }}
         volumeMounts:
-          - mountPath: /conf
+          - mountPath: /conf/nginx.template.conf
             name: nginx-conf
+            subPath: nginx.template.conf
             readOnly: true
+          - mountPath: /usr/src/entrypoint.sh
+            name: nginx-conf
+            subPath: entrypoint.sh
+          {{- if .Values.instrumentation.enabled }}
+          - mountPath: /etc/nginx/opentracing.json
+            name: nginx-conf
+            subPath: opentracing.json
+          {{- end }}
           - mountPath: /app/robots.txt
             name: robots-txt
             readOnly: true
@@ -101,6 +107,8 @@ spec:
             value: "linker-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
           {{- end }}
         envFrom:
+          - secretRef:
+              name: {{ template "sefaria.secrets.elasticUser" . }}
           - configMapRef:
               name: local-settings-nginx-{{ .Values.deployEnv }}
               optional: true
@@ -111,6 +119,7 @@ spec:
         - name:  nginx-conf
           configMap:
             name: nginx-conf-{{ .Values.deployEnv }}
+            defaultMode: 0755
         - name: robots-txt
           configMap:
             name: robots-txt-{{ .Values.deployEnv }}

helm-chart/sefaria-project/templates/rollout/web.yaml

@@ -98,6 +98,8 @@ spec:
           - name: HELM_REVISION
             value: "{{ .Release.Revision }}"
         envFrom:
+          - secretRef:
+              name: {{ template "sefaria.secrets.elasticUser" . }}  
           - secretRef:
               name: {{ .Values.secrets.localSettings.ref }}
               optional: true

helm-chart/sefaria-project/templates/secret/elastic-admin.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.secrets.elasticAdmin.data }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elastic-admin-{{ .Values.deployEnv }}
+  labels:
+    deployEnv: "{{ .Values.deployEnv }}"
+    {{- include "sefaria.labels" . | nindent 4 }}
+type: Opaque
+stringData: {{ .Values.secrets.elasticAdmin.data | toYaml | nindent 2 }}
+{{- end }}

helm-chart/sefaria-project/templates/secret/elastic-user.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.secrets.elasticUser.data }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elastic-user-{{ .Values.deployEnv }}
+  labels:
+    deployEnv: "{{ .Values.deployEnv }}"
+    {{- include "sefaria.labels" . | nindent 4 }}
+type: Opaque
+stringData: {{ .Values.secrets.elasticUser.data | toYaml | nindent 2 }}
+{{- end }}

helm-chart/sefaria-project/values.yaml

@@ -329,9 +329,7 @@ secrets:
     #   SEFARIA_DB:
     #   SEFARIA_DB_USER:
     #   SEFARIA_DB_PASSWORD:
-    #   SEARCH_ADMIN_USER:
-    #   SEARCH_ADMIN_PW:
-    #   SEARCH_ADMIN_K8S:
+    #   SEARCH_URL
     #   TURN_SECRET:
     #   TURN_USER:
     #   SEFARIA_BOT_API_KEY:
@@ -372,6 +370,16 @@ secrets:
     # should be commented out and vice-versa.
     ref: trello-secret
     # data:
+  elasticUser:
+    # If you're using a reference to an existing secret then the data: section
+    # should be commented out and vice-versa.
+    ref: elastic-user
+    # data:
+  elasticAdmin:
+    # If you're using a reference to an existing secret then the data: section
+    # should be commented out and vice-versa.
+    ref: elastic-admin
+    # data:
 
 
 # Settings for various cronjobs