Merge pull request #34 from Sefaria/dev

Dev

build/ci/production-values.yaml

@@ -152,6 +152,14 @@ cronJobs:
     enabled: true
     SEARCH_HOST_ES6: "contextus-es-default-0.elasticsearch.svc"
     SEARCH_HOST_ES8: "elasticsearch-8-es-default-0.elasticsearch.svc"
+  topicsIndexing:
+    enabled: true
+  trello:
+    enabled: true
+  trends:
+    enabled: true
+  weeklyEmailNotifications:
+     enabled: true
 secrets:
   localSettings:
     ref: local-settings-secrets-production
@@ -171,7 +179,7 @@ localSettings:
   }
   MONGO_HOST: "mongo"
   APSCHEDULER_NAME: "apscheduler-{{ .Values.deployEnv }}"
-  SEARCH_ADMIN: "http://contextus-es-http.elasticsearch.svc:9200"
+  SEARCH_URL: "http://contextus-es-http.elasticsearch.svc:9200"
   TURN_SERVER: ''
   USE_CLOUDFLARE: false
   FRONT_END_URL:  "http://contextus.org"
@@ -181,7 +189,6 @@ localSettings:
   GLOBAL_WARNING: false
   GLOBAL_WARNING_MESSAGE: "Contextus will be in <b>Read-Only</b> mode for scheduled maintenance from 4:45pm-6:45pm Pacific time. Edits will <b>not</b> be saved during that time."
   SITE_PACKAGE: "sites.s4d"
-  SEARCH_HOST: contextus-es-http.elasticsearch.svc
   DEFAULT_FROM_EMAIL: "Contextus <[email protected]>"
   SERVER_EMAIL: "[email protected]"
   MULTISERVER_ENABLED: "False"

helm-chart/sefaria-project/templates/cronjob/daily-email-notifications.yaml

@@ -40,7 +40,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/send_email_notifications.py daily"
+              "/app/run /app/scripts/scheduled/send_email_notifications.py daily"
             ]
           restartPolicy: Never
           volumes:

helm-chart/sefaria-project/templates/cronjob/index-from-queue.yaml

@@ -51,7 +51,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/index_from_queue.py"
+              "/app/run /app/scripts/scheduled/index_from_queue.py"
             ]
             resources:
               limits:

helm-chart/sefaria-project/templates/cronjob/metrics.yaml

@@ -40,7 +40,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/metrics.py"
+              "/app/run /app/scripts/scheduled/metrics.py"
             ]
           restartPolicy: OnFailure
           volumes:

helm-chart/sefaria-project/templates/cronjob/nation-builder-sync.yaml

@@ -47,7 +47,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/nation_builder_sync.py --sustainers-only"
+              "/app/run /app/scripts/scheduled/nation_builder_sync.py --sustainers-only"
             ]
           restartPolicy: OnFailure
   successfulJobsHistoryLimit: 1

helm-chart/sefaria-project/templates/cronjob/rambi-webpages-weekly.yaml

@@ -51,7 +51,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "pip install pymarc==4.2.2 && /app/run /app/scripts/parse_rambi_webpages.py"
+              "pip install pymarc==4.2.2 && /app/run /app/scripts/scheduled/parse_rambi_webpages.py"
             ]
             resources:
               requests:

helm-chart/sefaria-project/templates/cronjob/regenerate-long-cached-data.yaml

@@ -40,7 +40,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/regenerate_long_cached_data.py --all"
+              "/app/run /app/scripts/scheduled/regenerate_long_cached_data.py --all"
             ]
           restartPolicy: OnFailure
           volumes:

helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch-es6.yaml

@@ -33,7 +33,7 @@ spec:
                 memory: 7Gi
             env:
             - name: SEARCH_HOST
-              value: "{{ .Values.cronjobs.reindexElasticSearch.SEARCH_HOST_ES6 }}"
+              value: "{{ .Values.cronJobs.reindexElasticSearch.SEARCH_HOST_ES6 }}"
             - name: REDIS_HOST
               value: "redis-{{ .Values.deployEnv }}"
             - name: NODEJS_HOST
@@ -62,7 +62,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy && /app/run /app/scripts/reindex_elasticsearch_cronjob_ES6.py"
+              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy && /app/run /app/scripts/scheduled/reindex_elasticsearch_cronjob_ES6.py"
             ]
           restartPolicy: Never
           volumes:

helm-chart/sefaria-project/templates/cronjob/reindex-elasticsearch.yaml

@@ -64,7 +64,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy elasticsearch==8.8.2 git+https://github.com/Sefaria/[email protected]#egg=elasticsearch-dsl && /app/run /app/scripts/reindex_elasticsearch_cronjob.py"
+              "mkdir -p /log && touch /log/sefaria_book_errors.log && pip install numpy elasticsearch==8.8.2 git+https://github.com/Sefaria/[email protected]#egg=elasticsearch-dsl && /app/run /app/scripts/scheduled/reindex_elasticsearch_cronjob.py"
             ]
           restartPolicy: Never
           volumes:

helm-chart/sefaria-project/templates/cronjob/sitemaps.yaml

@@ -50,7 +50,7 @@ spec:
                 subPath: local_settings.py
                 readOnly: true
             command: ["bash"]
-            args: ["-c", "/app/run /app/scripts/generate_sitemaps.py -o /storage/"]
+            args: ["-c", "/app/run /app/scripts/scheduled/generate_sitemaps.py -o /storage/"]
           containers:
           - name: file-uploader
             image: google/cloud-sdk

helm-chart/sefaria-project/templates/cronjob/topics-indexing.yaml

@@ -18,7 +18,7 @@ spec:
           containers:
             - name: topics-indexer
               image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}"
-              args: [ "yes | pip3 install numpy && touch /log/sefaria_book_errors.log && python3 /app/scripts/recalculate_secondary_topic_data.py" ]
+              args: [ "yes | pip3 install numpy && touch /log/sefaria_book_errors.log && python3 /app/scripts/scheduled/recalculate_secondary_topic_data.py" ]
               env:
               - name: REDIS_HOST
                 value: "redis-{{ .Values.deployEnv }}"

helm-chart/sefaria-project/templates/cronjob/trello.yaml

@@ -52,7 +52,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/webpages_cronjob.py"
+              "/app/run /app/scripts/scheduled/webpages_cronjob.py"
             ]
             resources:
               limits:

helm-chart/sefaria-project/templates/cronjob/trends.yaml

@@ -0,0 +1,56 @@
+{{- if .Values.cronJobs.trends.enabled }}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ .Values.deployEnv }}-trends
+  labels:
+    {{- include "sefaria.labels" . | nindent 4 }}
+spec:
+  schedule: "0 1 * * 6"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        spec:
+          volumes:
+          - name: local-settings
+            configMap:
+              name: local-settings-file-{{ .Values.deployEnv }}
+              items:
+                - key: local_settings.py
+                  path: local_settings.py
+          containers:
+          - name: trends
+            image: "{{ .Values.web.containerImage.imageRegistry }}:{{ .Values.web.containerImage.tag }}"
+            env:
+            - name: REDIS_HOST
+              value: "redis-{{ .Values.deployEnv }}"
+            - name: NODEJS_HOST
+              value: "node-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
+            - name: VARNISH_HOST
+              value: "varnish-{{ .Values.deployEnv }}-{{ .Release.Revision }}"
+            envFrom:
+            - secretRef:
+                name: {{ .Values.secrets.localSettings.ref }}
+                optional: true
+            - secretRef:
+                name: local-settings-secrets-{{ .Values.deployEnv }}
+                optional: true
+            - configMapRef:
+                name: local-settings-{{ .Values.deployEnv }}
+            volumeMounts:
+              - mountPath: /app/sefaria/local_settings.py
+                name: local-settings
+                subPath: local_settings.py
+                readOnly: true
+            command: ["bash"]
+            args: [
+              "-c",
+              "/app/run /app/scripts/scheduled/recalculate_trends.py"
+            ]
+          restartPolicy: OnFailure
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 2
+{{- end }}

helm-chart/sefaria-project/templates/cronjob/weekly-email-notifications.yaml

@@ -40,7 +40,7 @@ spec:
             command: ["bash"]
             args: [
               "-c",
-              "/app/run /app/scripts/send_email_notifications.py weekly"
+              "/app/run /app/scripts/scheduled/send_email_notifications.py weekly"
             ]
           restartPolicy: Never
           volumes:

helm-chart/sefaria-project/values.yaml

@@ -387,6 +387,26 @@ cronJobs:
   # Settings for regenerating long cached data
   regenerate:
     enabled: true
+  dailyEmailNotifications:
+    enabled: false
+  indexFromQueue:
+    enabled: false
+  metrics:
+    enabled: false
+  nationBuilderSync:
+    enabled: false
+  rambiWebpagesWeekly:
+    enabled: false
+  reindexElasticSearch:
+    enabled: false
+    SEARCH_HOST_ES6: ""
+    SEARCH_HOST_ES8: ""
+  topicsIndexing:
+    enabled: false
+  trello:
+    enabled: false
+  weeklyEmailNotifications:
+    enabled: false
 
 localSettings:
   DEBUG: true

scripts/scheduled/recalculate_trends.py

@@ -0,0 +1,9 @@
+import django
+django.setup()
+from sefaria.model import *
+
+trend.setAllTrends()
+
+
+
+

scripts/scheduled/reindex_elasticsearch_cronjob_ES6.py

@@ -0,0 +1,49 @@
+"""
+This file is meant to be temporary while we are migrating to elasticsearch 8
+"""
+from datetime import datetime
+import requests
+import traceback
+import os
+import django
+django.setup()
+from sefaria.model import *
+from sefaria.search_ES6 import index_all
+from sefaria.local_settings import SEFARIA_BOT_API_KEY
+from sefaria.pagesheetrank import update_pagesheetrank
+
+"""
+Source sheets added after last_sheet_timestamp will be missing from the index process. We want to manually index all
+source sheets created after this. Depending on the database being used to index the timestamp will be different. If
+running against a production database, last_sheet_timestamp will be the time this script began running. Otherwise, this
+value will need to be set to the time at which the last mongo dump was created (assuming the database is using the most
+up-to-date mongo dump).
+"""
+# last_sheet_timestamp = datetime.fromtimestamp(os.path.getmtime("/var/data/sefaria_public/dump/sefaria")).isoformat()
+try:
+    last_sheet_timestamp = datetime.now().isoformat()
+    update_pagesheetrank()
+    index_all()
+    r = requests.post("https://www.sefaria.org/admin/index-sheets-by-timestamp", data={"timestamp": last_sheet_timestamp, "apikey": SEFARIA_BOT_API_KEY})
+    if "error" in r.text:
+        raise Exception("Error when calling admin/index-sheets-by-timestamp API: " + r.text)
+    else:
+        print("SUCCESS!", r.text)
+except Exception as e:
+    tb_str = traceback.format_exc()
+    print("Caught exception")
+    post_object = {
+        "icon_emoji": ":facepalm:",
+        "username": "Reindex ElasticSearch",
+        "channel": "#engineering-discuss",
+        "attachments": [
+            {
+                "fallback": tb_str,
+                "color": "#a30200",
+                "pretext": "Cronjob Error",
+                "text": tb_str
+            }
+        ]
+    }
+    requests.post(os.environ['SLACK_URL'], json=post_object)
+    raise e

static/js/TopicEditor.jsx

@@ -82,6 +82,14 @@ const TopicEditor = ({origData, onCreateSuccess, close, origWasCat}) => {
           alert(Sefaria._("Title must be provided."));
           return false;
         }
+        if (data.enImgCaption.length > 150) {
+            alert("English caption is too long.  It should not be more than 150 characters");
+            return false;
+        }
+        if (data.heImgCaption.length > 150) {
+            alert("Hebrew caption is too long.  It should not be more than 150 characters")
+            return false;
+        }
         if (sortedSubtopics.length > 0 && !isNew) {
             await saveReorderedSubtopics();  // make sure subtopics reordered before saving topic information below
         }