updated for open telemetry, trapi 1.5 and biolink 4.2.0

chp_api/Dockerfile

@@ -3,7 +3,7 @@
 ###########
 
 # first stage of build to pull repos
-FROM python:3.8 as intermediate
+FROM python:3.9 as intermediate
 
 # set work directory
 WORKDIR /usr/src/chp_api
@@ -25,7 +25,7 @@ RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_sp
 #########
 
 #pull official base image
-FROM python:3.8
+FROM python:3.9
 
 # add app user
 RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api

chp_api/gennifer/tasks.py

@@ -348,6 +348,7 @@ def create_task(task_pk):
         # Check in every 2 seconds
         time.sleep(5)
         status = get_status(algo, task_id)
+        print(status)
         if status["task_status"] == 'SUCCESS':
             return save_inference_task(task, status)
         if status["task_status"] == "FAILURE":

chp_api/manage.py

@@ -3,8 +3,14 @@
 import os
 import sys
 
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+
 def main():
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'chp_api.settings')
+
+    # This call is what makes the Django application be instrumented
+    DjangoInstrumentor().instrument()
+
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
@@ -16,4 +22,4 @@ def main():
     execute_from_command_line(sys.argv)
 
 if __name__ == '__main__':
-    main()
+    main()

chp_api/requirements.txt

@@ -1,21 +1,123 @@
-tqdm
-djangorestframework
-djangorestframework-simplejwt
-psycopg2-binary
-bmt
-reasoner_pydantic
-django-environ
-django-hosts
-gunicorn
-django
-requests
-requests-cache
-django-filter
-celery
-flower
-redis
-pandas
-django-cors-headers
-django-oauth-toolkit
-nltk
-pattern
+amqp==5.2.0
+asgiref==3.8.1
+async-timeout==4.0.3
+attrs==23.2.0
+autocommand==2.2.2
+backports.csv==1.0.7
+backports.tarfile==1.1.1
+beautifulsoup4==4.12.3
+billiard==4.2.0
+bmt==1.4.0
+cattrs==23.2.3
+celery==5.4.0
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+cheroot==10.0.1
+CherryPy==18.9.0
+click==8.1.7
+click-didyoumean==0.3.1
+click-plugins==1.1.1
+click-repl==0.3.0
+cryptography==42.0.5
+curies==0.7.9
+Deprecated==1.2.14
+deprecation==2.1.0
+Django==4.2.11
+django-cors-headers==4.3.1
+django-environ==0.11.2
+django-extensions==3.2.3
+django-filter==24.2
+django-hosts==6.0
+django-oauth-toolkit==2.3.0
+djangorestframework==3.15.1
+djangorestframework-simplejwt==5.3.1
+exceptiongroup==1.2.1
+feedparser==6.0.11
+flower==2.0.1
+future==1.0.0
+hbreader==0.9.1
+humanize==4.9.0
+idna==3.7
+importlib-metadata==7.0.0
+inflect==7.2.0
+iniconfig==2.0.0
+isodate==0.6.1
+jaraco.collections==5.0.1
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.0
+joblib==1.4.0
+json-flattener==0.1.9
+jsonasobj2==1.0.4
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jwcrypto==1.5.6
+kombu==5.3.7
+linkml-runtime==1.7.5
+lxml==5.2.1
+more-itertools==10.2.0
+mysqlclient==2.2.4
+nltk==3.8.1
+numpy==1.26.4
+oauthlib==3.2.2
+opentelemetry-api==1.24.0
+opentelemetry-instrumentation==0.45b0
+opentelemetry-instrumentation-django==0.45b0
+opentelemetry-instrumentation-wsgi==0.45b0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
+opentelemetry-util-http==0.45b0
+packaging==24.0
+pandas==2.2.2
+Pattern==3.6
+pdfminer.six==20231228
+platformdirs==4.2.1
+pluggy==1.5.0
+portend==3.2.0
+prefixcommons==0.1.12
+prefixmaps==0.2.4
+prometheus_client==0.20.0
+prompt-toolkit==3.0.43
+psycopg2-binary==2.9.9
+pycparser==2.22
+pydantic==1.10.12
+PyJWT==2.8.0
+pyparsing==3.1.2
+pytest==8.1.1
+pytest-logging==2015.11.4
+python-dateutil==2.9.0.post0
+python-docx==1.1.0
+PyTrie==0.4.0
+pytz==2024.1
+PyYAML==6.0.1
+rdflib==7.0.0
+reasoner-pydantic==5.0.2
+redis==5.0.4
+referencing==0.34.0
+regex==2024.4.16
+requests==2.31.0
+requests-cache==1.2.0
+rpds-py==0.18.0
+scipy==1.13.0
+sgmllib3k==1.0.0
+six==1.16.0
+sortedcontainers==2.4.0
+soupsieve==2.5
+sqlparse==0.5.0
+stringcase==1.2.0
+tempora==5.5.1
+tomli==2.0.1
+tornado==6.4
+tqdm==4.66.2
+typeguard==4.2.1
+typing_extensions==4.11.0
+tzdata==2024.1
+url-normalize==1.4.3
+urllib3==2.2.1
+uWSGI==2.0.25.1
+vine==5.1.0
+wcwidth==0.2.13
+wrapt==1.16.0
+zc.lockfile==3.0.post1
+zipp==3.18.1

chp_api/requirements.txt.base

@@ -0,0 +1,23 @@
+tqdm
+djangorestframework
+djangorestframework-simplejwt
+psycopg2-binary
+bmt
+reasoner_pydantic
+django-environ
+django-hosts
+django
+requests
+requests-cache
+django-filter
+celery
+flower
+redis
+pandas
+django-cors-headers
+django-oauth-toolkit
+nltk
+pattern
+opentelemetry-sdk
+opentelemetry-instrumentation-django
+uwsgi

compose.chp-api.yaml

@@ -66,7 +66,12 @@ services:
         # Uncomment this for production
         #- DJANGO_SETTINGS_MODULE=mysite.settings.production
         # Comment this for development
-        #- DJANGO_SETTINGS_MODULE=mysite.settings.base
+      - DJANGO_SETTINGS_MODULE=chp_api.settings
+        # For Open Telemetry
+      - OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true
+      - OTEL_TRACES_EXPORTER=jaeger
+      - OTEL_EXPORTER_JAEGER_AGENT_HOST=jaeger-otel-agent.sri
+      - OTEL_EXPORTER_JAEGER_AGENT_PORT=6831
     depends_on:
       - static-fs
       - db
@@ -78,7 +83,10 @@ services:
       retries: 3
     volumes:
       - static-files:/home/chp_api/staticfiles
-    command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application
+    #command: uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --logto /tmp/mylog.log --module chp_api.wsgi:application
+    #command: opentelemetry-instrument --traces_exporter console --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid  --module chp_api.wsgi:application
+    command: opentelemetry-instrument --traces_exporter jaeger --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --module chp_api.wsgi:application
+    #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:8000 chp_api.wsgi:application
     #command: python3 manage.py runserver 0.0.0.0:8000
 
   worker-api:

deploy/chp-api/templates/deployment.yaml

@@ -26,7 +26,7 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command: ["/bin/sh"]
-          args: ["-c", "python3 manage.py collectstatic --noinput && gunicorn -c gunicorn.config.py --log-file=- --env DJANGO_SETTINGS_MODULE=chp_api.settings chp_api.wsgi:application --bind 0.0.0.0:8000"]
+          args: ["-c", "opentelemetry-instrument --traces_exporter jaeger --metrics_exporter console uwsgi --http :8000 --max-requests=200 --master --pidfile=/tmp/project-master.pid --module chp_api.wsgi:application"]
           ports:
             - name: http-app
               containerPort: 8000
@@ -80,6 +80,14 @@ spec:
               value: "{{ .Values.app.djangoSuperuserUsername }}"
             - name: DJANGO_SUPERUSER_EMAIL
               value: "{{ .Values.app.djangoSuperuserEmail }}"
+            - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED
+              value: "{{ .Values.app.OtelPythonLoggingAutoInstrumentationEnabled }}"
+            - name: OTEL_TRACES_EXPORTER
+              value: "{{ .Values.app.OtelTracesExporter }}"
+            - name: OTEL_EXPORTER_JAEGER_AGENT_HOST
+              value: "{{ .Values.app.OtelExporterJaegerAgentHost }}"
+            - name: OTEL_EXPORTER_JAEGER_AGENT_PORT
+              value: "{{ .Values.app.OtelExporterJaegerAgentPort }}"
         - name: {{ .Chart.Name }}-nginx
           securityContext:
             {{- toYaml .Values.securityContextNginx | nindent 12 }}

deploy/chp-api/values.yaml

@@ -28,6 +28,10 @@ app:
   djangoSuperuserEmail: "[email protected]"
   staticfsFolder: "/var/www"
   staticfsDebug: "0"
+  OtelPythonLoggingAutoInstrumentationEnabled: "true"
+  OtelTracesExporter: "jaeger"
+  OtelExporterJaegerAgentHost: "jaeger-otel-agent.sri"
+  OtelExporterJaegerAgentPort: "6831"
 
 # database connection information
 db: