From 60f045385d15c668d15a7d102aeab0729673f3a5 Mon Sep 17 00:00:00 2001 From: Olivier Leger Date: Tue, 3 May 2022 13:59:02 -0400 Subject: [PATCH] Make pip dependencies consistent with KPI --- .github/workflows/pytest.yml | 2 +- .gitlab-ci.yml | 2 +- Dockerfile | 4 +- .../pip/{dev.in => dev_requirements.in} | 0 .../pip/{dev.txt => dev_requirements.txt} | 24 +- dependencies/pip/prod.in | 3 - dependencies/pip/prod.txt | 252 ------------------ dependencies/pip/requirements.in | 3 + dependencies/pip/requirements.txt | 2 + docker/init.bash | 12 +- 10 files changed, 28 insertions(+), 276 deletions(-) rename dependencies/pip/{dev.in => dev_requirements.in} (100%) rename dependencies/pip/{dev.txt => dev_requirements.txt} (92%) delete mode 100644 dependencies/pip/prod.in delete mode 100644 dependencies/pip/prod.txt diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 833645424..9521c8b8b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -45,7 +45,7 @@ jobs: - name: Install apt dependencies run: sudo apt-get install ghostscript libxml2-dev libxslt-dev python3-dev gdal-bin libproj-dev gettext postgresql-client openjdk-11-jre - name: Install Python dependencies - run: pip install -r dependencies/pip/dev.txt + run: pip install -r dependencies/pip/dev_requirements.txt - name: Run pytest run: pytest -vv -rf env: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee4a18f9a..e554e1b72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,7 +33,7 @@ test: alias: redis_cache script: - apt-get update && apt-get install -y ghostscript gdal-bin libproj-dev gettext openjdk-11-jre - - pip install -r dependencies/pip/dev.txt + - pip install -r dependencies/pip/dev_requirements.txt - pytest -vv -rf deploy-beta: diff --git a/Dockerfile b/Dockerfile index 1566339b4..6f63c48e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN python3 -m venv "$VIRTUAL_ENV" ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip install --quiet --upgrade pip && \ pip install --quiet pip-tools -COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +COPY ./dependencies/pip/requirements.txt "${TMP_DIR}/pip_dependencies.txt" RUN pip-sync "${TMP_DIR}/pip_dependencies.txt" 1>/dev/null FROM python:3.10-slim @@ -79,7 +79,7 @@ RUN adduser --disabled-password --gecos '' "$UWSGI_USER" # Copy virtualenv ENV PATH="$VIRTUAL_ENV/bin:$PATH" -COPY ./dependencies/pip/prod.txt "${TMP_DIR}/pip_dependencies.txt" +COPY ./dependencies/pip/requirements.txt "${TMP_DIR}/pip_dependencies.txt" COPY --from=build-python "$VIRTUAL_ENV" "$VIRTUAL_ENV" COPY . "${KOBOCAT_SRC_DIR}" diff --git a/dependencies/pip/dev.in b/dependencies/pip/dev_requirements.in similarity index 100% rename from dependencies/pip/dev.in rename to dependencies/pip/dev_requirements.in diff --git a/dependencies/pip/dev.txt b/dependencies/pip/dev_requirements.txt similarity index 92% rename from dependencies/pip/dev.txt rename to dependencies/pip/dev_requirements.txt index 28c81279e..e3673619c 100644 --- a/dependencies/pip/dev.txt +++ b/dependencies/pip/dev_requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile dependencies/pip/dev.in +# pip-compile dependencies/pip/dev_requirements.in # -e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest # via -r dependencies/pip/requirements.in @@ -152,16 +152,16 @@ executing==0.8.3 gdata-python3==3.0.1 # via -r dependencies/pip/requirements.in httmock==1.4.0 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in idna==3.3 # via requests iniconfig==1.1.1 # via pytest ipdb==0.13.9 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in ipython==8.2.0 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # ipdb jedi==0.18.1 # via ipython @@ -184,11 +184,11 @@ markdown==3.3.6 matplotlib-inline==0.1.3 # via ipython mock==4.0.3 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in modilabs-python-utils==0.1.5 # via -r dependencies/pip/requirements.in mongomock==4.0.0 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in numpy==1.22.3 # via pandas oauthlib==3.2.0 @@ -238,13 +238,13 @@ pyparsing==3.0.8 # via packaging pytest==7.1.2 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # pytest-django # pytest-env pytest-django==4.5.2 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in pytest-env==0.6.2 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in python-crontab==2.6.0 # via django-celery-beat python-dateutil==2.8.2 @@ -279,7 +279,7 @@ sentinels==1.0.0 sentry-sdk==1.5.10 # via -r dependencies/pip/requirements.in shell-command==0.1 - # via -r dependencies/pip/dev.in + # via -r dependencies/pip/dev_requirements.in simplejson==3.17.6 # via -r dependencies/pip/requirements.in six==1.16.0 @@ -292,7 +292,7 @@ six==1.16.0 # python-dateutil sqlparse==0.4.2 # via - # -r dependencies/pip/dev.in + # -r dependencies/pip/dev_requirements.in # django stack-data==0.2.0 # via ipython @@ -313,6 +313,8 @@ urllib3==1.26.9 # botocore # requests # sentry-sdk +uwsgi==2.0.20 + # via -r dependencies/pip/requirements.in vine==5.0.0 # via # amqp diff --git a/dependencies/pip/prod.in b/dependencies/pip/prod.in deleted file mode 100644 index bd6f83b94..000000000 --- a/dependencies/pip/prod.in +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.in - -uWSGI diff --git a/dependencies/pip/prod.txt b/dependencies/pip/prod.txt deleted file mode 100644 index f625196f6..000000000 --- a/dependencies/pip/prod.txt +++ /dev/null @@ -1,252 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# pip-compile dependencies/pip/prod.in -# --e git+https://github.com/dimagi/django-digest@419f7306443f9a800b07d832b2cc147941062d59#egg=django_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/jnm/django-storages@s3boto3_accurate_tell#egg=django_storages - # via -r dependencies/pip/requirements.in --e git+https://github.com/dimagi/python-digest@5c94bb74516b977b60180ee832765c0695ff2b56#egg=python_digest - # via -r dependencies/pip/requirements.in --e git+https://github.com/kobotoolbox/ssrf-protect@9b97d3f0fd8f737a38dd7a6b64efeffc03ab3cdd#egg=ssrf_protect - # via -r dependencies/pip/requirements.in -amqp==5.1.1 - # via - # -r dependencies/pip/requirements.in - # kombu -asgiref==3.5.0 - # via django -async-timeout==4.0.2 - # via redis -billiard==3.6.4.0 - # via celery -boto3==1.22.2 - # via -r dependencies/pip/requirements.in -botocore==1.25.2 - # via - # boto3 - # s3transfer -celery[redis]==5.2.6 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat -certifi==2021.10.8 - # via - # requests - # sentry-sdk -cffi==1.15.0 - # via cryptography -charset-normalizer==2.0.12 - # via requests -click==8.1.2 - # via - # celery - # click-didyoumean - # click-plugins - # click-repl -click-didyoumean==0.3.0 - # via celery -click-plugins==1.1.1 - # via celery -click-repl==0.2.0 - # via celery -cryptography==37.0.0 - # via jwcrypto -defusedxml==0.7.1 - # via - # djangorestframework-xml - # pyxform -deprecated==1.2.13 - # via - # jwcrypto - # redis -dict2xml==1.7.1 - # via -r dependencies/pip/requirements.in -django==3.2 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat - # django-cors-headers - # django-db-readonly - # django-extensions - # django-filter - # django-guardian - # django-oauth-toolkit - # django-render-block - # django-reversion - # django-storages - # django-taggit - # django-timezone-field - # djangorestframework - # djangorestframework-guardian - # jsonfield -django-celery-beat==2.2.1 - # via -r dependencies/pip/requirements.in -django-cors-headers==3.11.0 - # via -r dependencies/pip/requirements.in -django-db-readonly==0.7.0 - # via -r dependencies/pip/requirements.in -django-environ==0.8.1 - # via -r dependencies/pip/requirements.in -django-extensions==3.1.5 - # via -r dependencies/pip/requirements.in -django-filter==21.1 - # via -r dependencies/pip/requirements.in -django-guardian==2.4.0 - # via - # -r dependencies/pip/requirements.in - # djangorestframework-guardian -django-oauth-toolkit==2.0.0 - # via -r dependencies/pip/requirements.in -django-pure-pagination==0.3.0 - # via -r dependencies/pip/requirements.in -django-redis-sessions==0.6.2 - # via -r dependencies/pip/requirements.in -django-render-block==0.9.1 - # via django-templated-email -django-reversion==5.0.0 - # via -r dependencies/pip/requirements.in -django-taggit==2.1.0 - # via -r dependencies/pip/requirements.in -django-templated-email==3.0.0 - # via -r dependencies/pip/requirements.in -django-timezone-field==4.2.3 - # via - # -r dependencies/pip/requirements.in - # django-celery-beat -djangorestframework==3.13.1 - # via - # -r dependencies/pip/requirements.in - # djangorestframework-csv - # djangorestframework-guardian -djangorestframework-csv==2.1.1 - # via -r dependencies/pip/requirements.in -djangorestframework-guardian==0.3.0 - # via -r dependencies/pip/requirements.in -djangorestframework-jsonp==1.0.2 - # via -r dependencies/pip/requirements.in -djangorestframework-xml==2.0.0 - # via -r dependencies/pip/requirements.in -dpath==2.0.6 - # via -r dependencies/pip/requirements.in -ecdsa==0.17.0 - # via tlslite-ng -elaphe3==0.2.0 - # via -r dependencies/pip/requirements.in -et-xmlfile==1.1.0 - # via openpyxl -gdata-python3==3.0.1 - # via -r dependencies/pip/requirements.in -idna==3.3 - # via requests -jmespath==1.0.0 - # via - # boto3 - # botocore -jsonfield==3.1.0 - # via -r dependencies/pip/requirements.in -jwcrypto==1.0 - # via django-oauth-toolkit -kombu==5.2.4 - # via celery -lxml==4.8.0 - # via - # -r dependencies/pip/requirements.in - # gdata-python3 -markdown==3.3.6 - # via -r dependencies/pip/requirements.in -modilabs-python-utils==0.1.5 - # via -r dependencies/pip/requirements.in -numpy==1.22.3 - # via pandas -oauthlib==3.2.0 - # via django-oauth-toolkit -openpyxl==3.0.9 - # via - # -r dependencies/pip/requirements.in - # pyxform -packaging==21.3 - # via redis -pandas==1.4.2 - # via -r dependencies/pip/requirements.in -pillow==9.1.0 - # via - # -r dependencies/pip/requirements.in - # elaphe3 -prompt-toolkit==3.0.29 - # via click-repl -psycopg2==2.9.3 - # via -r dependencies/pip/requirements.in -pycparser==2.21 - # via cffi -pymongo==3.12.3 - # via -r dependencies/pip/requirements.in -pyparsing==3.0.8 - # via packaging -python-crontab==2.6.0 - # via django-celery-beat -python-dateutil==2.8.2 - # via - # botocore - # pandas - # python-crontab -pytz==2022.1 - # via - # celery - # django - # django-timezone-field - # djangorestframework - # pandas -pyxform==1.9.0 - # via -r dependencies/pip/requirements.in -redis==4.2.2 - # via - # -r dependencies/pip/requirements.in - # celery - # django-redis-sessions -requests==2.27.1 - # via django-oauth-toolkit -s3transfer==0.5.2 - # via boto3 -savreaderwriter @ https://bitbucket.org/fomcl/savreaderwriter/downloads/savReaderWriter-3.3.0.zip - # via -r dependencies/pip/requirements.in -sentry-sdk==1.5.10 - # via -r dependencies/pip/requirements.in -simplejson==3.17.6 - # via -r dependencies/pip/requirements.in -six==1.16.0 - # via - # click-repl - # djangorestframework-csv - # ecdsa - # python-dateutil -sqlparse==0.4.2 - # via django -tlslite-ng==0.7.6 - # via gdata-python3 -unicodecsv==0.14.1 - # via djangorestframework-csv -urllib3==1.26.9 - # via - # botocore - # requests - # sentry-sdk -uwsgi==2.0.20 - # via -r dependencies/pip/prod.in -vine==5.0.0 - # via - # amqp - # celery - # kombu -wcwidth==0.2.5 - # via prompt-toolkit -werkzeug==2.0.3 - # via -r dependencies/pip/requirements.in -wrapt==1.14.0 - # via deprecated -xlrd==2.0.1 - # via pyxform -xlwt==1.3.0 - # via -r dependencies/pip/requirements.in diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 676495e09..f1fa76880 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -87,3 +87,6 @@ sentry-sdk # mimetype detection Werkzeug<=2.0.3 + +# WSGI server +uWSGI diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 228055595..f5f0c33e5 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -233,6 +233,8 @@ urllib3==1.26.9 # botocore # requests # sentry-sdk +uwsgi==2.0.20 + # via -r dependencies/pip/requirements.in vine==5.0.0 # via # amqp diff --git a/docker/init.bash b/docker/init.bash index f9381b1a5..5e4ef8989 100755 --- a/docker/init.bash +++ b/docker/init.bash @@ -17,18 +17,18 @@ fi KOBOCAT_WEB_SERVER="${KOBOCAT_WEB_SERVER:-uWSGI}" if [[ "${KOBOCAT_WEB_SERVER,,}" == "uwsgi" ]]; then # `diff` returns exit code 1 if it finds a difference between the files - if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/requirements.txt" "${TMP_DIR}/pip_dependencies.txt" then echo "Syncing production pip dependencies..." - pip-sync dependencies/pip/prod.txt 1>/dev/null - cp "dependencies/pip/prod.txt" "${TMP_DIR}/pip_dependencies.txt" + pip-sync dependencies/pip/requirements.txt 1>/dev/null + cp "dependencies/pip/requirements.txt" "${TMP_DIR}/pip_dependencies.txt" fi else - if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + if ! diff -q "${KOBOCAT_SRC_DIR}/dependencies/pip/dev_requirements.txt" "${TMP_DIR}/pip_dependencies.txt" then echo "Syncing development pip dependencies..." - pip-sync dependencies/pip/dev.txt 1>/dev/null - cp "dependencies/pip/dev.txt" "${TMP_DIR}/pip_dependencies.txt" + pip-sync dependencies/pip/dev_requirements.txt 1>/dev/null + cp "dependencies/pip/dev_requirements.txt" "${TMP_DIR}/pip_dependencies.txt" fi fi