diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f6803e458..986c48289 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -486,6 +486,28 @@ If you use PyCharm Professional, then you may configure it according to the Static HTML files may be used simply by opening them in your favourite web browser. +Upgrading dependencies +^^^^^^^^^^^^^^^^^^^^^^ + +Galaxy uses pinned dependencies (aka lock-files) to provide reproducible +environment installations. These files are compiled from requirements files +by using ``pip-tools`` package. This package is not included into project +dependencies and should be installed manually. + +To update project requirements you should edit ``*.in`` files stored in ``requirements/`` directory. +Files ``requirements.in`` and ``dev-requirements.in`` are used for specifying +common and development requirements respectively. + +Then you should generate lock file for the requirements file you modified by +calling the following command: + +.. code-block:: none + + $ pip-tools --output requirements/.txt requirements/.in + +And then commit the both files to git. + + Commiting --------- diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4a88cf25e..000000000 --- a/requirements.txt +++ /dev/null @@ -1,64 +0,0 @@ -marshmallow -celery==3.1.24 -python-dateutil -gunicorn==19.7.1 -PyGithub -markdown==2.4.1 -requests -requests-oauthlib -bleach -rst2html5-tools -setuptools -PyYAML -ansible -pyOpenSSL -psycopg2-binary -semver -pytz - -# Linting -yamllint -flake8 -ansible-lint - -# Django -django>=1.11,<2.0 -djangorestframework>=3.7,<3.8 -django-allauth==0.34.0 - -# TODO(cutwater): Review usage of this package -django-bootstrap-form -django-celery - -# TODO(cutwater): Review usage of this package -django-filter - -# TODO(cutwater): Review usage of this package -django-debug-toolbar -dj-database-url - - -# Py3 compatibility -enum34 -six - -# Testing -mock -pytest -pytest-django -pytest-cov==2.5.1 -pytest-env - -# Docs -Sphinx<1.8.0 - -# logging related -# add request-id to django logs -django-log-request-id -# format log output as json -jog - -# Web analytics/metrics -prometheus_client -django-prometheus -influxdb diff --git a/requirements.txt b/requirements.txt new file mode 120000 index 000000000..6fa66c25d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +requirements/requirements.txt \ No newline at end of file diff --git a/requirements/dev-requirements.in b/requirements/dev-requirements.in new file mode 100644 index 000000000..9da639947 --- /dev/null +++ b/requirements/dev-requirements.in @@ -0,0 +1,14 @@ +# Include base requirements +-r requirements.in + +# Django +# TODO(cutwater): Review usage of this package +django-debug-toolbar + + +# Testing +mock +pytest +pytest-django +pytest-cov==2.5.1 +pytest-env diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt new file mode 100644 index 000000000..dff75bfad --- /dev/null +++ b/requirements/dev-requirements.txt @@ -0,0 +1,94 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/dev-requirements.txt requirements/dev-requirements.in +# +alabaster==0.7.12 # via sphinx +amqp==1.4.9 # via kombu +ansible-lint==3.5.1 +ansible==2.7.4 +anyjson==0.3.3 # via kombu +asn1crypto==0.24.0 # via cryptography +atomicwrites==1.2.1 # via pytest +attrs==18.2.0 # via pytest +babel==2.6.0 # via sphinx +bcrypt==3.1.4 # via paramiko +billiard==3.3.0.23 # via celery +bleach==3.0.2 +celery==3.1.24 +certifi==2018.11.29 # via requests +cffi==1.11.5 # via bcrypt, cryptography, pynacl +chardet==3.0.4 # via requests +coverage==4.5.2 # via pytest-cov +cryptography==2.4.2 # via ansible, paramiko, pyopenssl +defusedxml==0.5.0 # via python3-openid +deprecated==1.2.4 # via pygithub +dj-database-url==0.5.0 +django-allauth==0.34.0 +django-bootstrap-form==3.4 +django-celery==3.2.2 +django-debug-toolbar==1.11 +django-filter==2.0.0 +django-log-request-id==1.3.2 +django-prometheus==1.0.15 +django==1.11.17 +djangorestframework==3.7.7 +docutils==0.14 # via rst2html5-tools, sphinx +flake8==3.6.0 +gunicorn==19.7.1 +idna==2.7 # via cryptography, requests +imagesize==1.1.0 # via sphinx +influxdb==5.2.1 +jinja2==2.10 # via ansible, sphinx +jog==0.1.1 +kombu==3.0.37 # via celery +markdown==2.4.1 +markupsafe==1.1.0 # via jinja2 +marshmallow==2.16.3 +mccabe==0.6.1 # via flake8 +mock==2.0.0 +more-itertools==4.3.0 # via pytest +oauthlib==2.1.0 # via requests-oauthlib +packaging==18.0 # via sphinx +paramiko==2.4.2 # via ansible +pathspec==0.5.9 # via yamllint +pbr==5.1.1 # via mock +pluggy==0.8.0 # via pytest +prometheus-client==0.5.0 +psycopg2-binary==2.7.6.1 +py==1.7.0 # via pytest +pyasn1==0.4.4 # via paramiko +pycodestyle==2.4.0 # via flake8 +pycparser==2.19 # via cffi +pyflakes==2.0.0 # via flake8 +pygithub==1.43.3 +pygments==2.3.0 # via sphinx +pyjwt==1.7.1 # via pygithub +pynacl==1.3.0 # via paramiko +pyopenssl==18.0.0 +pyparsing==2.3.0 # via packaging +pytest-cov==2.5.1 +pytest-django==3.4.4 +pytest-env==0.6.2 +pytest==4.0.1 +python-dateutil==2.7.5 +python3-openid==3.1.0 # via django-allauth +pytz==2018.7 +pyyaml==3.13 +requests-oauthlib==1.0.0 +requests==2.20.1 +rst2html5-tools==0.5.3 +semver==2.8.1 +six==1.12.0 +snowballstemmer==1.2.1 # via sphinx +sphinx==1.7.9 +sphinxcontrib-websupport==1.1.0 # via sphinx +sqlparse==0.2.4 # via django-debug-toolbar +urllib3==1.24.1 # via requests +webencodings==0.5.1 # via bleach +wrapt==1.10.11 # via deprecated +yamllint==1.13.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 000000000..57804f383 --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,64 @@ +# Common +# NOTE(cutwater): Probably some of requirements below are obsolete. +# Review required. +ansible +bleach +gunicorn==19.7.1 +markdown==2.4.1 +marshmallow +PyGithub +pyOpenSSL +python-dateutil +pytz +PyYAML +requests +requests-oauthlib +rst2html5-tools +semver +setuptools + + +# Django +django>=1.11,<2.0 +django-allauth==0.34.0 +# TODO(cutwater): Review usage of this package +django-bootstrap-form +django-filter +# logging related +# add request-id to django logs +django-log-request-id +djangorestframework>=3.7,<3.8 +dj-database-url +# format log output as json +jog + + +# Database +psycopg2-binary +influxdb + + +# Tasking +celery==3.1.24 +# NOTE(cutwater): django-celery is deprecated and should be removed after +# upgrade to celery 4 +django-celery + + +# Python 3 compatibility +six + + +# Web analytics/metrics +prometheus_client +django-prometheus + + +# Content linting +ansible-lint +flake8 +yamllint + + +# Docs +Sphinx<1.8.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 000000000..e915f8c47 --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,80 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/requirements.txt requirements/requirements.in +# +alabaster==0.7.12 # via sphinx +amqp==1.4.9 # via kombu +ansible-lint==3.5.1 +ansible==2.7.4 +anyjson==0.3.3 # via kombu +asn1crypto==0.24.0 # via cryptography +babel==2.6.0 # via sphinx +bcrypt==3.1.4 # via paramiko +billiard==3.3.0.23 # via celery +bleach==3.0.2 +celery==3.1.24 +certifi==2018.11.29 # via requests +cffi==1.11.5 # via bcrypt, cryptography, pynacl +chardet==3.0.4 # via requests +cryptography==2.4.2 # via ansible, paramiko, pyopenssl +defusedxml==0.5.0 # via python3-openid +deprecated==1.2.4 # via pygithub +dj-database-url==0.5.0 +django-allauth==0.34.0 +django-bootstrap-form==3.4 +django-celery==3.2.2 +django-filter==2.0.0 +django-log-request-id==1.3.2 +django-prometheus==1.0.15 +django==1.11.17 +djangorestframework==3.7.7 +docutils==0.14 # via rst2html5-tools, sphinx +flake8==3.6.0 +gunicorn==19.7.1 +idna==2.7 # via cryptography, requests +imagesize==1.1.0 # via sphinx +influxdb==5.2.1 +jinja2==2.10 # via ansible, sphinx +jog==0.1.1 +kombu==3.0.37 # via celery +markdown==2.4.1 +markupsafe==1.1.0 # via jinja2 +marshmallow==2.16.3 +mccabe==0.6.1 # via flake8 +oauthlib==2.1.0 # via requests-oauthlib +packaging==18.0 # via sphinx +paramiko==2.4.2 # via ansible +pathspec==0.5.9 # via yamllint +prometheus-client==0.5.0 +psycopg2-binary==2.7.6.1 +pyasn1==0.4.4 # via paramiko +pycodestyle==2.4.0 # via flake8 +pycparser==2.19 # via cffi +pyflakes==2.0.0 # via flake8 +pygithub==1.43.3 +pygments==2.3.0 # via sphinx +pyjwt==1.7.1 # via pygithub +pynacl==1.3.0 # via paramiko +pyopenssl==18.0.0 +pyparsing==2.3.0 # via packaging +python-dateutil==2.7.5 +python3-openid==3.1.0 # via django-allauth +pytz==2018.7 +pyyaml==3.13 +requests-oauthlib==1.0.0 +requests==2.20.1 +rst2html5-tools==0.5.3 +semver==2.8.1 +six==1.12.0 +snowballstemmer==1.2.1 # via sphinx +sphinx==1.7.9 +sphinxcontrib-websupport==1.1.0 # via sphinx +urllib3==1.24.1 # via requests +webencodings==0.5.1 # via bleach +wrapt==1.10.11 # via deprecated +yamllint==1.13.0 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/scripts/docker/dev/Dockerfile b/scripts/docker/dev/Dockerfile index 2b112f3c6..3337796cc 100644 --- a/scripts/docker/dev/Dockerfile +++ b/scripts/docker/dev/Dockerfile @@ -24,7 +24,7 @@ RUN yarn global add @angular/cli@6.1.2 \ && ng set --global packageManager=yarn # Install python dependencies -COPY requirements.txt /tmp/requirements.txt +COPY requirements/dev-requirements.txt /tmp/requirements.txt ENV PIP_NO_CACHE_DIR off ENV VENV_BIN /var/lib/galaxy/venv/bin RUN mkdir -p /var/lib/galaxy/ \ diff --git a/scripts/docker/release/Dockerfile.base b/scripts/docker/release/Dockerfile.base index 367bf85f0..90abd63cd 100644 --- a/scripts/docker/release/Dockerfile.base +++ b/scripts/docker/release/Dockerfile.base @@ -11,7 +11,7 @@ RUN yum -y install epel-release \ && rm -rf /var/cache/yum # Install python dependencies -COPY requirements.txt /tmp/requirements.txt +COPY requirements/requirements.txt /tmp/requirements.txt RUN mkdir -p /var/lib/galaxy/ \ && python3.6 -m venv /var/lib/galaxy/venv \ && ${VENV_BIN}/pip install -U pip wheel \