Reproducible python environments

* Split base and dev requirements.
* Pin requirements versions with pip-tools package.
* Use pinned requirements in docker containers.
* Update CONTRIBUTING.rst guide.

CONTRIBUTING.rst

@@ -486,6 +486,28 @@ If you use PyCharm Professional, then you may configure it according to the
 
 Static HTML files may be used simply by opening them in your favourite web browser.
 
+Upgrading dependencies
+^^^^^^^^^^^^^^^^^^^^^^
+
+Galaxy uses pinned dependencies (aka lock-files) to provide reproducible
+environment installations. These files are compiled from requirements files
+by using ``pip-tools`` package. This package is not included into project
+dependencies and should be installed manually.
+
+To update project requirements you should edit ``*.in`` files stored in ``requirements/`` directory.
+Files ``requirements.in`` and ``dev-requirements.in`` are used for specifying
+common and development requirements respectively.
+
+Then you should generate lock file for the requirements file you modified by
+calling the following command:
+
+.. code-block:: none
+
+  $ pip-tools --output requirements/<filename>.txt requirements/<filename>.in
+
+And then commit the both files to git.
+
+
 Commiting
 ---------
 

requirements.txt

@@ -0,0 +1 @@
+requirements/requirements.txt

requirements/dev-requirements.in

@@ -0,0 +1,14 @@
+# Include base requirements
+-r requirements.in
+
+# Django
+# TODO(cutwater): Review usage of this package
+django-debug-toolbar
+
+
+# Testing
+mock
+pytest
+pytest-django
+pytest-cov==2.5.1
+pytest-env

requirements/dev-requirements.txt

@@ -0,0 +1,94 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/dev-requirements.txt requirements/dev-requirements.in
+#
+alabaster==0.7.12         # via sphinx
+amqp==1.4.9               # via kombu
+ansible-lint==3.5.1
+ansible==2.7.4
+anyjson==0.3.3            # via kombu
+asn1crypto==0.24.0        # via cryptography
+atomicwrites==1.2.1       # via pytest
+attrs==18.2.0             # via pytest
+babel==2.6.0              # via sphinx
+bcrypt==3.1.4             # via paramiko
+billiard==3.3.0.23        # via celery
+bleach==3.0.2
+celery==3.1.24
+certifi==2018.11.29       # via requests
+cffi==1.11.5              # via bcrypt, cryptography, pynacl
+chardet==3.0.4            # via requests
+coverage==4.5.2           # via pytest-cov
+cryptography==2.4.2       # via ansible, paramiko, pyopenssl
+defusedxml==0.5.0         # via python3-openid
+deprecated==1.2.4         # via pygithub
+dj-database-url==0.5.0
+django-allauth==0.34.0
+django-bootstrap-form==3.4
+django-celery==3.2.2
+django-debug-toolbar==1.11
+django-filter==2.0.0
+django-log-request-id==1.3.2
+django-prometheus==1.0.15
+django==1.11.17
+djangorestframework==3.7.7
+docutils==0.14            # via rst2html5-tools, sphinx
+flake8==3.6.0
+gunicorn==19.7.1
+idna==2.7                 # via cryptography, requests
+imagesize==1.1.0          # via sphinx
+influxdb==5.2.1
+jinja2==2.10              # via ansible, sphinx
+jog==0.1.1
+kombu==3.0.37             # via celery
+markdown==2.4.1
+markupsafe==1.1.0         # via jinja2
+marshmallow==2.16.3
+mccabe==0.6.1             # via flake8
+mock==2.0.0
+more-itertools==4.3.0     # via pytest
+oauthlib==2.1.0           # via requests-oauthlib
+packaging==18.0           # via sphinx
+paramiko==2.4.2           # via ansible
+pathspec==0.5.9           # via yamllint
+pbr==5.1.1                # via mock
+pluggy==0.8.0             # via pytest
+prometheus-client==0.5.0
+psycopg2-binary==2.7.6.1
+py==1.7.0                 # via pytest
+pyasn1==0.4.4             # via paramiko
+pycodestyle==2.4.0        # via flake8
+pycparser==2.19           # via cffi
+pyflakes==2.0.0           # via flake8
+pygithub==1.43.3
+pygments==2.3.0           # via sphinx
+pyjwt==1.7.1              # via pygithub
+pynacl==1.3.0             # via paramiko
+pyopenssl==18.0.0
+pyparsing==2.3.0          # via packaging
+pytest-cov==2.5.1
+pytest-django==3.4.4
+pytest-env==0.6.2
+pytest==4.0.1
+python-dateutil==2.7.5
+python3-openid==3.1.0     # via django-allauth
+pytz==2018.7
+pyyaml==3.13
+requests-oauthlib==1.0.0
+requests==2.20.1
+rst2html5-tools==0.5.3
+semver==2.8.1
+six==1.12.0
+snowballstemmer==1.2.1    # via sphinx
+sphinx==1.7.9
+sphinxcontrib-websupport==1.1.0  # via sphinx
+sqlparse==0.2.4           # via django-debug-toolbar
+urllib3==1.24.1           # via requests
+webencodings==0.5.1       # via bleach
+wrapt==1.10.11            # via deprecated
+yamllint==1.13.0
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements/requirements.in

@@ -0,0 +1,64 @@
+# Common
+# NOTE(cutwater): Probably some of requirements below are obsolete.
+#                 Review required.
+ansible
+bleach
+gunicorn==19.7.1
+markdown==2.4.1
+marshmallow
+PyGithub
+pyOpenSSL
+python-dateutil
+pytz
+PyYAML
+requests
+requests-oauthlib
+rst2html5-tools
+semver
+setuptools
+
+
+# Django
+django>=1.11,<2.0
+django-allauth==0.34.0
+# TODO(cutwater): Review usage of this package
+django-bootstrap-form
+django-filter
+# logging related
+# add request-id to django logs
+django-log-request-id
+djangorestframework>=3.7,<3.8
+dj-database-url
+# format log output as json
+jog
+
+
+# Database
+psycopg2-binary
+influxdb
+
+
+# Tasking
+celery==3.1.24
+# NOTE(cutwater): django-celery is deprecated and should be removed after
+#                 upgrade to celery 4
+django-celery
+
+
+# Python 3 compatibility
+six
+
+
+# Web analytics/metrics
+prometheus_client
+django-prometheus
+
+
+# Content linting
+ansible-lint
+flake8
+yamllint
+
+
+# Docs
+Sphinx<1.8.0

requirements/requirements.txt

@@ -0,0 +1,80 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements/requirements.txt requirements/requirements.in
+#
+alabaster==0.7.12         # via sphinx
+amqp==1.4.9               # via kombu
+ansible-lint==3.5.1
+ansible==2.7.4
+anyjson==0.3.3            # via kombu
+asn1crypto==0.24.0        # via cryptography
+babel==2.6.0              # via sphinx
+bcrypt==3.1.4             # via paramiko
+billiard==3.3.0.23        # via celery
+bleach==3.0.2
+celery==3.1.24
+certifi==2018.11.29       # via requests
+cffi==1.11.5              # via bcrypt, cryptography, pynacl
+chardet==3.0.4            # via requests
+cryptography==2.4.2       # via ansible, paramiko, pyopenssl
+defusedxml==0.5.0         # via python3-openid
+deprecated==1.2.4         # via pygithub
+dj-database-url==0.5.0
+django-allauth==0.34.0
+django-bootstrap-form==3.4
+django-celery==3.2.2
+django-filter==2.0.0
+django-log-request-id==1.3.2
+django-prometheus==1.0.15
+django==1.11.17
+djangorestframework==3.7.7
+docutils==0.14            # via rst2html5-tools, sphinx
+flake8==3.6.0
+gunicorn==19.7.1
+idna==2.7                 # via cryptography, requests
+imagesize==1.1.0          # via sphinx
+influxdb==5.2.1
+jinja2==2.10              # via ansible, sphinx
+jog==0.1.1
+kombu==3.0.37             # via celery
+markdown==2.4.1
+markupsafe==1.1.0         # via jinja2
+marshmallow==2.16.3
+mccabe==0.6.1             # via flake8
+oauthlib==2.1.0           # via requests-oauthlib
+packaging==18.0           # via sphinx
+paramiko==2.4.2           # via ansible
+pathspec==0.5.9           # via yamllint
+prometheus-client==0.5.0
+psycopg2-binary==2.7.6.1
+pyasn1==0.4.4             # via paramiko
+pycodestyle==2.4.0        # via flake8
+pycparser==2.19           # via cffi
+pyflakes==2.0.0           # via flake8
+pygithub==1.43.3
+pygments==2.3.0           # via sphinx
+pyjwt==1.7.1              # via pygithub
+pynacl==1.3.0             # via paramiko
+pyopenssl==18.0.0
+pyparsing==2.3.0          # via packaging
+python-dateutil==2.7.5
+python3-openid==3.1.0     # via django-allauth
+pytz==2018.7
+pyyaml==3.13
+requests-oauthlib==1.0.0
+requests==2.20.1
+rst2html5-tools==0.5.3
+semver==2.8.1
+six==1.12.0
+snowballstemmer==1.2.1    # via sphinx
+sphinx==1.7.9
+sphinxcontrib-websupport==1.1.0  # via sphinx
+urllib3==1.24.1           # via requests
+webencodings==0.5.1       # via bleach
+wrapt==1.10.11            # via deprecated
+yamllint==1.13.0
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

scripts/docker/dev/Dockerfile

@@ -24,7 +24,7 @@ RUN yarn global add @angular/[email protected] \
     && ng set --global packageManager=yarn
 
 # Install python dependencies
-COPY requirements.txt /tmp/requirements.txt
+COPY requirements/dev-requirements.txt /tmp/requirements.txt
 ENV PIP_NO_CACHE_DIR off
 ENV VENV_BIN /var/lib/galaxy/venv/bin
 RUN mkdir -p /var/lib/galaxy/ \

scripts/docker/release/Dockerfile.base

@@ -11,7 +11,7 @@ RUN yum -y install epel-release \
     && rm -rf /var/cache/yum
 
 # Install python dependencies
-COPY requirements.txt /tmp/requirements.txt
+COPY requirements/requirements.txt /tmp/requirements.txt
 RUN mkdir -p /var/lib/galaxy/ \
     && python3.6 -m venv /var/lib/galaxy/venv \
     && ${VENV_BIN}/pip install -U pip wheel \