Merge pull request #1180 from consideRatio/binderhub-image-freeze

dev-requirements.txt

@@ -1,10 +1,11 @@
 beautifulsoup4
+chartpress==0.6.*
+click
 codecov
 html5lib
+jupyterhub
 pytest
 pytest-asyncio
 pytest-cov
 requests
 ruamel.yaml>=0.15
-chartpress==0.6.*
-jupyterhub

doc/conf.py

@@ -100,20 +100,18 @@
 
 # -- Options for HTML output ----------------------------------------------
 
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
 html_theme = 'pydata_sphinx_theme'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
 html_theme_options = {
   "use_edit_page_button": True,
   "github_url": "https://github.com/jupyterhub/binderhub",
   "twitter_url": "https://twitter.com/mybinderteam",
 }
+html_context = {
+    "github_user": "jupyterhub",
+    "github_repo": "binderhub",
+    "github_version": "master",
+    "doc_path": "doc",
+}
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,

doc/doc-requirements.txt

@@ -8,25 +8,27 @@
 # first. This is only relevant if sphinx==1.* is already installed in the
 # environment, which it is on ReadTheDocs.
 
-# documentation specific extra packages
+# Documentation specific packages
 myst-parser
 pydata-sphinx-theme
 sphinx-copybutton
-traitlets
-pandas
-ruamel.yaml
 
-# install BinderHub dependencies. We manually list them here because some
-# dependencies (like pycurl) can't be installed on ReadTheDocs and aren't
-# needed to build the docs.
-kubernetes
-escapism
-traitlets
+# sphinx.ext.autodoc as configured in conf.py and autodo_traits.py is
+# automatically building documentation based on inspection of the binderhub
+# package, which means we need to install it on RTD so it is available for
+# inspection.
+#
+# The binderhub package dependencies include pycurl though, and pycurl cannot be
+# installed on RTD, so due to that we maintain a copy of binderhub's
+# requirements.txt where we comment out pyrcurl.
 docker
+escapism
 jinja2
+jsonschema
+jupyterhub
+kubernetes
 prometheus_client
+#pycurl
 python-json-logger
-jupyterhub
-jsonschema
 tornado>=5.1
-#pycurl Do not install for docs as it breaks the RTD build. Its primary use is for mocks in testing .
+traitlets

helm-chart/images/binderhub/Dockerfile

@@ -4,7 +4,7 @@ FROM buildpack-deps:$DIST as build-stage
 # ARG has to occur twice to be used in both contents and FROM
 ARG DIST=buster
 
-RUN echo "deb http://deb.nodesource.com/node_12.x $DIST main" > /etc/apt/sources.list.d/nodesource.list \
+RUN echo "deb http://deb.nodesource.com/node_14.x $DIST main" > /etc/apt/sources.list.d/nodesource.list \
  && curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
 
 RUN apt-get update && \
@@ -24,7 +24,7 @@ RUN python3 setup.py bdist_wheel
 
 # The final stage
 # ---------------
-FROM python:3.7-$DIST
+FROM python:3.8-$DIST
 WORKDIR /
 
 # Copy the built binderhub python wheel from the build-stage
@@ -38,6 +38,14 @@ RUN pip install --no-cache-dir \
         *.whl \
         -r requirements.txt
 
+# when building the image used to compute the "frozen"
+# dependencies we add `pip-tools` which is for computing
+# the dependencies.
+# It is not added when chartpress builds the image used by
+# the helm chart when deploying BinderHub
+ARG PIP_TOOLS=
+RUN test -z "$PIP_TOOLS" || pip install --no-cache pip-tools==$PIP_TOOLS
+
 ENTRYPOINT ["python3", "-m", "binderhub"]
 CMD ["--config", "/etc/binderhub/config/binderhub_config.py"]
 ENV PYTHONUNBUFFERED=1

helm-chart/images/binderhub/dependencies

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""automatically manage requirements.txt dependencies with pip-tools
+
+This is an adjusted script from z2jh by @minrk to mount the binderhub python
+package requirements.txt file as a requirements.in file to freeze it to
+requirements.txt in helm-chart/images/binderhub.
+
+See
+
+    ./dependencies --help for commands and arguments
+
+How it works:
+
+- the image used in the helm chart installs a frozen environment from
+  requirements.txt
+- `pip-compile` is used to generate frozen `requirements.txt` from our actual
+  requirements in /requirements.txt next to setup.py for the binderhub package.
+- `pip list --outdated` is used to report available updates for packages in the
+  frozen environment
+- pip-compile etc. are run *inside the image* to ensure consistent behavior,
+  rather than running on host systems, which can vary.
+- When building the image to be used for running dependency-management commands,
+  chartpress configuration is loaded to ensure the environment is the same as
+  when chartpress builds the tagged image to be published.
+"""
+
+import json
+import os
+from functools import lru_cache
+from subprocess import check_call, check_output
+
+import click
+from ruamel.yaml import YAML
+
+yaml = YAML()
+here = os.path.dirname(os.path.abspath(__file__))
+chartpress_yaml = os.path.join(here, os.pardir, os.pardir, "chartpress.yaml")
+values_yaml = os.path.join(here, os.pardir, os.pardir, "jupyterhub", "values.yaml")
+root_dir = os.path.join(here, os.pardir, os.pardir, os.pardir)
+dependencies_image = 'binderhub-dependencies'
+pip_tools_version="5.*"
+
+
+@lru_cache()
+def build_args(image_name='binderhub'):
+    """retrieve docker build arguments from chartpress.yaml config file
+
+    Args:
+
+    image_name (str):
+        the name of the image to be built in chartpress.yaml
+    """
+    with open(chartpress_yaml) as f:
+        chartpress_config = yaml.load(f)
+    chart = chartpress_config['charts'][0]
+    image_config = chart['images'][image_name]
+    return image_config.get('buildArgs', {})
+
+
+def build_image():
+    """Build the docker image used for computing dependencies
+
+    This runs the chartpress build of the current image
+    with the addition of pip-tools, used for computing dependencies.
+
+    The image is built with the current frozen environment in requirements.txt
+    and pip-tools commands are available for updating requirements.txt from requirements.in.
+    """
+    click.echo(f"Building docker image {dependencies_image}")
+    build_arg_dict = build_args()
+    build_arg_list = ["--build-arg", f"PIP_TOOLS={pip_tools_version}"]
+    for key in sorted(build_arg_dict):
+        value = build_arg_dict[key]
+        build_arg_list.append("--build-arg")
+        build_arg_list.append(f"{key}={value}")
+    check_call(["docker", "build", "-t", dependencies_image, "-f", here + "/Dockerfile"] + build_arg_list + [root_dir])
+
+
+@click.group()
+def cli():
+    """Manage the Python dependencies in this image."""
+    pass
+
+
+@click.command()
+@click.option(
+    '--build/--no-build',
+    help="add --no-build to skip building the dependencies image prior to upgrading",
+    default=True,
+)
+@click.option(
+    '--upgrade/--no-upgrade',
+    help="--upgrade to upgrade all dependencies within the range specified in requirements.in",
+    default=False,
+)
+@click.option(
+    '--upgrade-package',
+    help="specify individual packages to upgrade within the range specified in requirements.in",
+    multiple=True,
+)
+def freeze(build, upgrade, upgrade_package):
+    """Freeze the environment, updating requirements.txt from requirements.in
+
+    Individual packages can be updated, or the whole environment.
+
+    This command:
+
+    1. builds the image with the current frozen environment
+    2. runs pip-compile in the image to update requirements.txt from requirements.in,
+       passing through additional arguments to pip-compile
+    """
+    if build:
+        build_image()
+    click.echo("freezing dependencies with pip-compile")
+    upgrade_args = []
+    if upgrade:
+        upgrade_args.append("--upgrade")
+    for pkg in upgrade_package:
+        upgrade_args.append("--upgrade-package")
+        upgrade_args.append(pkg)
+    check_call(
+        [
+            "docker", "run",
+            "--rm", "-it",
+            "-e", "CUSTOM_COMPILE_COMMAND=./dependencies freeze --upgrade",
+            "--volume", f"{here}:/io",
+            "--volume", f"{root_dir}/requirements.txt:/io/binderhub.in",
+            "--workdir", "/io",
+            "--entrypoint", "",
+            dependencies_image,
+            # run the following within the image
+            "pip-compile", "requirements.in", "binderhub.in",
+            "--output-file", "requirements.txt",
+        ]
+        + upgrade_args
+    )
+    # remove remnant empty file
+    os.remove("binderhub.in")
+
+
+cli.add_command(freeze)
+
+
+@click.command()
+@click.option('--build/--no-build', default=True)
+def outdated(build):
+    """Check for outdated dependencies with pip.
+
+    This command:
+
+    1. builds the image with the current frozen environment
+    2. runs `pip list --outdated` to report any outdated packages
+       that could be candidates for upgrade
+    """
+    if build:
+        build_image()
+    click.echo("Checking for outdated dependencies with pip.")
+    outdated_json = check_output(
+        [
+            "docker", "run",
+            "--rm", "-it",
+            dependencies_image,
+            # run the following within the image
+            "pip", "list",
+            "--outdated",
+            "--format=json",
+        ]
+    ).decode("utf8")
+    outdated = json.loads(outdated_json)
+    have_outdated = False
+    for pkg in outdated:
+        name = pkg['name']
+        # ignore some common packages that aren't relevant to our requirements.txt
+        if name in {'pip', 'setuptools', 'wheel'}:
+            continue
+        have_outdated = True
+        version = pkg['version']
+        latest = pkg['latest_version']
+        # TODO: parser requirements.in to check if latest is in-range?
+        # If they are in-range, running freeze again is enough,
+        # but if they are outside the range, requirements.in needs to be updated
+        # first to pick them up
+        # for now, print as much so humans can decide
+        print(f"Have {name}=={version}, latest is {name}=={latest}")
+
+    if have_outdated:
+        print("There are outdated dependencies!")
+        print(
+            "To pick up any versions outside the range(s) specified in requirements.in,"
+        )
+        print("update the pinning(s) in that file.")
+        print(
+            "To update the whole environment within the given ranges, run `./dependencies freeze --upgrade`"
+        )
+        print(
+            "To update one or more specific packages, run `./dependencies freeze --upgrade-package pkg1 [--upgrade-package pkg2]`"
+        )
+    else:
+        print("Everything appears to be up-to-date!")
+
+
+cli.add_command(outdated)
+
+
+if __name__ == '__main__':
+    cli()

helm-chart/images/binderhub/requirements.in

@@ -0,0 +1,6 @@
+# The dependencies in this file, together with binderhub's dependencies, can be
+# used to update requirements.txt for this Docker image:
+#
+#    ./dependencies freeze --upgrade
+#
+google-cloud-logging

helm-chart/images/binderhub/requirements.txt

@@ -1,7 +1,62 @@
-pycurl==7.43.0.1
-tornado==6.0.*
-kubernetes==9.0.*
-jupyterhub==1.1.0
-jsonschema==2.6.0
-# Logging sinks to send eventlogging events to
-google-cloud-logging==1.8.*
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    ./dependencies freeze --upgrade
+#
+alembic==1.4.3            # via jupyterhub
+async-generator==1.10     # via jupyterhub
+attrs==20.2.0             # via jsonschema
+cachetools==4.1.1         # via google-auth
+certifi==2020.6.20        # via kubernetes, requests
+certipy==0.1.3            # via jupyterhub
+cffi==1.14.3              # via cryptography
+chardet==3.0.4            # via requests
+cryptography==3.2.1       # via pyopenssl
+docker==4.3.1             # via -r binderhub.in
+entrypoints==0.3          # via jupyterhub
+escapism==1.0.1           # via -r binderhub.in
+google-api-core[grpc]==1.23.0  # via google-cloud-core, google-cloud-logging
+google-auth==1.22.1       # via google-api-core, kubernetes
+google-cloud-core==1.4.3  # via google-cloud-logging
+google-cloud-logging==1.15.1  # via -r requirements.in
+googleapis-common-protos==1.52.0  # via google-api-core
+grpcio==1.33.2            # via google-api-core
+idna==2.10                # via requests
+ipython-genutils==0.2.0   # via traitlets
+jinja2==2.11.2            # via -r binderhub.in, jupyterhub
+jsonschema==3.2.0         # via -r binderhub.in, jupyter-telemetry
+jupyter-telemetry==0.1.0  # via jupyterhub
+jupyterhub==1.2.0         # via -r binderhub.in
+kubernetes==12.0.0        # via -r binderhub.in
+mako==1.1.3               # via alembic
+markupsafe==1.1.1         # via jinja2, mako
+oauthlib==3.1.0           # via jupyterhub, requests-oauthlib
+pamela==1.0.0             # via jupyterhub
+prometheus-client==0.8.0  # via -r binderhub.in, jupyterhub
+protobuf==3.13.0          # via google-api-core, googleapis-common-protos
+pyasn1-modules==0.2.8     # via google-auth
+pyasn1==0.4.8             # via pyasn1-modules, rsa
+pycparser==2.20           # via cffi
+pycurl==7.43.0.6          # via -r binderhub.in
+pyopenssl==19.1.0         # via certipy
+pyrsistent==0.17.3        # via jsonschema
+python-dateutil==2.8.1    # via alembic, jupyterhub, kubernetes
+python-editor==1.0.4      # via alembic
+python-json-logger==2.0.1  # via -r binderhub.in, jupyter-telemetry
+pytz==2020.1              # via google-api-core
+pyyaml==5.3.1             # via kubernetes
+requests-oauthlib==1.3.0  # via kubernetes
+requests==2.24.0          # via docker, google-api-core, jupyterhub, kubernetes, requests-oauthlib
+rsa==4.6                  # via google-auth
+ruamel.yaml.clib==0.2.2   # via ruamel.yaml
+ruamel.yaml==0.16.12      # via jupyter-telemetry
+six==1.15.0               # via cryptography, docker, google-api-core, google-auth, grpcio, jsonschema, kubernetes, protobuf, pyopenssl, python-dateutil, websocket-client
+sqlalchemy==1.3.20        # via alembic, jupyterhub
+tornado==6.0.4            # via -r binderhub.in, jupyterhub
+traitlets==5.0.5          # via -r binderhub.in, jupyter-telemetry, jupyterhub
+urllib3==1.25.11          # via kubernetes, requests
+websocket-client==0.57.0  # via docker, kubernetes
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools