Merge branch 'master' into stable

Pipfile

@@ -19,7 +19,7 @@ girder-client = "==2.4.0"
 girder-worker = "==0.5.0"
 redis = "==3.5.3"
 requests = "*"
-girderfs = {git = "https://github.com/whole-tale/girderfs",ref = "master"}
+girderfs = {git = "https://github.com/whole-tale/girderfs",ref = "v1.2rc1"}
 pyOpenSSL = "*"
 python-dateutil = "*"
 PyYAML = "*"
@@ -28,6 +28,7 @@ lxml = "*"
 "dataone.common" = "==3.4.7"
 "dataone.libclient" = "==3.4.7"
 "dataone.cli" = "==3.4.7"
+packaging = "*"
 
 [requires]
 python_version = "3.8"

gwvolman/build_utils.py

@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+from packaging import version
 import shutil
 import tempfile
 from urllib.parse import urlparse
@@ -16,7 +17,7 @@ class DockerHelper:
     def __init__(self):
         username = os.environ.get("REGISTRY_USER", "fido")
         password = os.environ.get("REGISTRY_PASS")
-        self.cli = docker.from_env(version='1.28')
+        self.cli = docker.from_env(version="1.28")
         self.cli.login(
             username=username, password=password, registry=DEPLOYMENT.registry_url
         )
@@ -35,6 +36,19 @@ def build_context(self):
             self._build_context = self._create_build_context()
         return self._build_context
 
+    @property
+    def engine(self):
+        # See https://github.com/whole-tale/repo2docker_wholetale/pull/44
+        tag = self.container_config.repo2docker_version.rsplit(":")[-1]
+        r2d_version = version.parse(tag[1:])
+
+        if isinstance(
+            r2d_version,
+            version.LegacyVersion,  # i.e. not something following v{version}
+        ) or r2d_version >= version.Version("1.2dev0"):
+            return "--engine dockercli"
+        return ""
+
     def __init__(self, gc, imageId=None, tale=None):
         if (imageId is None) == (tale is None):
             raise ValueError("Only one of 'imageId' and 'tale' can be set")
@@ -61,7 +75,9 @@ def pull_r2d(self):
 
     def _create_build_context(self):
         temp_dir = tempfile.mkdtemp(dir=os.environ.get("HOSTDIR", "/host") + "/tmp")
-        logging.info("Downloading r2d files to %s (taleId:%s)", temp_dir, self.tale["_id"])
+        logging.info(
+            "Downloading r2d files to %s (taleId:%s)", temp_dir, self.tale["_id"]
+        )
         extra_build_files = self.tale["config"].get("extra_build_files", [])
         workspaceId = self.tale.get("workspaceId")
         if workspaceId:
@@ -148,9 +164,9 @@ def run_r2d(
         """
 
         # Extra arguments for r2d
-        extra_args = ''
+        extra_args = ""
         if self.container_config.buildpack == "MatlabBuildPack":
-            extra_args = ' --build-arg FILE_INSTALLATION_KEY={} '.format(
+            extra_args = " --build-arg FILE_INSTALLATION_KEY={} ".format(
                 os.environ.get("MATLAB_FILE_INSTALLATION_KEY")
             )
         elif self.container_config.buildpack == "StataBuildPack":
@@ -160,39 +176,37 @@ def run_r2d(
             source_path = _get_stata_license_path()
             with open("/host/" + source_path, "r") as license_file:
                 stata_license = license_file.read()
-                encoded = base64.b64encode(stata_license.encode("ascii")).decode("ascii")
+                encoded = base64.b64encode(stata_license.encode("ascii")).decode(
+                    "ascii"
+                )
                 extra_args = " --build-arg STATA_LICENSE_ENCODED='{}' ".format(encoded)
 
         op = "--no-build" if dry_run else "--no-run"
         target_repo_dir = os.path.join(self.container_config.target_mount, "workspace")
         r2d_cmd = (
-            "jupyter-repo2docker "
+            f"jupyter-repo2docker {self.engine} "
             "--config='/wholetale/repo2docker_config.py' "
             f"--target-repo-dir='{target_repo_dir}' "
             f"--user-id=1000 --user-name={self.container_config.container_user} "
             f"--no-clean {op} --debug {extra_args} "
             f"--image-name {tag} {build_dir}"
         )
 
-        logging.info('Calling %s', r2d_cmd)
+        logging.info("Calling %s", r2d_cmd)
 
         volumes = {
-            '/var/run/docker.sock': {
-                'bind': '/var/run/docker.sock', 'mode': 'rw'
-            },
-            '/tmp': {
-                'bind': '/host/tmp', 'mode': 'ro'
-            }
+            "/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"},
+            "/tmp": {"bind": "/host/tmp", "mode": "ro"},
         }
 
         container = self.dh.cli.containers.run(
             image=self.container_config.repo2docker_version,
             command=r2d_cmd,
-            environment=['DOCKER_HOST=unix:///var/run/docker.sock'],
+            environment=["DOCKER_HOST=unix:///var/run/docker.sock"],
             privileged=True,
             detach=True,
             remove=True,
-            volumes=volumes
+            volumes=volumes,
         )
 
         # Job output must come from stdout/stderr

gwvolman/constants.py

@@ -24,7 +24,7 @@
 
 REPO2DOCKER_VERSION = os.environ.get(
     "REPO2DOCKER_VERSION",
-    "wholetale/repo2docker_wholetale:latest"
+    "wholetale/repo2docker_wholetale:v1.2rc1"
 )
 CPR_VERSION = os.environ.get("CPR_VERSION", "wholetale/wt-cpr:latest")
 
@@ -73,6 +73,7 @@
     "Dockerfile",
 )
 
+
 class InstanceStatus(object):
     LAUNCHING = 0
     RUNNING = 1
@@ -84,6 +85,7 @@ class TaleStatus(object):
     READY = 1
     ERROR = 2
 
+
 class RunStatus(object):
     UNKNOWN = 0
     STARTING = 1

gwvolman/tests/test_build_image.py

@@ -232,7 +232,8 @@ def test_r2d_calls(depl, dapicli):
         image_builder = ImageBuilder(gc, tale=tale)
         stata_expected_call = mock.call(
             image=REPO2DOCKER_VERSION,
-            command="jupyter-repo2docker --config='/wholetale/repo2docker_config.py'"
+            command="jupyter-repo2docker --engine dockercli"
+            " --config='/wholetale/repo2docker_config.py'"
             " --target-repo-dir='/home/jovyan/work/workspace'"
             " --user-id=1000 --user-name=jovyan --no-clean --no-run --debug"
             f"  --build-arg STATA_LICENSE_ENCODED='{base64.b64encode(b'blah').decode()}'"
@@ -257,7 +258,8 @@ def test_r2d_calls(depl, dapicli):
         image_builder = ImageBuilder(gc, tale=tale)
         matlab_expected_call = mock.call(
             image=REPO2DOCKER_VERSION,
-            command="jupyter-repo2docker --config='/wholetale/repo2docker_config.py'"
+            command="jupyter-repo2docker --engine dockercli"
+            " --config='/wholetale/repo2docker_config.py'"
             " --target-repo-dir='/home/jovyan/work/workspace'"
             " --user-id=1000 --user-name=jovyan --no-clean --no-run --debug"
             "  --build-arg FILE_INSTALLATION_KEY=fake-key"

gwvolman/tests/test_recorded_run.py

@@ -38,7 +38,7 @@ def mock_gc_get(path, parameters=None):
 
 CONTAINER_CONFIG = ContainerConfig(
     buildpack="JupyterBuildPack",
-    repo2docker_version="wholetale/repo2docker_wholetale:latest",
+    repo2docker_version="wholetale/repo2docker_wholetale:v1.2rc1",
     image="abc123",
     command="test",
     mem_limit=2,

requirements.txt

@@ -5,6 +5,7 @@ fusepy==3.0.1
 girder-client==2.4.0
 girder-worker==0.5.0
 redis==3.5.3
+packaging
 pyOpenSSL
 requests
 pyyaml
@@ -13,6 +14,6 @@ markdown
 dataone.common==3.4.7
 dataone.libclient==3.4.7
 dataone.cli==3.4.7
-git+https://github.com/whole-tale/girderfs@master#egg=girderfs
+git+https://github.com/whole-tale/girderfs@v1.2rc1#egg=girderfs
 python-dateutil
 pyparsing<3,>=2.0.2  # for packaging 21.2 ... 

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='gwvolman',
-    version='1.1.1',
+    version='1.2rc1',
     description='An example girder worker extension',
     author='Kacper Kowalik',
     author_email='[email protected]',
@@ -25,6 +25,7 @@
         'docker>=2.3.0',
         'requests',
         'markdown',
+        'lxml',
     ],
     packages=find_packages(),
     zip_safe=False