Merge pull request #204 from Xarthisius/remote_builder

enh: add remote docker images builder

.github/workflows/build-test.yaml

@@ -32,7 +32,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .tox
-        key: ${{ matrix.python-version }}-tox-${{ hashFiles('setup.py', 'requirements.txt') }}
+        key: ${{ matrix.python-version }}-tox-${{ hashFiles('setup.py', 'requirements-dev.txt') }}
     - name: Run Linter
       run: tox -e lint -- gwvolman
     - name: Run Tests with coverage

Dockerfile

@@ -34,7 +34,6 @@ RUN echo "ubuntu ALL=(ALL)    NOPASSWD: /usr/bin/mount, /usr/bin/umount" >> /etc
 USER ubuntu
 WORKDIR /gwvolman
 
-COPY --chown=ubuntu:ubuntu requirements.txt /gwvolman/requirements.txt
 COPY --chown=ubuntu:ubuntu setup.py /gwvolman/setup.py
 COPY --chown=ubuntu:ubuntu gwvolman /gwvolman/gwvolman
 
@@ -64,7 +63,5 @@ RUN echo "use_locks 0" >> /etc/davfs2/davfs2.conf && \
   echo "gui_optimize 1" >> /etc/davfs2/davfs2.conf
 
 COPY docker-entrypoint.sh /docker-entrypoint.sh
-COPY scheduler-entrypoint.sh /scheduler-entrypoint.sh
 RUN chmod +x /docker-entrypoint.sh
-RUN chmod +x /scheduler-entrypoint.sh
 ENTRYPOINT ["/usr/bin/tini", "--", "/docker-entrypoint.sh"]

Dockerfile.server

@@ -0,0 +1,9 @@
+FROM xarthisius/gwvolman:latest
+
+COPY --chown=ubuntu:ubuntu setup.py /gwvolman/setup.py
+COPY --chown=ubuntu:ubuntu gwvolman /gwvolman/gwvolman
+COPY ./server-dev.sh /server-dev.sh
+
+RUN . /home/ubuntu/venv/bin/activate && pip install fastapi uvicorn
+RUN chmod +x /server-dev.sh
+ENTRYPOINT ["/server-dev.sh"]

MANIFEST.in

@@ -1,6 +1,6 @@
 prune tests
 exclude tests
-prune Dockerfile docker-entrypoint.sh MANIFEST.in README.md requirements-dev.txt setup.cfg tox.ini
+prune Dockerfile docker-entrypoint.sh MANIFEST.in README.md requirements-dev.txt setup.cfg tox.ini Dockerfile.server
 prune .gitignore .github .coveragerc
-exclude Dockerfile docker-entrypoint.sh MANIFEST.in README.md requirements-dev.txt setup.cfg tox.ini
+exclude Dockerfile docker-entrypoint.sh MANIFEST.in README.md requirements-dev.txt setup.cfg tox.ini Dockerfile.server
 exclude .gitignore .github .coveragerc

docker-entrypoint.sh

@@ -7,7 +7,7 @@ girder-worker-config set celery broker ${CELERY_BROKER:-redis://redis/}
 girder-worker-config set girder_worker tmp_root /tmp
 
 if [[ -n "$DEV" ]] ; then
-  python3 -m pip install -r /gwvolman/requirements.txt -e /gwvolman
+  python3 -m pip install -e /gwvolman
 fi
 
 # If GOSU_CHOWN environment variable set, recursively chown all specified directories

gwvolman/r2d/__init__.py

@@ -1,8 +1,17 @@
+import logging
 import os
+
 from .docker import DockerImageBuilder  # noqa
 from .kaniko import KanikoImageBuilder  # noqa
+from .remote import RemoteImageBuilder  # noqa
+
+logger = logging.getLogger(__name__)
 
-if os.environ.get("DEPLOYMENT", "docker") == "k8s":
+if os.environ.get("BUILDER_URL"):
+    ImageBuilder = RemoteImageBuilder
+elif os.environ.get("DEPLOYMENT", "docker") == "k8s":
     ImageBuilder = KanikoImageBuilder
 else:
     ImageBuilder = DockerImageBuilder
+
+logger.warning(f"gwvolman:init: Using {ImageBuilder.__name__} as image builder")

gwvolman/r2d/docker.py

@@ -14,24 +14,39 @@
 
 
 class DockerHelper:
-    def __init__(self, auth=True):
-        username = os.environ.get("REGISTRY_USER", "fido")
-        password = os.environ.get("REGISTRY_PASS")
+    def __init__(
+        self, registry_user=None, registry_password=None, registry_url=None, auth=True
+    ):
+        username = registry_user or os.environ.get("REGISTRY_USER", "fido")
+        password = registry_password or os.environ.get("REGISTRY_PASS")
+        registry_url = registry_url or DEPLOYMENT.registry_url
         self.cli = docker.from_env(version="1.28")
         self.apicli = docker.APIClient(base_url="unix://var/run/docker.sock")
         if auth:
-            self.cli.login(
-                username=username, password=password, registry=DEPLOYMENT.registry_url
-            )
+            self.cli.login(username=username, password=password, registry=registry_url)
             self.apicli.login(
-                username=username, password=password, registry=DEPLOYMENT.registry_url
+                username=username, password=password, registry=registry_url
             )
 
 
 class DockerImageBuilder(ImageBuilderBase):
-    def __init__(self, gc, imageId=None, tale=None, auth=True):
+    def __init__(
+        self,
+        gc,
+        imageId=None,
+        tale=None,
+        registry_user=None,
+        registry_password=None,
+        registry_url=None,
+        auth=True,
+    ):
         super().__init__(gc, imageId=imageId, tale=tale, auth=auth)
-        self.dh = DockerHelper(auth)
+        self.dh = DockerHelper(
+            registry_user=registry_user,
+            registry_password=registry_password,
+            registry_url=registry_url,
+            auth=auth,
+        )
 
     def pull_r2d(self):
         try:

gwvolman/r2d/remote.py

@@ -0,0 +1,95 @@
+import json
+import os
+
+import requests
+
+from .builder import ImageBuilderBase
+
+
+class RemoteImageBuilder(ImageBuilderBase):
+    def __init__(
+        self,
+        gc,
+        imageId=None,
+        tale=None,
+        builder_url=None,
+        registry_user=None,
+        registry_password=None,
+        registry_url=None,
+        auth=True,
+    ):
+        super().__init__(gc, imageId=imageId, tale=tale, auth=auth)
+        self.builder_url = builder_url or os.environ.get(
+            "BUILDER_URL", "https://builder.local.xarthisius.xyz"
+        )
+        self.registry_url = registry_url or "https://registry.local.xarthisius.xyz"
+        self.registry_user = registry_user or os.environ.get("REGISTRY_USER", "fido")
+        self.registry_password = registry_password or os.environ.get("REGISTRY_PASS")
+
+    def pull_r2d(self):
+        response = requests.put(
+            f"{self.builder_url}/pull",
+            params={
+                "repo2docker_version": self.container_config.repo2docker_version,
+            },
+            stream=True,
+        )
+        for chunk in response.iter_lines():  # Adjust chunk size as needed
+            try:
+                msg = json.loads(chunk)
+                try:
+                    print(msg["status"])
+                except KeyError:
+                    raise json.jsonJSONDecodeError
+            except json.JSONDecodeError:
+                print(chunk)
+
+    def push_image(self, image):
+        """Push image to the registry"""
+        repository, tag = image.split(":", 1)
+        response = requests.put(
+            f"{self.builder_url}/push",
+            params={
+                "image": image,
+                "registry_user": self.registry_user,
+                "registry_password": self.registry_password,
+                "registry_url": self.registry_url,
+            },
+            stream=True,
+        )
+        for chunk in response.iter_lines():  # Adjust chunk size as needed
+            print(chunk)
+
+    def run_r2d(self, tag, dry_run=False, task=None):
+        """
+        Run repo2docker on the workspace using a shared temp directory. Note that
+        this uses the "local" provider.  Use the same default user-id and
+        user-name as BinderHub
+        """
+        response = requests.post(
+            f"{self.builder_url}/build",
+            params={
+                "taleId": self.tale["_id"],
+                "apiUrl": self.gc.urlBase,
+                "token": self.gc.token,
+                "registry_url": self.registry_url,
+                "dry_run": dry_run,
+                "tag": tag,
+            },
+            stream=True,
+        )
+        for chunk in response.iter_lines():
+            try:
+                msg = json.loads(chunk)
+                if "message" in msg:
+                    msg = msg["message"]
+                    if isinstance(msg, dict) and "error" in msg.keys():
+                        return {"StatusCode": 1, "error": msg["error"]}, None
+                    print(msg)
+                elif "return" in msg:
+                    data = msg["return"]
+                    return data["ret"], data["digest"]
+                elif "error" in msg:
+                    return {"StatusCode": 1, "error": msg["error"]}, None
+            except json.JSONDecodeError:
+                print(chunk)

gwvolman/remote_builder/server.py

@@ -0,0 +1,135 @@
+import hashlib
+import json
+import logging
+import os
+import tempfile
+
+import docker
+from fastapi import FastAPI, HTTPException, Query
+from fastapi.responses import StreamingResponse
+from girder_client import GirderClient
+
+from ..r2d.docker import DockerImageBuilder
+
+app = FastAPI()
+client = docker.from_env()
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+@app.put("/pull")
+async def pull_docker_r2d_image(
+    repo2docker_version: str = Query(
+        ..., description="Repository and version of the image"
+    ),
+):
+    async def pull_stream():
+        try:
+            for line in client.api.pull(repository=repo2docker_version, stream=True):
+                line = json.loads(line.decode("utf-8").strip())
+                yield json.dumps(line) + "\n"
+        except Exception as e:
+            yield json.dumps({"error": str(e)}) + "\n"
+
+    return StreamingResponse(pull_stream(), media_type="application/json")
+
+
+@app.put("/push")
+async def push_tale_image(
+    image: str = Query(..., description="Repository and version of the image"),
+    registry_url: str = Query(..., description="Docker registry URL"),
+    registry_user: str = Query(..., description="Docker registry username"),
+    registry_password: str = Query(..., description="Docker registry password"),
+):
+    async def push_stream():
+        try:
+            repository, tag = image.split(":", 1)
+            client.api.login(
+                registry=registry_url,
+                username=registry_user,
+                password=registry_password,
+            )
+            for line in client.api.push(repository, tag=tag, stream=True, decode=True):
+                yield json.dumps(line) + "\n"
+        except Exception as e:
+            yield json.dumps({"error": str(e)}) + "\n"
+
+    return StreamingResponse(push_stream(), media_type="application/json")
+
+
+@app.post("/build")
+async def build_tale(
+    taleId: str = Query(..., description="Tale identifier"),
+    apiUrl: str = Query(..., description="Girder API URL"),
+    token: str = Query(..., description="Girder authentication token"),
+    registry_url: str = Query(..., description="Docker registry URL"),
+    dry_run: bool = Query(..., description="If true, do not build the image"),
+    tag: str = Query(..., description="Repository and version of the image"),
+):
+    girder_client = GirderClient(apiUrl=apiUrl)
+    girder_client.token = token
+    try:
+        tale = girder_client.get("tale/%s" % taleId)
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid Tale ID")
+
+    image_builder = DockerImageBuilder(
+        girder_client, tale=tale, registry_url=registry_url, auth=False
+    )
+
+    async def build_stream():
+        try:
+            yield json.dumps({"message": f"Building image {tag}"}) + "\n"
+            r2d_cmd = image_builder.r2d_command(tag, dry_run=dry_run)
+            r2d_context_dir = os.path.relpath(
+                image_builder.build_context, tempfile.gettempdir()
+            )
+            host_r2d_context_dir = os.path.join("/tmp", r2d_context_dir)
+
+            volumes = {
+                host_r2d_context_dir: {
+                    "bind": image_builder.build_context,
+                    "mode": "rw",
+                },
+                "/var/run/docker.sock": {
+                    "bind": "/var/run/docker.sock",
+                    "mode": "rw",
+                },
+            }
+
+            container = client.containers.run(
+                image=image_builder.container_config.repo2docker_version,
+                command=r2d_cmd,
+                environment={"DOCKER_HOST": "unix:///var/run/docker.sock"},
+                privileged=True,
+                detach=True,
+                remove=False,
+                volumes=volumes,
+            )
+
+            yield json.dumps({"message": f"Calling {r2d_cmd}"}) + "\n"
+
+            h = hashlib.md5("R2D ouptut".encode())
+            for line in container.logs(stream=True):
+                output = line.decode("utf-8").strip()
+                if not output.startswith("Using local repo"):
+                    h.update(output.encode("utf-8"))
+                    logger.info(output)
+                if not dry_run:
+                    yield json.dumps({"message": output}) + "\n"
+
+            try:
+                ret = container.wait(timeout=10)
+            except (docker.errors.TimeoutError, docker.errors.NotFound):
+                ret = {"StatusCode": -123}
+
+            if ret["StatusCode"] != 0:
+                yield json.dumps({"error": f"Error building image {tag}"}) + "\n"
+
+            yield json.dumps({"return": {"ret": ret, "digest": h.hexdigest()}}) + "\n"
+        except Exception as e:
+            yield json.dumps({"error": str(e)}) + "\n"
+
+    return StreamingResponse(build_stream(), media_type="application/json")