From 8e2b63fcd7361eb4abdbddcefeea1e7a59ddefa3 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Wed, 22 Nov 2023 23:14:52 +0800 Subject: [PATCH 01/18] tmp dockerfile Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 plugins/flytekit-flyin/Dockerfile diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile new file mode 100644 index 0000000000..8fde6f559c --- /dev/null +++ b/plugins/flytekit-flyin/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.9-slim-buster +USER root +WORKDIR /root +ENV PYTHONPATH /root + +# Install necessary packages +RUN apt-get update && apt-get install build-essential -y +RUN apt-get install git -y + +# Install additional dependencies +RUN apt-get -y install curl +RUN mkdir -p /tmp/ +RUN mkdir -p /tmp/code-server +RUN curl -kfL -o /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz +RUN tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code-server/ + +ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" + +# RUN code-server --install-extension https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix +# RUN code-server --install-extension https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix + + + +# Install Python packages +RUN pip install -U git+https://github.com/Future-Outlier/flytekit.git@77dd7dd767feae3636fbe541b40a43a1bd11e216#subdirectory=plugins/flytekit-flyin +# RUN pip install flytekitplugins-flyin From 76eab0dc274aedb4763afc299bda53676abf1ca4 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Wed, 22 Nov 2023 23:41:57 +0800 Subject: [PATCH 02/18] docker file with code-server Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 8fde6f559c..3396251e99 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -16,9 +16,12 @@ RUN tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" -# RUN code-server --install-extension https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -# RUN code-server --install-extension https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix +RUN wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server +RUN wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server + +RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix +RUN code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix # Install Python packages From aeb441feb56210c89a47c3841d3403d9dca50a5d Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 23 Nov 2023 09:56:43 +0800 Subject: [PATCH 03/18] finish Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 3396251e99..9de4f0c744 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -7,8 +7,9 @@ ENV PYTHONPATH /root RUN apt-get update && apt-get install build-essential -y RUN apt-get install git -y -# Install additional dependencies +# Install code-server and its extensions RUN apt-get -y install curl +RUN apt-get -y install wget RUN mkdir -p /tmp/ RUN mkdir -p /tmp/code-server RUN curl -kfL -o /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz @@ -17,13 +18,11 @@ RUN tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" RUN wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server - RUN wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix RUN code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix - -# Install Python packages -RUN pip install -U git+https://github.com/Future-Outlier/flytekit.git@77dd7dd767feae3636fbe541b40a43a1bd11e216#subdirectory=plugins/flytekit-flyin -# RUN pip install flytekitplugins-flyin +# Install Flytekit and Flyin packages +RUN pip install flytekitplugins-flyin +ENV FLYTE_SDK_LOGGING_LEVEL=20 From c394aee6666cd3f0596e7bb84300f86c70f73c6d Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 23 Nov 2023 11:24:12 +0800 Subject: [PATCH 04/18] dockerfile complete version Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 19 ++++++++++++++----- plugins/flytekit-flyin/setup.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 9de4f0c744..8fc6cf1c80 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -1,10 +1,12 @@ -FROM python:3.9-slim-buster -USER root +ARG PYTHON_VERSION +FROM python:${PYTHON_VERSION}-slim-buster +MAINTAINER Flyte Team +LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit WORKDIR /root ENV PYTHONPATH /root # Install necessary packages -RUN apt-get update && apt-get install build-essential -y +RUN apt-get update && apt-get install build-essential libmagic1 -y RUN apt-get install git -y # Install code-server and its extensions @@ -23,6 +25,13 @@ RUN wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai. RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix RUN code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix -# Install Flytekit and Flyin packages -RUN pip install flytekitplugins-flyin +# Install Flyin packages +COPY . ../flytekit-flyin +RUN pip install -e /flytekit-flyin + ENV FLYTE_SDK_LOGGING_LEVEL=20 + +RUN useradd -u 1000 flytekit +RUN chown flytekit: /root +RUN chown flytekit: /home +USER flytekit diff --git a/plugins/flytekit-flyin/setup.py b/plugins/flytekit-flyin/setup.py index 5215e07e77..d4c89f94a0 100644 --- a/plugins/flytekit-flyin/setup.py +++ b/plugins/flytekit-flyin/setup.py @@ -15,7 +15,7 @@ author_email="admin@flyte.org", description="This package holds the flyin plugins for flytekit", namespace_packages=["flytekitplugins"], - packages=[f"flytekitplugins.{PLUGIN_NAME}"], + packages=[f"flytekitplugins.{PLUGIN_NAME}", f"flytekitplugins.{PLUGIN_NAME}.vscode_lib"], install_requires=plugin_requires, license="apache2", python_requires=">=3.8", From bc19147de1f41992825a8bad97fb06966bcd49ad Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 23 Nov 2023 23:19:26 +0800 Subject: [PATCH 05/18] dockerfile success Signed-off-by: Future Outlier --- .github/workflows/pythonpublish.yml | 42 +++++++++++++++++++++++++++++ plugins/flytekit-flyin/Dockerfile | 42 ++++++++++++++--------------- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 6dc4bb4459..19b33fce47 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -225,3 +225,45 @@ jobs: file: ./plugins/flytekit-spark/Dockerfile cache-from: type=gha cache-to: type=gha,mode=max + + build-and-push-flyin-images: + runs-on: ubuntu-latest + needs: deploy + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Login to GitHub Container Registry + if: ${{ github.event_name == 'release' }} + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: "${{ secrets.FLYTE_BOT_USERNAME }}" + password: "${{ secrets.FLYTE_BOT_PAT }}" + - name: Prepare flyin Image Names + id: flyin-names + uses: docker/metadata-action@v3 + with: + images: | + ghcr.io/${{ github.repository_owner }}/flytekit + tags: | + flyin-latest + flyin-${{ github.sha }} + flyin-${{ needs.deploy.outputs.version }} + - name: Push Flyin Image to GitHub Registry + uses: docker/build-push-action@v2 + with: + context: "./plugins/flytekit-flyin/" + platforms: linux/arm64, linux/amd64 + push: ${{ github.event_name == 'release' }} + tags: ${{ steps.flyin-names.outputs.tags }} + build-args: | + VERSION=${{ needs.deploy.outputs.version }} + file: ./plugins/flytekit-flyin/Dockerfile + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 8fc6cf1c80..dfe6630ba2 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -6,32 +6,32 @@ WORKDIR /root ENV PYTHONPATH /root # Install necessary packages -RUN apt-get update && apt-get install build-essential libmagic1 -y -RUN apt-get install git -y - -# Install code-server and its extensions -RUN apt-get -y install curl -RUN apt-get -y install wget -RUN mkdir -p /tmp/ -RUN mkdir -p /tmp/code-server -RUN curl -kfL -o /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz -RUN tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code-server/ +RUN apt-get update \ + && apt-get install build-essential libmagic1 git wget -y \ + && mkdir -p /tmp/ \ + && mkdir -p /tmp/code-server \ + && wget --no-check-certificate -O /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz \ + && tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code-server/ \ + && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ + && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" -RUN wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server -RUN wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server +RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ + && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix \ + pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION -RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix -RUN code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix +ENV FLYTE_SDK_LOGGING_LEVEL=20 -# Install Flyin packages -COPY . ../flytekit-flyin -RUN pip install -e /flytekit-flyin +# Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 +RUN apt-get clean autoclean \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ -ENV FLYTE_SDK_LOGGING_LEVEL=20 +RUN useradd -u 1000 flytekit \ + && chown flytekit: /root \ + && chown flytekit: /home \ + && chown -R flytekit:flytekit /tmp \ + && chmod 755 /tmp -RUN useradd -u 1000 flytekit -RUN chown flytekit: /root -RUN chown flytekit: /home USER flytekit From a00e81f95aa93ef03d504be1a4b159d346c97e28 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 23 Nov 2023 23:24:09 +0800 Subject: [PATCH 06/18] Add annotation Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index dfe6630ba2..1dc1d64c29 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -28,6 +28,8 @@ RUN apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ +# Some packages will create config file under /home by default, so we need to make sure it's writable +# change the permission of /tmp, so that others can run command on it RUN useradd -u 1000 flytekit \ && chown flytekit: /root \ && chown flytekit: /home \ @@ -35,3 +37,5 @@ RUN useradd -u 1000 flytekit \ && chmod 755 /tmp USER flytekit + +ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE" From 2b9d1317b533a2e20dc5b7c22197fc1f1c39a876 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 23 Nov 2023 23:32:55 +0800 Subject: [PATCH 07/18] update RUN Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 1dc1d64c29..c1914fac32 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -16,21 +16,18 @@ RUN apt-get update \ && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" - -RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ - && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix \ - pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION - ENV FLYTE_SDK_LOGGING_LEVEL=20 # Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 -RUN apt-get clean autoclean \ - && apt-get autoremove --yes \ - && rm -rf /var/lib/{apt,dpkg,cache,log}/ - # Some packages will create config file under /home by default, so we need to make sure it's writable # change the permission of /tmp, so that others can run command on it -RUN useradd -u 1000 flytekit \ +RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ + && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix \ + && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ + && apt-get clean autoclean \ + && apt-get autoremove --yes \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ + && useradd -u 1000 flytekit \ && chown flytekit: /root \ && chown flytekit: /home \ && chown -R flytekit:flytekit /tmp \ From c394eb7cfd027761894397fc421f14aff63f45fa Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Sat, 25 Nov 2023 09:45:01 +0800 Subject: [PATCH 08/18] better comments for more readable Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index c1914fac32..95db3b3f26 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -6,6 +6,9 @@ WORKDIR /root ENV PYTHONPATH /root # Install necessary packages +# 1. Update the necessary packages for flytekit +# 2. Install code-server +# 3. Download code-server extensions via wget RUN apt-get update \ && apt-get install build-essential libmagic1 git wget -y \ && mkdir -p /tmp/ \ @@ -15,12 +18,15 @@ RUN apt-get update \ && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server +# Set the environment variable for code-server ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" ENV FLYTE_SDK_LOGGING_LEVEL=20 -# Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 -# Some packages will create config file under /home by default, so we need to make sure it's writable -# change the permission of /tmp, so that others can run command on it +# 1. Use code-server to install the extensions +# 2. Install flytekit and flytekit-flyin with no cache +# 3. Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 +# 4. Some packages will create config file under /home by default, so we need to make sure it's writable +# 5. Change the permission of /tmp, so that others can run command on it RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix \ && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ From 43a595904a4a7fef7c102d7dd1d8949da3263df6 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Mon, 27 Nov 2023 22:32:27 +0800 Subject: [PATCH 09/18] remove logger env Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 95db3b3f26..87d13fffa1 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -20,7 +20,6 @@ RUN apt-get update \ # Set the environment variable for code-server ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" -ENV FLYTE_SDK_LOGGING_LEVEL=20 # 1. Use code-server to install the extensions # 2. Install flytekit and flytekit-flyin with no cache From ede3184ffbbd9b183be853b6e81fa72179e10ebc Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 13:10:19 +0800 Subject: [PATCH 10/18] remove libmagic1 Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 87d13fffa1..b97cdaa8b9 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -10,7 +10,7 @@ ENV PYTHONPATH /root # 2. Install code-server # 3. Download code-server extensions via wget RUN apt-get update \ - && apt-get install build-essential libmagic1 git wget -y \ + && apt-get install build-essential git wget -y \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ && wget --no-check-certificate -O /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz \ From d6ee4814d136500c2eac2d4a654abe00f1f5e6ec Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 20:17:58 +0800 Subject: [PATCH 11/18] succeeded Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 41 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index b97cdaa8b9..5c09f1e8e8 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -5,39 +5,42 @@ LABEL org.opencontainers.image.source https://github.com/flyteorg/flytekit WORKDIR /root ENV PYTHONPATH /root -# Install necessary packages +ARG VERSION +ARG BUILDARCH + # 1. Update the necessary packages for flytekit # 2. Install code-server -# 3. Download code-server extensions via wget +# 3. Download code-server extensions for Python and Jupyter via wget +# 4. Install flytekit and flytekit-flyin with no cache +# 5. Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 +# 6. Some packages will create config file under /home by default, so we need to make sure it's writable +# 7. Change the permission of /tmp, so that others can run command on it RUN apt-get update \ && apt-get install build-essential git wget -y \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ - && wget --no-check-certificate -O /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-amd64.tar.gz \ - && tar -xzf /tmp/code-server/code-server-4.18.0-linux-amd64.tar.gz -C /tmp/code-server/ \ + && wget --no-check-certificate -O /tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-${BUILDARCH}.tar.gz \ + && tar -xzf /tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}.tar.gz -C /tmp/code-server/ \ && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ - && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server - -# Set the environment variable for code-server -ENV PATH="/tmp/code-server/code-server-4.18.0-linux-amd64/bin:${PATH}" - -# 1. Use code-server to install the extensions -# 2. Install flytekit and flytekit-flyin with no cache -# 3. Delete apt cache. Reference: https://gist.github.com/marvell/7c812736565928e602c4 -# 4. Some packages will create config file under /home by default, so we need to make sure it's writable -# 5. Change the permission of /tmp, so that others can run command on it -RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ - && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix \ - && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ + && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ + && pip install --no-cache-dir -U git+https://github.com/Future-Outlier/flytekit.git@ede3184ffbbd9b183be853b6e81fa72179e10ebc#subdirectory=plugins/flytekit-flyin \ && apt-get clean autoclean \ && apt-get autoremove --yes \ - && rm -rf /var/lib/{apt,dpkg,cache,log}/ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ && useradd -u 1000 flytekit \ && chown flytekit: /root \ && chown flytekit: /home \ && chown -R flytekit:flytekit /tmp \ && chmod 755 /tmp + # && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ + +# Set the environment variable for code-server +ENV PATH="/tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}/bin:${PATH}" USER flytekit -ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE" +# Install extensions using code-server +# Execution is performed here as code-server configuration depends on the USER setting +# Now, the config of code-server will be stored in /home/flytekit/.config/code-server/config.yaml +RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ + && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix From 3e63123125012c915f22f50d4ea9c3c750452b82 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 22:28:15 +0800 Subject: [PATCH 12/18] support arm64 and amd64 Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 5c09f1e8e8..d36d006476 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -19,8 +19,8 @@ RUN apt-get update \ && apt-get install build-essential git wget -y \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ - && wget --no-check-certificate -O /tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.18.0/code-server-4.18.0-linux-${BUILDARCH}.tar.gz \ - && tar -xzf /tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}.tar.gz -C /tmp/code-server/ \ + && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${BUILDARCH}.tar.gz \ + && tar -xzf /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz -C /tmp/code-server/ \ && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ && pip install --no-cache-dir -U git+https://github.com/Future-Outlier/flytekit.git@ede3184ffbbd9b183be853b6e81fa72179e10ebc#subdirectory=plugins/flytekit-flyin \ @@ -35,12 +35,13 @@ RUN apt-get update \ # && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ # Set the environment variable for code-server -ENV PATH="/tmp/code-server/code-server-4.18.0-linux-${BUILDARCH}/bin:${PATH}" +ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}/bin:${PATH}" USER flytekit # Install extensions using code-server # Execution is performed here as code-server configuration depends on the USER setting # Now, the config of code-server will be stored in /home/flytekit/.config/code-server/config.yaml + RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix From 606bc8aa47c99cbfdfeb9cdd1a9b4f9ecd832f83 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 22:29:23 +0800 Subject: [PATCH 13/18] add flyin package Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index d36d006476..c288fc32ac 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -17,6 +17,7 @@ ARG BUILDARCH # 7. Change the permission of /tmp, so that others can run command on it RUN apt-get update \ && apt-get install build-essential git wget -y \ + && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${BUILDARCH}.tar.gz \ @@ -32,7 +33,7 @@ RUN apt-get update \ && chown flytekit: /home \ && chown -R flytekit:flytekit /tmp \ && chmod 755 /tmp - # && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ + # Set the environment variable for code-server ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}/bin:${PATH}" From f51367571b247dfa966a7fc62d510eb4820472fa Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 22:29:44 +0800 Subject: [PATCH 14/18] add flyin package Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index c288fc32ac..71c428546c 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -17,14 +17,13 @@ ARG BUILDARCH # 7. Change the permission of /tmp, so that others can run command on it RUN apt-get update \ && apt-get install build-essential git wget -y \ - && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${BUILDARCH}.tar.gz \ && tar -xzf /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz -C /tmp/code-server/ \ && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ - && pip install --no-cache-dir -U git+https://github.com/Future-Outlier/flytekit.git@ede3184ffbbd9b183be853b6e81fa72179e10ebc#subdirectory=plugins/flytekit-flyin \ + && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ From bf3c2f16d2091a97c0ed2404a0e5138cf4e8e6b5 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Tue, 28 Nov 2023 22:30:06 +0800 Subject: [PATCH 15/18] remove space Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 71c428546c..92c180e145 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -31,8 +31,7 @@ RUN apt-get update \ && chown flytekit: /root \ && chown flytekit: /home \ && chown -R flytekit:flytekit /tmp \ - && chmod 755 /tmp - + && chmod 755 /tmp # Set the environment variable for code-server ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}/bin:${PATH}" From 7470baf933cd8b0da1bc27184ed6dfbb56a25dc2 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 30 Nov 2023 10:16:02 +0800 Subject: [PATCH 16/18] merge Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 92c180e145..eef6366454 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -6,7 +6,7 @@ WORKDIR /root ENV PYTHONPATH /root ARG VERSION -ARG BUILDARCH +ARG TARGETARCH # 1. Update the necessary packages for flytekit # 2. Install code-server @@ -19,8 +19,8 @@ RUN apt-get update \ && apt-get install build-essential git wget -y \ && mkdir -p /tmp/ \ && mkdir -p /tmp/code-server \ - && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${BUILDARCH}.tar.gz \ - && tar -xzf /tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}.tar.gz -C /tmp/code-server/ \ + && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${TARGETARCH}.tar.gz \ + && tar -xzf /tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}.tar.gz -C /tmp/code-server/ \ && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ @@ -34,13 +34,13 @@ RUN apt-get update \ && chmod 755 /tmp # Set the environment variable for code-server -ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${BUILDARCH}/bin:${PATH}" +ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}/bin:${PATH}" USER flytekit # Install extensions using code-server # Execution is performed here as code-server configuration depends on the USER setting +# If we install it as ROOT, the config will be stored in /root/.config/code-server/config.yaml # Now, the config of code-server will be stored in /home/flytekit/.config/code-server/config.yaml - RUN code-server --install-extension /tmp/code-server/ms-python.python-2023.20.0.vsix \ && code-server --install-extension /tmp/code-server/ms-toolsai.jupyter-2023.9.100.vsix From 930d928f9f79e27743d9ba167993ecbc711ba7b5 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 30 Nov 2023 13:32:06 +0800 Subject: [PATCH 17/18] no check certificate Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index eef6366454..6a2eb81543 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -21,8 +21,8 @@ RUN apt-get update \ && mkdir -p /tmp/code-server \ && wget --no-check-certificate -O /tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}.tar.gz https://github.com/coder/code-server/releases/download/v4.19.0/code-server-4.19.0-linux-${TARGETARCH}.tar.gz \ && tar -xzf /tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}.tar.gz -C /tmp/code-server/ \ - && wget https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ - && wget https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ + && wget --no-check-certificate https://open-vsx.org/api/ms-python/python/2023.20.0/file/ms-python.python-2023.20.0.vsix -P /tmp/code-server \ + && wget --no-check-certificate https://open-vsx.org/api/ms-toolsai/jupyter/2023.9.100/file/ms-toolsai.jupyter-2023.9.100.vsix -P /tmp/code-server \ && pip install --no-cache-dir -U flytekit-flyin==$VERSION flytekit==$VERSION \ && apt-get clean autoclean \ && apt-get autoremove --yes \ From ac7d690adedc1979f20fca1862df081f026d4217 Mon Sep 17 00:00:00 2001 From: Future Outlier Date: Thu, 30 Nov 2023 13:32:58 +0800 Subject: [PATCH 18/18] blank Signed-off-by: Future Outlier --- plugins/flytekit-flyin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 6a2eb81543..213ac1a665 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -31,7 +31,7 @@ RUN apt-get update \ && chown flytekit: /root \ && chown flytekit: /home \ && chown -R flytekit:flytekit /tmp \ - && chmod 755 /tmp + && chmod 755 /tmp # Set the environment variable for code-server ENV PATH="/tmp/code-server/code-server-4.19.0-linux-${TARGETARCH}/bin:${PATH}"