update

turingmotors · Mar 29, 2024 · da3e62a · da3e62a
1 parent 02621a4
commit da3e62a
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 50 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -5,7 +5,7 @@ on:
       - 'main'
 
 jobs:
-  py-3-11-bullseye:
+  python311-bullseye:
     name: 3.11-bullseye
     runs-on: ubuntu-latest
     permissions:
@@ -33,32 +33,3 @@ jobs:
           context: 3.11/bullseye
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/${{ env.REPOSITORY }}:3.11-bullseye
-
-  py-3-11-bookworm:
-    name: 3.11-bookworm
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write # needed to interact with GitHub's OIDC Token endpoint.
-      contents: read
-    env:
-      REPOSITORY: jupyter
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::767397804933:role/github-actions-image-publishing-role-dpzyqzl6r1zt
-          aws-region: us-west-2
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
-
-      - name: Build and Push
-        uses: docker/build-push-action@v5
-        with:
-          context: 3.11/bookworm
-          push: true
-          tags: ${{ steps.login-ecr.outputs.registry }}/${{ env.REPOSITORY }}:3.11-bookworm
diff --git a/3.11/bullseye/Dockerfile b/3.11/bullseye/Dockerfile
@@ -1,5 +1,47 @@
+# Define global args
+ARG FFMPEG_DIR=/opt/ffmpeg
+
+# Stage 1 - build ffmpeg
+FROM --platform=linux/amd64 public.ecr.aws/docker/library/buildpack-deps:bullseye as ffmpeg_builder
+
+RUN apt-get update && apt-get install -y \
+      cmake \
+      yasm \
+      libx264-dev \
+      libx265-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG FFMPEG_DIR
+
+WORKDIR /usr/local/src
+
+ARG FFMPEG_VERSION=6.1
+ARG PKG_CONFIG_PATH="${FFMPEG_DIR}/lib/pkgconfig"
+
+RUN git clone --depth=1 -b release/${FFMPEG_VERSION} https://github.com/FFmpeg/FFmpeg . \
+    && ./configure \
+      --prefix="${FFMPEG_DIR}" \
+      --bindir="${FFMPEG_DIR}/bin" \
+      --enable-libx264 --enable-gpl --enable-nonfree \
+      --enable-libx265 \
+      --enable-shared \
+    && make -j $(nproc) \
+    && make install
+
+# Stage 2 - build final container
 FROM --platform=linux/amd64 public.ecr.aws/docker/library/python:3.11.8-bullseye
 
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Copy ffmpeg
+ARG FFMPEG_DIR
+COPY --from=ffmpeg_builder ${FFMPEG_DIR} ${FFMPEG_DIR}
+ENV LD_LIBRARY_PATH="${FFMPEG_DIR}/lib"
+ENV FFMPEG_BINARY=${FFMPEG_DIR}/bin/ffmpeg
+
+RUN ln -s ${FFMPEG_DIR}/bin/ffmpeg /usr/local/bin/ffmpeg
+RUN ln -s ${FFMPEG_DIR}/bin/ffprobe /usr/local/bin/ffprobe
+
 ARG NB_USER="sagemaker-user"
 ARG NB_UID=1000
 ARG NB_GID=100
@@ -11,32 +53,28 @@ RUN apt-get update && \
     echo "${NB_USER}    ALL=(ALL)    NOPASSWD:    ALL" >> /etc/sudoers && \
     rm -rf /var/lib/apt/lists/*
 
-# Install JupyterLab & AWS CLI
-RUN python3 -m pip install jupyterlab==4.1.0 \
-                           sagemaker==2.207.1 \
-                           https://github.com/aws/aws-cli/archive/v2.tar.gz
-
-# Docker - https://docs.aws.amazon.com/sagemaker/latest/dg/studio-updated-local.html
-
-# Add Docker's official GPG key:
-RUN install -m 0755 -d /etc/apt/keyrings && \
-    curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \
-    chmod a+r /etc/apt/keyrings/docker.asc
+# Install docker-cli and github-cli
+COPY docker-cli-install.sh github-cli-install.sh /tmp/
+RUN /tmp/docker-cli-install.sh \
+    && /tmp/github-cli-install.sh
 
-# Add the repository to Apt sources:
-RUN echo \
-  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
-  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
-  tee /etc/apt/sources.list.d/docker.list > /dev/null
+RUN apt-get update && apt-get install -y \
+      libgl1-mesa-dev \
+      libx264-dev \
+      libx265-dev \
+    && rm -rf /var/lib/apt/lists/*
 
-# Install docker (Studio only supports Docker version 20.10.X.)
-RUN apt-get update && \
-    apt-get install -y docker-ce=5:20.10.* docker-ce-cli=5:20.10.* containerd.io docker-buildx-plugin docker-compose-plugin && \
-    rm -rf /var/lib/apt/lists/*
+WORKDIR /home/$NB_USER
 
 USER $NB_USER
 
+# Install poetry and dependences
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
+
+ENV PATH=/home/$NB_USER/.local/bin:$PATH
 ENV SHELL=/bin/bash
 
+
 ENTRYPOINT ["jupyter-lab"]
 CMD ["--ServerApp.ip=0.0.0.0", "--ServerApp.port=8888", "--ServerApp.allow_origin=*", "--ServerApp.token=''", "--ServerApp.base_url=/jupyterlab/default"]
diff --git a/3.11/bullseye/docker-cli-install.sh b/3.11/bullseye/docker-cli-install.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# https://github.com/aws-samples/amazon-sagemaker-local-mode/blob/main/sagemaker_studio_docker_cli_install/sagemaker-debian-bullseye-cli-install.sh
+
+set -e
+
+apt-get update
+apt-get install ca-certificates curl
+install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+chmod a+r /etc/apt/keyrings/docker.asc
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
+  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+  tee /etc/apt/sources.list.d/docker.list > /dev/null
+apt-get update
+
+# pick the latest patch from:
+# apt-cache madison docker-ce | awk '{ print $3 }' | grep -i 20.10
+VERSION_STRING=5:20.10.24~3-0~debian-bullseye
+apt-get install docker-ce-cli=$VERSION_STRING docker-compose-plugin -y
diff --git a/3.11/bullseye/github-cli-install.sh b/3.11/bullseye/github-cli-install.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# https://github.com/cli/cli/blob/trunk/docs/install_linux.md
+
+set -e
+
+mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+apt update
+apt install gh -y
diff --git a/3.11/bullseye/requirements.txt b/3.11/bullseye/requirements.txt
@@ -0,0 +1,9 @@
+black==24.2.0
+isort==5.13.2
+jupyterlab==4.1.5
+jupyterlab-code-formatter==2.2.1
+opencv-python==4.9.0.80
+poetry==1.8.2
+sagemaker==2.213.0
+sagemaker[local]
+https://github.com/aws/aws-cli/archive/v2.tar.gz