Merge pull request #7 from khiron/Build-container-images-in-workflow

Build container images in workflow

.devcontainer/DockerFile → .devcontainer/Dockerfile

@@ -1,7 +1,9 @@
 # Use continuumio/miniconda3 as the base image
 FROM continuumio/miniconda3
 
-LABEL maintainer="Richard Morris <[email protected]>"
+LABEL maintainer="khiron <https://github.com/khiron>"
+
+RUN conda update --all 
 
 # Install required packages
 RUN apt-get update -q && \
@@ -12,6 +14,7 @@ RUN apt-get update -q && \
         curl \
         sudo \
         openssh-client \
+        time \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -21,7 +24,7 @@ RUN useradd -m user && \
 
 # Switch to the non-root user
 USER user
-WORKDIR /home/user
+WORKDIR /home/user 
 
 # Set up zsh as the default shell
 SHELL ["/bin/zsh", "-c"]
@@ -43,31 +46,36 @@ RUN source /opt/conda/etc/profile.d/conda.sh && \
     /opt/conda/bin/conda create -n c312 python=3.12 -y && \
     conda activate c312
 
-
 # Initialize conda for zsh shell
 RUN echo "source /opt/conda/etc/profile.d/conda.sh" >> ~/.zshrc && \
     echo "conda activate c312" >> ~/.zshrc
 
 # Clone the repositories using SSH
 WORKDIR /home/user/repos
-RUN git clone --branch develop https://github.com/cogent3/cogent3.git /home/$USERNAME/repos/cogent3 && \
-    git clone --branch master https://github.com/cogent3/EnsemblLite.git /home/$USERNAME/repos/EnsemblLite
+RUN git clone --branch develop https://github.com/cogent3/cogent3.git /home/user/repos/cogent3 && \
+    git clone --branch develop https://github.com/cogent3/EnsemblLite.git /home/user/repos/EnsemblLite
 
 # Install flit in the conda environment
 RUN source /opt/conda/etc/profile.d/conda.sh && \
     conda activate c312 && \
     conda install -c conda-forge flit -y
-    
-# Install the repositories using flit
+
+# Install the repositories using flit --deps production to avoid dev dependencies that conflict (black/isort/click)
 RUN source /opt/conda/etc/profile.d/conda.sh && \
     conda activate c312 && \
     cd /home/user/repos/cogent3 && \
-    flit install -s && \
+    flit install --deps production -s && \
     cd /home/user/repos/EnsemblLite && \
-    flit install -s
+    flit install --deps production -s
 
-# Start in the home directory     
+# Start in the home directory
 WORKDIR /home/user
 
+# copy the data directory to ~/data
+COPY data /home/user/data
+
+# set ownership of ~/data to user
+RUN sudo chown -R user:user /home/user/
+
 # Set zsh as the default shell for the container
 CMD [ "zsh" ]

.devcontainer/Dockerfile.apes_112

@@ -0,0 +1,25 @@
+# Use the base image
+FROM ghcr.io/khiron/cogent3-workshop:latest
+
+LABEL maintainer="khiron <https://github.com/khiron>"
+
+# Populate data
+USER user
+
+# Ensure the data directory exists
+RUN mkdir -p /home/user/data
+
+# copy the config from the build context into the user data folder
+COPY data/apes_112.cfg /home/user/data/
+
+RUN echo "Populating data..." && \
+    /usr/bin/time -v /home/user/.conda/envs/c312/bin/elt download -c /home/user/data/apes_112.cfg 
+    # && \
+    # /usr/bin/time -v /home/user/.conda/envs/c312/bin/elt install -d /home/user/data/apes_112/download_112 -np 2 -f && \
+    # /usr/bin/time -v /home/user/.conda/envs/c312/bin/elt homologs -i /home/user/data/apes_112/install_112 --ref human --limit 20 -np 2 -o sampled-20
+
+# Start in the home directory
+WORKDIR /home/user
+
+# Set zsh as the default shell for the container
+CMD [ "zsh" ]

.devcontainer/README.md

@@ -1,36 +1,63 @@
-# Cogent3 workshop Development container for cogent3 and EnsemblLite
+# Cogent3 Workshop Development Container for cogent3 and EnsemblLite
 
-This folder contains a Docker configuration file for setting up the workshop environment for working with `cogent3` and `EnsemblLite` in a docker container, and a `devcontainer.json` configuration so that VS Code can manage the container.  The container installs the develop branch of `cogent3`, and the master branch of `EnsemblLite`.  It also pre-populates VS code extensions for jupiter notebooks and for working with python.
+This folder contains Docker configuration files for setting up the workshop environment for working with `cogent3` and `EnsemblLite` in a Docker container, 
+and a `devcontainer.json` configuration so that VS Code can manage the container. The container installs the develop branch of `cogent3`, 
+and the master branch of `EnsemblLite`. 
+It also pre-populates VS Code extensions for Jupyter notebooks and for working with Python.
+
+For instructions on building this Docker image into a container to run the workshop, see the [Computer Setup Instructions](https://github.com/cogent3/Cogent3Workshop/wiki/Computer-Setup).
+
+## Choosing the Container Version
+
+You can choose whether to have the data pre-populated in the `~/data` folder by selecting the appropriate container from the GitHub Container Registry (GHCR).
+
+- **Default Container**: This container does not include prepopulated data. Use the default `devcontainer.json` file with the image set to `ghcr.io/khiron/cogent3-workshop:latest`.
+- **Container prepopulated with apes_112**: This container includes the data prepopulated in the `~/data` folder. Change the image in the `devcontainer.json` file to `ghcr.io/khiron/cogent3-workshop:apes_112`.
+
+### Steps to Switch Containers
+
+1. Open the `devcontainer.json` file in your editor.
+2. To use the default container, ensure the image is set to:
+   ``"image": "ghcr.io/khiron/cogent3-workshop:latest"``
+3. To use the prepopulated data container, change the image to:
+   ``"image": "ghcr.io/khiron/cogent3-workshop:apes_112"``
+
+4. Add the following `updateContentCommand` to ensure the latest image is pulled:
+   ``"updateContentCommand": "docker pull ghcr.io/khiron/cogent3-workshop:latest"``
+   or
+   ``"updateContentCommand": "docker pull ghcr.io/khiron/cogent3-workshop:apes_112"`` if you are pre-populating the container with data 
 
-For instructions for building this docker image into a container to run the workshop, see the [Computer setup instructions](https://github.com/cogent3/Cogent3Workshop/wiki/Computer-Setup).
 
 ## Configuring Resource Allocation
 
-You can configure the resource used for the development container by modifying the `devcontainer.json` file. This file allows you to specify the number of CPUs and the amount of RAM the container can use.
+You can configure the resources used for the development container by modifying the `devcontainer.json` file. This file allows you to specify the number of CPUs and the amount of RAM the container can use.
 
-Steps
+### Steps
 
-- Open the devcontainer.json file in your editor.
-- Look for the runArgs property. by default it is set to `    "runArgs": ["--cpus", "2", "-m", "4g"],`, which is 2 cpu and 4GB of RAM.
-- To limit the number of CPUs the container can use, add "--cpus" and your desired number of CPUs to the runArgs array.
-- To limit the amount of RAM the container can use, add "-m" and "4g" (or your desired amount of RAM) to the runArgs array.
+- Open the `devcontainer.json` file in your editor.
+- Look for the `runArgs` property. By default, it is set to ``"runArgs": ["--cpus", "2", "-m", "4g"],``, which allocates 2 CPUs and 4GB of RAM.
+- To limit the number of CPUs the container can use, add ``"--cpus"`` and your desired number of CPUs to the `runArgs` array.
+- To limit the amount of RAM the container can use, add ``"-m"`` and your desired amount of RAM (e.g., ``"4g"``) to the `runArgs` array.
 
-Please note that these settings will only limit the maximum resources a container can use. The container will use less resources if it doesn't need the maximum amount.  Also, please make sure that your Docker host has enough resources to allocate to the container. If the host doesn't have enough resources, the container may not start.
+Please note that these settings will only limit the maximum resources a container can use. 
+The container will use fewer resources if it doesn't need the maximum amount. 
+Also, please ensure that your Docker host has enough resources to allocate to the container. 
+If the host doesn't have enough resources, the container may not start.
 
-## Container description
+## Container Description
 
-This development container is designed to provide a consistent and reproducible development environment for working with the    `cogent3` and `EnsemblLite` Python packages. It is based on a slim Debian image and includes a variety of tools and configurations to facilitate development.
+This development container is designed to provide a consistent and reproducible development environment for working with the `cogent3` and `EnsemblLite` Python packages. 
+It is based on a slim Debian image and includes a variety of tools and configurations to facilitate development.
 
-- Base Image: The base image is continuumIO/miniconda3 which is based on debian:bookworm-slim, a slim version of the latest Debian release.
-- Python: Python 3.12 is installed and set as the default Python version.
-- Non-root User: A non-root user named user is created for running the container. This is a good security practice.
-- Repositories: The cogent3 and EnsemblLite repositories are cloned into the /home/user/repos directory.
-- Conda Environment: A new conda environment named c312 is created with Python 3.12. This environment is activated when a new shell is started.
-- Packages: Flit is installed in the c312 environment 
-- `cogent3` and `EnsemblLite`: The `cogent3` and `EnsemblLite` packages are installed from the cloned repositories in the c312 environment using flit.
-- Zsh and Oh My Zsh: The Zsh shell is installed and set as the default shell. Oh My Zsh is installed for additional shell features and configurations. Several plugins are enabled, including git, zsh-autosuggestions, zsh-syntax-highlighting, and autojump.
+- **Base Image**: The base image is `continuumio/miniconda3`, which is based on `debian:bookworm-slim`, a slim version of the latest Debian release.
+- **Python**: Python 3.12 is installed and set as the default Python version.
+- **Non-root User**: A non-root user with sudo access named `user` is created in the container. 
+- **Repositories**: The `cogent3` and `EnsemblLite` repositories are cloned into the `/home/user/repos` directory.
+- **Conda Environment**: A new conda environment named `c312` is created with Python 3.12. This environment is activated when a new shell is started.
+- **Packages**: Flit is installed in the `c312` environment.
+- **`cogent3` and `EnsemblLite`**: The `cogent3` and `EnsemblLite` packages are installed from the cloned repositories in the `c312` environment using flit.
+- **Zsh and Oh My Zsh**: The Zsh shell is installed and set as the default shell. Oh My Zsh is installed for additional shell features and configurations. Several plugins are enabled, including `git`, `zsh-autosuggestions`, `zsh-syntax-highlighting`, and `autojump`.
 
 ## Using the Container
 
-To use the container, you can load your clone of the workshop repository in VS-Code and you will be asked if you want to run in a devcontainer.  When you select yes, VS-Code will rebuild the container, start it, remote connect into the container, and open a new VS-Code window in the container.  You can then open a terminal in the container and you should automatically be in the zsh shell, in the mamba c312 environment, logged in as the user `User`, and in the directory /workspaces/$workshopname$ with the workshop files.  You can navigate to the the /home/user/repos directory to access the cogent3 and EnsemblLite repositories.
-
+To use the container, you can load your clone of the workshop repository in VS Code and you will be asked if you want to run in a devcontainer. When you select "yes", VS Code will rebuild the container, start it, remote connect into the container, and open a new VS Code window in the container. You can then open a terminal in the container and you should automatically be in the Zsh shell, in the mamba `c312` environment, logged in as the user `user`, and in the directory `/workspaces/$workshopname$` with the workshop files. You can navigate to the `/home/user/repos` directory to access the `cogent3` and `EnsemblLite` repositories.

.devcontainer/devcontainer.json

@@ -1,24 +1,30 @@
 {
-    "name": "Cogent3 Workshop",
-    "build": {
-      "dockerfile": "Dockerfile",
-      "context": ".."
-    },    
-    "runArgs": ["--cpus", "2", "-m", "4g"],
-    "
-    settings": {
-      "terminal.integrated.defaultProfile.linux": "zsh",
-      "terminal.integrated.profiles.linux": {
-        "zsh": {
-          "path": "zsh"
-        }
-      },
-      "python.defaultInterpreterPath": "/home/user/.conda/envs/c312/bin/python",
-    },
-    "extensions": [
-      "ms-python.python",
-      "ms-toolsai.jupyter"
-    ],
-    "remoteUser": "user",
-    "postStartCommand": "echo 'Cogent3 Workshop 2024 environment is ready!'"
-  }
+  "name": "Cogent3 Workshop",
+  "image": "ghcr.io/khiron/cogent3-workshop:latest",
+  "runArgs": ["--cpus", "2", "-m", "4g"],
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-toolsai.jupyter"
+      ],
+      "settings": {
+        "terminal.integrated.defaultProfile.linux": "zsh",
+        "terminal.integrated.profiles.linux": {
+          "zsh": {
+            "path": "zsh"
+          }
+        },
+        "python.defaultInterpreterPath": "/home/user/.conda/envs/c312/bin/python"
+      }
+    }
+  },
+  "remoteUser": "user",
+  "workspaceFolder": "/home/user/workshop",
+  "postStartCommand": "chown -R user:user /home/user/.ssh && echo 'Cogent3 Workshop 2024 environment is ready!'",
+  "mounts": [
+    "source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh,target=/home/user/.ssh,type=bind,consistency=cached",
+    "source=${localWorkspaceFolder},target=/home/user/workshop,type=bind,consistency=cached"
+  ],
+  "updateContentCommand": "docker pull ghcr.io/khiron/cogent3-workshop:latest"
+}

.github/workflows/README.md

@@ -0,0 +1,4 @@
+To run this workflow there has to be a secret stored named `GHCR_PAT` that contains a GitHub Personal Access Token with the following permissions set:
+- read:packages
+  - write:packages
+- delete:packages

.github/workflows/publish_containers.yml

@@ -1,36 +1,30 @@
-# This is a basic workflow to help you get started with Actions
+name: Build and Push Docker images
 
-name: CI
-
-# Controls when the workflow will run
 on:
-  # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
+    branches:
+      - main
+      - develop
+      - Build-container-images-in-workflow
 
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
-  build:
-    # The type of runner that the job will run on
+  build-and-push:
     runs-on: ubuntu-latest
-
-    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v2
 
-      # Runs a single command using the runners shell
-      - name: Run a one-line script
-        run: echo Hello, world!
+      - name: Log in to GitHub Container Registry
+        run: echo "${{ secrets.GHCR_PAT }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+
+      - name: Build and Push Base Docker image
+        run: |
+          docker build -f .devcontainer/Dockerfile . \
+            -t ghcr.io/${{ github.repository_owner }}/cogent3-workshop:latest
+          docker push ghcr.io/${{ github.repository_owner }}/cogent3-workshop:latest
 
-      # Runs a set of commands using the runners shell
-      - name: Run a multi-line script
+      - name: Build and Push Docker image prepopulated with apes_112
         run: |
-          echo Add other actions to build,
-          echo test, and deploy your project.
+          docker build -f .devcontainer/Dockerfile.apes_112 . \
+            -t ghcr.io/${{ github.repository_owner }}/cogent3-workshop:apes_112
+          docker push ghcr.io/${{ github.repository_owner }}/cogent3-workshop:apes_112

data/apes_112.cfg

@@ -0,0 +1,17 @@
+[remote path]
+host=ftp.ensembl.org
+path=pub
+[local path]
+staging_path=apes_112/download_112
+install_path=apes_112/install_112
+[release]
+release=112
+[Human]
+db=core
+[Chimpanzee]
+db=core
+[Gorilla]
+db=core
+[compara]
+align_names=10_primates.epo
+homologies=