Merge branch 'main' of https://github.com/boettiger-lab/k8s

.github/workflows/jupyter-image.yml

@@ -1,4 +1,4 @@
-name: Docker Image CI
+name: Jupyter Image
 on:
   workflow_dispatch: null
   push:

.github/workflows/pytorch-image.yml

@@ -0,0 +1,34 @@
+name: PyTorch-notebook Image
+on:
+  workflow_dispatch: null
+  push:
+    paths: ['images/*']
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+    # For biggish images, github actions runs out of disk space.
+    # So we cleanup some unwanted things in the disk image, and reclaim that space for our docker use
+    # https://github.com/actions/virtual-environments/issues/2606#issuecomment-772683150
+    # and https://github.com/easimon/maximize-build-space/blob/b4d02c14493a9653fe7af06cc89ca5298071c66e/action.yml#L104
+    # This gives us a total of about 52G of free space, which should be enough for now
+      - name: cleanup disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc
+          df -h
+      - uses: actions/checkout@v3
+      - name: Login to GitHub Container Registry
+        if: github.repository == 'boettiger-lab/k8s'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Build the Docker image
+        if: github.repository == 'boettiger-lab/k8s'
+        run: docker build images/ -f images/Dockerfile.pytorch --tag ghcr.io/boettiger-lab/pytorch:latest
+      - name: Publish
+        if: github.repository == 'boettiger-lab/k8s'
+        run: docker push ghcr.io/boettiger-lab/pytorch:latest
+

github-actions/.gitignore

@@ -0,0 +1,2 @@
+helm-secrets.sh
+

images/Dockerfile.dual

@@ -0,0 +1,57 @@
+## devcontainer-focused Rocker
+FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3
+
+## latest version of geospatial libs
+RUN /rocker_scripts/experimental/install_dev_osgeo.sh
+RUN apt-get update -qq && apt-get -y install vim texlive
+
+# standard python/jupyter setup
+ENV NB_USER=rstudio
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
+RUN wget https://github.com/rocker-org/rocker-versioned2/raw/master/scripts/install_jupyter.sh && \
+  bash -e install_jupyter.sh && \
+  rm install_jupyter.sh && \
+  chown ${NB_USER}:staff -R ${VIRTUAL_ENV}
+
+# Set up conda
+ENV CONDA_ENV=/opt/miniforge3
+ENV PATH=${PATH}:$CONDA_ENV/bin
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+    bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_ENV} && \
+    chown ${NB_USER}:staff -R ${CONDA_ENV}
+
+# podman doesn't not understand group permissions
+RUN chown ${NB_USER}:staff -R ${R_HOME}/site-library
+
+# some teaching preferences
+RUN git config --system pull.rebase false && \
+    git config --system credential.helper 'cache --timeout=36000'
+
+## codeserver
+RUN curl -fsSL https://code-server.dev/install.sh | sh
+
+
+## Openscapes-specific configs
+USER rstudio
+WORKDIR /home/rstudio
+RUN usermod -s /bin/bash rstudio
+
+# install into the default environment
+COPY requirements.txt requirements.txt
+RUN python -m pip install -r requirements.txt && rm requirements.txt
+COPY install.R install.R
+RUN Rscript install.R && rm install.R
+
+# Create a conda-based env and install into it without using conda init/conda activate
+# (this yaml file doesn't include everything from pangeo, consider a different one...)
+ENV MY_ENV=${CONDA_ENV}/envs/openscapes
+RUN wget https://github.com/NASA-Openscapes/corn/raw/main/ci/environment.yml && \
+    conda env create -p ${MY_ENV} -f environment.yml
+
+# This won't be the default enviornment but we register it
+RUN ${MY_ENV}/bin/python -m pip install ipykernel && \
+    ${MY_ENV}/bin/python -m ipykernel install --prefix /opt/venv --name=openscapes
+
+
+

images/Dockerfile.gpu

@@ -1,52 +1,35 @@
-FROM docker.io/rocker/ml
+FROM quay.io/jupyter/pytorch-notebook:cuda12-ubuntu-24.04
 
-ENV NB_USER rstudio
-ENV VIRTUAL_ENV /opt/venv
-ENV PATH ${VIRTUAL_ENV}/bin:$PATH
+COPY jupyter-ai.yml environment.yml
+RUN conda update -n base -c conda-forge conda && \
+    conda env update --file environment.yml
 
-RUN /rocker_scripts/install_jupyter.sh
-RUN python3 -m pip install numpy
-RUN chown -R ${NB_USER}:staff ${VIRTUAL_ENV} && chmod -R g+rw ${VIRTUAL_ENV}
+USER root
+RUN curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o ollama-linux-amd64.tgz && tar -C /usr -xzf ollama-linux-amd64.tgz
 
-RUN /rocker_scripts/experimental/install_dev_osgeo.sh
-RUN chown -R ${NB_USER}:staff ${VIRTUAL_ENV} && chmod -R g+rw ${VIRTUAL_ENV}
+RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache
+RUN git config --system pull.rebase false && \
+    echo '"\e[5~": history-search-backward' >> /etc/inputrc && \
+    echo '"\e[6~": history-search-forward' >> /etc/inputrc
+
+#git config --system credential.helper 'cache --timeout=30000' && \
 
-COPY apt.txt apt.txt
-RUN apt-get update -qq && xargs sudo apt-get -y install < apt.txt
+RUN apt-get update -qq && apt-get -y install vim git-lfs
 
 # install codeserver
 RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache
-#RUN wget https://github.com/coder/code-server/releases/download/v4.89.1/code-server_4.89.1_amd64.deb && dpkg -i code-server*.deb && rm code-server*.deb
 
 # ollama
-# RUN curl -fsSL https://ollama.com/install.sh | sh
-RUN curl -L https://ollama.com/download/ollama-linux-amd64 -o /usr/local/bin/ollama && chmod +x /usr/local/bin/ollama
-
-
-# some preferences
-RUN git config --system pull.rebase false && \
-    git config --system credential.helper 'cache --timeout=30000' && \
-    echo '"\e[5~": history-search-backward' >> /etc/inputrc && \
-    echo '"\e[6~": history-search-forward' >> /etc/inputrc
+RUN curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o ollama-linux-amd64.tgz && tar -C /usr -xzf ollama-linux-amd64.tgz
 
 USER ${NB_USER}
-WORKDIR /home/${NB_USER}
-
-RUN usermod -s /bin/bash ${NB_USER}
-COPY spatial-requirements.txt /tmp/spatial-requirements.txt
-RUN python3 -m pip install --no-cache-dir -r /tmp/spatial-requirements.txt
-COPY rl-requirements.txt /tmp/rl-requirements.txt
-RUN python3 -m pip install --no-cache-dir -r /tmp/rl-requirements.txt
-COPY jupyter-requirements.txt /tmp/jupyter-requirements.txt
-RUN python3 -m pip install --no-cache-dir -r /tmp/jupyter-requirements.txt
-
-## Register the environment with ipykernel, mostly for vscode to find it
-RUN python3 -m ipykernel install --user --name=venv
+COPY spatial-env.yml spatial-env.yml
+RUN conda update -n base -c conda-forge conda && \
+    conda env update --file spatial-env.yml
 
+COPY rl-env.yml rl-env.yml
+RUN conda update -n base -c conda-forge conda && \
+    conda env update --file rl-env.yml
 
-#USER root 
-#COPY install.R /tmp/install.R
-#RUN Rscript /tmp/install.R && rm /tmp/install.R && chown -R ${NB_USER}:staff ${R_HOME}/site-library 
-#USER ${NB_USER}
 
 

images/Dockerfile.pytorch

@@ -0,0 +1,15 @@
+FROM quay.io/jupyter/pytorch-notebook:cuda12-ubuntu-24.04
+COPY jupyter-ai.yml environment.yml
+
+RUN conda update -n base -c conda-forge conda && \
+    conda env update --file environment.yml
+
+USER root
+RUN curl -L https://ollama.com/download/ollama-linux-amd64.tgz -o ollama-linux-amd64.tgz && tar -C /usr -xzf ollama-linux-amd64.tgz
+
+RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache
+RUN git config --system pull.rebase false && \
+    git config --system credential.helper 'cache --timeout=30000' && \
+    echo '"\e[5~": history-search-backward' >> /etc/inputrc && \
+    echo '"\e[6~": history-search-forward' >> /etc/inputrc
+

images/jupyter-ai.yml

@@ -0,0 +1,15 @@
+name: base 
+channels:
+  - conda-forge
+dependencies:
+  - ibis-duckdb
+  - jupyterlab-myst
+  - jupyter-ai
+  - jupyter-resource-usage
+  - jupyter-vscode-proxy
+  - pypdf
+  - pip
+  - seaborn
+  - pip:
+    - jupyter-tensorboard-proxy
+

images/rl-env.yml

@@ -0,0 +1,20 @@
+name: base 
+channels:
+  - conda-forge
+dependencies:
+  - darts
+  - gymnasium
+  - gputil
+  - huggingface_hub
+  - nox
+  - nvitop
+  - plotnine
+  - PyYaml
+  - pytest
+  - stable-baselines3
+  - sb3-contrib
+  - scikit-optimize
+  - pip:
+    - ray[rllib,tune]
+    - huggingface_sb3
+

images/spatial-env.yml

@@ -0,0 +1,38 @@
+name: base 
+channels:
+  - conda-forge
+dependencies:
+  - awscliv2
+  - distributed
+  - earthaccess
+  - fiona
+  - fsspec
+  - geopandas
+  - geocube
+  - leafmap[maplibregl]
+  - localtileserver
+  - mapclassify
+  - maplibre
+  - minio
+  - netCDF4
+  - odc-geo
+  - odc-stac
+  - planetary-computer
+  - pmtiles
+  - polars
+  - pyarrow
+  - pydeck
+  - pyogrio
+  - pystac
+  - pystac-client
+  - rasterio
+  - rasterstats
+  - requests
+  - rio-cogeo
+  - rioxarray
+  - stackstac
+  - streamlit
+  - tqdm
+  - xarray
+  - zarr
+