Merge pull request #3 from lsst/tickets/DM-46257

DM-46257: Add Docker support for ppdb-replication

.github/workflows/docker.yaml

@@ -0,0 +1,33 @@
+name: Docker
+on:
+  pull_request: {}
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Needed to fetch tags, used by Python install process to
+          # figure out version number
+          fetch-depth: 0
+
+      - uses: lsst-sqre/build-and-push-to-ghcr@v1
+        id: build
+        with:
+          image: "lsst/ppdb-replication"
+          dockerfile: "docker/Dockerfile.replication"
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          push: true
+
+      - run: echo Pushed ghcr.io/lsst/ppdb-replication:${{ steps.build.outputs.tag }}

.gitignore

@@ -13,6 +13,10 @@ config.log
 version.py
 bin/
 
+# pip 
+build/
+python/lsst_dax_ppdb.egg-info/
+
 # Pytest
 tests/.tests
 pytest_session.txt

docker/Dockerfile.replication

@@ -0,0 +1,44 @@
+FROM python:3.11.6-slim-bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update and install OS dependencies
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get -y install --no-install-recommends git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install required python build dependencies
+RUN pip install --upgrade --no-cache-dir pip setuptools wheel uv
+
+# Create the build directory
+WORKDIR /build
+COPY . /build
+
+# Install requirements
+RUN uv pip install --no-cache-dir --system cassandra-driver psycopg2-binary
+RUN uv pip install --no-cache-dir --system -r requirements.txt
+
+# Install the package
+RUN uv pip install --no-cache-dir --system --no-deps .
+
+# Setup the application scripts
+WORKDIR /app
+
+# Install sdm_schemas
+# Change this using: -e SDM_SCHEMAS_REF=branch_or_tag_name
+ENV SDM_SCHEMAS_REF=main
+COPY ./docker/scripts/download-sdm-schemas.sh .
+RUN ./download-sdm-schemas.sh && rm download-sdm-schemas.sh
+ENV SDM_SCHEMAS_DIR=/app/sdm_schemas
+
+# Copy the entrypoint script
+COPY docker/scripts/entrypoint-replication.sh .
+RUN chmod +x /app/entrypoint-replication.sh
+
+# Remove the build directory
+RUN rm -rf /build
+
+# Run the wrapper script for the ppdb-replication command
+CMD ["/app/entrypoint-replication.sh"]

docker/scripts/download-sdm-schemas.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+_url=https://github.com/lsst/sdm_schemas.git
+
+# Make sure SDM_SCHEMAS_REF is set with default
+if [ -z "$SDM_SCHEMAS_REF" ]; then
+    SDM_SCHEMAS_REF=main
+fi
+
+echo "Cloning SDM schemas from $_url at $SDM_SCHEMAS_REF"
+
+# Determine if SDM_SCHEMAS_REF is a branch or a tag
+if git ls-remote --heads "$_url" "$SDM_SCHEMAS_REF" | grep -q "$SDM_SCHEMAS_REF"; then
+    echo "$SDM_SCHEMAS_REF is a branch"
+    git clone --depth=1 --branch "$SDM_SCHEMAS_REF" "$_url"
+elif git ls-remote --tags "$_url" "$SDM_SCHEMAS_REF" | grep -q "refs/tags/$SDM_SCHEMAS_REF"; then
+    echo "$SDM_SCHEMAS_REF is a tag"
+    git clone --depth=1 "$_url" && \
+        pushd sdm_schemas && \
+        git checkout tags/"$SDM_SCHEMAS_REF" && \
+        popd
+else
+    echo "Error: $SDM_SCHEMAS_REF is neither a branch nor a tag."
+    exit 1
+fi

docker/scripts/entrypoint-replication.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# This a wrapper script for the ppdb-replication script, intended to be as the
+# entrypoint to a Docker container. Command line configuration is managed by
+# environment variables, which are defined in the Phalanx application.
+###############################################################################
+
+# Bash "strict mode", to help catch problems and bugs in the shell script.
+# Every bash script you write should include this. See
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for
+# details.
+set -euo pipefail
+
+# Check if the command is found
+command -v ppdb-replication >/dev/null 2>&1 || { echo "ppdb-replication command not found"; exit 1; }
+echo "Found ppdb-replication command"
+
+# Function to check if an environment variable is set
+check_env_var() {
+    local var_name="$1"
+    local var_value="${!var_name}"
+    if [ -z "${var_value:-}" ]; then
+        echo "$var_name is a required environment variable"
+        exit 1
+    fi
+}
+
+# Check if the required environment variables are set
+check_env_var "PPDB_REPLICATION_APDB_CONFIG"
+check_env_var "PPDB_REPLICATION_PPDB_CONFIG"
+
+# Build the command from the environment variables
+_CMD="ppdb-replication"
+[ -n "${PPDB_REPLICATION_MON_LOGGER:-}" ] && _CMD="$_CMD --mon-logger $PPDB_REPLICATION_MON_LOGGER"
+[ -n "${PPDB_REPLICATION_MON_RULES:-}" ] && _CMD="$_CMD --mon-rules $PPDB_REPLICATION_MON_RULES"
+[ -n "${PPDB_REPLICATION_LOG_LEVEL:-}" ] && _CMD="$_CMD -l $PPDB_REPLICATION_LOG_LEVEL"
+_CMD="$_CMD run"
+[ "${PPDB_REPLICATION_UPDATE_EXISTING:-}" = "true" ] && _CMD="$_CMD --update"
+[ -n "${PPDB_REPLICATION_MIN_WAIT_TIME:-}" ] && _CMD="$_CMD --min-wait-time $PPDB_REPLICATION_MIN_WAIT_TIME"
+[ -n "${PPDB_REPLICATION_MAX_WAIT_TIME:-}" ] && _CMD="$_CMD --max-wait-time $PPDB_REPLICATION_MAX_WAIT_TIME"
+[ -n "${PPDB_REPLICATION_CHECK_INTERVAL:-}" ] && _CMD="$_CMD --check-interval $PPDB_REPLICATION_CHECK_INTERVAL"
+_CMD="$_CMD $PPDB_REPLICATION_APDB_CONFIG"
+_CMD="$_CMD $PPDB_REPLICATION_PPDB_CONFIG"
+
+# Run the command
+echo "Running: $_CMD"
+$_CMD

pyproject.toml

@@ -54,6 +54,10 @@ license-files = ["COPYRIGHT", "LICENSE"]
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }
 
+[project.scripts]
+ppdb-replication = "lsst.dax.ppdb.cli.ppdb_replication:main"
+ppdb-cli = "lsst.dax.ppdb.cli.ppdb_cli:main"
+
 [tool.black]
 line-length = 110
 target-version = ["py311"]