diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000..5237eaa --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,33 @@ +name: Docker +on: + pull_request: {} + push: + branches: + - main + tags: + - "*" + +jobs: + docker: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + packages: write + + steps: + - uses: actions/checkout@v4 + with: + # Needed to fetch tags, used by Python install process to + # figure out version number + fetch-depth: 0 + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build + with: + image: "lsst/ppdb-replication" + dockerfile: "docker/Dockerfile.replication" + github_token: ${{ secrets.GITHUB_TOKEN }} + push: true + + - run: echo Pushed ghcr.io/lsst/ppdb-replication:${{ steps.build.outputs.tag }} diff --git a/.gitignore b/.gitignore index bb98353..a9fc08d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ config.log version.py bin/ +# pip +build/ +python/lsst_dax_ppdb.egg-info/ + # Pytest tests/.tests pytest_session.txt diff --git a/docker/Dockerfile.replication b/docker/Dockerfile.replication new file mode 100644 index 0000000..ba86f5b --- /dev/null +++ b/docker/Dockerfile.replication @@ -0,0 +1,44 @@ +FROM python:3.11.6-slim-bookworm + +ENV DEBIAN_FRONTEND=noninteractive + +# Update and install OS dependencies +RUN apt-get -y update && \ + apt-get -y upgrade && \ + apt-get -y install --no-install-recommends git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install required python build dependencies +RUN pip install --upgrade --no-cache-dir pip setuptools wheel uv + +# Create the build directory +WORKDIR /build +COPY . /build + +# Install requirements +RUN uv pip install --no-cache-dir --system cassandra-driver psycopg2-binary +RUN uv pip install --no-cache-dir --system -r requirements.txt + +# Install the package +RUN uv pip install --no-cache-dir --system --no-deps . + +# Setup the application scripts +WORKDIR /app + +# Install sdm_schemas +# Change this using: -e SDM_SCHEMAS_REF=branch_or_tag_name +ENV SDM_SCHEMAS_REF=main +COPY ./docker/scripts/download-sdm-schemas.sh . +RUN ./download-sdm-schemas.sh && rm download-sdm-schemas.sh +ENV SDM_SCHEMAS_DIR=/app/sdm_schemas + +# Copy the entrypoint script +COPY docker/scripts/entrypoint-replication.sh . +RUN chmod +x /app/entrypoint-replication.sh + +# Remove the build directory +RUN rm -rf /build + +# Run the wrapper script for the ppdb-replication command +CMD ["/app/entrypoint-replication.sh"] diff --git a/docker/scripts/download-sdm-schemas.sh b/docker/scripts/download-sdm-schemas.sh new file mode 100755 index 0000000..5b42496 --- /dev/null +++ b/docker/scripts/download-sdm-schemas.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +_url=https://github.com/lsst/sdm_schemas.git + +# Make sure SDM_SCHEMAS_REF is set with default +if [ -z "$SDM_SCHEMAS_REF" ]; then + SDM_SCHEMAS_REF=main +fi + +echo "Cloning SDM schemas from $_url at $SDM_SCHEMAS_REF" + +# Determine if SDM_SCHEMAS_REF is a branch or a tag +if git ls-remote --heads "$_url" "$SDM_SCHEMAS_REF" | grep -q "$SDM_SCHEMAS_REF"; then + echo "$SDM_SCHEMAS_REF is a branch" + git clone --depth=1 --branch "$SDM_SCHEMAS_REF" "$_url" +elif git ls-remote --tags "$_url" "$SDM_SCHEMAS_REF" | grep -q "refs/tags/$SDM_SCHEMAS_REF"; then + echo "$SDM_SCHEMAS_REF is a tag" + git clone --depth=1 "$_url" && \ + pushd sdm_schemas && \ + git checkout tags/"$SDM_SCHEMAS_REF" && \ + popd +else + echo "Error: $SDM_SCHEMAS_REF is neither a branch nor a tag." + exit 1 +fi diff --git a/docker/scripts/entrypoint-replication.sh b/docker/scripts/entrypoint-replication.sh new file mode 100755 index 0000000..e7ced8a --- /dev/null +++ b/docker/scripts/entrypoint-replication.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +############################################################################### +# This a wrapper script for the ppdb-replication script, intended to be as the +# entrypoint to a Docker container. Command line configuration is managed by +# environment variables, which are defined in the Phalanx application. +############################################################################### + +# Bash "strict mode", to help catch problems and bugs in the shell script. +# Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Check if the command is found +command -v ppdb-replication >/dev/null 2>&1 || { echo "ppdb-replication command not found"; exit 1; } +echo "Found ppdb-replication command" + +# Function to check if an environment variable is set +check_env_var() { + local var_name="$1" + local var_value="${!var_name}" + if [ -z "${var_value:-}" ]; then + echo "$var_name is a required environment variable" + exit 1 + fi +} + +# Check if the required environment variables are set +check_env_var "PPDB_REPLICATION_APDB_CONFIG" +check_env_var "PPDB_REPLICATION_PPDB_CONFIG" + +# Build the command from the environment variables +_CMD="ppdb-replication" +[ -n "${PPDB_REPLICATION_MON_LOGGER:-}" ] && _CMD="$_CMD --mon-logger $PPDB_REPLICATION_MON_LOGGER" +[ -n "${PPDB_REPLICATION_MON_RULES:-}" ] && _CMD="$_CMD --mon-rules $PPDB_REPLICATION_MON_RULES" +[ -n "${PPDB_REPLICATION_LOG_LEVEL:-}" ] && _CMD="$_CMD -l $PPDB_REPLICATION_LOG_LEVEL" +_CMD="$_CMD run" +[ "${PPDB_REPLICATION_UPDATE_EXISTING:-}" = "true" ] && _CMD="$_CMD --update" +[ -n "${PPDB_REPLICATION_MIN_WAIT_TIME:-}" ] && _CMD="$_CMD --min-wait-time $PPDB_REPLICATION_MIN_WAIT_TIME" +[ -n "${PPDB_REPLICATION_MAX_WAIT_TIME:-}" ] && _CMD="$_CMD --max-wait-time $PPDB_REPLICATION_MAX_WAIT_TIME" +[ -n "${PPDB_REPLICATION_CHECK_INTERVAL:-}" ] && _CMD="$_CMD --check-interval $PPDB_REPLICATION_CHECK_INTERVAL" +_CMD="$_CMD $PPDB_REPLICATION_APDB_CONFIG" +_CMD="$_CMD $PPDB_REPLICATION_PPDB_CONFIG" + +# Run the command +echo "Running: $_CMD" +$_CMD diff --git a/pyproject.toml b/pyproject.toml index b6d5d2f..7979d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,10 @@ license-files = ["COPYRIGHT", "LICENSE"] [tool.setuptools.dynamic] version = { attr = "lsst_versions.get_lsst_version" } +[project.scripts] +ppdb-replication = "lsst.dax.ppdb.cli.ppdb_replication:main" +ppdb-cli = "lsst.dax.ppdb.cli.ppdb_cli:main" + [tool.black] line-length = 110 target-version = ["py311"]