Skip to content

Commit

Permalink
Imports correctly.
Browse files Browse the repository at this point in the history
  • Loading branch information
athornton committed Aug 13, 2024
1 parent f13f8c4 commit a87fef1
Show file tree
Hide file tree
Showing 25 changed files with 1,569 additions and 288 deletions.
Empty file removed CHANGELOG.
Empty file.
23 changes: 10 additions & 13 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@
# environment is the only thing copied into the runtime image.
# runtime-image
# - Copies the virtual environment into place.
# - Runs a non-root user.
# - Sets up the entrypoint and port.
# - Sets up the entrypoint.
#
# Note that the scratchpurger will typically run as root, because its job
# is to use its privilege to clean up after people who did not clean up
# after themselves.

FROM python:3.12.1-slim-bullseye as base-image
FROM python:3.12.5-slim-bookworm as base-image

# Update system packages
COPY scripts/install-base-packages.sh .
Expand All @@ -33,31 +36,25 @@ RUN python -m venv $VIRTUAL_ENV
# Make sure we use the virtualenv.
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

# Put the latest pip and setuptools in the virtualenv.
RUN pip install --upgrade --no-cache-dir pip setuptools wheel
# Put the latest uv in the virtualenv.
RUN pip install --upgrade --no-cache-dir uv

# Install the app's Python runtime dependencies.
COPY requirements/main.txt ./requirements.txt
RUN pip install --quiet --no-cache-dir -r requirements.txt
RUN uv pip install --quiet --no-cache-dir -r requirements.txt

# Install the Python package.
COPY . /workdir
WORKDIR /workdir
RUN pip install --no-cache-dir .
RUN uv pip install --no-cache-dir .

FROM base-image AS runtime-image

# Create a non-root user.
RUN useradd --create-home appuser

# Copy the virtualenv.
COPY --from=install-image /opt/venv /opt/venv

# Make sure we use the virtualenv.
ENV PATH="/opt/venv/bin:$PATH"

# Switch to the non-root user.
USER appuser

# Run something innocuous
CMD ["/bin/true"]
14 changes: 7 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
.PHONY: update-deps
update-deps:
pip install --upgrade pip-tools pip setuptools
pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in
pip install --upgrade uv tox-uv
uv pip compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
uv pip compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in

.PHONY: init
init:
pip install --upgrade pip setuptools wheel
pip install --editable .
pip install --upgrade -r requirements/main.txt -r requirements/dev.txt
pip install --upgrade uv tox-uv
uv pip install --editable .
uv pip install --upgrade -r requirements/main.txt -r requirements/dev.txt
rm -rf .tox
pip install --upgrade tox
uv pip install --upgrade tox
pre-commit install

.PHONY: update
Expand Down
3 changes: 2 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ dependencies = []
dynamic = ["version"]

[project.scripts]
purge_backups = "rsp_scratchpurger.cli:purge"
rsp_purge = "rsp_scratchpurger.cli:purge"
rsp_report = "rsp_scratchpurger.cli:report"

[project.urls]
Homepage = "https://phalanx.lsst.io"
Expand Down
10 changes: 9 additions & 1 deletion requirements/dev.in
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
-c main.txt

# Make tox faster
tox-uv

# Type checking
mypy
types-pyyaml

# Linting
ruff
pre-commit

# Testing
pytest
pytest-asyncio
pytest-cov

311 changes: 311 additions & 0 deletions requirements/dev.txt

Large diffs are not rendered by default.

341 changes: 341 additions & 0 deletions requirements/main.txt

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions src/rsp_scratchpurger/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
from .purger import Purger

__all__ = ["Purger"]
113 changes: 84 additions & 29 deletions src/rsp_scratchpurger/cli.py
Original file line number Diff line number Diff line change
@@ -1,48 +1,103 @@
"""Command-line interface for Google Filestore tools."""
"""CLI for the filesystem purger."""

import argparse
import asyncio
import os
from pathlib import Path

import yaml
from pydantic import ValidationError
from safir.logging import LogLevel, Profile

from .constants import ENV_PREFIX
from .models.config import Config
from .models.v1.policy import Policy
from .purger import Purger
from .constants import POLICY_FILE, ENV_PREFIX


def _add_options() -> argparse.ArgumentParser:
"""Add options applicable to any filestore tool."""
parser = argparse.ArgumentParser()
def _add_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
parser.add_argument(
"-f",
"--file",
"-c",
"--config-file",
"--config",
help="Application configuration file",
)
parser.add_argument(
"-p",
"--policy-file",
help="Policy file for purger",
default=os.environ.get(f"{ENV_PREFIX}FILE", POLICY_FILE),
type=Path,
required=True,
"--policy",
help="Purger policy configuration file",
)
parser.add_argument(
"-x",
"--dry-run",
help="Do not perform actions, but print what would be done",
type=bool,
default=bool(os.environ.get(f"{ENV_PREFIX}DRY_RUN", "")),
"-d", "--debug", action="store_true", help="Enable debug logging"
)
parser.add_argument(
"-d",
"--debug",
"--verbose",
default=bool(os.environ.get(f"{ENV_PREFIX}DEBUG", "")),
type=bool,
help="Verbose debugging output",
"-x",
"--dry-run",
action="store_true",
help="Do not act, but report what would be done",
)

return parser

def purge() -> None:
"""Purge the target filesystems."""
args = _get_options().parse_args()
purger = Purger(
policy_file=args.policy,
dry_run=args.dry_run,
debug=args.debug

def _postprocess_args_to_config(raw_args: argparse.Namespace) -> Config:
config: Config | None = None
override_cf = raw_args.config_file or os.getenv(
ENV_PREFIX + "CONFIG_FILE", ""
)
if override_cf:
config_file = Path(override_cf)
try:
config_obj = yaml.safe_load(config_file.read_text())
config = Config.model_validate(config_obj)
except (FileNotFoundError, UnicodeDecodeError, ValidationError):
config = Config()
else:
config = Config()
# Validate policy. If the file is specified, use that; if not, use
# defaults from config.
override_pf = raw_args.policy_file or os.getenv(
ENV_PREFIX + "POLICY_FILE", ""
)
policy_file = Path(override_pf) if override_pf else config.policy_file
policy_obj = yaml.safe_load(policy_file.read_text())
Policy.model_validate(policy_obj)
# If we get this far, it's a legal policy file.
config.policy_file = policy_file
# For dry-run and debug, if specified, use that, and if not, do whatever
# the config says.
if raw_args.debug is not None:
if raw_args.debug:
config.logging.log_level = LogLevel.DEBUG
config.logging.profile = Profile.development
else:
# User asked for no debug, so let's override the config.
# I guess?
config.logging.log_level = LogLevel.INFO
config.logging.profile = Profile.production
if raw_args.dry_run is not None:
config.dry_run = raw_args.dry_run
return config


def _get_executor(desc: str) -> Purger:
parser = argparse.ArgumentParser(description=desc)
parser = _add_args(parser)
args = parser.parse_args()
config = _postprocess_args_to_config(args)
return Purger(config=config)


def report() -> None:
"""Report what files would be purged."""
reporter = _get_executor("Report what files would be purged.")
asyncio.run(reporter.plan())
asyncio.run(reporter.report())


def purge() -> None:
"""Purge files."""
purger = _get_executor("Purge files.")
asyncio.run(purger.plan())
asyncio.run(purger.purge())
7 changes: 4 additions & 3 deletions src/rsp_scratchpurger/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from pathlib import Path

ENV_PREFIX="RSP_SCRATCHPURGER_"
POLICY_FILE=Path("/etc/purger/config.yaml")

CONFIG_FILE = Path("/etc/purger/config.yaml")
ENV_PREFIX = "RSP_SCRATCHPURGER_"
POLICY_FILE = Path("/etc/purger/policy.yaml")
ROOT_LOGGER = "rsp_scratchpurger"
11 changes: 11 additions & 0 deletions src/rsp_scratchpurger/exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
"""Exceptions for the purger."""

from safir.slack.blockkit import SlackException


class PlanNotReadyError(SlackException):
"""An operation needing a Plan was requested, but no Plan is ready."""


class PolicyNotFoundError(SlackException):
"""No Policy matching the given directory was found."""
Loading

0 comments on commit a87fef1

Please sign in to comment.