feat: rerender in one spot using admin token and VMs for isolation

.github/workflows/tests.yml

@@ -120,17 +120,35 @@ jobs:
         env:
           GH_TOKEN: ${{ steps.generate_token.outputs.token }}
 
-      - name: run package upload tests
+      - name: run rerender tests
         shell: bash -l {0}
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          export CF_WEBSERVICES_TEST=1
-          ./scripts/run_cfep13_tests.sh
+          if [[ "${GITHUB_HEAD_REF}" != "" ]]; then
+            branch="${GITHUB_HEAD_REF}"
+          else
+            branch="${GITHUB_REF_NAME}"
+          fi
+
+          export CF_FEEDSTOCK_OPS_CONTAINER_NAME=condaforge/webservices-dispatch-action
+          export CF_FEEDSTOCK_OPS_CONTAINER_TAG="prod"
+
+          cd tests
+          pytest -vvs --branch=${branch} test_live_rerender.py
         env:
-          GH_TOKEN: ${{ steps.generate_token.outputs.token }}
-          PROD_BINSTAR_TOKEN: ${{ secrets.PROD_BINSTAR_TOKEN }}
-          STAGING_BINSTAR_TOKEN: ${{ secrets.HEROKU_ONLY_STAGING_BINSTAR_TOKEN }}
-          CF_WEBSERVICES_TOKEN: ${{ secrets.CF_WEBSERVICES_TOKEN }}
-          CF_WEBSERVICES_APP_ID: ${{ secrets.CF_CURATOR_APP_ID }}
-          CF_WEBSERVICES_PRIVATE_KEY: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
-          ACTION_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
+          GH_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}
+
+      # - name: run package upload tests
+      #   shell: bash -l {0}
+      #   if: ${{ !github.event.pull_request.head.repo.fork }}
+      #   run: |
+      #     export CF_WEBSERVICES_TEST=1
+      #     ./scripts/run_cfep13_tests.sh
+      #   env:
+      #     GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+      #     PROD_BINSTAR_TOKEN: ${{ secrets.PROD_BINSTAR_TOKEN }}
+      #     STAGING_BINSTAR_TOKEN: ${{ secrets.HEROKU_ONLY_STAGING_BINSTAR_TOKEN }}
+      #     CF_WEBSERVICES_TOKEN: ${{ secrets.CF_WEBSERVICES_TOKEN }}
+      #     CF_WEBSERVICES_APP_ID: ${{ secrets.CF_CURATOR_APP_ID }}
+      #     CF_WEBSERVICES_PRIVATE_KEY: ${{ secrets.CF_CURATOR_PRIVATE_KEY }}
+      #     ACTION_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"

.github/workflows/webservices-workflow-dispatch.yml

@@ -0,0 +1,112 @@
+name: webservices-workflow-dispatch
+
+on:
+  workflow_dispatch:
+    inputs:
+      task:
+        description: 'the task to perform'
+        required: true
+        type: string
+      repo:
+        description: 'the repository to run on'
+        required: true
+        type: string
+      pr_number:
+        description: 'the pull request number'
+        required: true
+        type: string
+      ref:
+        description: 'the conda-forge-webservices branch to use'
+        required: false
+        type: string
+        default: 'main'
+
+env:
+  PY_COLORS: 1
+
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+permissions: {}
+
+jobs:
+  run-task:
+    name: run-task
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: setup conda
+        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822
+        with:
+          environment-file: conda-lock.yml
+          environment-name: webservices
+          condarc: |
+            show_channel_urls: true
+            channel_priority: strict
+            channels:
+              - conda-forge
+
+      - name: install code
+        run: |
+          pip install --no-deps --no-build-backend -e .
+
+      - name: run task
+        run: |
+          conda-forge-webservices-run-task \
+            --task=${{ inputs.task }} \
+            --repo=${{ inputs.repo }} \
+            --pr-number=${{ inputs.pr_number }} \
+            --task-data-dir=${{ github.workspace }}/task-data
+
+      - name: upload task data
+        id: upload-task-data
+        uses: actions/upload-artifact@v4
+        with:
+          name: task-data-${{ inputs.task }}-${{ inputs.repo }}-${{ inputs.pr_number }}-${{ github.run_id }}-${{ github.run_number }}
+          path: ${{ github.workspace }}/task-data
+          retention-days: 2
+          include-hidden-files: true
+
+  finalize-task:
+    name: finalize-task
+    runs-on: ubuntu-latest
+    needs:
+      - run-task
+    steps:
+      - name: checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: setup conda
+        uses: mamba-org/setup-micromamba@f8b8a1e23a26f60a44c853292711bacfd3eac822
+        with:
+          environment-file: conda-lock.yml
+          environment-name: webservices
+          condarc: |
+            show_channel_urls: true
+            channel_priority: strict
+            channels:
+              - conda-forge
+
+      - name: install code
+        run: |
+          pip install --no-deps --no-build-backend -e .
+
+      - name: download task data
+        uses: actions/download-artifact@v4
+        with:
+          name: task-data-${{ inputs.task }}-${{ inputs.repo }}-${{ inputs.pr_number }}-${{ github.run_id }}-${{ github.run_number }}
+          path: ${{ github.workspace }}/task-data
+
+      - name: finalize task
+        run: |
+          conda-forge-webservices-finalize-task \
+            --task-data-dir=${{ github.workspace }}/task-data
+        env:
+          GH_TOKEN: ${{ secrets.CF_ADMIN_GITHUB_TOKEN }}

.gitignore

@@ -11,3 +11,4 @@ recipe/*
 .ruff_cache/
 built_dists/
 conda_forge_webservices/_version.py
+recipe/

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
   rev: v0.6.3
   hooks:
     - id: ruff
-      args: [ --fix ]    
+      args: [ --fix ]
     - id: ruff-format
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: v1.11.2
@@ -15,6 +15,7 @@ repos:
     - types-python-dateutil
     - types-cachetools
     - types-mock
+    - types-pyyaml
 ci:
     autofix_commit_msg: |
         [pre-commit.ci] auto fixes from pre-commit.com hooks

conda_forge_webservices/github_actions_integration/__init__.py

@@ -0,0 +1,6 @@
+# flake8: noqa
+from .env_management import SensitiveEnv
+
+global_sensitive_env = SensitiveEnv()
+global_sensitive_env.hide_env_vars()
+sensitive_env = global_sensitive_env.sensitive_env

conda_forge_webservices/github_actions_integration/__main__.py

@@ -0,0 +1,100 @@
+import json
+import logging
+import os
+import pprint
+import subprocess
+import sys
+
+import click
+from git import Repo
+
+# this is the only import that should go here
+# everything else should be in the functions
+# this import hides the env vars and has to run first-ish
+import conda_forge_webservices.github_actions_integration  # noqa: F401
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _pull_docker_image():
+    try:
+        print("::group::docker image pull", flush=True)
+        subprocess.run(
+            [
+                "docker",
+                "pull",
+                f"{os.environ['CF_FEEDSTOCK_OPS_CONTAINER_NAME']}:{os.environ['CF_FEEDSTOCK_OPS_CONTAINER_TAG']}",
+            ],
+        )
+        sys.stderr.flush()
+        sys.stdout.flush()
+    finally:
+        print("::endgroup::", flush=True)
+
+
+@click.command(name="conda-forge-webservices-run-task")
+@click.option("--task", required=True, type="str")
+@click.option("--repo", required=True, type="str")
+@click.option("--pr-number", required=True, type="str")
+@click.option("--task-data-dir", required=True, type="str")
+def main_run_task(task, repo, pr_number, task_data_dir):
+    from .rerendering import rerender
+
+    logging.basicConfig(level=logging.INFO)
+
+    LOGGER.info("running task %s for conda-forge/%s#%s", task, repo, pr_number)
+
+    feedstock_dir = os.path.join(
+        task_data_dir,
+        repo,
+    )
+    os.makedirs(feedstock_dir, exist_ok=True)
+    repo_url = f"https://github.com/conda-forge/{repo}.git"
+    git_repo = Repo.clone_from(
+        repo_url,
+        feedstock_dir,
+    )
+    git_repo.remotes.origin.fetch([f"pull/{pr_number}/head:pull/{pr_number}/head"])
+    git_repo.git.switch(f"pull/{pr_number}/head")
+
+    task_data = {"task": task, "repo": repo, "pr_number": pr_number, "task_results": {}}
+
+    if task == "rerender":
+        _pull_docker_image()
+        changed, rerender_error, info_message = rerender(git_repo)
+        task_data["task_results"]["changed"] = changed
+        task_data["task_results"]["rerender_error"] = rerender_error
+        task_data["task_results"]["info_message"] = info_message
+    else:
+        raise ValueError(f"Task `{task}` is not valid!")
+
+    with open(os.path.join(task_data_dir, "task_data.json"), "w") as f:
+        json.dump(task_data, f)
+
+    subprocess.run(
+        ["rm", "-rf", os.path.join(feedstock_dir, ".git")],
+        check=True,
+        capture_output=True,
+    )
+
+
+@click.command(name="conda-forge-webservices-finalize-task")
+@click.option("--task-data-dir", required=True, type="str")
+def main_finalize_task(task_data_dir):
+    from .utils import flush_logger
+
+    logging.basicConfig(level=logging.INFO)
+
+    with open(os.path.join(task_data_dir, "task_data.json")) as f:
+        task_data = json.load(f)
+
+    task = task_data["task"]
+    repo = task_data["repo"]
+    pr_number = task_data["pr_number"]
+    task_results = task_data["task_results"]
+
+    LOGGER.info("running task %s for conda-forge/%s#%s", task, repo, pr_number)
+    LOGGER.info("task results:")
+    flush_logger(LOGGER)
+    print(pprint.pformat(task_results), flush=True)
+    flush_logger(LOGGER)

conda_forge_webservices/github_actions_integration/api_sessions.py

@@ -0,0 +1,58 @@
+import os
+from functools import lru_cache
+
+import requests
+import urllib3.util.retry
+from github import Github
+
+from . import sensitive_env
+
+
+def create_api_sessions():
+    """Create API sessions for GitHub.
+
+    Returns
+    -------
+    session : requests.Session
+        A `requests` session w/ the beta `check_run` API configured.
+    gh : github.MainClass.Github
+        A `Github` object from the PyGithub package.
+    """
+    with sensitive_env():
+        return _create_api_sessions(os.environ["GH_TOKEN"])
+
+
+@lru_cache(maxsize=1)
+def _create_api_sessions(github_token):
+    # based on
+    #  https://alexwlchan.net/2019/03/
+    #    creating-a-github-action-to-auto-merge-pull-requests/
+    # with lots of edits
+    sess = requests.Session()
+    sess.headers = {
+        "Accept": "; ".join(
+            [
+                "application/vnd.github.v3+json",
+                # special beta api for check_suites endpoint
+                "application/vnd.github.antiope-preview+json",
+            ]
+        ),
+        "Authorization": f"Bearer {github_token}",
+        "User-Agent": f"GitHub Actions script in {__file__}",
+    }
+
+    def raise_for_status(resp, *args, **kwargs):
+        try:
+            resp.raise_for_status()
+        except Exception as e:
+            print("ERROR:", resp.text)
+            raise e
+
+    sess.hooks["response"].append(raise_for_status)
+
+    # build a github object too
+    gh = Github(
+        github_token, retry=urllib3.util.retry.Retry(total=10, backoff_factor=0.1)
+    )
+
+    return sess, gh

conda_forge_webservices/github_actions_integration/env_management.py

@@ -0,0 +1,41 @@
+import os
+from contextlib import contextmanager
+
+
+class SensitiveEnv:
+    SENSITIVE_KEYS = (
+        "USERNAME",
+        "PASSWORD",
+        "GITHUB_TOKEN",
+        "GH_TOKEN",
+        "CF_ADMIN_GITHUB_TOKEN",
+    )
+
+    def __init__(self):
+        self.classified_info = {}
+
+    def hide_env_vars(self):
+        """Remove sensitive env vars"""
+        self.classified_info.update(
+            {
+                k: os.environ.pop(k, self.classified_info.get(k, None))
+                for k in self.SENSITIVE_KEYS
+            },
+        )
+
+    def reveal_env_vars(self):
+        """Restore sensitive env vars"""
+        os.environ.update(
+            **{k: v for k, v in self.classified_info.items() if v is not None}
+        )
+
+    @contextmanager
+    def sensitive_env(self):
+        """Add sensitive keys to environ if needed, when ctx is finished
+        remove keys and update the sensitive env in case any were updated
+        inside the ctx"""
+        self.reveal_env_vars()
+        try:
+            yield os.environ
+        finally:
+            self.hide_env_vars()