Merge branch 'main' into ww/yank-only

.github/workflows/ci.yml

@@ -8,7 +8,6 @@ on:
     types: [checks_requested]
   workflow_dispatch:  # generally only for the "combine-prs" workflow
 permissions:
-  id-token: write
   contents: read
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -19,9 +18,13 @@ jobs:
     runs-on: depot-ubuntu-22.04-arm
     outputs:
       buildId: ${{ steps.build.outputs.build-id}}
+    permissions:
+      id-token: write
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Set up Depot CLI
         uses: depot/setup-action@v1
       - name: Build image
@@ -59,6 +62,8 @@ jobs:
       image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
       env:
         BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://stripe:12111 api_version=2020-08-27
+    permissions:
+      id-token: write
     services:
       postgres:
         image: ${{ (matrix.name == 'Tests') && 'postgres:16.1' || '' }}
@@ -81,6 +86,8 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Cache mypy results
         if: ${{ (matrix.name == 'Lint') }}
         uses: actions/cache@v4
@@ -98,6 +105,8 @@ jobs:
     continue-on-error: true
     container:
       image: registry.depot.dev/rltf7cln5v:${{ needs.build.outputs.buildId }}
+    permissions:
+      id-token: write
     services:
       postgres:
         image: postgres:16.1
@@ -112,6 +121,8 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - name: Dotenv Action
         # We need to load the environment variables to run the CLI
         id: dotenv

.github/workflows/codeql-analysis.yml

@@ -39,6 +39,8 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Set up Python
       if: matrix.language == 'python'

.github/workflows/combine-prs.yml

@@ -11,23 +11,22 @@ on:
         required: true
         default: 'blocked'
 
-# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
-permissions:
-  contents: write
-  pull-requests: write
-  checks: read
-  actions: write
-
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   # This workflow contains a single job called "combine-prs"
   combine-prs:
+    # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
+    permissions:
+      contents: write
+      pull-requests: write
+      checks: read
+      actions: write
     # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
     steps:
       - name: Use GitHub App Token
-        uses: wow-actions/use-app-token@v2.0.2
+        uses: wow-actions/use-app-token@v2.1.1
         id: generate_token
         with:
           app_id: ${{ secrets.COMBINE_PRS_APP_ID }}
@@ -36,7 +35,7 @@ jobs:
 
       - name: combine-prs
         id: combine-prs
-        uses: github/combine-prs@v5.1.0
+        uses: github/combine-prs@v5.2.0
         with:
           github_token: ${{ steps.generate_token.outputs.BOT_TOKEN }}
           ignore_label: ${{ github.event.inputs.ignoreLabel || 'blocked' }}

.github/workflows/dev-env-test.yml

@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
       - run: make build
       - run: docker compose up -d
       - run: docker compose ps

.github/workflows/node-ci.yml

@@ -28,9 +28,11 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
+        with:
+            persist-credentials: false
       - uses: actions/setup-node@v4
         with:
-          node-version: 22.3.0
+          node-version: 23.1.0
           cache: 'npm'
       - name: Install Node dependencies
         run: npm ci

.github/workflows/zizmor.yml

@@ -0,0 +1,35 @@
+# https://github.com/woodruffw/zizmor
+name: GitHub Actions Security Analysis with Zizmor
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["*"]
+
+jobs:
+  zizmor:
+    name: Zizmor latest via Cargo
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Setup Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Get zizmor
+        run: cargo install zizmor
+      - name: Run zizmor
+        run: zizmor --format sarif . > results.sarif
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: results.sarif
+          # Optional category for the results
+          # Used to differentiate multiple results for one commit
+          category: zizmor

.readthedocs.yml

@@ -6,7 +6,7 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
     python: "3.12"
   commands:

Dockerfile

@@ -1,6 +1,6 @@
 # First things first, we build an image which is where we're going to compile
 # our static assets with. We use this stage in development.
-FROM node:22.9.0-bookworm AS static-deps
+FROM node:23.1.0-bookworm AS static-deps
 
 WORKDIR /opt/warehouse/src/
 

docs/dev/development/database-migrations.rst

@@ -21,10 +21,8 @@ to the old version of Warehouse from being shut down. This means that each
 migration *must* be compatible with the current ``main`` branch of Warehouse.
 
 This makes it more difficult to make breaking changes, since you must phase
-them in over time (for example, to rename a column you must add the column in
-one migration + start writing to that column/reading from both, then you must
-make a migration that backfills all of the data, then switch the code to stop
-using the old column all together, then finally you can remove the old column).
+them in over time. See :ref:`destructive-migrations` for tips on doing
+migrations that involve column deletions or renames.
 
 To help protect against an accidentally long running migration from taking down
 PyPI, by default a migration will timeout if it is waiting more than 4s to
@@ -48,10 +46,61 @@ add:
 
 To your migration.
 
-
 For more information on what kind of operations are safe in a high availability
 environment like PyPI, there is related reading available at:
 
 - `PostgreSQL at Scale: Database Schema Changes Without Downtime <https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680>`_
 - `Move fast and migrate things: how we automated migrations in Postgres <https://benchling.engineering/move-fast-and-migrate-things-how-we-automated-migrations-in-postgres-d60aba0fc3d4>`_
 - `PgHaMigrations <https://github.com/braintree/pg_ha_migrations>`_
+
+.. _destructive-migrations:
+
+Destructive migrations
+----------------------
+
+.. warning::
+
+  Read this section and its respective sub-sections **completely** before
+  attempting to follow them! Failure to do so can result in serious
+  deployment errors and outages.
+
+Migrations that do column renames or deletions need to be performed
+with special care, due to how Warehouse is deployed. Performing a
+migration without these steps will cause errors during deployment,
+and may require a full revert.
+
+.. _removing-a-column:
+
+Removing a column
+=================
+
+To remove a column:
+
+1. Perform the Python-level code changes, i.e. remove usages of the
+   column/attribute within Warehouse itself. Do **not** generate
+   an accompanying migration.
+2. Submit the changes as a PR. Tag the PR with ``skip-db-check`` to allow
+   it to pass CI without accompanying migrations.
+3. Prepare a second PR containing just the generated migrations.
+4. Merge the first PR and ensure its deployment before merging the second.
+
+This will ensure that the "old" version of Warehouse (prior to the new migration
+has no references to the column being deleted).
+
+Renaming a column
+=================
+
+Renaming a column is more complex than deleting a column, since it involves
+a data migration. To rename a column:
+
+1. Create an initial migration that adds the new column, and add code that
+   writes to the new column while reading from both it and the old column.
+2. Deploy the initial migration.
+3. Prepare a second migration that performs a backfill of the old column to
+   the new column.
+4. Deploy the second migration.
+5. Follow the :ref:`removing-a-column` steps *in entirety* to remove the old
+   column.
+
+In total, this requires three separate migrations: one to add the new column,
+one to backfill to it, and a third to remove the old column.

docs/dev/development/patterns.rst

@@ -133,6 +133,8 @@ Adding a new service
 3. (Optional) Create other implementations of the interface. For instance, many services in ``warehouse``
    also provide a ``NullService`` version used for development. These Null implementations only
    provide basic functionalities without verifications and reduce the need for stubs in tests.
+   When implementing a warning for using the service, subclass the exception from
+   ``warehouse.utils.exceptions.DevelopmentModeWarning`` to minimize warnings in the test logs.
 
    Any new implementation must implement the complete interface, including all its methods and attributes.
 

docs/user/project_metadata.md

@@ -26,7 +26,7 @@ PyPI currently supports several ways of verifying project URLs. When a URL is ve
 
 !!! warning
 
-    An URL being verified only attests that the URL is under control of the
+    A URL being verified only attests that the URL is under control of the
     PyPI package owner at the time of verification, and does not imply any
     additional safety about that URL or any other relationship to the project
     in question.
@@ -102,7 +102,7 @@ will be recognized.
 | Homepage         | :fontawesome-solid-house-chimney:         | For the project homepage  |                                                                                                          |
 | Download         | :fontawesome-solid-cloud:                 | A download link           |                                                                                                          |
 | Changelog        | :fontawesome-solid-scroll:                | Changelog information     | Change log, Changes, Release notes, News, What's new, History                                            |
-| Documentation^*^ | :fontawesome-solid-book:                  | Project documentation     | Docs^*^ , an URL pointing to [Read the Docs] domains or an URL starting with `docs.` or `documentation.` |
+| Documentation^*^ | :fontawesome-solid-book:                  | Project documentation     | Docs^*^ , a URL pointing to [Read the Docs] domains or a URL starting with `docs.` or `documentation.` |
 | Bug^*^           | :fontawesome-solid-bug:                   | Bug/Issue report location | Issue^*^, Tracker^*^, Report^*^                                                                          |
 | Funding^*^       | :fontawesome-solid-circle-dollar-to-slot: | Sponsoring information    | Sponsor^*^, Donation^*^, Donate^*^                                                                       |
 

package.json

@@ -23,7 +23,7 @@
     "@hotwired/stimulus": "^3.2.1",
     "@hotwired/stimulus-webpack-helpers": "^1.0.1",
     "admin-lte": "^3.2.0",
-    "cookie": "^0.5.0",
+    "cookie": "^0.7.0",
     "date-fns": "^2.30.0",
     "debounce": "^1.2.1",
     "jquery": "^3.7.0",

pyproject.toml

@@ -50,18 +50,15 @@ cache_dir = "dev/.mypy_cache"
 [[tool.mypy.overrides]]
 # These modules do not yet have types available.
 module = [
-    "alembic_postgresql_enum.*",
-    "automat.*",
     "bpython.*",  # https://github.com/bpython/bpython/issues/892
     "b2sdk.*",  # https://github.com/Backblaze/b2-sdk-python/issues/148
     "celery.app.backends.*",
     "celery.backends.redis.*",
-    "github_reserved_names.*",
+    "github_reserved_names.*",  # https://github.com/Julian/github-reserved-names/pull/10
     "google.cloud.*",
     "forcediphttpsadapter.*",
     "IPython.*",  # has types, but only installed in dev
-    "linehaul.*",
-    "packaging_legacy.*",
+    "packaging_legacy.*",  # https://github.com/di/packaging_legacy/pull/5
     "paginate.*",
     "paginate_sqlalchemy.*",
     "premailer.*",
@@ -73,11 +70,11 @@ module = [
     "pyramid_rpc.*",
     "pyqrcode.*",
     "requests_aws4auth.*",  # https://github.com/tedder/requests-aws4auth/issues/53
-    "rfc3986.*",
+    "rfc3986.*",  # https://github.com/python-hyper/rfc3986/issues/122
     "transaction.*",
     "ua_parser.*",  # https://github.com/ua-parser/uap-python/issues/110
     "venusian.*",
-    "whitenoise.*",
+    "whitenoise.*",  # https://github.com/evansd/whitenoise/pull/410
     "zope.sqlalchemy.*",
 ]
 ignore_missing_imports = true
@@ -96,9 +93,7 @@ markers = [
     'functional: Slower running tests which test the entire system is functioning.',
 ]
 filterwarnings = [
-    'ignore::warehouse.admin.services.InsecureStorageWarning',
-    'ignore::warehouse.utils.exceptions.InsecureOIDCPublisherWarning',
-    'ignore::warehouse.packaging.services.InsecureStorageWarning',
+    'ignore::warehouse.utils.exceptions.DevelopmentModeWarning',
     'error:SELECT statement has a cartesian product:sqlalchemy.exc.SAWarning',
 ]
 

requirements/deploy.in

@@ -1,2 +1,2 @@
 gunicorn==23.0.0
-ddtrace==2.14.0
+ddtrace==2.14.4