Merge branch 'main' into gs/gdrive-create-discussion-relations

pubpub · Feb 27, 2025 · b868ef2 · b868ef2
2 parents 492939a + 626e97e
commit b868ef2
Show file tree

Hide file tree

Showing 98 changed files with 6,190 additions and 1,243 deletions.
diff --git a/.env.docker-compose.dev b/.env.docker-compose.dev
@@ -0,0 +1,13 @@
+MINIO_ROOT_USER=pubpub-minio-admin
+MINIO_ROOT_PASSWORD=pubpub-minio-admin
+
+ASSETS_BUCKET_NAME=assets.v7.pubpub.org
+ASSETS_UPLOAD_KEY=pubpubuser
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_REGION=us-east-1
+
+POSTGRES_PORT=54322
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+
diff --git a/.env.docker-compose.test b/.env.docker-compose.test
@@ -0,0 +1,40 @@
+MINIO_ROOT_USER=pubpub-minio-admin
+MINIO_ROOT_PASSWORD=pubpub-minio-admin
+
+ASSETS_BUCKET_NAME=byron.v7.pubpub.org
+ASSETS_UPLOAD_KEY=pubpubuserrr
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_REGION=us-east-1
+ASSETS_STORAGE_ENDPOINT=http://localhost:9000
+
+POSTGRES_PORT=54323
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+POSTGRES_HOST=db
+
+# annoying duplication because jobs uses this version
+PGHOST=db
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD=postgres
+PGDATABASE=postgres
+
+# this needs to be db:5432 bc that's what it is in the app-network
+# if you are running this from outside the docker network, you need to use
+# @localhost:${POSTGRES_PORT} instead
+DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
+
+
+JWT_SECRET=xxx
+MAILGUN_SMTP_PASSWORD=xxx
+GCLOUD_KEY_FILE=xxx
+
+MAILGUN_SMTP_HOST=inbucket
+MAILGUN_SMTP_PORT=2500
+# this needs to be localhost:54324 instead of inbucket:9000 bc we are almost always running the integration tests from outside the docker network
+INBUCKET_URL=http://localhost:54324
+MAILGUN_SMTP_USERNAME=omitted
+OTEL_SERVICE_NAME=core.core
+PUBPUB_URL=http://localhost:3000
+API_KEY=xxx
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,55 @@
+# yaml-language-server: $schema=https://json.schemastore.org/dependabot-2.0.json
+# Dependabot configuration file
+# See documentation: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+    # package.json + pnpm catalog updates
+    - package-ecosystem: "npm"
+      directory: "/"
+      schedule:
+          interval: "weekly"
+          day: "monday"
+      open-pull-requests-limit: 10
+      labels:
+          - "dependencies"
+          - "npm"
+      commit-message:
+          prefix: "npm"
+          include: "scope"
+      # group all minor and patch updates together
+      groups:
+          minor-patch-dependencies:
+              patterns:
+                  - "*"
+              update-types:
+                  - "minor"
+                  - "patch"
+
+    # GitHub Actions updates
+    - package-ecosystem: "github-actions"
+      directory: "/"
+      schedule:
+          interval: "weekly"
+          day: "monday"
+      open-pull-requests-limit: 5
+      labels:
+          - "dependencies"
+          - "github-actions"
+      commit-message:
+          prefix: "github-actions"
+          include: "scope"
+
+    # docker updates
+    - package-ecosystem: "docker"
+      directory: "/"
+      schedule:
+          interval: "weekly"
+          day: "monday"
+      open-pull-requests-limit: 5
+      labels:
+          - "dependencies"
+          - "docker"
+      commit-message:
+          prefix: "docker"
+          include: "scope"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ jobs:
         runs-on: ubuntu-latest
         env:
             COMPOSE_FILE: docker-compose.test.yml
+            ENV_FILE: .env.docker-compose.test
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -55,8 +56,8 @@ jobs:
                   restore-keys: |
                       ${{ runner.os }}-turbo-
 
-            - name: Start up DB
-              run: docker compose --profile test up -d
+            - name: Start test dependencies
+              run: pnpm test:setup
 
             - name: Install dependencies
               run: pnpm install --frozen-lockfile --prefer-offline
@@ -66,8 +67,6 @@ jobs:
 
             - name: Run migrations
               run: pnpm --filter core migrate-test
-              env:
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
 
             - name: Run prettier
               run: pnpm format

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -20,6 +20,8 @@ jobs:
     integration-tests:
         name: Integration tests
         runs-on: ubuntu-latest
+        env:
+            ENV_FILE: .env.docker-compose.test
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -48,28 +50,6 @@ jobs:
                   restore-keys: |
                       ${{ runner.os }}-pnpm-store-
 
-            - name: Install dependencies
-              run: pnpm install --frozen-lockfile --prefer-offline
-
-            - name: Start up DB
-              run: docker compose -f docker-compose.test.yml --profile test up -d
-
-            - name: p:build
-              run: pnpm p:build
-
-            - name: Run migrations
-              run: pnpm --filter core prisma migrate deploy
-              env:
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
-
-            - name: seed db
-              run: pnpm --filter core prisma db seed
-              env:
-                  # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests
-                  MINIMAL_SEED: true
-                  SKIP_VALIDATION: true
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
-
             - name: Configure AWS credentials
               uses: aws-actions/configure-aws-credentials@v4
               with:
@@ -97,10 +77,26 @@ jobs:
                   echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT
                   echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile --prefer-offline
+
+            - name: Start up db images
+              run: pnpm test:setup
+
+            - name: p:build
+              run: pnpm p:build
+
+            - name: Run migrations and seed
+              run: pnpm --filter core db:test:reset
+              env:
+                  # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests
+                  MINIMAL_SEED: true
+                  SKIP_VALIDATION: true
+
             - run: pnpm --filter core exec playwright install chromium --with-deps
 
-            - name: Start up core
-              run: docker compose -f docker-compose.test.yml --profile integration up -d
+            - name: Start up core etc
+              run: pnpm integration:setup
               env:
                   INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}}
                   JOBS_IMAGE: ${{steps.label.outputs.jobs_label}}
@@ -118,7 +114,7 @@ jobs:
                   INTEGRATION_TEST_HOST: localhost
 
             - name: Print container logs
-              if: failure()
+              if: ${{failure() || cancelled()}}
               run: docker compose -f docker-compose.test.yml --profile integration logs
 
             - name: Upload playwright snapshots artifact

diff --git a/.github/workflows/ecrbuild-all.yml b/.github/workflows/ecrbuild-all.yml
@@ -9,6 +9,20 @@ on:
                 required: true
             AWS_SECRET_ACCESS_KEY:
                 required: true
+        inputs:
+            publish_to_ghcr:
+                type: boolean
+                default: false
+        outputs:
+            core-image:
+                description: "Core image SHA"
+                value: ${{ jobs.build-core.outputs.image-sha}}
+            base-image:
+                description: "Base image SHA"
+                value: ${{ jobs.build-base.outputs.image-sha }}
+            jobs-image:
+                description: "Jobs image SHA"
+                value: ${{ jobs.build-jobs.outputs.image-sha }}
 
 jobs:
     emit-sha-tag:
@@ -26,6 +40,9 @@ jobs:
 
     build-base:
         uses: ./.github/workflows/ecrbuild-template.yml
+        with:
+            publish_to_ghcr: ${{ inputs.publish_to_ghcr }}
+            ghcr_image_name: platform-migrations
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -36,6 +53,8 @@ jobs:
         #   - build-base
         with:
             package: core
+            publish_to_ghcr: ${{ inputs.publish_to_ghcr }}
+            ghcr_image_name: platform
             # we require a bigger lad
             # We are now public, default public runner is big enough
             # runner: ubuntu-latest-m
@@ -50,6 +69,8 @@ jobs:
         with:
             package: jobs
             target: jobs
+            publish_to_ghcr: ${{ inputs.publish_to_ghcr }}
+            ghcr_image_name: platform-jobs
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/ecrbuild-template.yml b/.github/workflows/ecrbuild-template.yml
@@ -12,6 +12,16 @@ on:
                 default: ubuntu-latest
             target:
                 type: string
+            publish_to_ghcr:
+                type: boolean
+                default: false
+            ghcr_image_name:
+                type: string
+                required: false
+        outputs:
+            image-sha:
+                description: "Image SHA"
+                value: ${{ jobs.build.outputs.image-sha }}
         secrets:
             AWS_ACCESS_KEY_ID:
                 required: true
@@ -28,6 +38,8 @@ jobs:
     build:
         name: Build
         runs-on: ${{ inputs.runner }}
+        outputs:
+            image-sha: ${{ steps.label.outputs.label }}
 
         steps:
             - name: Checkout
@@ -45,6 +57,13 @@ jobs:
               id: login-ecr
               uses: aws-actions/amazon-ecr-login@v2
 
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
               # necessary in order to upload build source maps to sentry
             - name: Get sentry token
               id: sentry-token
@@ -75,6 +94,16 @@ jobs:
                     echo "target=${TARGET:-next-app-${PACKAGE}}" >> $GITHUB_OUTPUT
                   fi
                   echo "label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX$package_suffix:$sha_short" >> $GITHUB_OUTPUT
+                  if [[ ${{ inputs.publish_to_ghcr }} == "true" && -n ${{ inputs.ghcr_image_name }} ]]
+                  then
+                    TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+                    echo "ghcr_latest_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:latest" >> $GITHUB_OUTPUT
+
+                    echo "ghcr_sha_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$sha_short" >> $GITHUB_OUTPUT
+
+                    echo "ghcr_timestamp_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$TIMESTAMP" >> $GITHUB_OUTPUT
+                  fi
 
             - name: Check if SENTRY_AUTH_TOKEN is set
               run: |
@@ -103,6 +132,10 @@ jobs:
                   secrets: |
                       SENTRY_AUTH_TOKEN=${{ env.SENTRY_AUTH_TOKEN }}
                   target: ${{ steps.label.outputs.target }}
-                  tags: ${{ steps.label.outputs.label }}
+                  tags: |
+                      ${{ steps.label.outputs.label }}
+                      ${{ steps.label.outputs.ghcr_latest_label }}
+                      ${{ steps.label.outputs.ghcr_sha_label }}
+                      ${{ steps.label.outputs.ghcr_timestamp_label }}
                   platforms: linux/amd64
                   push: true
diff --git a/.github/workflows/on_main.yml b/.github/workflows/on_main.yml
@@ -14,6 +14,8 @@ jobs:
     build-all:
         needs: ci
         uses: ./.github/workflows/ecrbuild-all.yml
+        with:
+            publish_to_ghcr: true
         secrets:
             AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
             AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml
@@ -14,6 +14,7 @@ env:
 jobs:
     ci:
         uses: ./.github/workflows/ci.yml
+
     build-all:
         uses: ./.github/workflows/ecrbuild-all.yml
         secrets:

diff --git a/.gitignore b/.gitignore
@@ -69,4 +69,6 @@ core/supabase/.temp
 *storybook.log
 storybook-static
 
-./playwright
+./playwright
+
+.local_data
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -3,6 +3,8 @@
 		"ms-playwright.playwright",
 		"YoavBls.pretty-ts-errors",
 		"esbenp.prettier-vscode",
-		"dbaeumer.vscode-eslint"
+		"dbaeumer.vscode-eslint",
+		// for yaml autocompletion using the # yaml-language-server: $schema=... directive
+		"redhat.vscode-yaml"
 	]
 }
diff --git a/core/.env.development b/core/.env.development
@@ -6,8 +6,11 @@ PUBPUB_URL="http://localhost:3000"
 ASSETS_BUCKET_NAME="assets.v7.pubpub.org"
 ASSETS_REGION="us-east-1"
 
-ASSETS_UPLOAD_KEY="xxx"
-ASSETS_UPLOAD_SECRET_KEY="xxx"
+# mninio defaults
+ASSETS_UPLOAD_KEY="pubpubuser"
+ASSETS_UPLOAD_SECRET_KEY="pubpubpass"
+ASSETS_STORAGE_ENDPOINT="http://localhost:9000"
+
 MAILGUN_SMTP_PASSWORD="xxx"
 MAILGUN_SMTP_USERNAME="xxx"