(self-host|3): self-hostable platform!

.env.docker-compose.dev

@@ -0,0 +1,13 @@
+MINIO_ROOT_USER=pubpub-minio-admin
+MINIO_ROOT_PASSWORD=pubpub-minio-admin
+
+ASSETS_BUCKET_NAME=assets.v7.pubpub.org
+ASSETS_UPLOAD_KEY=pubpubuser
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_REGION=us-east-1
+
+POSTGRES_PORT=54322
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+

.env.docker-compose.test

@@ -0,0 +1,40 @@
+MINIO_ROOT_USER=pubpub-minio-admin
+MINIO_ROOT_PASSWORD=pubpub-minio-admin
+
+ASSETS_BUCKET_NAME=byron.v7.pubpub.org
+ASSETS_UPLOAD_KEY=pubpubuserrr
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_REGION=us-east-1
+ASSETS_STORAGE_ENDPOINT=http://localhost:9000
+
+POSTGRES_PORT=54323
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+POSTGRES_HOST=db
+
+# annoying duplication because jobs uses this version
+PGHOST=db
+PGPORT=5432
+PGUSER=postgres
+PGPASSWORD=postgres
+PGDATABASE=postgres
+
+# this needs to be db:5432 bc that's what it is in the app-network
+# if you are running this from outside the docker network, you need to use
+# @localhost:${POSTGRES_PORT} instead
+DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
+
+
+JWT_SECRET=xxx
+MAILGUN_SMTP_PASSWORD=xxx
+GCLOUD_KEY_FILE=xxx
+
+MAILGUN_SMTP_HOST=inbucket
+MAILGUN_SMTP_PORT=2500
+# this needs to be localhost:54324 instead of inbucket:9000 bc we are almost always running the integration tests from outside the docker network
+INBUCKET_URL=http://localhost:54324
+MAILGUN_SMTP_USERNAME=omitted
+OTEL_SERVICE_NAME=core.core
+PUBPUB_URL=http://localhost:3000
+API_KEY=xxx

.github/workflows/ci.yml

@@ -19,6 +19,7 @@ jobs:
         runs-on: ubuntu-latest
         env:
             COMPOSE_FILE: docker-compose.test.yml
+            ENV_FILE: .env.docker-compose.test
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -55,8 +56,8 @@ jobs:
                   restore-keys: |
                       ${{ runner.os }}-turbo-
 
-            - name: Start up DB
-              run: docker compose --profile test up -d
+            - name: Start test dependencies
+              run: pnpm test:setup
 
             - name: Install dependencies
               run: pnpm install --frozen-lockfile --prefer-offline
@@ -66,8 +67,6 @@ jobs:
 
             - name: Run migrations
               run: pnpm --filter core migrate-test
-              env:
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
 
             - name: Run prettier
               run: pnpm format

.github/workflows/e2e.yml

@@ -20,6 +20,8 @@ jobs:
     integration-tests:
         name: Integration tests
         runs-on: ubuntu-latest
+        env:
+            ENV_FILE: .env.docker-compose.test
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -48,28 +50,6 @@ jobs:
                   restore-keys: |
                       ${{ runner.os }}-pnpm-store-
 
-            - name: Install dependencies
-              run: pnpm install --frozen-lockfile --prefer-offline
-
-            - name: Start up DB
-              run: docker compose -f docker-compose.test.yml --profile test up -d
-
-            - name: p:build
-              run: pnpm p:build
-
-            - name: Run migrations
-              run: pnpm --filter core prisma migrate deploy
-              env:
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
-
-            - name: seed db
-              run: pnpm --filter core prisma db seed
-              env:
-                  # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests
-                  MINIMAL_SEED: true
-                  SKIP_VALIDATION: true
-                  DATABASE_URL: postgresql://postgres:postgres@localhost:5433/postgres
-
             - name: Configure AWS credentials
               uses: aws-actions/configure-aws-credentials@v4
               with:
@@ -97,10 +77,26 @@ jobs:
                   echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT
                   echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT
 
+            - name: Install dependencies
+              run: pnpm install --frozen-lockfile --prefer-offline
+
+            - name: Start up db images
+              run: pnpm test:setup
+
+            - name: p:build
+              run: pnpm p:build
+
+            - name: Run migrations and seed
+              run: pnpm --filter core db:test:reset
+              env:
+                  # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests
+                  MINIMAL_SEED: true
+                  SKIP_VALIDATION: true
+
             - run: pnpm --filter core exec playwright install chromium --with-deps
 
-            - name: Start up core
-              run: docker compose -f docker-compose.test.yml --profile integration up -d
+            - name: Start up core etc
+              run: pnpm integration:setup
               env:
                   INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}}
                   JOBS_IMAGE: ${{steps.label.outputs.jobs_label}}
@@ -118,7 +114,7 @@ jobs:
                   INTEGRATION_TEST_HOST: localhost
 
             - name: Print container logs
-              if: failure()
+              if: ${{failure() || cancelled()}}
               run: docker compose -f docker-compose.test.yml --profile integration logs
 
             - name: Upload playwright snapshots artifact

.gitignore

@@ -69,4 +69,6 @@ core/supabase/.temp
 *storybook.log
 storybook-static
 
-./playwright
+./playwright
+
+.local_data

core/.env.development

@@ -6,8 +6,11 @@ PUBPUB_URL="http://localhost:3000"
 ASSETS_BUCKET_NAME="assets.v7.pubpub.org"
 ASSETS_REGION="us-east-1"
 
-ASSETS_UPLOAD_KEY="xxx"
-ASSETS_UPLOAD_SECRET_KEY="xxx"
+# mninio defaults
+ASSETS_UPLOAD_KEY="pubpubuser"
+ASSETS_UPLOAD_SECRET_KEY="pubpubpass"
+ASSETS_STORAGE_ENDPOINT="http://localhost:9000"
+
 MAILGUN_SMTP_PASSWORD="xxx"
 MAILGUN_SMTP_USERNAME="xxx"
 

core/.env.test

@@ -1,13 +1,15 @@
-DATABASE_URL=postgresql://postgres:postgres@localhost:5433/postgres
+DATABASE_URL=postgresql://postgres:postgres@localhost:54323/postgres
 PUBPUB_URL=http://localhost:3000
 MAILGUN_SMTP_HOST=localhost
 MAILGUN_SMTP_PORT=54325
 API_KEY="super_secret_key"
-ASSETS_BUCKET_NAME="assets.v7.pubpub.org"
-ASSETS_REGION="us-east-1"
 
-ASSETS_UPLOAD_KEY="xxx"
-ASSETS_UPLOAD_SECRET_KEY="xxx"
+ASSETS_BUCKET_NAME=byron.v7.pubpub.org
+ASSETS_UPLOAD_KEY=pubpubuserrr
+ASSETS_UPLOAD_SECRET_KEY=pubpubpass
+ASSETS_REGION=us-east-1
+ASSETS_STORAGE_ENDPOINT="http://localhost:9000"
+
 MAILGUN_SMTP_PASSWORD="xxx"
 MAILGUN_SMTP_USERNAME="xxx"
 
@@ -17,3 +19,4 @@ HONEYCOMB_API_KEY="xxx"
 KYSELY_DEBUG="true"
 
 GCLOUD_KEY_FILE='xxx'
+

core/app/components/forms/FileUpload.tsx

@@ -21,7 +21,7 @@ export function FileUploadPreview({ files }: { files: FileUpload }) {
 									The file is <strong>{file.fileSize}</strong> bytes in size. Its
 									MIME type is <strong>{file.fileType}</strong>.
 								</p>
-								<Button variant="secondary">
+								<Button variant="secondary" asChild>
 									<a target="_blank" href={file.fileUploadUrl}>
 										Open file in new tab
 									</a>

core/app/components/forms/elements/FileUploadElement.tsx

@@ -34,7 +34,8 @@ export const FileUploadElement = ({
 	const signedUploadUrl = (fileName: string) => {
 		return upload(pubId, fileName);
 	};
-	const { control, getValues } = useFormContext();
+	const { control, getValues, formState } = useFormContext();
+
 	const formElementToggle = useFormElementToggleContext();
 	const isEnabled = formElementToggle.isEnabled(slug);
 	const files = getValues()[slug];
@@ -58,7 +59,7 @@ export const FileUploadElement = ({
 									{...field}
 									disabled={!isEnabled}
 									upload={signedUploadUrl}
-									onUpdateFiles={(event: any[]) => {
+									onUpdateFiles={(event) => {
 										field.onChange(event);
 									}}
 									id={slug}

core/lib/env/env.mjs

@@ -25,6 +25,7 @@ export const env = createEnv({
 		ASSETS_REGION: z.string(),
 		ASSETS_UPLOAD_KEY: z.string(),
 		ASSETS_UPLOAD_SECRET_KEY: z.string(),
+		ASSETS_STORAGE_ENDPOINT: z.string().url().optional(),
 		/**
 		 * Whether or not to verbosely log `memoize` cache hits and misses
 		 */

core/lib/server/assets.ts

@@ -12,11 +12,13 @@ export const generateSignedAssetUploadUrl = async (pubId: PubsId, fileName: stri
 	const bucket = env.ASSETS_BUCKET_NAME;
 
 	const client = new S3Client({
+		endpoint: env.ASSETS_STORAGE_ENDPOINT,
 		region: region,
 		credentials: {
 			accessKeyId: key,
 			secretAccessKey: secret,
 		},
+		forcePathStyle: !!env.ASSETS_STORAGE_ENDPOINT, // Required for MinIO
 	});
 	const command = new PutObjectCommand({
 		Bucket: bucket,

core/package.json

@@ -12,7 +12,8 @@
 		"db:migrate-deploy": "pnpm migrate-deploy",
 		"db:migrate-diff": "pnpm migrate-diff",
 		"db:migrate-docker": "pnpm migrate-docker",
-		"db:migrate-test": "pnpm migrate-test",
+		"db:test:migrate": "pnpm migrate-test",
+		"db:test:reset": "dotenv -e .env.test -- prisma migrate reset --preview-feature --force | pino-pretty",
 		"db:prisma": "pnpm prisma",
 		"db:studio": "pnpm prisma studio",
 		"db:generate-history-table": "pnpm exec tsx prisma/scripts/history-tables/generate-history-table.mts",

core/playwright.config.ts

@@ -17,6 +17,8 @@ export default defineConfig({
 	expect: {
 		timeout: process.env.CI ? 5_000 : 60_000,
 	},
+	// don't continue going after 3 tests have failed, that's too many
+	maxFailures: process.env.CI ? 3 : undefined,
 	// a
 	webServer: [
 		{

core/playwright/fileUpload.spec.ts

@@ -0,0 +1,117 @@
+import type { Page } from "@playwright/test";
+
+import { expect, test } from "@playwright/test";
+
+import { FieldsPage } from "./fixtures/fields-page";
+import { LoginPage } from "./fixtures/login-page";
+import { PubTypesPage } from "./fixtures/pub-types-page";
+import { PubsPage } from "./fixtures/pubs-page";
+import { createCommunity } from "./helpers";
+
+const now = new Date().getTime();
+const COMMUNITY_SLUG = `playwright-test-community-${now}`;
+
+test.describe.configure({ mode: "serial" });
+
+let page: Page;
+
+test.beforeAll(async ({ browser }) => {
+	page = await browser.newPage();
+
+	const loginPage = new LoginPage(page);
+	await loginPage.goto();
+	await loginPage.loginAndWaitForNavigation();
+
+	await createCommunity({
+		page,
+		community: { name: `test community ${now}`, slug: COMMUNITY_SLUG },
+	});
+
+	/**
+	 * Fill out everything required to make an external form:
+	 * 1. Fields
+	 * 2. Form with fields
+	 * 3. A pub
+	 */
+	// Populate the fields page with options
+	const fieldsPage = new FieldsPage(page, COMMUNITY_SLUG);
+	await fieldsPage.goto();
+	await fieldsPage.addFieldsOfEachType();
+
+	const pubTypePage = new PubTypesPage(page, COMMUNITY_SLUG);
+	await pubTypePage.goto();
+	await pubTypePage.addType("File Upload Test", "", ["title", "fileupload"]);
+});
+
+test.afterAll(async () => {
+	await page.close();
+});
+
+test.describe("File upload", () => {
+	test("should upload a file", async ({ context }) => {
+		const pubsPage = new PubsPage(page, COMMUNITY_SLUG);
+		await pubsPage.goTo();
+		const pubId = await pubsPage.createPub({
+			pubType: "File Upload Test",
+			values: { title: "The Activity of Slugs" },
+		});
+
+		const pubEditUrl = `/c/${COMMUNITY_SLUG}/pubs/${pubId}/edit`;
+		await page.goto(pubEditUrl);
+
+		await page.setInputFiles("input[type='file']", [
+			new URL("fixtures/test-assets/test-diagram.png", import.meta.url).pathname,
+		]);
+
+		await page.getByRole("button", { name: "Upload 1 file", exact: true }).click({
+			timeout: 2_000,
+		});
+
+		await page.getByText("Complete", { exact: true }).waitFor({
+			timeout: 10_000,
+		});
+
+		await page.getByRole("button", { name: "Save" }).click();
+		await page.getByText("Pub successfully updated", { exact: true }).waitFor({
+			timeout: 2_000,
+		});
+
+		await page.getByRole("link", { name: "View Pub", exact: true }).click();
+		await page.waitForURL(`/c/${COMMUNITY_SLUG}/pubs/${pubId}`);
+
+		const fileUploadValue = await page
+			.getByTestId(`FileUpload-value`)
+			.getByRole("button")
+			.click({
+				timeout: 1_000,
+			});
+
+		const describeThing = await page.getByText("Its MIME type is").first().textContent({
+			timeout: 2_000,
+		});
+
+		/**
+		 * So we know the actual file is being uploaded
+		 */
+		const parsedDescription = describeThing?.match(
+			/The file is (\d+) bytes in size. Its MIME type is (\w+\/\w+)./
+		);
+		expect(parsedDescription).not.toBeNull();
+
+		const fileSize = parsedDescription?.[1];
+		const mimeType = parsedDescription?.[2];
+
+		expect(fileSize).toBeDefined();
+		expect(parseInt(fileSize!)).toBe(108_839);
+		expect(mimeType).toBe("image/png");
+
+		const link = page.getByRole("link", { name: "Open file in new tab" });
+
+		const url = await link.getAttribute("href", { timeout: 1_000 });
+
+		expect(url).toBeDefined();
+		await page.goto(url!);
+
+		await page.waitForURL(/localhost:9000/);
+	});
+});