Merge branch 'main' into tfk/pub-filter

core/lib/server/assets.db.test.ts

@@ -0,0 +1,48 @@
+import fs from "fs/promises";
+
+import { beforeAll, describe, expect, it } from "vitest";
+
+import { mockServerCode } from "~/lib/__tests__/utils";
+import { env } from "../env/env.mjs";
+
+const { createForEachMockedTransaction } = await mockServerCode();
+
+const { getTrx } = createForEachMockedTransaction();
+
+beforeAll(async () => {
+	// check if minio is up
+
+	if (!env.ASSETS_STORAGE_ENDPOINT) {
+		throw new Error(
+			"You should only run this test against a local minio instance, not to prod S3"
+		);
+	}
+
+	const check = await fetch(env.ASSETS_STORAGE_ENDPOINT, {
+		method: "OPTIONS",
+	});
+
+	if (!check.ok) {
+		throw new Error(
+			"Minio is not running. Please setup the test environment properly by running `pnpm -w test:setup`"
+		);
+	}
+});
+
+describe("assets upload", () => {
+	it("should be able to upload a file to the minio bucket from the server", async () => {
+		const { uploadFileToS3 } = await import("./assets");
+
+		const file = await fs.readFile(new URL("./assets.ts", import.meta.url));
+
+		const url = await uploadFileToS3("test", "test.ts", file, {
+			contentType: "text/plain",
+		});
+
+		expect(url).toBeDefined();
+
+		const text = await fetch(url).then((res) => res.text());
+
+		expect(text).toEqual(file.toString("utf8"));
+	});
+});

core/lib/server/assets.ts

@@ -1,17 +1,24 @@
 import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 
 import type { PubsId } from "db/public";
+import { logger } from "logger";
 
 import { env } from "../env/env.mjs";
 
-export const generateSignedAssetUploadUrl = async (pubId: PubsId, fileName: string) => {
+let s3Client: S3Client;
+
+export const getS3Client = () => {
+	if (s3Client) {
+		return s3Client;
+	}
+
 	const region = env.ASSETS_REGION;
 	const key = env.ASSETS_UPLOAD_KEY;
 	const secret = env.ASSETS_UPLOAD_SECRET_KEY;
-	const bucket = env.ASSETS_BUCKET_NAME;
 
-	const client = new S3Client({
+	s3Client = new S3Client({
 		endpoint: env.ASSETS_STORAGE_ENDPOINT,
 		region: region,
 		credentials: {
@@ -20,9 +27,72 @@ export const generateSignedAssetUploadUrl = async (pubId: PubsId, fileName: stri
 		},
 		forcePathStyle: !!env.ASSETS_STORAGE_ENDPOINT, // Required for MinIO
 	});
+
+	return s3Client;
+};
+
+export const generateSignedAssetUploadUrl = async (pubId: PubsId, fileName: string) => {
+	const client = getS3Client();
+
+	const bucket = env.ASSETS_BUCKET_NAME;
 	const command = new PutObjectCommand({
 		Bucket: bucket,
 		Key: `${pubId}/${fileName}`,
 	});
+
 	return await getSignedUrl(client, command, { expiresIn: 3600 });
 };
+
+/**
+ * Uploads a file to the S3 bucket using the S3 client directly
+ * @param id - id under which the file will be stored. eg for a pub, the pubId. for community assets like the logo, the communityId. for user avatars, the userId.
+ * @param fileName - name of the file to be stored
+ * @param fileData - the file data to upload (Buffer or Uint8Array)
+ * @param contentType - MIME type of the file (e.g., 'image/jpeg')
+ * @returns the URL of the uploaded file
+ */
+export const uploadFileToS3 = async (
+	id: string,
+	fileName: string,
+	fileData: Buffer | Uint8Array,
+	{
+		contentType,
+		queueSize,
+		partSize,
+		progressCallback,
+	}: {
+		contentType: string;
+		queueSize?: number;
+		partSize?: number;
+		progressCallback?: (progress: any) => void;
+	}
+): Promise<string> => {
+	const client = getS3Client();
+	const bucket = env.ASSETS_BUCKET_NAME;
+	const key = `${id}/${fileName}`;
+
+	const parallelUploads3 = new Upload({
+		client,
+		params: {
+			Bucket: bucket,
+			Key: key,
+			Body: fileData,
+			ContentType: contentType,
+		},
+		queueSize: queueSize ?? 3, // optional concurrency configuration
+		partSize: partSize ?? 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
+		leavePartsOnError: false, // optional manually handle dropped parts
+	});
+
+	parallelUploads3.on(
+		"httpUploadProgress",
+		progressCallback ??
+			((progress) => {
+				logger.debug(progress);
+			})
+	);
+
+	const result = await parallelUploads3.done();
+
+	return result.Location!;
+};

core/package.json

@@ -54,6 +54,7 @@
 	},
 	"dependencies": {
 		"@aws-sdk/client-s3": "^3.445.0",
+		"@aws-sdk/lib-storage": "^3.750.0",
 		"@aws-sdk/s3-request-presigner": "^3.445.0",
 		"@dagrejs/dagre": "^1.0.4",
 		"@dnd-kit/core": "^6.1.0",

docker-compose.test.yml

@@ -26,6 +26,7 @@ services:
         networks:
             - app-network
         profiles:
+            - test
             - integration
 
     minio-init:
@@ -39,6 +40,7 @@ services:
         networks:
             - app-network
         profiles:
+            - test
             - integration
 
     inbucket: