feat(file-manager): multipart upload and progress bar (#3232)

Author: Pavel910

packages/api-file-manager-s3/src/multiPartUpload/CompleteMultiPartUploadUseCase.ts

@@ -0,0 +1,80 @@
+import S3 from "aws-sdk/clients/s3";
+
+interface CompleteMultiPartUploadParams {
+    fileKey: string;
+    uploadId: string;
+}
+
+interface GetAllUploadPartsParams {
+    Bucket: string;
+    Key: string;
+    UploadId: string;
+}
+
+export class CompleteMultiPartUploadUseCase {
+    private readonly s3: S3;
+    private readonly bucket: string;
+
+    constructor(bucket: string, s3Client: S3) {
+        this.bucket = bucket;
+        this.s3 = s3Client;
+    }
+
+    async execute(params: CompleteMultiPartUploadParams) {
+        const uploadParams = {
+            Bucket: this.bucket,
+            Key: params.fileKey,
+            UploadId: params.uploadId
+        };
+
+        const allParts = await this.getAllUploadParts(uploadParams);
+
+        const s3Params = {
+            ...uploadParams,
+            MultipartUpload: {
+                Parts: allParts
+            }
+        };
+
+        return new Promise<void>((resolve, reject) => {
+            this.s3.completeMultipartUpload(s3Params, (err, data) => {
+                if (err) {
+                    console.error(err);
+                    reject(err);
+                    return;
+                }
+
+                console.log(data);
+                resolve();
+            });
+        });
+    }
+
+    private async getAllUploadParts(params: GetAllUploadPartsParams) {
+        const parts: S3.Parts = [];
+
+        let marker: number | undefined = undefined;
+        while (true) {
+            const { Parts, PartNumberMarker }: S3.ListPartsOutput = await this.s3
+                .listParts({
+                    ...params,
+                    PartNumberMarker: marker
+                })
+                .promise();
+
+            if (Parts) {
+                Parts.forEach(part => parts.push(part));
+            }
+
+            marker = PartNumberMarker || undefined;
+            if (!marker) {
+                break;
+            }
+        }
+
+        return parts.map(part => ({
+            ETag: part.ETag as string,
+            PartNumber: part.PartNumber as number
+        }));
+    }
+}

packages/api-file-manager-s3/src/multiPartUpload/CreateMultiPartUploadUseCase.ts

@@ -0,0 +1,52 @@
+import S3 from "aws-sdk/clients/s3";
+import { prepareFileData } from "~/utils/prepareFileData";
+
+interface CreateMultiPartUploadParams {
+    file: {
+        name: string;
+        type: string;
+        size: number;
+    };
+    numberOfParts: number;
+}
+
+export class CreateMultiPartUploadUseCase {
+    private readonly s3: S3;
+    private readonly bucket: string;
+
+    constructor(bucket: string, s3Client: S3) {
+        this.bucket = bucket;
+        this.s3 = s3Client;
+    }
+
+    async execute(params: CreateMultiPartUploadParams) {
+        const file = prepareFileData(params.file);
+
+        const s3Params = { Bucket: this.bucket, Key: file.key };
+
+        const { UploadId } = await this.s3.createMultipartUpload(s3Params).promise();
+
+        const parts = await Promise.all(
+            Array.from({ length: params.numberOfParts }).map((_, index) => {
+                return this.s3
+                    .getSignedUrlPromise("uploadPart", {
+                        ...s3Params,
+                        UploadId,
+                        PartNumber: index + 1,
+                        // URL expires after 24 hours.
+                        Expires: 86400
+                    })
+                    .then(url => ({
+                        url,
+                        partNumber: index + 1
+                    }));
+            })
+        );
+
+        return {
+            file,
+            uploadId: UploadId,
+            parts
+        };
+    }
+}

packages/api-file-manager-s3/src/plugins/graphqlFileStorageS3.ts

@@ -1,10 +1,13 @@
+import S3 from "aws-sdk/clients/s3";
 import { GraphQLSchemaPlugin } from "@webiny/handler-graphql/types";
 import { ErrorResponse, Response } from "@webiny/handler-graphql/responses";
 import { FileManagerContext } from "@webiny/api-file-manager/types";
 import { getPresignedPostPayload } from "~/utils/getPresignedPostPayload";
 import WebinyError from "@webiny/error";
 import { checkPermission } from "~/plugins/checkPermission";
 import { PresignedPostPayloadData } from "~/types";
+import { CreateMultiPartUploadUseCase } from "~/multiPartUpload/CreateMultiPartUploadUseCase";
+import { CompleteMultiPartUploadUseCase } from "~/multiPartUpload/CompleteMultiPartUploadUseCase";
 
 const plugin: GraphQLSchemaPlugin<FileManagerContext> = {
     type: "graphql-schema",
@@ -36,6 +39,22 @@ const plugin: GraphQLSchemaPlugin<FileManagerContext> = {
                 data: GetPreSignedPostPayloadResponseData
             }
 
+            type MultiPartUploadFilePart {
+                partNumber: Int!
+                url: String!
+            }
+
+            type CreateMultiPartUploadResponseData {
+                file: GetPreSignedPostPayloadResponseDataFile!
+                uploadId: String!
+                parts: [MultiPartUploadFilePart!]!
+            }
+
+            type CompleteMultiPartUploadResponse {
+                data: Boolean
+                error: FileError
+            }
+
             type GetPreSignedPostPayloadsResponse {
                 error: FileError
                 data: [GetPreSignedPostPayloadResponseData!]!
@@ -49,6 +68,28 @@ const plugin: GraphQLSchemaPlugin<FileManagerContext> = {
                     data: [PreSignedPostPayloadInput]!
                 ): GetPreSignedPostPayloadsResponse
             }
+
+            type CreateMultiPartUploadResponse {
+                data: CreateMultiPartUploadResponseData
+                error: FileError
+            }
+
+            input MultiPartUploadFilePartInput {
+                partNumber: Int!
+                etag: String!
+            }
+
+            extend type FmMutation {
+                createMultiPartUpload(
+                    data: PreSignedPostPayloadInput!
+                    numberOfParts: Number!
+                ): CreateMultiPartUploadResponse
+
+                completeMultiPartUpload(
+                    fileKey: String!
+                    uploadId: String!
+                ): CompleteMultiPartUploadResponse
+            }
         `,
         resolvers: {
             FmQuery: {
@@ -104,6 +145,64 @@ const plugin: GraphQLSchemaPlugin<FileManagerContext> = {
                         });
                     }
                 }
+            },
+            FmMutation: {
+                createMultiPartUpload: async (_, args, context) => {
+                    await checkPermission(context, { rwd: "w" });
+
+                    const s3Client = new S3({
+                        region: process.env.AWS_REGION,
+                        signatureVersion: "v4"
+                    });
+
+                    try {
+                        const useCase = new CreateMultiPartUploadUseCase(
+                            String(process.env.S3_BUCKET),
+                            s3Client
+                        );
+
+                        const multiPartUpload = await useCase.execute({
+                            file: args.data,
+                            numberOfParts: args.numberOfParts
+                        });
+
+                        return new Response(multiPartUpload);
+                    } catch (e) {
+                        return new ErrorResponse({
+                            message: e.message,
+                            code: e.code,
+                            data: e.data
+                        });
+                    }
+                },
+                completeMultiPartUpload: async (_, args, context) => {
+                    await checkPermission(context, { rwd: "w" });
+
+                    const s3Client = new S3({
+                        region: process.env.AWS_REGION,
+                        signatureVersion: "v4"
+                    });
+
+                    try {
+                        const useCase = new CompleteMultiPartUploadUseCase(
+                            String(process.env.S3_BUCKET),
+                            s3Client
+                        );
+
+                        await useCase.execute({
+                            fileKey: args.fileKey,
+                            uploadId: args.uploadId
+                        });
+
+                        return new Response(true);
+                    } catch (e) {
+                        return new ErrorResponse({
+                            message: e.message,
+                            code: e.code,
+                            data: e.data
+                        });
+                    }
+                }
             }
         }
     }

packages/api-file-manager-s3/src/types.ts

@@ -9,13 +9,15 @@ export interface PresignedPostPayloadData {
     keyPrefix?: string;
 }
 
+export interface FileData {
+    id: string;
+    name: string;
+    key: string;
+    type: string;
+    size: number;
+}
+
 export interface PresignedPostPayloadDataResponse {
     data: S3.PresignedPost;
-    file: {
-        id: string;
-        name: string;
-        key: string;
-        type: string;
-        size: number;
-    };
+    file: FileData;
 }

packages/api-file-manager-s3/src/utils/getPresignedPostPayload.ts

@@ -1,11 +1,8 @@
-// @ts-ignore `mdbid` has no type declarations
-import mdbid from "mdbid";
-import sanitizeFilename from "sanitize-filename";
 import S3 from "aws-sdk/clients/s3";
 import { validation } from "@webiny/validation";
-import { PresignedPostPayloadData, PresignedPostPayloadDataResponse } from "~/types";
 import { FileManagerSettings } from "@webiny/api-file-manager/types";
-import { mimeTypes } from "./mimeTypes";
+import { prepareFileData } from "~/utils/prepareFileData";
+import { PresignedPostPayloadData, PresignedPostPayloadDataResponse } from "~/types";
 
 const S3_BUCKET = process.env.S3_BUCKET;
 const UPLOAD_MAX_FILE_SIZE_DEFAULT = 1099511627776; // 1TB
@@ -24,37 +21,7 @@ export const getPresignedPostPayload = (
     data: PresignedPostPayloadData,
     settings: FileManagerSettings
 ): PresignedPostPayloadDataResponse => {
-    // If type is missing, let's use the default "application/octet-stream" type,
-    // which is also the default type that the Amazon S3 would use.
-    if (!data.type) {
-        data.type = "application/octet-stream";
-    }
-
-    const contentType = data.type;
-    if (!contentType) {
-        throw Error(`File's content type could not be resolved.`);
-    }
-
-    const id = data.id || mdbid();
-    let key = data.key || sanitizeFilename(data.name);
-
-    // We must prefix file key with file ID.
-    if (!key.startsWith(id)) {
-        key = id + "/" + key;
-    }
-
-    if (data.keyPrefix) {
-        key = data.keyPrefix + key;
-    }
-
-    // Replace all whitespace.
-    key = key.replace(/\s/g, "");
-
-    // Make sure file key contains a file extension
-    const extensions = mimeTypes[contentType];
-    if (!extensions.some(ext => key.endsWith(`.${ext}`))) {
-        key = key + `.${extensions[0]}`;
-    }
+    const file = prepareFileData(data);
 
     const uploadMinFileSize = sanitizeFileSizeValue(settings.uploadMinFileSize, 0);
     const uploadMaxFileSize = sanitizeFileSizeValue(
@@ -67,8 +34,8 @@ export const getPresignedPostPayload = (
         Bucket: S3_BUCKET,
         Conditions: [["content-length-range", uploadMinFileSize, uploadMaxFileSize]],
         Fields: {
-            "Content-Type": contentType,
-            key
+            "Content-Type": file.type,
+            key: file.key
         }
     };
 
@@ -81,12 +48,6 @@ export const getPresignedPostPayload = (
 
     return {
         data: payload,
-        file: {
-            id,
-            name: data.name,
-            key,
-            type: contentType,
-            size: data.size
-        }
+        file
     };
 };