914 [Feature] - "Add remaining CRUD endpoints to SST" (#943)

* Implemented skeleton; still need to fill in handler implementation * Work in progress * Added type hints * Implemented get_trainspace endpoint * Progress * Progress * Resolved ticket 914 * Implemented DynamoDB built-in pagination consideration to get all trainspace endpoint * Installed yarn uuid package * Installed dependencies to serverless directory * Added types/uuid * Implemented PR feedback * T * Fixed stuff * Implemented PR changes: Narrowewd scope of SST handler permissions, cleaned up code, tested * 🎨 Auto-generated directory tree for repository in Architecture.md * Removed useless comment * Split creation of different trainspaces into different API endpoints * Implemented Daniel Wu's PR comments * Fixed SonarCloud comment changes * Renamed file * Fixed path issues * FF * Implemented Daniel Wu's PR comments * 🎨 Auto-generated directory tree for repository in Architecture.md * PR Changes * Implemented PR comments * Removed unused file * 🎨 Auto-generated directory tree for repository in Architecture.md --------- Co-authored-by: alantao912 <[email protected]>
DSGT-DLP · Nov 11, 2023 · 35b6972 · 35b6972
1 parent f9b7d92
commit 35b6972
Show file tree

Hide file tree

Showing 14 changed files with 1,916 additions and 5,977 deletions.
diff --git a/dlp-cli b/dlp-cli
diff --git a/frontend/src/features/Train/features/Tabular/redux/tabularApi.ts b/frontend/src/features/Train/features/Tabular/redux/tabularApi.ts
@@ -12,6 +12,7 @@ const tabularApi = backendApi.injectEndpoints({
         method: "POST",
         body: {
           name: trainspaceData.name,
+          data_source: trainspaceData.dataSource,
           target: trainspaceData.parameterData.targetCol,
           features: trainspaceData.parameterData.features,
           default: trainspaceData.datasetData.isDefaultDataset

diff --git a/package-lock.json b/package-lock.json
diff --git a/serverless/package.json b/serverless/package.json
@@ -23,6 +23,10 @@
     "typescript": "^5.2.2"
   },
   "dependencies": {
-    "@aws-sdk/s3-request-presigner": "^3.438.0"
+    "@aws-sdk/client-dynamodb": "^3.418.0",
+    "@aws-sdk/lib-dynamodb": "^3.418.0",
+    "@aws-sdk/s3-request-presigner": "^3.391.0",
+    "@aws-sdk/types": "^3.418.0",
+    "@types/uuid": "^9.0.4"
   }
 }
diff --git a/serverless/packages/functions/package.json b/serverless/packages/functions/package.json
@@ -7,9 +7,12 @@
     "typecheck": "tsc -noEmit"
   },
   "devDependencies": {
-    "@types/node": "^20.5.0",
     "@types/aws-lambda": "^8.10.119",
-    "vitest": "^0.34.1",
-    "sst": "^2.24.3"
+    "@types/node": "^20.5.0",
+    "sst": "^2.24.3",
+    "vitest": "^0.34.1"
+  },
+  "dependencies": {
+    "uuid": "^9.0.1"
   }
-}
+}
diff --git a/serverless/packages/functions/src/datasets/user/delete_url.ts b/serverless/packages/functions/src/datasets/user/delete_url.ts
@@ -0,0 +1,29 @@
+import { APIGatewayProxyHandlerV2 } from "aws-lambda";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { S3Client, DeleteObjectCommand } from "@aws-sdk/client-s3";
+import parseJwt from "@dlp-sst-app/core/parseJwt";
+
+export const handler: APIGatewayProxyHandlerV2 = async (event) => {
+  if (event?.pathParameters?.type && event.pathParameters.filename) {
+    const client = new S3Client();
+    const user_id: string = parseJwt(event.headers.authorization ?? "")[
+      "user_id"
+    ];
+    // Generate the presigned URL for a putObject operation
+    const command = new DeleteObjectCommand({
+      Bucket: "dlp-upload-bucket",
+      Key: `${user_id}/${event.pathParameters.type}/${event.pathParameters.filename}`,
+    });
+    const url = await getSignedUrl(client, command, {
+      expiresIn: 15 * 60,
+    });
+    return {
+      statusCode: 200,
+      body: JSON.stringify({ data: url, message: "Success" }),
+    };
+  }
+  return {
+    statusCode: 404,
+    body: JSON.stringify({ message: "Not Found" }),
+  };
+};
diff --git a/serverless/packages/functions/src/trainspace/constants.ts b/serverless/packages/functions/src/trainspace/constants.ts
@@ -0,0 +1,9 @@
+
+export enum TrainStatus {
+    QUEUED = "QUEUED",
+    STARTING="STARTING",
+    UPLOADING="UPLOADING",
+    TRAINING="TRAINING",
+    SUCCESS="SUCCESS",
+    ERROR="ERROR"
+}
diff --git a/serverless/packages/functions/src/trainspace/create_image_trainspace.ts b/serverless/packages/functions/src/trainspace/create_image_trainspace.ts
@@ -0,0 +1,63 @@
+import { APIGatewayProxyHandlerV2 } from "aws-lambda";
+import parseJwt from "@dlp-sst-app/core/parseJwt";
+import { v4 as uuidv4 } from 'uuid';
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, PutCommand, PutCommandInput } from '@aws-sdk/lib-dynamodb';
+import { TrainStatus } from './constants';
+
+export const handler: APIGatewayProxyHandlerV2 = async (event) => {
+    if (event) {
+        const user_id: string = parseJwt(event.headers.authorization ?? "")["user_id"];
+        const eventBody = JSON.parse(event.body? event.body : "");
+
+        const trainspaceId = uuidv4();
+        const putCommandInput: PutCommandInput = {
+            TableName: "trainspace",
+            Item:
+            {
+                trainspace_id: trainspaceId,
+                uid: user_id,
+                created: Date.now().toString(),
+                data_source: 'IMAGE',
+                dataset_data: JSON.stringify(eventBody['dataset_data']),
+                name: eventBody['name'],
+                parameters_data: JSON.stringify(eventBody['parameters_data']),
+                review_data: eventBody['review_data'],
+                status: TrainStatus.QUEUED
+            }
+        }
+
+        if (putCommandInput == null)
+        {
+            return {
+                statusCode: 400,
+                body: JSON.stringify({ message: "Invalid request body" })
+            }
+        }
+
+        const client = new DynamoDBClient({});
+        const docClient = DynamoDBDocumentClient.from(client);
+
+        const command = new PutCommand(putCommandInput);
+        const response = await docClient.send(command);
+
+        if (response.$metadata.httpStatusCode != 200) {
+            client.destroy();
+
+            return {
+                statusCode: 500,
+                body: JSON.stringify({ message: "Internal server error."})
+            };
+        }
+
+        client.destroy();
+        return {
+            statusCode: 200,
+            body: JSON.stringify({ trainspaceId: trainspaceId, message: "Successfully created a new trainspace."})
+        };
+      }
+    return {
+        statusCode: 404,
+        body: JSON.stringify({ message: "Not Found" }),
+    };
+};
diff --git a/serverless/packages/functions/src/trainspace/create_tabular_trainspace.ts b/serverless/packages/functions/src/trainspace/create_tabular_trainspace.ts
@@ -0,0 +1,67 @@
+import { APIGatewayProxyHandlerV2 } from "aws-lambda";
+import parseJwt from "@dlp-sst-app/core/parseJwt";
+import { v4 as uuidv4 } from 'uuid';
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, PutCommand, PutCommandInput } from '@aws-sdk/lib-dynamodb';
+import { TrainStatus } from './constants';
+
+export const handler: APIGatewayProxyHandlerV2 = async (event) => {
+    if (event) {
+        const user_id: string = parseJwt(event.headers.authorization ?? "")["user_id"];
+        const eventBody = JSON.parse(event.body? event.body : "");
+
+        const trainspaceId = uuidv4();
+        let putCommandInput: PutCommandInput = {
+            TableName: "trainspace",
+            Item:
+            {
+                trainspace_id: trainspaceId,
+                uid: user_id,
+                created: Date.now().toString(),
+                data_source: 'TABULAR',
+                dataset_data: JSON.stringify(eventBody['dataset_data']),
+                name: eventBody['name'],
+                parameters_data: { 
+                    criterion: eventBody['criterion'],
+                    optimizer_name: eventBody['optimizer_name'],
+                    shuffle: eventBody['shuffle'],
+                    epochs: eventBody['epochs'],
+                    batch_size: eventBody['batch_size'],
+                    user_arch: eventBody['user_arch']
+                },
+                review_data: "",
+                status: TrainStatus.QUEUED
+            }
+        }
+
+        if (putCommandInput == null)
+        {
+            return {
+                statusCode: 400,
+                body: JSON.stringify({ message: "Invalid request body" })
+            }
+        }
+
+        const client = new DynamoDBClient({});
+        const docClient = DynamoDBDocumentClient.from(client);
+
+        const command = new PutCommand(putCommandInput);
+        const response = await docClient.send(command);
+
+        if (response.$metadata.httpStatusCode != 200) {
+            return {
+                statusCode: 500,
+                body: JSON.stringify({ message: "Internal server error."})
+            };
+        }
+
+        return {
+            statusCode: 200,
+            body: JSON.stringify({ trainspaceId: trainspaceId, message: "Successfully created a new trainspace."})
+        };
+      }
+    return {
+        statusCode: 404,
+        body: JSON.stringify({ message: "Not Found" }),
+    };
+};
diff --git a/serverless/packages/functions/src/trainspace/delete_trainspace.ts b/serverless/packages/functions/src/trainspace/delete_trainspace.ts
@@ -0,0 +1,46 @@
+import { APIGatewayProxyHandlerV2 } from "aws-lambda";
+
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, DeleteCommand } from '@aws-sdk/lib-dynamodb';
+
+export const handler: APIGatewayProxyHandlerV2 = async (event) => {
+    let queryParams = null;
+    if (event && (queryParams = event['pathParameters']) != null) {
+        const trainspaceId: string | undefined = queryParams['id'];
+
+        if (trainspaceId == undefined) {
+            return {
+                statusCode: 400,
+                body: JSON.stringify({ message : "Malformed request content - trainspace ID missing." }),
+            };
+        }
+
+        const client = new DynamoDBClient({});
+        const docClient = DynamoDBDocumentClient.from(client);
+
+        const command = new DeleteCommand({
+            TableName : "trainspace",
+            Key :
+            {
+                trainspace_id: trainspaceId
+            }
+        });
+
+        const response = await docClient.send(command);
+        if (response.$metadata.httpStatusCode == undefined || response.$metadata.httpStatusCode != 200) 
+        {
+            return {
+                statusCode: 404,
+                body: JSON.stringify({ message : "Delete operation failed" })
+            }
+        }
+        return {
+            statusCode: 200,
+            body: "Successfully deleted trainspace with id " + trainspaceId
+        }
+    }
+    return {
+        statusCode: 400,
+        body: JSON.stringify({ message : "Malformed request content" }),
+    };
+};
diff --git a/serverless/packages/functions/src/trainspace/get_all_trainspace.ts b/serverless/packages/functions/src/trainspace/get_all_trainspace.ts
@@ -0,0 +1,50 @@
+import { APIGatewayProxyHandlerV2 } from "aws-lambda";
+import parseJwt from "@dlp-sst-app/core/parseJwt";
+import { DynamoDBClient, QueryCommand } from '@aws-sdk/client-dynamodb';
+
+export const handler: APIGatewayProxyHandlerV2 = async (event) => {
+    if (event) {
+        const user_id: string = parseJwt(event.headers.authorization ?? "")[
+            "user_id"
+        ];
+
+        const client = new DynamoDBClient({});
+
+        const fetchedTrainspaceIds: Array<string> = [];
+        let lastEvaluatedKey = undefined;
+
+        do {
+            const getCommand: QueryCommand = new QueryCommand({
+                TableName: "trainspace",
+                IndexName: "uid",
+                KeyConditionExpression: "uid = :uid",
+                ExpressionAttributeValues: {
+                    ":uid" : 
+                    { 
+                        S: user_id
+                    }
+                },
+                ExclusiveStartKey: lastEvaluatedKey
+            });
+
+            const results = await client.send(getCommand);
+            lastEvaluatedKey = results.LastEvaluatedKey;
+
+            if (results['Items']) {
+                const page: Array<string | undefined> = results['Items']?.map(trainspace => trainspace['trainspace_id'].S);
+                page.forEach(id => { if (id) fetchedTrainspaceIds.push(id); });
+            }
+
+
+        } while (lastEvaluatedKey !== undefined);
+        return { 
+            statusCode: 200, 
+            body: JSON.stringify({ trainspace_ids : fetchedTrainspaceIds}) 
+        };
+    }
+
+    return {
+        statusCode: 404,
+        body: JSON.stringify({ message: "Not Found" }),
+    };
+};
diff --git a/serverless/packages/functions/src/trainspace/get_trainspace.ts b/serverless/packages/functions/src/trainspace/get_trainspace.ts
@@ -0,0 +1,46 @@
+import { APIGatewayProxyHandlerV2, APIGatewayProxyEventV2 } from "aws-lambda";
+import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
+import { DynamoDBDocumentClient, GetCommand } from '@aws-sdk/lib-dynamodb';
+
+export const handler: APIGatewayProxyHandlerV2 = async (event : APIGatewayProxyEventV2) => {
+    const queryParams = event['pathParameters'];
+    if (queryParams != null)
+    {
+        const trainspaceId: string | undefined = queryParams['id'];
+
+        if (trainspaceId == undefined) {
+            return {
+                statusCode: 400,
+                body: JSON.stringify({message: "Malformed request content - trainspace ID missing."})
+            };
+        }
+
+        const client: DynamoDBClient = new DynamoDBClient({});
+        const docClient = DynamoDBDocumentClient.from(client);
+
+        const command : GetCommand = new GetCommand({
+            TableName : "trainspace",
+            Key : 
+            {
+                trainspace_id : trainspaceId
+            }
+        });
+
+        const response = await docClient.send(command);
+        if (!response.Item)
+        {
+            return {
+                statusCode: 404,
+                body: JSON.stringify({message: "Provided trainspaceId does not exist"})
+            }
+        }
+        return {
+            statusCode: 200,
+            body: JSON.stringify({message: "Successfully retrieved trainspace data", trainspace: response.Item})
+        }
+    }
+    return {
+        statusCode: 400,
+        body: JSON.stringify({message: "Malformed request content"})
+    };
+};