Merge pull request #276 from bento-platform/qa/v2.1

v2.1 QA
bento-platform · Aug 8, 2023 · b0fd4be · b0fd4be
2 parents c558d05 + c56a62a
commit b0fd4be
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 40 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -6,6 +6,7 @@ on:
   pull_request:
     branches:
       - master
+      - qa/**
       - releases/**
   push:
     branches:

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "bento_web",
-  "version": "2.1.1",
+  "version": "2.1.2",
   "description": "Bento platform front-end",
   "main": "src/index.js",
   "dependencies": {

diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js
@@ -102,60 +102,79 @@ const buildKeyFromRecord = (record) => `${record.dataType}-${record.tableId}`;
 
 const fileNameFromPath = (path) => path.split("/").at(-1);
 
-const getFileInputsFromWorkflowMetadata = (workflowMetadata) => {
-    return workflowMetadata.inputs
-        .filter(input => input.type === "file" || input.type === "file[]")
-        .map(input => `${workflowMetadata.id}.${input.id}`);
-};
+const getFileInputsFromWorkflow = (workflowId, {inputs}) =>
+    inputs
+        .filter(input => ["file", "file[]"].includes(input.type))
+        .map(input => `${workflowId}.${input.id}`);
 
 const processIngestions = (data, currentTables) => {
     const currentTableIds = new Set((currentTables || []).map((table) => table.table_id));
 
     const ingestionsByDataType = data.reduce((ingestions, run) => {
-        if (run.state === "COMPLETE") {
-            const dataType = run.details.request.tags.workflow_metadata.data_type;
-            const tableId = run.details.request.tags.table_id;
+        if (run.state !== "COMPLETE") {
+            return ingestions;
+        }
 
-            if (tableId === undefined || !currentTableIds.has(tableId)) {
-                return ingestions;
-            }
-            const fileNameKey = getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata);
-            const filePaths = fileNameKey.flatMap(key =>
-                Array.isArray(run.details.request.workflow_params[key])
-                    ? run.details.request.workflow_params[key]
-                    : [run.details.request.workflow_params[key]],
-            );
-            const fileNames = Array.isArray(filePaths)
-                ? filePaths.map(fileNameFromPath)
-                : [fileNameFromPath(filePaths)];
-
-            const date = Date.parse(run.details.run_log.end_time);
-
-            const currentIngestion = { date, dataType, tableId, fileNames };
-            const dataTypeAndTableId = buildKeyFromRecord(currentIngestion);
-
-            if (ingestions[dataTypeAndTableId]) {
-                const existingDate = ingestions[dataTypeAndTableId].date;
-                if (date > existingDate) {
-                    ingestions[dataTypeAndTableId].date = date;
-                }
-                ingestions[dataTypeAndTableId].fileNames.push(...fileNames);
-            } else {
-                ingestions[dataTypeAndTableId] = currentIngestion;
+        const {
+            workflow_id: workflowId,
+            workflow_metadata: workflowMetadata,
+            table_id: tableId,
+        } = run.details.request.tags;
+
+        if (tableId === undefined || !currentTableIds.has(tableId)) {
+            return ingestions;
+        }
+
+        const fileNames =
+            getFileInputsFromWorkflow(workflowId ?? workflowMetadata.id, workflowMetadata)
+                .flatMap(key => {
+                    const paramValue = run.details.request.workflow_params[key];
+                    if (!paramValue) {
+                        // Key isn't in workflow params or is null
+                        // - possibly optional field or something else going wrong
+                        return [];
+                    }
+                    return Array.isArray(paramValue) ? paramValue : [paramValue];
+                })
+                .map(fileNameFromPath);
+
+        const date = Date.parse(run.details.run_log.end_time);
+
+        const currentIngestion = { date, dataType: workflowMetadata.data_type, tableId, fileNames };
+        const dataTypeAndTableId = buildKeyFromRecord(currentIngestion);
+
+        if (ingestions[dataTypeAndTableId]) {
+            const existingDate = ingestions[dataTypeAndTableId].date;
+            if (date > existingDate) {
+                ingestions[dataTypeAndTableId].date = date;
             }
+            ingestions[dataTypeAndTableId].fileNames.push(...fileNames);
+        } else {
+            ingestions[dataTypeAndTableId] = currentIngestion;
         }
+
         return ingestions;
     }, {});
 
     return Object.values(ingestionsByDataType).sort((a, b) => Date.parse(b.date) - Date.parse(a.date));
 };
 
 const LastIngestionTable = () => {
-    const runs = useSelector((state) => state.runs.items);
-    const currentTables = useSelector((state) => state.projectTables.items);
+    const servicesFetching = useSelector(state => state.services.isFetchingAll);
+    const {items: runs, isFetching: runsFetching} = useSelector((state) => state.runs);
+    const {
+        items: currentTables,
+        isFetching: projectTablesFetching,
+    } = useSelector((state) => state.projectTables);
     const ingestions = useMemo(() => processIngestions(runs, currentTables), [runs, currentTables]);
 
-    return <Table bordered={true} columns={COLUMNS_LAST_CONTENT} dataSource={ingestions} rowKey={buildKeyFromRecord} />;
+    return <Table
+        bordered={true}
+        columns={COLUMNS_LAST_CONTENT}
+        loading={servicesFetching || runsFetching || projectTablesFetching}
+        dataSource={ingestions}
+        rowKey={buildKeyFromRecord}
+    />;
 };
 
 export default LastIngestionTable;