From 8e48bf159aba3d183f08c0a41a3be6ab695ce5e7 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 12:46:01 -0400 Subject: [PATCH 01/10] refact: invert if logic for lest indenting --- src/components/manager/runs/RunLastContent.js | 61 ++++++++++--------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 9eace6bec..5073a64b7 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -112,38 +112,41 @@ const processIngestions = (data, currentTables) => { const currentTableIds = new Set((currentTables || []).map((table) => table.table_id)); const ingestionsByDataType = data.reduce((ingestions, run) => { - if (run.state === "COMPLETE") { - const dataType = run.details.request.tags.workflow_metadata.data_type; - const tableId = run.details.request.tags.table_id; + if (run.state !== "COMPLETE") { + return ingestions; + } - if (tableId === undefined || !currentTableIds.has(tableId)) { - return ingestions; - } - const fileNameKey = getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata); - const filePaths = fileNameKey.flatMap(key => - Array.isArray(run.details.request.workflow_params[key]) - ? run.details.request.workflow_params[key] - : [run.details.request.workflow_params[key]], - ); - const fileNames = Array.isArray(filePaths) - ? filePaths.map(fileNameFromPath) - : [fileNameFromPath(filePaths)]; - - const date = Date.parse(run.details.run_log.end_time); - - const currentIngestion = { date, dataType, tableId, fileNames }; - const dataTypeAndTableId = buildKeyFromRecord(currentIngestion); - - if (ingestions[dataTypeAndTableId]) { - const existingDate = ingestions[dataTypeAndTableId].date; - if (date > existingDate) { - ingestions[dataTypeAndTableId].date = date; - } - ingestions[dataTypeAndTableId].fileNames.push(...fileNames); - } else { - ingestions[dataTypeAndTableId] = currentIngestion; + const dataType = run.details.request.tags.workflow_metadata.data_type; + const tableId = run.details.request.tags.table_id; + + if (tableId === undefined || !currentTableIds.has(tableId)) { + return ingestions; + } + const fileNameKey = getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata); + const filePaths = fileNameKey.flatMap(key => + Array.isArray(run.details.request.workflow_params[key]) + ? run.details.request.workflow_params[key] + : [run.details.request.workflow_params[key]], + ); + const fileNames = Array.isArray(filePaths) + ? filePaths.map(fileNameFromPath) + : [fileNameFromPath(filePaths)]; + + const date = Date.parse(run.details.run_log.end_time); + + const currentIngestion = { date, dataType, tableId, fileNames }; + const dataTypeAndTableId = buildKeyFromRecord(currentIngestion); + + if (ingestions[dataTypeAndTableId]) { + const existingDate = ingestions[dataTypeAndTableId].date; + if (date > existingDate) { + ingestions[dataTypeAndTableId].date = date; } + ingestions[dataTypeAndTableId].fileNames.push(...fileNames); + } else { + ingestions[dataTypeAndTableId] = currentIngestion; } + return ingestions; }, {}); From afc4d7f4caa420f10a4efe7b20adb855ecf20de0 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 12:52:16 -0400 Subject: [PATCH 02/10] fix: handle undefined file params in workflow run last content component --- src/components/manager/runs/RunLastContent.js | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 5073a64b7..bbdccb559 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -116,25 +116,28 @@ const processIngestions = (data, currentTables) => { return ingestions; } - const dataType = run.details.request.tags.workflow_metadata.data_type; - const tableId = run.details.request.tags.table_id; + const { workflow_metadata: workflowMetadata, table_id: tableId } = run.details.request.tags; if (tableId === undefined || !currentTableIds.has(tableId)) { return ingestions; } - const fileNameKey = getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata); - const filePaths = fileNameKey.flatMap(key => - Array.isArray(run.details.request.workflow_params[key]) - ? run.details.request.workflow_params[key] - : [run.details.request.workflow_params[key]], - ); - const fileNames = Array.isArray(filePaths) - ? filePaths.map(fileNameFromPath) - : [fileNameFromPath(filePaths)]; + + const fileNames = + getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata) + .flatMap(key => { + const paramValue = run.details.request.workflow_params[key]; + if (!paramValue) { + // Key isn't in workflow params or is null + // - possibly optional field or something else going wrong + return []; + } + return Array.isArray(paramValue) ? paramValue : [paramValue]; + }) + .map(fileNameFromPath); const date = Date.parse(run.details.run_log.end_time); - const currentIngestion = { date, dataType, tableId, fileNames }; + const currentIngestion = { date, dataType: workflowMetadata.data_type, tableId, fileNames }; const dataTypeAndTableId = buildKeyFromRecord(currentIngestion); if (ingestions[dataTypeAndTableId]) { From d33824e6dfa087bab6047ecda165cd8ef05952d6 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 12:59:38 -0400 Subject: [PATCH 03/10] ci: build qa branches --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 307707eae..e918f8388 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,7 @@ on: pull_request: branches: - master + - qa/** - releases/** push: branches: From 1b01f36cd9e1758f9034b6841de90d0505dcdd28 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 13:10:17 -0400 Subject: [PATCH 04/10] fix: workflow ID access logic --- src/components/manager/runs/RunLastContent.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index bbdccb559..482a55ecc 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -102,11 +102,10 @@ const buildKeyFromRecord = (record) => `${record.dataType}-${record.tableId}`; const fileNameFromPath = (path) => path.split("/").at(-1); -const getFileInputsFromWorkflowMetadata = (workflowMetadata) => { - return workflowMetadata.inputs - .filter(input => input.type === "file" || input.type === "file[]") - .map(input => `${workflowMetadata.id}.${input.id}`); -}; +const getFileInputsFromWorkflow = (workflowId, {inputs}) => + inputs + .filter(input => ["file", "file[]"].includes(input.type)) + .map(input => `${workflowId}.${input.id}`); const processIngestions = (data, currentTables) => { const currentTableIds = new Set((currentTables || []).map((table) => table.table_id)); @@ -116,14 +115,18 @@ const processIngestions = (data, currentTables) => { return ingestions; } - const { workflow_metadata: workflowMetadata, table_id: tableId } = run.details.request.tags; + const { + workflow_id: workflowId, + workflow_metadata: workflowMetadata, + table_id: tableId, + } = run.details.request.tags; if (tableId === undefined || !currentTableIds.has(tableId)) { return ingestions; } const fileNames = - getFileInputsFromWorkflowMetadata(run.details.request.tags.workflow_metadata) + getFileInputsFromWorkflow(workflowId ?? workflowMetadata.id, workflowMetadata) .flatMap(key => { const paramValue = run.details.request.workflow_params[key]; if (!paramValue) { From 019ea59c1dcd40fdf22c638d60841e7ca9453aa8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 14:51:08 -0400 Subject: [PATCH 05/10] fix: missing loading indicator for last ingestions table --- src/components/manager/runs/RunLastContent.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 482a55ecc..3b232338a 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -160,11 +160,18 @@ const processIngestions = (data, currentTables) => { }; const LastIngestionTable = () => { - const runs = useSelector((state) => state.runs.items); + const servicesFetching = useSelector(state => state.services.isFetchingAll); + const {items: runs, isFetching: runsFetching} = useSelector((state) => state.runs); const currentTables = useSelector((state) => state.projectTables.items); const ingestions = useMemo(() => processIngestions(runs, currentTables), [runs, currentTables]); - return ; + return
; }; export default LastIngestionTable; From fd39e19950e29be569850f622da8a78b6c0ad020 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 15:00:48 -0400 Subject: [PATCH 06/10] more data sources for loading in runlastcontent --- src/components/manager/runs/RunLastContent.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 3b232338a..c23711dbd 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -162,13 +162,16 @@ const processIngestions = (data, currentTables) => { const LastIngestionTable = () => { const servicesFetching = useSelector(state => state.services.isFetchingAll); const {items: runs, isFetching: runsFetching} = useSelector((state) => state.runs); - const currentTables = useSelector((state) => state.projectTables.items); + const { + items: currentTables, + isFetching: projectTablesFetching, + } = useSelector((state) => state.projectTables.items); const ingestions = useMemo(() => processIngestions(runs, currentTables), [runs, currentTables]); return
; From 8f7f027f7fbeee4ac38623a1addee4b278ac5963 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 15:01:14 -0400 Subject: [PATCH 07/10] debug logging --- src/components/manager/runs/RunLastContent.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index c23711dbd..5dfa838bc 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -129,6 +129,7 @@ const processIngestions = (data, currentTables) => { getFileInputsFromWorkflow(workflowId ?? workflowMetadata.id, workflowMetadata) .flatMap(key => { const paramValue = run.details.request.workflow_params[key]; + console.log(key, paramValue); if (!paramValue) { // Key isn't in workflow params or is null // - possibly optional field or something else going wrong From 66429d11920b3cfae1a2bd33c4d26c3419f75d17 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 15:09:09 -0400 Subject: [PATCH 08/10] fix --- src/components/manager/runs/RunLastContent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 5dfa838bc..aa73c760c 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -166,7 +166,7 @@ const LastIngestionTable = () => { const { items: currentTables, isFetching: projectTablesFetching, - } = useSelector((state) => state.projectTables.items); + } = useSelector((state) => state.projectTables); const ingestions = useMemo(() => processIngestions(runs, currentTables), [runs, currentTables]); return
Date: Tue, 8 Aug 2023 15:19:43 -0400 Subject: [PATCH 09/10] rm debug log --- src/components/manager/runs/RunLastContent.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index aa73c760c..7f6163684 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -129,7 +129,6 @@ const processIngestions = (data, currentTables) => { getFileInputsFromWorkflow(workflowId ?? workflowMetadata.id, workflowMetadata) .flatMap(key => { const paramValue = run.details.request.workflow_params[key]; - console.log(key, paramValue); if (!paramValue) { // Key isn't in workflow params or is null // - possibly optional field or something else going wrong From c123098706a85aea90322d92dd05e0659fe85777 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 8 Aug 2023 15:21:40 -0400 Subject: [PATCH 10/10] chore: bump version to 2.1.2 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4d0fd7ce8..d7bd4af7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "bento_web", - "version": "2.1.1", + "version": "2.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "bento_web", - "version": "2.1.1", + "version": "2.1.2", "license": "LGPL-3.0-only", "dependencies": { "ajv": "^8.12.0", diff --git a/package.json b/package.json index b152976da..1a11c917e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bento_web", - "version": "2.1.1", + "version": "2.1.2", "description": "Bento platform front-end", "main": "src/index.js", "dependencies": {