fix: [Importer] In job status response, include count of successful f…

…iles and total files imported

packages/spacecat-shared-data-access/docs/schema.json

@@ -1000,6 +1000,22 @@
           "Projection": {
             "ProjectionType": "ALL"
           }
+        },
+        {
+          "IndexName": "spacecat-services-all-import-jobs-by-date-range",
+          "KeyAttributes": {
+            "PartitionKey": {
+              "AttributeName": "GSI1PK",
+              "AttributeType": "S"
+            },
+            "SortKey": {
+              "AttributeName": "startTime",
+              "AttributeType": "S"
+            }
+          },
+          "Projection": {
+            "ProjectionType": "ALL"
+          }
         }
       ],
       "DataAccess": {

packages/spacecat-shared-data-access/src/dto/import-job.js

@@ -35,8 +35,6 @@ export const ImportJobDto = {
     duration: importJob.getDuration(),
     status: importJob.getStatus(),
     urlCount: importJob.getUrlCount(),
-    successCount: importJob.getSuccessCount(),
-    failedCount: importJob.getFailedCount(),
     importQueueId: importJob.getImportQueueId(),
     initiatedBy: importJob.getInitiatedBy(),
     GSI1PK: 'ALL_IMPORT_JOBS',
@@ -57,12 +55,11 @@ export const ImportJobDto = {
       duration: dynamoItem.duration,
       status: dynamoItem.status,
       urlCount: dynamoItem.urlCount,
-      successCount: dynamoItem.successCount,
-      failedCount: dynamoItem.failedCount,
       importQueueId: dynamoItem.importQueueId,
       initiatedBy: dynamoItem.initiatedBy,
     };
 
     return createImportJob(importJobData);
   },
+
 };

packages/spacecat-shared-data-access/src/index.d.ts

@@ -10,6 +10,22 @@
  * governing permissions and limitations under the License.
  */
 
+// see packages/spacecat-shared-data-access/src/models/importer/import-constants.js
+export declare const ImportJobStatus: {
+  readonly RUNNING: 'RUNNING';
+  readonly COMPLETE: 'COMPLETE';
+  readonly FAILED: 'FAILED';
+};
+
+// packages/spacecat-shared-data-access/src/models/importer/import-constants.js
+export declare const ImportUrlStatus: {
+  readonly PENDING: 'PENDING';
+  readonly REDIRECT: 'REDIRECT';
+  readonly RUNNING: 'RUNNING';
+  readonly COMPLETE: 'COMPLETE';
+  readonly FAILED: 'FAILED';
+};
+
 // TODO: introduce AuditType interface or Scores interface
 
 /**
@@ -463,7 +479,7 @@ export interface ImportJob {
   /**
    * Retrieves the status of the import job.
    */
-  getStatus: () => string;
+  getStatus: () => typeof ImportJobStatus;
 
   /**
    * Retrieves the baseURL of the import job.
@@ -515,28 +531,38 @@ export interface ImportJob {
    */
   getInitiatedBy: () => object;
 
+  /**
+   * Retrieves the progress of the import job.
+   */
+  getProgress: () => {
+    completed: number;
+    failed: number;
+    pending: number;
+    running: number;
+    redirect: number;
+  };
 }
 
 export interface ImportUrl {
   /**
    * Retrieves the ID of the import URL.
    */
-    getId: () => string;
+  getId: () => string;
 
   /**
    * Retrieves the status of the import URL.
    */
-    getStatus: () => string;
+  getStatus: () => typeof ImportUrlStatus;
 
   /**
    * Retrieves the URL of the import URL.
    */
-    getUrl: () => string;
+  getUrl: () => string;
 
   /**
    * Retrieves the job ID of the import URL.
    */
-    getJobId: () => string;
+  getJobId: () => string;
 
   /**
    * The reason that the import of a URL failed.
@@ -867,13 +893,3 @@ export function createDataAccess(
   config: DataAccessConfig,
   logger: object,
 ): DataAccess;
-
-export interface ImportJobStatus {
-  RUNNING: string,
-  COMPLETE: string,
-  FAILED: string,
-}
-
-export interface ImportUrlStatus extends ImportJobStatus {
-  PENDING: string,
-}

packages/spacecat-shared-data-access/src/models/importer/import-constants.js

@@ -15,6 +15,10 @@
 export const ImportOptions = {
   ENABLE_JAVASCRIPT: 'enableJavascript',
   PAGE_LOAD_TIMEOUT: 'pageLoadTimeout',
+  SAVE_AS_DOCX: 'saveAsDocx',
+  HAS_CUSTOM_HEADERS: 'hasCustomHeaders',
+  HAS_CUSTOM_IMPORT_JS: 'hasCustomImportJs',
+  SCROLL_TO_BOTTOM: 'scrollToBottom',
 };
 
 /**

packages/spacecat-shared-data-access/src/models/importer/import-job.js

@@ -16,6 +16,19 @@ import {
 import { Base } from '../base.js';
 import { ImportJobStatus, ImportOptions } from './import-constants.js';
 
+export const jobToApiMap = [
+  ['id', 'id'],
+  ['baseURL', 'baseURL'],
+  ['options', 'options'],
+  ['startTime', 'startTime'],
+  ['endTime', 'endTime'],
+  ['duration', 'duration'],
+  ['status', 'status'],
+  ['urlCount', 'urlCount'],
+  ['initiatedBy', 'initiatedBy'],
+  ['progress', 'progress'],
+];
+
 /**
  * Creates a new ImportJob object.
  *
@@ -34,11 +47,18 @@ const ImportJob = (data) => {
   self.getDuration = () => self.state.duration;
   self.getStatus = () => self.state.status;
   self.getUrlCount = () => self.state.urlCount;
-  self.getSuccessCount = () => self.state.successCount;
-  self.getFailedCount = () => self.state.failedCount;
   self.getImportQueueId = () => self.state.importQueueId;
   self.getInitiatedBy = () => self.state.initiatedBy;
 
+  // the progress is a derived property from querying the status of the import urls table
+  self.getProgress = () => self.state.progress || {
+    running: 0,
+    failed: 0,
+    completed: 0,
+    pending: 0,
+    redirect: 0,
+  };
+
   /**
    * Updates the state of the ImportJob.
    * @param key - The key to update.
@@ -97,27 +117,6 @@ const ImportJob = (data) => {
     }
   });
 
-  /**
-   * Updates the success count of the ImportJob.
-   * @param {number} successCount - The new success count.
-   */
-  self.updateSuccessCount = (successCount) => updateState('successCount', successCount, (value) => {
-    if (!isInteger(value)) {
-      throw new Error(`Invalid success count during update: ${value}`);
-    }
-  });
-
-  /**
-   * Updates the failed count of the ImportJob.
-   * @param {number} failedCount - The new failed count.
-   * @returns {{ImportJob}} The updated ImportJob object.
-   */
-  self.updateFailedCount = (failedCount) => updateState('failedCount', failedCount, (value) => {
-    if (!isInteger(value)) {
-      throw new Error(`Invalid failed count during update: ${value}`);
-    }
-  });
-
   /**
    * Updates the import queue id of the ImportJob.
    * @param {string} importQueueId - The new import queue id.
@@ -184,8 +183,10 @@ export const createImportJob = (data) => {
       throw new Error(`Invalid options: ${newState.options}`);
     }
 
-    const invalidOptions = Object.keys(newState.options)
-      .filter((key) => !Object.values(ImportOptions).includes(key));
+    const invalidOptions = Object.keys(newState.options).filter(
+      (key) => !Object.values(ImportOptions)
+        .some((value) => value.toLowerCase() === key.toLowerCase()),
+    );
 
     if (invalidOptions.length > 0) {
       throw new Error(`Invalid options: ${invalidOptions}`);

packages/spacecat-shared-data-access/src/models/importer/import-model-mapper.js

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Map the properties of the source object into the target object based on the mapping array.
+ *
+ * @param {object} sourceObj The source object.
+ * @param {array} mapping An array of arrays in the format of either
+ * ['modelPropertyKey', ['apiPropertyKey'] or ['modelPropertyKey', modelToApiFn()]
+ */
+function mappingReducer(sourceObj, mapping) {
+  const sourceMapIndex = 0;
+  const targetMapIndex = 1;
+
+  return mapping.reduce((targetObj, mapEntry) => {
+    if (sourceObj[mapEntry[sourceMapIndex]] === undefined) {
+      return targetObj;
+    }
+
+    if (typeof mapEntry[targetMapIndex] === 'function') {
+      const result = mapEntry[targetMapIndex](sourceObj[mapEntry[sourceMapIndex]]);
+      Object.assign(targetObj, result);
+    } else {
+      // eslint-disable-next-line no-param-reassign
+      targetObj[mapEntry[targetMapIndex]] = sourceObj[mapEntry[sourceMapIndex]];
+    }
+
+    return targetObj;
+  }, {});
+}
+
+/**
+ * Map the model object to a plain old object based on the model map. If the source
+ * object does not contain a key that is specified in the mapping array, that key
+ * will not be included in the resulting object.
+ *
+ * @param {object} model The model to convert to a POJO
+ * @param {array} modelMap Map mapping array that contains multiple arrays. The nested arrays
+ * containing key mappings between objects.
+ * Inner array format can be either ['modelPropertyKey', ['apiPropertyKey'] or
+ * ['modelPropertyKey', modelToApiFn()]
+ */
+export function map(model, modelMap) {
+  return mappingReducer(model, modelMap);
+}

packages/spacecat-shared-data-access/src/service/import-job/accessPatterns.js

@@ -10,9 +10,11 @@
  * governing permissions and limitations under the License.
  */
 
-import { isObject } from '@adobe/spacecat-shared-utils';
+import { isArray, isObject } from '@adobe/spacecat-shared-utils';
 import { ImportJobDto } from '../../dto/import-job.js';
 import { createImportJob } from '../../models/importer/import-job.js';
+import { getImportUrlsByJobId } from '../import-url/accessPatterns.js';
+import { ImportUrlStatus } from '../../models/importer/import-constants.js';
 
 /**
  * Get all Import Jobs within a specific date range
@@ -40,19 +42,58 @@ export const getImportJobsByDateRange = async (dynamoClient, config, log, startD
 };
 
 /**
- * Get Import Job by ID
+ * Get Import Job by ID.
  * @param {DynamoClient} dynamoClient
  * @param {Object} config
  * @param {Logger} log
  * @param {string} id
  * @returns {Promise<ImportJobDto> | null}
  */
 export const getImportJobByID = async (dynamoClient, config, log, id) => {
-  const item = await dynamoClient.getItem(
+  const jobEntity = await dynamoClient.getItem(
     config.tableNameImportJobs,
     { id },
   );
-  return item ? ImportJobDto.fromDynamoItem(item) : null;
+
+  if (!jobEntity) {
+    return null;
+  }
+
+  const jobModel = ImportJobDto.fromDynamoItem(jobEntity);
+
+  const importUrls = await getImportUrlsByJobId(dynamoClient, config, log, id);
+
+  if (isArray(importUrls)) {
+    jobModel.state.progress = importUrls.reduce((acc, importUrl) => {
+      // eslint-disable-next-line default-case
+      switch (importUrl.state.status) {
+        case ImportUrlStatus.PENDING:
+          acc.pending += 1;
+          break;
+        case ImportUrlStatus.REDIRECT:
+          acc.redirect += 1;
+          break;
+        case ImportUrlStatus.RUNNING:
+          acc.running += 1;
+          break;
+        case ImportUrlStatus.COMPLETE:
+          acc.completed += 1;
+          break;
+        case ImportUrlStatus.FAILED:
+          acc.failed += 1;
+          break;
+      }
+      return acc;
+    }, {
+      pending: 0,
+      redirect: 0,
+      running: 0,
+      completed: 0,
+      failed: 0,
+    });
+  }
+
+  return jobModel;
 };
 
 /**

packages/spacecat-shared-data-access/src/service/import-url/accessPatterns.js

@@ -64,7 +64,7 @@ export const updateImportUrl = async (dynamoClient, config, log, importUrl) => {
   );
 
   if (!isObject(existingImportUrl)) {
-    throw new Error(`Import Url with ID:${importUrl.getId()} does not exist`);
+    throw new Error(`Import Url with ID: ${importUrl.getId()} does not exist`);
   }
 
   await dynamoClient.putItem(config.tableNameImportUrls, ImportUrlDto.toDynamoItem(importUrl));
@@ -103,7 +103,7 @@ export const getImportUrlsByJobIdAndStatus = async (dynamoClient, config, log, j
  * @param {Object} config
  * @param {Logger} log
  * @param {string} jobId
- * @returns {Promise<ImportUrl[]>}
+ * @returns {Promise<ImportUrlDto[]>}
  */
 export const getImportUrlsByJobId = async (dynamoClient, config, log, jobId) => {
   const items = await dynamoClient.query({
@@ -114,5 +114,6 @@ export const getImportUrlsByJobId = async (dynamoClient, config, log, jobId) =>
       ':jobId': jobId,
     },
   });
-  return items.map((item) => ImportUrlDto.fromDynamoItem(item));
+
+  return items ? items.map(ImportUrlDto.fromDynamoItem) : null;
 };