fix: SITES-23133 [Importer] Report on the progress of a import job vi…

…a a new jobs endpoint

packages/spacecat-shared-data-access/docs/schema.json

@@ -975,6 +975,10 @@
           "AttributeName": "successCount",
           "AttributeType": "N"
         },
+        {
+          "AttributeName": "redirectCount",
+          "AttributeType": "N"
+        },
         {
           "AttributeName": "urlCount",
           "AttributeType": "N"

packages/spacecat-shared-data-access/src/index.d.ts

@@ -530,17 +530,6 @@ export interface ImportJob {
    * Retrieves the initiatedBy metadata (name, imsOrgId, imsUserId, userAgent) of the import job.
    */
   getInitiatedBy: () => object;
-
-  /**
-   * Retrieves the progress of the import job.
-   */
-  getProgress: () => {
-    completed: number;
-    failed: number;
-    pending: number;
-    running: number;
-    redirect: number;
-  };
 }
 
 export interface ImportUrl {

packages/spacecat-shared-data-access/src/models/importer/import-constants.js

@@ -16,12 +16,6 @@
 export const ImportOptions = {
   ENABLE_JAVASCRIPT: 'enableJavascript',
   PAGE_LOAD_TIMEOUT: 'pageLoadTimeout',
-
-  // the following 4 options need to be removed
-  SAVE_AS_DOCX: 'saveAsDocx',
-  HAS_CUSTOM_HEADERS: 'hasCustomHeaders',
-  HAS_CUSTOM_IMPORT_JS: 'hasCustomImportJs',
-  SCROLL_TO_BOTTOM: 'scrollToBottom',
 };
 
 /**

packages/spacecat-shared-data-access/src/models/importer/import-job.js

@@ -16,19 +16,6 @@ import {
 import { Base } from '../base.js';
 import { ImportJobStatus, ImportOptions } from './import-constants.js';
 
-export const jobToApiMap = [
-  ['id', 'id'],
-  ['baseURL', 'baseURL'],
-  ['options', 'options'],
-  ['startTime', 'startTime'],
-  ['endTime', 'endTime'],
-  ['duration', 'duration'],
-  ['status', 'status'],
-  ['urlCount', 'urlCount'],
-  ['initiatedBy', 'initiatedBy'],
-  ['progress', 'progress'],
-];
-
 /**
  * Creates a new ImportJob object.
  *
@@ -50,15 +37,6 @@ const ImportJob = (data) => {
   self.getImportQueueId = () => self.state.importQueueId;
   self.getInitiatedBy = () => self.state.initiatedBy;
 
-  // the progress is a derived property from querying the status of the import urls table
-  self.getProgress = () => self.state.progress || {
-    running: 0,
-    failed: 0,
-    completed: 0,
-    pending: 0,
-    redirect: 0,
-  };
-
   /**
    * Updates the state of the ImportJob.
    * @param key - The key to update.

packages/spacecat-shared-data-access/src/service/import-job/accessPatterns.js

@@ -10,11 +10,9 @@
  * governing permissions and limitations under the License.
  */
 
-import { isArray, isObject } from '@adobe/spacecat-shared-utils';
+import { isObject } from '@adobe/spacecat-shared-utils';
 import { ImportJobDto } from '../../dto/import-job.js';
 import { createImportJob } from '../../models/importer/import-job.js';
-import { getImportUrlsByJobId } from '../import-url/accessPatterns.js';
-import { ImportUrlStatus } from '../../models/importer/import-constants.js';
 
 /**
  * Get all Import Jobs within a specific date range
@@ -50,50 +48,12 @@ export const getImportJobsByDateRange = async (dynamoClient, config, log, startD
  * @returns {Promise<ImportJobDto> | null}
  */
 export const getImportJobByID = async (dynamoClient, config, log, id) => {
-  const jobEntity = await dynamoClient.getItem(
+  const item = await dynamoClient.getItem(
     config.tableNameImportJobs,
     { id },
   );
 
-  if (!jobEntity) {
-    return null;
-  }
-
-  const jobModel = ImportJobDto.fromDynamoItem(jobEntity);
-
-  const importUrls = await getImportUrlsByJobId(dynamoClient, config, log, id);
-
-  if (isArray(importUrls)) {
-    jobModel.state.progress = importUrls.reduce((acc, importUrl) => {
-      // eslint-disable-next-line default-case
-      switch (importUrl.state.status) {
-        case ImportUrlStatus.PENDING:
-          acc.pending += 1;
-          break;
-        case ImportUrlStatus.REDIRECT:
-          acc.redirect += 1;
-          break;
-        case ImportUrlStatus.RUNNING:
-          acc.running += 1;
-          break;
-        case ImportUrlStatus.COMPLETE:
-          acc.completed += 1;
-          break;
-        case ImportUrlStatus.FAILED:
-          acc.failed += 1;
-          break;
-      }
-      return acc;
-    }, {
-      pending: 0,
-      redirect: 0,
-      running: 0,
-      completed: 0,
-      failed: 0,
-    });
-  }
-
-  return jobModel;
+  return item ? ImportJobDto.fromDynamoItem(item) : null;
 };
 
 /**

packages/spacecat-shared-data-access/src/service/import-url/accessPatterns.js

@@ -98,7 +98,7 @@ export const getImportUrlsByJobIdAndStatus = async (dynamoClient, config, log, j
 };
 
 /**
- * Get Import Urls by Job ID
+ * Get Import Urls by Job ID, if no urls exist an empty array is returned.
  * @param {DynamoClient} dynamoClient
  * @param {Object} config
  * @param {Logger} log
@@ -115,5 +115,5 @@ export const getImportUrlsByJobId = async (dynamoClient, config, log, jobId) =>
     },
   });
 
-  return items ? items.map(ImportUrlDto.fromDynamoItem) : null;
+  return items ? items.map((item) => ImportUrlDto.fromDynamoItem(item)) : [];
 };

packages/spacecat-shared-data-access/test/it/db.test.js

@@ -1055,13 +1055,6 @@ describe('DynamoDB Integration Test', async () => {
         expect(updatedJob.getDuration()).to.be.equal(1234);
         expect(updatedJob.getUrlCount()).to.be.equal(100);
         expect(updatedJob.getImportQueueId()).to.be.equal('Q-456');
-        expect(updatedJob.getProgress()).to.deep.equal({
-          running: 0,
-          failed: 0,
-          completed: 0,
-          pending: 0,
-          redirect: 0,
-        });
         expect(updatedJob.getOptions()).to.deep.equal({
           [ImportOptions.ENABLE_JAVASCRIPT]: true,
         });
@@ -1079,13 +1072,6 @@ describe('DynamoDB Integration Test', async () => {
         const job = await createNewImportJob();
         const jobEntry = await dataAccess.getImportJobByID(job.getId());
         expect(job.getId()).to.be.equal(jobEntry.getId());
-        expect(job.getProgress()).to.deep.equal({
-          running: 0,
-          failed: 0,
-          completed: 0,
-          pending: 0,
-          redirect: 0,
-        });
       });
 
       it('Verify getImportJobsByDateRange', async () => {

packages/spacecat-shared-data-access/test/unit/models/importer/import-job.test.js

@@ -183,15 +183,5 @@ describe('ImportJob Model tests', () => {
     it('retrieves the startTime of the import job', () => {
       expect(importJob.getStartTime()).to.equal('2024-05-29T14:26:00.000Z');
     });
-
-    it('retrieves the progress of the job', () => {
-      expect(importJob.getProgress()).to.deep.equal({
-        running: 0,
-        failed: 0,
-        completed: 0,
-        pending: 0,
-        redirect: 0,
-      });
-    });
   });
 });

packages/spacecat-shared-data-access/test/unit/service/import-job/index.test.js

@@ -85,11 +85,6 @@ describe('Import Job Tests', () => {
         const result = await exportedFunctions.getImportJobByID('test-id');
 
         expect(result.state.id).to.equal('test-id');
-        expect(result.state.progress.pending).to.equal(1);
-        expect(result.state.progress.redirect).to.equal(1);
-        expect(result.state.progress.running).to.equal(1);
-        expect(result.state.progress.completed).to.equal(1);
-        expect(result.state.progress.failed).to.equal(1);
       });
 
       it('should return null if item is not found', async () => {