#25873 fix cr issues

flotiq · Jan 24, 2025 · 8109e1d · 8109e1d
1 parent ecdccda
commit 8109e1d
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 55 deletions.
diff --git a/commands/importer.js b/commands/importer.js
@@ -429,7 +429,7 @@ async function handler(argv) {
 
     const flotiqApi = new FlotiqApi(`${config.apiUrl}/api/v1`,  argv.flotiqApiKey, {
         batchSize: 100,
-        internalWpsLimit: 10
+        writePerSecondLimit: 10
     });
 
     let [featuredImages, CTDs] = await importer(
@@ -451,7 +451,7 @@ async function handler(argv) {
         directory,
         flotiqApi,
         mediaApi,
-        internalWpsLimit = 10
+        writePerSecondLimit = 10
     );
 
     await featuredImagesImport(

diff --git a/src/configuration/config.js b/src/configuration/config.js
@@ -1,7 +1,10 @@
 require('dotenv').config();
 
 let settings = {
-    apiUrl: process.env.FLOTIQ_API_URL || 'https://api.flotiq.com',
+    // apiUrl: process.env.FLOTIQ_API_URL || 'https://api.flotiq.com',
+    apiUrl: process.env.FLOTIQ_API_URL || 'https://api-staging.flotiq.com',
+    // apiUrl: process.env.FLOTIQ_API_URL || 'https://new-cms-staging.api.dev.cdwv.pl',
+    // apiUrl: process.env.FLOTIQ_API_URL || 'http://localhost:8069',
     gatsbyCli:'./node_modules/.bin/gatsby'
 }
 

diff --git a/src/flotiq-api.js b/src/flotiq-api.js
@@ -20,7 +20,7 @@ module.exports = class FlotiqApi {
     this.flotiqApiKey = flotiqApiKey;
     this.batchSizeRead = options.batchSizeRead || options.batchSize || 1000;
     this.batchSize = options.batchSize || 100;
-    this.internalWpsLimit = options.internalWpsLimit || 10;
+    this.interval = 1000 / (options.writePerSecondLimit || 10);
 
     this.headers = {
       "Content-type": "application/json;charset=utf-8",
@@ -118,53 +118,31 @@ module.exports = class FlotiqApi {
   async persistContentObjectBatch(ctd, obj) {
     assert(typeof ctd, 'string');
     assert(Array.isArray(obj));
-    const interval = 1000 / this.internalWpsLimit;
 
     const bar = new ProgressBar(`Persisting ${ctd} [:bar] :percent ETA :etas`, { total: obj.length });
     const uri = `/content/${ctd}/batch?updateExisting=true`;
 
-    for (let i = 0; i < obj.length; i += this.batchSize) {
-      const batch = obj.slice(i, i + this.batchSize);
-      await this._sendRequest(uri, batch, 'POST');
-
-      bar.tick(this.batchSize);
-
-      await new Promise(resolve => setTimeout(resolve, interval));
-    }
+    await this._sendRequest(uri, obj, 'POST', bar);
   }
 
 
   async patchContentObjectBatch(ctd, obj) {
     assert(typeof ctd === 'string');
     assert(Array.isArray(obj));
-    const interval = 1000 / this.internalWpsLimit;
 
     const bar = new ProgressBar(`Updating ${ctd} [:bar] :percent ETA :etas`, { total: obj.length });
     const uri = `/content/${ctd}/batch`;
 
-    for (let i = 0; i < obj.length; i += this.batchSize) {
-      const batch = obj.slice(i, i + this.batchSize);
-      await this._sendRequest(uri, batch, 'PATCH');
-
-      bar.tick(this.batchSize);
-
-      await new Promise(resolve => setTimeout(resolve, interval));
-    }
+    await this._sendRequest(uri, obj, 'PATCH', bar);
   }
 
   async deleteContentObjectBatch(ctd, obj) {
     assert(typeof ctd === 'string');
     assert(Array.isArray(obj));
-    const interval = 1000 / this.internalWpsLimit;
 
     const uri = `/content/${ctd}/batch-delete`;
-
-    for (let i = 0; i < obj.length; i += this.batchSize) {
-      const batch = obj.slice(i, i + this.batchSize).map(item => item.id);
-      await this._sendRequest(uri, batch, 'DELETE');
 
-      await new Promise(resolve => setTimeout(resolve, interval));
-    }
+    await this._sendRequest(uri, obj, 'DELETE');
   }
 
 
@@ -292,25 +270,32 @@ module.exports = class FlotiqApi {
     })
   }
 
-  async _sendRequest(uri, batch, method) {
-    try {
-      switch (method) {
-        case 'POST':
-          return await this.middleware.post(uri, batch);
-        case 'PATCH':
-          return await this.middleware.patch(uri, batch);
-        case 'DELETE':
-          return await this.middleware.post(uri, batch);
+  async _sendRequest(uri, obj, method, bar) {
+    for (let i = 0; i < obj.length; i += this.batchSize) {
+      const batch = obj.slice(i, i + this.batchSize);
+      const actions = {
+        POST: async () => await this.middleware.post(uri, batch),
+        PATCH: async () => await this.middleware.patch(uri, batch),
+        DELETE: async () => await this.middleware.post(uri, batch),
+      };
+
+      try {
+        await actions[method]();
+      } catch (e) {
+        if (e.response && e.response.status === 429) {
+          logger.info(this.tooManyRequestsMessage);
+          await new Promise(resolve => setTimeout(resolve, 1000)); // Retry after 1 second
+          return this._sendRequest(uri, batch, method); // Retry request
+        } else {
+          console.dir(e.response.data.errors, { depth: undefined });
+          throw new Error(e.message);
+        }
       }
-    } catch (e) {
-      if (e.response && e.response.status === 429) {
-        logger.info(this.tooManyRequestsMessage);
-        await new Promise(resolve => setTimeout(resolve, 1000)); // Retry after 1 second
-        return this._sendRequest(uri, batch, method); // Retry request
-      } else {
-        console.dir(e.response.data.errors, { depth: undefined });
-        throw new Error(e.message);
+      if (bar) {
+        bar.tick(this.batchSize);
       }
+
+      await new Promise(resolve => setTimeout(resolve, this.interval));
     }
   }
 };
diff --git a/src/media.js b/src/media.js
@@ -4,7 +4,7 @@ const logger = require('./logger')
 const { Blob } = require('buffer');
 const {readCTDs, shouldUpdate } = require("./util");
 
-async function mediaImporter (directory, flotiqApi, mediaApi, internalWpsLimit = 10) {
+async function mediaImporter (directory, flotiqApi, mediaApi, writePerSecondLimit = 10) {
     const checkIfMediaUsed = true;
 
     const flotiqDefinitions = await flotiqApi.fetchContentTypeDefs();
@@ -64,7 +64,7 @@ async function mediaImporter (directory, flotiqApi, mediaApi, internalWpsLimit =
         form.append('type', file.type);
         form.append('file', blob, file.fileName);
 
-        const mediaEntity = await postMedia('', form, mediaApi, internalWpsLimit);
+        const mediaEntity = await postMedia('', form, mediaApi, writePerSecondLimit);
 
         replacements.push([file, mediaEntity]);
     }
@@ -95,8 +95,8 @@ async function mediaImporter (directory, flotiqApi, mediaApi, internalWpsLimit =
     return replacements;
 }
 
-const postMedia = async (url, form, mediaApi, internalWpsLimit) => {
-    const interval = 1000 / internalWpsLimit;
+const postMedia = async (url, form, mediaApi, writePerSecondLimit) => {
+    const interval = 1000 / writePerSecondLimit;
 
     try {
         await new Promise(resolve => setTimeout(resolve, interval));
@@ -110,7 +110,7 @@ const postMedia = async (url, form, mediaApi, internalWpsLimit) => {
             logger.info(`Received 429 on media upload, retrying after 1 second...`);
             // Wait for 1 second before retrying
             await new Promise(resolve => setTimeout(resolve, 1000));
-            return postMedia(url, form, mediaApi, internalWpsLimit);
+            return postMedia(url, form, mediaApi, writePerSecondLimit);
         } else {
             throw new Error(e.message);
         }

diff --git a/tests/flotiq-api.test.js b/tests/flotiq-api.test.js
@@ -19,7 +19,7 @@ describe('FlotiqApi', () => {
 
         const flotiqApi = new FlotiqApi(`${mockApiUrl}/api/v1`, mockApiKey, {
             batchSize: 100,
-            internalWpsLimit: 5,
+            writePerSecondLimit: 5,
         });
 
         const obj = new Array(100).fill({});
@@ -42,7 +42,7 @@ describe('FlotiqApi', () => {
 
         const flotiqApi = new FlotiqApi(`${mockApiUrl}/api/v1`, mockApiKey, {
             batchSize: 100,
-            internalWpsLimit: 5,
+            writePerSecondLimit: 5,
         });
 
         const obj = new Array(100).fill({});

diff --git a/tests/media.test.js b/tests/media.test.js
@@ -83,7 +83,7 @@ describe('mediaImporter', () => {
         expect(mockMediaApi.post).toHaveBeenCalledTimes(2);
     });
 
-    it('should respect internalWpsLimit and throttle uploads', async () => {
+    it('should respect writePerSecondLimit and throttle uploads', async () => {
         const flotiqApi = new FlotiqApi(`${mockApiUrl}/api/v1`,  mockApiKey, {
             batchSize: 100,
         });
@@ -97,7 +97,7 @@ describe('mediaImporter', () => {
         };
 
         const start = Date.now();
-        await mediaImporter(mockDirectory, flotiqApi, mockMediaApi, 1); // internalWpsLimit = 1
+        await mediaImporter(mockDirectory, flotiqApi, mockMediaApi, 1); // writePerSecondLimit = 1
 
         const end = Date.now();
         const elapsed = end - start;