handle the pagination state properly

rakhimundhada15 · Dec 21, 2023 · 6c852b7 · 6c852b7
1 parent 95467fe
commit 6c852b7
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 19 deletions.
diff --git a/collectors/googlestackdriver/collector.js b/collectors/googlestackdriver/collector.js
@@ -161,25 +161,34 @@ timestamp < "${state.until}"`;
     }
 
     _getNextCollectionState(curState, nextPage) {
-        // Reset the page size for next collection as log collection completed for throttling interval
-        if (nextPage && nextPage.pageSize && nextPage.pageSize < MAX_PAGE_SIZE) {
-            nextPage.pageSize = MAX_PAGE_SIZE;
-        } else if (curState.pageSize && curState.pageSize < MAX_PAGE_SIZE) {
-            curState.pageSize = MAX_PAGE_SIZE;
+        // Reset the page size for the next collection if it's less than the maximum
+        const pageSize = Math.max(MAX_PAGE_SIZE, nextPage?.pageSize || curState.pageSize || MAX_PAGE_SIZE);
+
+        const { stream, since, until } = curState;
+
+        if (nextPage) {
+            // Case: Continue with the next page
+            return {
+                since,
+                nextPage: { ...nextPage, pageSize },
+                stream,
+                until,
+                poll_interval_sec: 1
+            };
+        } else {
+            // Case: Start a new collection
+            const untilMoment = moment(curState.until);
+
+            const { nextUntilMoment, nextSinceMoment, nextPollInterval } = calcNextCollectionInterval('day-week-progression', untilMoment, this.pollInterval);
+
+            return {
+                since: nextSinceMoment.toISOString(),
+                nextPage,
+                stream,
+                until: nextUntilMoment.toISOString(),
+                poll_interval_sec: nextPollInterval
+            };
         }
-        const {stream} = curState;
-
-        const untilMoment = moment(curState.until);
-
-        const { nextUntilMoment, nextSinceMoment, nextPollInterval } = calcNextCollectionInterval('day-week-progression', untilMoment, this.pollInterval);
-
-        return  {
-            since: nextSinceMoment.toISOString(),
-            nextPage,
-            stream,
-            until: nextUntilMoment.toISOString(),
-            poll_interval_sec: nextPollInterval
-        };
     }
 
     // TODO: probably need to actually decode hte protobuf payload on these logs

diff --git a/collectors/googlestackdriver/package.json b/collectors/googlestackdriver/package.json
@@ -1,6 +1,6 @@
 {
   "name": "googlestackdriver-collector",
-  "version": "1.2.6",
+  "version": "1.2.7",
   "description": "Alert Logic AWS based Googlestackdriver Log Collector",
   "repository": {},
   "private": true,