Handle the throttle error for okta collector

rakhimundhada15 · Aug 6, 2023 · 3a752a0 · 3a752a0
1 parent b5da3e1
commit 3a752a0
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 10 deletions.
diff --git a/collectors/okta/okta_collector.js b/collectors/okta/okta_collector.js
@@ -19,6 +19,7 @@ const calcNextCollectionInterval = require('@alertlogic/paws-collector').calcNex
 const packageJson = require('./package.json');
 
 const THROTTLING_ERROR_REGEXP = /rateLimit/g;
+const MAX_POLL_INTERVAL = 900;
 
 const typeIdPaths = [
     { path: ['eventType'] },
@@ -76,8 +77,18 @@ class OktaCollector extends PawsCollector {
         .catch((error) => {
             error.errorCode = this._isNoErrorCode(error);
             if (this._isThrottlingError(error)) {
-                collector.reportApiThrottling(function() {
-                    return callback(error);
+                // if x-rate-limit-reset value return by api then accordingly delay the api call to avoid throttle error again other wise increase the delay by 1min till max 15min.
+                let resetSeconds = state.poll_interval_sec;
+                if (error['headers'] && error['headers']['x-rate-limit-reset']) {
+                    const retryEpochSeconds = parseInt((error['headers']['x-rate-limit-reset']), 10);
+                    const currentEpochSeconds = moment().unix();
+                    resetSeconds = retryEpochSeconds - currentEpochSeconds;
+                }
+                const delaySeconds = resetSeconds && resetSeconds < MAX_POLL_INTERVAL ? resetSeconds + 60 : MAX_POLL_INTERVAL;
+                state.poll_interval_sec = delaySeconds;
+                AlLogger.info(`OKTA000003 API limit Exceeded. The quota will be reset at ${moment().add(delaySeconds, 'seconds').toISOString()}`);
+                collector.reportApiThrottling(function () {
+                    return callback(null, [], state, state.poll_interval_sec);
                 });
             } else {
                 return callback(error);

diff --git a/collectors/okta/package.json b/collectors/okta/package.json
@@ -1,6 +1,6 @@
 {
   "name": "okta-collector",
-  "version": "1.2.17",
+  "version": "1.2.18",
   "description": "Alert Logic AWS based Okta Log Collector",
   "repository": {},
   "private": true,

diff --git a/collectors/okta/test/okta_test.js b/collectors/okta/test/okta_test.js
@@ -211,28 +211,65 @@ describe('Unit Tests', function() {
             });
         });
 
-        it('gets logs throttling', function(done) {
-            const {Client} = okta;
+        it('it should return the same state with pollinterval delay if get api return throttle error', function (done) {
+            const { Client } = okta;
+            const error = { "name": "OktaApiError", "status": 429, "errorCode": "E0000047", "errorSummary": "API call exceeded rate limit due to too many requests.", "errorCauses": [], "errorLink": "E0000047", "errorId": "oaeJacBsJ0pQES61B_uegmlzA", "url": "https://alertlogic-admin.okta.com/api/v1/logs?since=2023-06-03T08%3A32%3A20.000Z&until=2023-06-03T08%3A33%3A20.000Z", "headers": {}, "message": "Okta HTTP 429 E0000047 API call exceeded rate limit due to too many requests.. " };
             const oktaSdkMock = sinon.stub(Client.prototype, 'getLogs').callsFake(() => {
                 return {
                     each: () => {
                         return new Promise((res, rej) => {
-                            rej(new Error('HTTP request time exceeded okta.client.rateLimit.requestTimeout'));
+                            rej(error);
                         });
                     }
                 };
             });
-            OktaCollector.load().then(function(creds) {
+            OktaCollector.load().then(function (creds) {
                 var collector = new OktaCollector(ctx, creds);
                 const startDate = moment().subtract(1, 'days').toISOString();
                 const mockState = {
                     since: startDate,
-                    until: moment().toISOString()
+                    until: moment().toISOString(),
+                    poll_interval_sec: 60
                 };
                 var reportSpy = sinon.spy(collector, 'reportApiThrottling');
-
-                collector.pawsGetLogs(mockState, (err) => {
+
+                collector.pawsGetLogs(mockState, (err, logs, state, pollIntervalSec) => {
+                    assert.equal(true, reportSpy.calledOnce);
+                    assert.equal(err, null);
+                    // if header not return rate-limit-resect-sec then add the 60 sec in existing pollinterval seconds
+                    assert.equal(pollIntervalSec, 120);
+                    oktaSdkMock.restore();
+                    done();
+                });
+            });
+        });
+        it('It should set the delay second if there is throttle error and header contain X-Rate-Limit-Reset', function (done) {
+            const { Client } = okta;
+            const resetSecs = moment().add(120, 'seconds').unix();
+            const error = { "name": "OktaApiError", "status": 429, "errorCode": "E0000047", "errorSummary": "API call exceeded rate limit due to too many requests.", "url": "https://alertlogic-admin.okta.com/api/v1/logs?since=2023-06-03T08%3A32%3A20.000Z&until=2023-06-03T08%3A33%3A20.000Z", "headers": { "x-rate-limit-reset": resetSecs }, "message": "Okta HTTP 429 E0000047 API call exceeded rate limit due to too many requests.. " };
+            const oktaSdkMock = sinon.stub(Client.prototype, 'getLogs').callsFake(() => {
+                return {
+                    each: () => {
+                        return new Promise((res, rej) => {
+                            rej(error);
+                        });
+                    }
+                };
+            });
+            OktaCollector.load().then(function (creds) {
+                var collector = new OktaCollector(ctx, creds);
+                const startDate = moment().subtract(1, 'days').toISOString();
+                const mockState = {
+                    since: startDate,
+                    until: moment().toISOString(),
+                    poll_interval_sec: 60
+                };
+                var reportSpy = sinon.spy(collector, 'reportApiThrottling');
+
+                collector.pawsGetLogs(mockState, (err, logs, state, poll_interval_sec) => {
                     assert.equal(true, reportSpy.calledOnce);
+                    assert.equal(err, null);
+                    assert.equal(poll_interval_sec, 180);
                     oktaSdkMock.restore();
                     done();
                 });