From 93b750b5cf67d972aeb6198640055d304f7db96b Mon Sep 17 00:00:00 2001 From: Cariel Cohen Date: Wed, 6 Nov 2024 21:01:10 -0500 Subject: [PATCH 1/2] Bug FIx, Validate Services Exist before running the scan for it --- collectors/aws/collector.js | 30 ++++++++++++++++++++------- collectors/aws/collector_multipart.js | 16 ++++++++++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/collectors/aws/collector.js b/collectors/aws/collector.js index 6f7fa7b99a..f4db9a3d98 100644 --- a/collectors/aws/collector.js +++ b/collectors/aws/collector.js @@ -10,8 +10,8 @@ - api_calls: (Optional) If provided, will only query these APIs. - Example: { - "skip_regions": ["us-east-2", "eu-west-1"], - "api_calls": ["EC2:describeInstances", "S3:listBuckets"] + "skip_regions": ["us-east-2", "eu-west-1"], + "api_calls": ["EC2:describeInstances", "S3:listBuckets"] } - callback: Function to call when the collection is complete *********************/ @@ -84,6 +84,9 @@ var collect = function(AWSConfig, settings, callback) { var serviceLower = service.toLowerCase(); if (!collection[serviceLower]) collection[serviceLower] = {}; + // Log service being processed + console.log(`[INFO] Processing service: ${serviceName}`); + // Loop through each of the service's functions async.eachOfLimit(call, 15, function(callObj, callKey, callCb) { if (settings.api_calls && settings.api_calls.indexOf(serviceName + ':' + callKey) === -1) return callCb(); @@ -136,6 +139,11 @@ var collect = function(AWSConfig, settings, callback) { } }); } else { + if (!AWS[serviceName]) { + console.error(`[ERROR] Service ${serviceName} does not exist in AWS SDK.`); + regionCb(); + return; + } var executor = debugMode ? (AWSXRay.captureAWSClient(new AWS[serviceName](LocalAWSConfig))) : new AWS[serviceName](LocalAWSConfig); var paginating = false; var executorCb = function(err, data) { @@ -244,6 +252,9 @@ var collect = function(AWSConfig, settings, callback) { if (!collection[serviceLower]) collection[serviceLower] = {}; + // Log service being processed + console.log(`[INFO] Processing postcall for service: ${serviceName}`); + async.eachOfLimit(serviceObj, 1, function(callObj, callKey, callCb) { if (settings.api_calls && settings.api_calls.indexOf(serviceName + ':' + callKey) === -1) return callCb(); @@ -278,10 +289,10 @@ var collect = function(AWSConfig, settings, callback) { if (callObj.reliesOnCall && (!collection[callObj.reliesOnService] || - !collection[callObj.reliesOnService][callObj.reliesOnCall] || - !collection[callObj.reliesOnService][callObj.reliesOnCall][region] || - !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data || - !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data.length)) + !collection[callObj.reliesOnService][callObj.reliesOnCall] || + !collection[callObj.reliesOnService][callObj.reliesOnCall][region] || + !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data || + !collection[callObj.reliesOnService][callObj.reliesOnCall][region].data.length)) return regionCb(); var LocalAWSConfig = JSON.parse(JSON.stringify(AWSConfig)); @@ -305,6 +316,11 @@ var collect = function(AWSConfig, settings, callback) { } }); } else { + if (!AWS[serviceName]) { + console.error(`[ERROR] Service ${serviceName} does not exist in AWS SDK.`); + regionCb(); + return; + } var executor = debugMode ? (AWSXRay.captureAWSClient(new AWS[serviceName](LocalAWSConfig))) : new AWS[serviceName](LocalAWSConfig); if (!collection[callObj.reliesOnService][callObj.reliesOnCall][LocalAWSConfig.region] || @@ -424,4 +440,4 @@ var collect = function(AWSConfig, settings, callback) { }); }; -module.exports = collect; \ No newline at end of file +module.exports = collect; diff --git a/collectors/aws/collector_multipart.js b/collectors/aws/collector_multipart.js index e9960d4aee..a19c0ad3cd 100644 --- a/collectors/aws/collector_multipart.js +++ b/collectors/aws/collector_multipart.js @@ -113,6 +113,9 @@ var collect = function(AWSConfig, settings, callback) { var serviceLower = service.toLowerCase(); if (!collection[serviceLower]) collection[serviceLower] = {}; + // Log service being processed + console.log(`[INFO] Processing service: ${serviceName}`); + // Loop through each of the service's functions async.eachOfLimit(call, 15, function(callObj, callKey, callCb) { if (settings.api_calls && settings.api_calls.indexOf(serviceName + ':' + callKey) === -1) return callCb(); @@ -165,6 +168,11 @@ var collect = function(AWSConfig, settings, callback) { } }); } else { + if (!AWS[serviceName]) { + console.error(`[ERROR] Service ${serviceName} does not exist in AWS SDK.`); + regionCb(); + return; + } var executor = debugMode ? (AWSXRay.captureAWSClient(new AWS[serviceName](LocalAWSConfig))) : new AWS[serviceName](LocalAWSConfig); var paginating = false; var executorCb = function(err, data) { @@ -284,6 +292,9 @@ var collect = function(AWSConfig, settings, callback) { if (!collection[serviceLower]) collection[serviceLower] = {}; + // Log service being processed + console.log(`[INFO] Processing service: ${serviceName}`); + async.eachOfLimit(serviceObj, 1, function(callObj, callKey, callCb) { if (settings.api_calls && settings.api_calls.indexOf(serviceName + ':' + callKey) === -1) return callCb(); @@ -345,6 +356,11 @@ var collect = function(AWSConfig, settings, callback) { } }); } else { + if (!AWS[serviceName]) { + console.error(`[ERROR] Service ${serviceName} does not exist in AWS SDK.`); + regionCb(); + return; + } var executor = debugMode ? (AWSXRay.captureAWSClient(new AWS[serviceName](LocalAWSConfig))) : new AWS[serviceName](LocalAWSConfig); if (!collection[callObj.reliesOnService][callObj.reliesOnCall][LocalAWSConfig.region] || From 2c5b3eff925555deaf77468faf5faaddaf3f2b66 Mon Sep 17 00:00:00 2001 From: Cariel Cohen Date: Wed, 6 Nov 2024 21:07:02 -0500 Subject: [PATCH 2/2] SDK update --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 741c0ba80f..9cba731b71 100644 --- a/package.json +++ b/package.json @@ -41,16 +41,16 @@ "dependencies": { "@alicloud/pop-core": "^1.7.10", "@azure/data-tables": "^13.2.2", + "@azure/storage-blob": "^12.14.0", "@azure/storage-file-share": "^12.14.0", "@azure/storage-queue": "^12.13.0", - "@azure/storage-blob": "^12.14.0", "@octokit/auth-app": "^6.0.3", "@octokit/request": "^8.1.6", "@octokit/rest": "^20.0.2", "ali-oss": "^6.15.2", "argparse": "^2.0.0", "async": "^2.6.1", - "aws-sdk": "^2.1506.0", + "aws-sdk": "^2.1692.0", "azure-storage": "^2.10.3", "csv-write-stream": "^2.0.0", "fast-safe-stringify": "^2.0.6",