fix(aws-lambda): reduced lambda runtimes for large number of spans

refs https://jsw.ibm.com/browse/INSTA-13498

Author: kirrg001

packages/aws-lambda/src/identity_provider.js

@@ -11,6 +11,10 @@ exports.init = function init(arnInfo) {
   qualifiedArn = arnInfo.arn;
 };
 
+exports.getName = function getName() {
+  return 'aws-lambda';
+};
+
 exports.getHostHeader = function getHostHeader() {
   return qualifiedArn;
 };

packages/aws-lambda/test/multiple_data/test.js

@@ -94,7 +94,7 @@ describe('multiple data lambda handler', function () {
     });
   });
 
-  describe('[batching disabled] with 100 iterations', function () {
+  describe('with 100 iterations and high transmission config values', function () {
     this.timeout(config.getTestTimeout() * 2);
     let control;
 
@@ -106,7 +106,9 @@ describe('multiple data lambda handler', function () {
         env: {
           INSTANA_AGENT_KEY: instanaAgentKey,
           WITH_CONFIG: 'true',
-          INSTANA_NUMBER_OF_ITERATIONS: 100
+          INSTANA_NUMBER_OF_ITERATIONS: 100,
+          INSTANA_FORCE_TRANSMISSION_STARTING_AT: 500,
+          INSTANA_TRACING_INITIAL_TRANSMISSION_DELAY: 1000
         }
       });
 
@@ -160,7 +162,9 @@ describe('multiple data lambda handler', function () {
     });
   });
 
-  describe('[batching enabled] with 100 iterations', function () {
+  // TODO: This test does not use any batchable spans.
+  //       We could also extend the test to enable batching.
+  describe('with 100 iterations and default behavior', function () {
     this.timeout(config.getTestTimeout() * 2);
     let control;
 
@@ -172,10 +176,7 @@ describe('multiple data lambda handler', function () {
         env: {
           INSTANA_AGENT_KEY: instanaAgentKey,
           WITH_CONFIG: 'true',
-          INSTANA_NUMBER_OF_ITERATIONS: 100,
-          INSTANA_SPANBATCHING_ENABLED: 'true',
-          INSTANA_FORCE_TRANSMISSION_STARTING_AT: 10,
-          INSTANA_DEV_MIN_DELAY_BEFORE_SENDING_SPANS: 100
+          INSTANA_NUMBER_OF_ITERATIONS: 100
         }
       });
 

packages/collector/src/pidStore/index.js

@@ -36,6 +36,10 @@ Object.defineProperty(exports, 'pid', {
   }
 });
 
+exports.getName = function getName() {
+  return 'default-identity';
+};
+
 exports.getEntityId = function getEntityId() {
   return internalPidStore.pid;
 };

packages/core/src/index.js

@@ -68,7 +68,7 @@ function preInit() {
 function init(config, downstreamConnection, processIdentityProvider) {
   log.init(/** @type {log.LoggerConfig} */ (config));
   util.hasThePackageBeenInitializedTooLate();
-  config = normalizeConfig(config);
+  config = normalizeConfig(config, processIdentityProvider);
   secrets.init(/** @type {secrets.SecretOption} */ (config));
   util.requireHook.init(config);
   tracing.init(config, downstreamConnection, processIdentityProvider);

packages/core/src/tracing/spanBuffer.js

@@ -22,16 +22,8 @@ let isActive = false;
 /** @type {number} */
 let activatedAt = null;
 
-let minDelayBeforeSendingSpans = 1000;
-if (process.env.INSTANA_DEV_MIN_DELAY_BEFORE_SENDING_SPANS != null) {
-  minDelayBeforeSendingSpans = parseInt(process.env.INSTANA_DEV_MIN_DELAY_BEFORE_SENDING_SPANS, 10);
-  if (isNaN(minDelayBeforeSendingSpans)) {
-    minDelayBeforeSendingSpans = 1000;
-  }
-}
-
 /** @type {number} */
-let initialDelayBeforeSendingSpans;
+let initialTransmissionDelay;
 /** @type {number} */
 let transmissionDelay;
 /** @type {number} */
@@ -98,7 +90,19 @@ exports.init = function init(config, _downstreamConnection) {
   forceTransmissionStartingAt = config.tracing.forceTransmissionStartingAt;
   transmissionDelay = config.tracing.transmissionDelay;
   batchingEnabled = config.tracing.spanBatchingEnabled;
-  initialDelayBeforeSendingSpans = Math.max(transmissionDelay, minDelayBeforeSendingSpans);
+  initialTransmissionDelay = config.tracing.initialTransmissionDelay;
+
+  // TODO: drop undocumented env variable in v4 major release
+  if (process.env.INSTANA_DEV_MIN_DELAY_BEFORE_SENDING_SPANS != null) {
+    initialTransmissionDelay = parseInt(process.env.INSTANA_DEV_MIN_DELAY_BEFORE_SENDING_SPANS, 10);
+    if (isNaN(initialTransmissionDelay)) {
+      initialTransmissionDelay = config.tracing.initialTransmissionDelay;
+    }
+  }
+
+  // TODO: remove in v4 and make initialTransmissionDelay configurable instead
+  initialTransmissionDelay = Math.max(transmissionDelay, initialTransmissionDelay);
+
   isFaaS = false;
   transmitImmediate = false;
 
@@ -146,7 +150,7 @@ exports.activate = function activate(extraConfig) {
   //       On AWS Lambda we wait till the handler finishes and then transmit all collected spans via
   //       `sendBundle`. Any detected span will be sent directly to the BE.
   if (!isFaaS) {
-    transmissionTimeoutHandle = setTimeout(transmitSpans, initialDelayBeforeSendingSpans);
+    transmissionTimeoutHandle = setTimeout(transmitSpans, initialTransmissionDelay);
     transmissionTimeoutHandle.unref();
   }
 
@@ -201,8 +205,8 @@ exports.addSpan = function (span) {
       addToBucket(span);
     }
 
-    // NOTE: we send out spans directly if the number of spans reaches > 500 [default] and if the min delay is reached.
-    if (spans.length >= forceTransmissionStartingAt && Date.now() - minDelayBeforeSendingSpans > activatedAt) {
+    // NOTE: we send out spans directly if the number of spans reaches > X [default] and if the min delay is reached.
+    if (spans.length >= forceTransmissionStartingAt && Date.now() - initialTransmissionDelay > activatedAt) {
       transmitSpans();
     }
   }
@@ -402,6 +406,11 @@ function batchingBucketKey(span) {
 }
 
 /**
+ * IMPORTANT: Only some instrumentations are enabled to be batchable.
+ *            e.g. mysql, pg, redis, elasticsearch, etc.
+ *            Batching spans means to collect multiple spans of the same type
+ *            and merge them into one span.
+ *
  * @param {import('./cls').InstanaBaseSpan} span
  * @returns {boolean}
  */

packages/core/src/util/normalizeConfig.js

@@ -8,7 +8,7 @@
 'use strict';
 
 const supportedTracingVersion = require('../tracing/supportedVersion');
-
+const deepMerge = require('../util/deepMerge');
 const constants = require('../tracing/constants');
 
 /**
@@ -18,6 +18,7 @@ const constants = require('../tracing/constants');
  * @property {boolean} [automaticTracingEnabled]
  * @property {boolean} [activateImmediately]
  * @property {number} [forceTransmissionStartingAt]
+ * @property {number} [initialTransmissionDelay]
  * @property {number} [maxBufferedSpans]
  * @property {number} [transmissionDelay]
  * @property {number} [stackTraceLength]
@@ -91,7 +92,7 @@ logger = require('../logger').getLogger('configuration', newLogger => {
 });
 
 /** @type {InstanaConfig} */
-const defaults = {
+const defaultsBase = {
   serviceName: null,
   packageJsonPath: null,
 
@@ -108,6 +109,7 @@ const defaults = {
     forceTransmissionStartingAt: 500,
     maxBufferedSpans: 1000,
     transmissionDelay: 1000,
+    initialTransmissionDelay: 1000,
     http: {
       extraHttpHeadersToCapture: []
     },
@@ -126,8 +128,17 @@ const defaults = {
   }
 };
 
-const validKafkaHeaderFormats = ['binary', 'string', 'both'];
+/** @type {Record<string, InstanaConfig>} */
+const identityProviderDefaults = {
+  'aws-lambda': {
+    tracing: { forceTransmissionStartingAt: 10, transmissionDelay: 100, initialTransmissionDelay: 100 }
+  }
+};
+
+/** @type InstanaConfig */
+let defaults = {};
 
+const validKafkaHeaderFormats = ['binary', 'string', 'both'];
 const validSecretsMatcherModes = ['equals-ignore-case', 'equals', 'contains-ignore-case', 'contains', 'regex', 'none'];
 
 /**
@@ -136,13 +147,20 @@ const validSecretsMatcherModes = ['equals-ignore-case', 'equals', 'contains-igno
 
 /**
  * @param {InstanaConfig} [config]
+ * @param {import('../../../collector/src/pidStore')} [identityProvider]
  * @returns {InstanaConfig}
  */
-module.exports = function normalizeConfig(config) {
+module.exports = function normalizeConfig(config, identityProvider) {
   if (config == null) {
     config = {};
   }
 
+  if (identityProvider && identityProvider.getName && identityProviderDefaults[identityProvider.getName()]) {
+    defaults = deepMerge(defaultsBase, identityProviderDefaults[identityProvider.getName()]);
+  } else {
+    defaults = defaultsBase;
+  }
+
   normalizeServiceName(config);
   normalizePackageJsonPath(config);
   normalizeMetricsConfig(config);
@@ -338,6 +356,13 @@ function normalizeTracingTransmission(config) {
     'config.tracing.forceTransmissionStartingAt',
     'INSTANA_FORCE_TRANSMISSION_STARTING_AT'
   );
+
+  config.tracing.initialTransmissionDelay = normalizeSingleValue(
+    config.tracing.initialTransmissionDelay,
+    defaults.tracing.initialTransmissionDelay,
+    'config.tracing.initialTransmissionDelay',
+    'INSTANA_TRACING_INITIAL_TRANSMISSION_DELAY'
+  );
 }
 
 /**