test(instrumentation): acceptance testing instrumentation (CLI-347) (s…

…nyk#5295)

* test(instrumentation): adds accpetance testing for instrumentation

* chore: linting

.gitleaksignore

@@ -83,3 +83,4 @@ c2de35484dcad696a6ee32f2fa317d5cfaffc133:test/fixtures/code/sample-analyze-folde
 4c12242de73be79ebd768468e065790f0b9d23a7:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98
 25f37b4c609380452b0b96c3853b69e4dc29bb48:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98
 ccd03cce97470452766ab397f2ba770dbb2e002e:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98
+test/jest/acceptance/instrumentation.spec.ts:snyk-api-token:19

test/acceptance/fake-server.ts

@@ -604,6 +604,13 @@ export const fakeServer = (basePath: string, snykToken: string): FakeServer => {
     res.status(200).send({});
   });
 
+  app.post(
+    basePath.replace('v1', 'hidden') + `/orgs/:orgId/analytics`,
+    (req, res) => {
+      res.status(201).send({});
+    },
+  );
+
   app.post(`/rest/orgs/:orgId/sbom_tests`, (req, res) => {
     const response = {
       data: {

test/jest/acceptance/instrumentation.spec.ts

@@ -0,0 +1,176 @@
+import { fakeServer } from '../../acceptance/fake-server';
+import { createProjectFromWorkspace } from '../util/createProject';
+import { runSnykCLI } from '../util/runSnykCLI';
+import { getServerPort } from '../util/getServerPort';
+import { matchers } from 'jest-json-schema';
+
+expect.extend(matchers);
+
+const INSTRUMENTATION_SCHEMA = require('../../schemas/instrumentationSchema.json');
+
+jest.setTimeout(1000 * 30);
+
+describe('instrumentation module', () => {
+  let server;
+  let env: Record<string, string>;
+  const fixtureName = 'npm-package';
+  const baseApi = '/api/v1';
+  const port = getServerPort(process);
+  const snykOrg = '11111111-2222-3333-4444-555555555555';
+  const defaultEnvVars = {
+    SNYK_API: 'http://localhost:' + port + baseApi,
+    SNYK_HOST: 'http://localhost:' + port,
+    SNYK_TOKEN: '123456789',
+    SNYK_CFG_ORG: snykOrg,
+  };
+
+  beforeAll((done) => {
+    env = {
+      ...process.env,
+      ...defaultEnvVars,
+    };
+    server = fakeServer(baseApi, env.SNYK_TOKEN);
+    server.listen(port, () => {
+      done();
+    });
+  });
+
+  afterEach(() => {
+    server.restore();
+  });
+
+  afterAll((done) => {
+    server.close(() => {
+      done();
+    });
+  });
+
+  describe('CLI support', () => {
+    it('sends instrumentation data for the CLI', async () => {
+      const project = await createProjectFromWorkspace(fixtureName);
+      const { code } = await runSnykCLI('test --debug', {
+        cwd: project.path(),
+        env,
+      });
+
+      expect(code).toBe(0);
+
+      // find the instrumentation request
+      const instrumentationRequest = server
+        .getRequests()
+        .filter((value) =>
+          (value.url as string).includes(
+            `/api/hidden/orgs/${snykOrg}/analytics`,
+          ),
+        )
+        .pop();
+
+      expect(instrumentationRequest?.body).toMatchSchema(
+        INSTRUMENTATION_SCHEMA,
+      );
+    });
+
+    it('sends instrumentation data even if disable analytics is set via SNYK_DISABLE_ANALYTICS', async () => {
+      const project = await createProjectFromWorkspace(fixtureName);
+      const { code } = await runSnykCLI(`test --debug`, {
+        env: {
+          cwd: project.path(),
+          ...env,
+          SNYK_DISABLE_ANALYTICS: '1',
+        },
+      });
+      expect(code).toBe(0);
+
+      // v1 analytics should not be sent
+      const v1AnalyticsRequest = server
+        .getRequests()
+        .filter((value) => value.url == '/api/v1/analytics/cli')
+        .pop();
+
+      // but instrumentation should
+      const instrumentationRequest = server
+        .getRequests()
+        .filter((value) =>
+          (value.url as string).includes(
+            `/api/hidden/orgs/${snykOrg}/analytics`,
+          ),
+        )
+        .pop();
+
+      expect(v1AnalyticsRequest).toBeUndefined();
+      expect(instrumentationRequest?.body).toMatchSchema(
+        INSTRUMENTATION_SCHEMA,
+      );
+    });
+  });
+
+  describe.each(['VS_CODE', 'JETBRAINS_IDE', 'VISUAL_STUDIO', 'ECLIPSE'])(
+    'IDE support',
+    (ide) => {
+      describe('analytics command not called from IDE', () => {
+        it(`does not send instrumentation data for the ${ide} IDE`, async () => {
+          const project = await createProjectFromWorkspace(fixtureName);
+          const { code } = await runSnykCLI('test --debug', {
+            cwd: project.path(),
+            env: {
+              ...process.env,
+              ...defaultEnvVars,
+              SNYK_INTEGRATION_NAME: ide,
+            },
+          });
+
+          expect(code).toBe(0);
+
+          const instrumentationRequest = server
+            .getRequests()
+            .filter((value) =>
+              (value.url as string).includes(
+                `/api/hidden/orgs/${snykOrg}/analytics`,
+              ),
+            )
+            .pop();
+
+          // we should not expect to find any requests to the analytics API instrumentation endpoint
+          expect(instrumentationRequest).toBeUndefined();
+        });
+      });
+
+      describe('analytics command called from IDE', () => {
+        // we need to remove all whitepace here due to how we split the CLI args in runSnykCLI()
+        const v1Data =
+          '{"data":{"type":"analytics","attributes":{"path":"/path/to/test","device_id":"unique-uuid","application":"snyk-cli","application_version":"1.1233.0","os":"macOS","arch":"ARM64","integration_name":"IntelliJ","integration_version":"2.5.5","integration_environment":"Pycharm","integration_environment_version":"2023.1","event_type":"Scandone","status":"Succeeded","scan_type":"SnykOpenSource","unique_issue_count":{"critical":15,"high":10,"medium":1,"low":2},"duration_ms":"1000","timestamp_finished":"2023-09-01T12:00:00Z"}}}';
+
+        it(`sends instrumentation data for the ${ide} IDE`, async () => {
+          const project = await createProjectFromWorkspace(fixtureName);
+          const { code } = await runSnykCLI(
+            `analytics report --experimental --debug --inputData ${v1Data}`,
+            {
+              cwd: project.path(),
+              env: {
+                ...process.env,
+                ...defaultEnvVars,
+                SNYK_INTEGRATION_NAME: ide,
+              },
+            },
+          );
+
+          expect(code).toBe(0);
+
+          // find the intrumentation request
+          const intrumentationRequest = server
+            .getRequests()
+            .filter((value) =>
+              (value.url as string).includes(
+                `/api/hidden/orgs/${snykOrg}/analytics`,
+              ),
+            )
+            .pop();
+
+          expect(intrumentationRequest?.body).toMatchSchema(
+            INSTRUMENTATION_SCHEMA,
+          );
+        });
+      });
+    },
+  );
+});