diff --git a/.gitleaksignore b/.gitleaksignore index 7490114050..c99df68d86 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -83,3 +83,4 @@ c2de35484dcad696a6ee32f2fa317d5cfaffc133:test/fixtures/code/sample-analyze-folde 4c12242de73be79ebd768468e065790f0b9d23a7:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98 25f37b4c609380452b0b96c3853b69e4dc29bb48:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98 ccd03cce97470452766ab397f2ba770dbb2e002e:test/jest/unit/lib/iac/drift/fixtures/all.console:aws-access-token:98 +test/jest/acceptance/instrumentation.spec.ts:snyk-api-token:19 diff --git a/test/acceptance/fake-server.ts b/test/acceptance/fake-server.ts index 609e47048e..8d36e6b10b 100644 --- a/test/acceptance/fake-server.ts +++ b/test/acceptance/fake-server.ts @@ -604,6 +604,13 @@ export const fakeServer = (basePath: string, snykToken: string): FakeServer => { res.status(200).send({}); }); + app.post( + basePath.replace('v1', 'hidden') + `/orgs/:orgId/analytics`, + (req, res) => { + res.status(201).send({}); + }, + ); + app.post(`/rest/orgs/:orgId/sbom_tests`, (req, res) => { const response = { data: { diff --git a/test/jest/acceptance/instrumentation.spec.ts b/test/jest/acceptance/instrumentation.spec.ts new file mode 100644 index 0000000000..b5d591843a --- /dev/null +++ b/test/jest/acceptance/instrumentation.spec.ts @@ -0,0 +1,176 @@ +import { fakeServer } from '../../acceptance/fake-server'; +import { createProjectFromWorkspace } from '../util/createProject'; +import { runSnykCLI } from '../util/runSnykCLI'; +import { getServerPort } from '../util/getServerPort'; +import { matchers } from 'jest-json-schema'; + +expect.extend(matchers); + +const INSTRUMENTATION_SCHEMA = require('../../schemas/instrumentationSchema.json'); + +jest.setTimeout(1000 * 30); + +describe('instrumentation module', () => { + let server; + let env: Record; + const fixtureName = 'npm-package'; + const baseApi = '/api/v1'; + const port = getServerPort(process); + const snykOrg = '11111111-2222-3333-4444-555555555555'; + const defaultEnvVars = { + SNYK_API: 'http://localhost:' + port + baseApi, + SNYK_HOST: 'http://localhost:' + port, + SNYK_TOKEN: '123456789', + SNYK_CFG_ORG: snykOrg, + }; + + beforeAll((done) => { + env = { + ...process.env, + ...defaultEnvVars, + }; + server = fakeServer(baseApi, env.SNYK_TOKEN); + server.listen(port, () => { + done(); + }); + }); + + afterEach(() => { + server.restore(); + }); + + afterAll((done) => { + server.close(() => { + done(); + }); + }); + + describe('CLI support', () => { + it('sends instrumentation data for the CLI', async () => { + const project = await createProjectFromWorkspace(fixtureName); + const { code } = await runSnykCLI('test --debug', { + cwd: project.path(), + env, + }); + + expect(code).toBe(0); + + // find the instrumentation request + const instrumentationRequest = server + .getRequests() + .filter((value) => + (value.url as string).includes( + `/api/hidden/orgs/${snykOrg}/analytics`, + ), + ) + .pop(); + + expect(instrumentationRequest?.body).toMatchSchema( + INSTRUMENTATION_SCHEMA, + ); + }); + + it('sends instrumentation data even if disable analytics is set via SNYK_DISABLE_ANALYTICS', async () => { + const project = await createProjectFromWorkspace(fixtureName); + const { code } = await runSnykCLI(`test --debug`, { + env: { + cwd: project.path(), + ...env, + SNYK_DISABLE_ANALYTICS: '1', + }, + }); + expect(code).toBe(0); + + // v1 analytics should not be sent + const v1AnalyticsRequest = server + .getRequests() + .filter((value) => value.url == '/api/v1/analytics/cli') + .pop(); + + // but instrumentation should + const instrumentationRequest = server + .getRequests() + .filter((value) => + (value.url as string).includes( + `/api/hidden/orgs/${snykOrg}/analytics`, + ), + ) + .pop(); + + expect(v1AnalyticsRequest).toBeUndefined(); + expect(instrumentationRequest?.body).toMatchSchema( + INSTRUMENTATION_SCHEMA, + ); + }); + }); + + describe.each(['VS_CODE', 'JETBRAINS_IDE', 'VISUAL_STUDIO', 'ECLIPSE'])( + 'IDE support', + (ide) => { + describe('analytics command not called from IDE', () => { + it(`does not send instrumentation data for the ${ide} IDE`, async () => { + const project = await createProjectFromWorkspace(fixtureName); + const { code } = await runSnykCLI('test --debug', { + cwd: project.path(), + env: { + ...process.env, + ...defaultEnvVars, + SNYK_INTEGRATION_NAME: ide, + }, + }); + + expect(code).toBe(0); + + const instrumentationRequest = server + .getRequests() + .filter((value) => + (value.url as string).includes( + `/api/hidden/orgs/${snykOrg}/analytics`, + ), + ) + .pop(); + + // we should not expect to find any requests to the analytics API instrumentation endpoint + expect(instrumentationRequest).toBeUndefined(); + }); + }); + + describe('analytics command called from IDE', () => { + // we need to remove all whitepace here due to how we split the CLI args in runSnykCLI() + const v1Data = + '{"data":{"type":"analytics","attributes":{"path":"/path/to/test","device_id":"unique-uuid","application":"snyk-cli","application_version":"1.1233.0","os":"macOS","arch":"ARM64","integration_name":"IntelliJ","integration_version":"2.5.5","integration_environment":"Pycharm","integration_environment_version":"2023.1","event_type":"Scandone","status":"Succeeded","scan_type":"SnykOpenSource","unique_issue_count":{"critical":15,"high":10,"medium":1,"low":2},"duration_ms":"1000","timestamp_finished":"2023-09-01T12:00:00Z"}}}'; + + it(`sends instrumentation data for the ${ide} IDE`, async () => { + const project = await createProjectFromWorkspace(fixtureName); + const { code } = await runSnykCLI( + `analytics report --experimental --debug --inputData ${v1Data}`, + { + cwd: project.path(), + env: { + ...process.env, + ...defaultEnvVars, + SNYK_INTEGRATION_NAME: ide, + }, + }, + ); + + expect(code).toBe(0); + + // find the intrumentation request + const intrumentationRequest = server + .getRequests() + .filter((value) => + (value.url as string).includes( + `/api/hidden/orgs/${snykOrg}/analytics`, + ), + ) + .pop(); + + expect(intrumentationRequest?.body).toMatchSchema( + INSTRUMENTATION_SCHEMA, + ); + }); + }); + }, + ); +}); diff --git a/test/schemas/instrumentationSchema.json b/test/schemas/instrumentationSchema.json new file mode 100644 index 0000000000..108e67abb3 --- /dev/null +++ b/test/schemas/instrumentationSchema.json @@ -0,0 +1,384 @@ +{ + "type": "object", + "required": ["data"], + "properties": { + "data": { + "$ref": "#/schemas/AnalyticsData" + } + }, + "schemas": { + "AnalyticsData": { + "type": "object", + "required": ["type", "attributes"], + "properties": { + "type": { + "type": "string", + "description": "The type of data ('analytics')." + }, + "attributes": { + "$ref": "#/schemas/AnalyticsAttributes" + } + } + }, + "AnalyticsAttributes": { + "type": "object", + "required": ["interaction"], + "properties": { + "interaction": { + "$ref": "#/schemas/Interaction" + }, + "runtime": { + "$ref": "#/schemas/Runtime" + } + } + }, + "Interaction": { + "type": "object", + "required": ["id", "timestamp_ms", "type", "status", "target"], + "properties": { + "id": { + "type": "string", + "format": "uri", + "description": "The client-generated ID of the interaction event in the form of 'urn:snyk:interaction:00000000-0000-0000-0000-000000000000'\n" + }, + "timestamp_ms": { + "type": "integer", + "format": "int64", + "description": "The timestamp in epoch milliseconds when the interaction was started in UTC (Zulu time)." + }, + "stage": { + "type": "string", + "description": "The stage of the SDLC where the Interaction occurred, such as 'dev'|'cicd'|'prchecks'|'unknown'.", + "enum": ["dev", "cicd", "prchecks", "unknown"] + }, + "type": { + "type": "string", + "description": "The type of interaction, could be 'Scan done'. Scan Done indicates that a test was run no matter if the CLI or IDE ran it, other types can be freely chosen types.\n" + }, + "categories": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The category vector used to describe the interaction in detail, 'oss','test'." + }, + "status": { + "type": "string", + "description": "Status would be 'success' or 'failure', where success means the action was executed, while failure means it didn't run.\n" + }, + "results": { + "type": "array", + "description": "The result of the interaction. Could be a something like this [{'name': 'critical', 'count': 0}].\nOnly strings, integers, and boolean values are allowed.\n", + "items": { + "type": "object", + "additionalProperties": true + } + }, + "target": { + "$ref": "#/schemas/Target" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/schemas/InteractionError" + } + }, + "extension": { + "type": "object", + "description": "Optional additional extension.\n\nOnly strings, integers, and boolean values are allowed.\n", + "additionalProperties": true + } + } + }, + "Runtime": { + "type": "object", + "properties": { + "application": { + "$ref": "#/schemas/Application" + }, + "integration": { + "$ref": "#/schemas/Integration" + }, + "environment": { + "$ref": "#/schemas/Environment" + }, + "platform": { + "$ref": "#/schemas/Platform" + }, + "performance": { + "$ref": "#/schemas/Performance" + } + } + }, + "Target": { + "type": "object", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "format": "uri", + "description": "A purl is a URL composed of seven components.\nscheme:type/namespace/name@version?qualifiers#subpath\n\nThe purl specification is available here:\n\n'https://github.com/package-url/purl-spec'\n\nSome purl examples\n\n'pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c'\n\n'pkg:npm/%40angular/animation@12.3.1'\n\n'pkg:pypi/django@1.11.1'\n" + } + } + }, + "InteractionError": { + "type": "object", + "required": ["id"], + "properties": { + "id": { + "type": "string", + "description": "Error identifier corresponding to the errors defined in the error catalog.\n\n'https://docs.snyk.io/scan-with-snyk/error-catalog'\n" + }, + "code": { + "type": "string", + "description": "The HTTP specific error code." + } + } + }, + "Application": { + "type": "object", + "required": ["name", "version"], + "description": "The application name, e.g. snyk-ls. The version of the integration.\n", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "Integration": { + "type": "object", + "required": ["name", "version"], + "description": "TODO UPDATE with correct samples of integration name. The name of the integration, could be a plugin or extension (e.g. Snyk Security plugin for intelliJ). The version of the integration (2.3.4).\n", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "Environment": { + "type": "object", + "required": ["name", "version"], + "description": "The environment for the integration (e.g., IntelliJ Ultimate, Pycharm). The version of the integration environment (e.g. 2023.3)\n", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + }, + "Platform": { + "type": "object", + "required": ["os", "arch"], + "description": "The operating system and the architecture (AMD64, ARM64, 386, ALPINE).", + "properties": { + "os": { + "type": "string" + }, + "arch": { + "type": "string" + } + } + }, + "Performance": { + "type": "object", + "required": ["duration_ms"], + "description": "The scan duration in milliseconds", + "properties": { + "duration_ms": { + "type": "integer", + "format": "int64" + } + } + }, + "JsonApi": { + "type": "object", + "properties": { + "version": { + "type": "string", + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$", + "description": "Version of the JSON API specification this server supports.", + "example": "1.0" + } + }, + "required": ["version"], + "additionalProperties": false, + "example": { + "version": "1.0" + } + }, + "ErrorLink": { + "type": "object", + "description": "A link that leads to further details about this particular occurrence of the problem.", + "properties": { + "about": { + "$ref": "#/schemas/LinkProperty" + } + }, + "additionalProperties": false, + "example": { + "about": "https://example.com/about_this_error" + } + }, + "LinkProperty": { + "oneOf": [ + { + "type": "string", + "description": "A string containing the link’s URL.", + "example": "https://example.com/api/resource" + }, + { + "type": "object", + "properties": { + "href": { + "type": "string", + "description": "A string containing the link’s URL.", + "example": "https://example.com/api/resource" + }, + "meta": { + "$ref": "#/schemas/Meta" + } + }, + "required": ["href"], + "additionalProperties": false, + "example": { + "href": "https://example.com/api/resource" + } + } + ], + "example": "https://example.com/api/resource" + }, + "Meta": { + "type": "object", + "description": "Free-form object that may contain non-standard information.", + "example": { + "key1": "value1", + "key2": { + "sub_key": "sub_value" + }, + "key3": ["array_value1", "array_value2"] + }, + "additionalProperties": true, + "properties": {} + }, + "ErrorDocument": { + "type": "object", + "properties": { + "jsonapi": { + "$ref": "#/schemas/JsonApi" + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/schemas/Error" + }, + "minItems": 1, + "example": [ + { + "status": "403", + "detail": "Permission denied for this resource" + } + ] + } + }, + "additionalProperties": false, + "required": ["jsonapi", "errors"], + "example": { + "jsonapi": { + "version": "1.0" + }, + "errors": [ + { + "status": "403", + "detail": "Permission denied for this resource" + } + ] + } + }, + "Error": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid", + "description": "A unique identifier for this particular occurrence of the problem.", + "example": "f16c31b5-6129-4571-add8-d589da9be524" + }, + "links": { + "$ref": "#/schemas/ErrorLink" + }, + "status": { + "type": "string", + "pattern": "^[45]\\d\\d$", + "description": "The HTTP status code applicable to this problem, expressed as a string value.", + "example": "400" + }, + "detail": { + "type": "string", + "description": "A human-readable explanation specific to this occurrence of the problem.", + "example": "The request was missing these required fields: ..." + }, + "code": { + "type": "string", + "description": "An application-specific error code, expressed as a string value.", + "example": "entity-not-found" + }, + "title": { + "type": "string", + "description": "A short, human-readable summary of the problem that SHOULD NOT change from occurrence to occurrence of the problem, except for purposes of localization.", + "example": "Bad request" + }, + "source": { + "type": "object", + "properties": { + "pointer": { + "type": "string", + "description": "A JSON Pointer [RFC6901] to the associated entity in the request document.", + "example": "/data/attributes" + }, + "parameter": { + "type": "string", + "description": "A string indicating which URI query parameter caused the error.", + "example": "param1" + } + }, + "additionalProperties": false, + "example": { + "pointer": "/data/attributes" + } + }, + "meta": { + "type": "object", + "additionalProperties": true, + "example": { + "key": "value" + }, + "properties": {} + } + }, + "required": ["status", "detail"], + "additionalProperties": false, + "example": { + "status": "404", + "detail": "Not Found" + } + }, + "QueryVersion": { + "type": "string", + "description": "Requested API version", + "pattern": "^(wip|work-in-progress|experimental|beta|((([0-9]{4})-([0-1][0-9]))-((3[01])|(0[1-9])|([12][0-9]))(~(wip|work-in-progress|experimental|beta))?))$" + }, + "ActualVersion": { + "type": "string", + "description": "Resolved API version", + "pattern": "^((([0-9]{4})-([0-1][0-9]))-((3[01])|(0[1-9])|([12][0-9]))(~(wip|work-in-progress|experimental|beta))?)$" + } + } +}