[test visibility] Add total code coverage measure to vitest (#4570)

integration-tests/ci-visibility/vitest-tests/coverage-test.mjs

@@ -0,0 +1,8 @@
+import { describe, test, expect } from 'vitest'
+import { sum } from './sum'
+
+describe('code coverage', () => {
+  test('passes', () => {
+    expect(sum(1, 2)).to.equal(3)
+  })
+})

integration-tests/vitest.config.mjs

@@ -1,9 +1,19 @@
 import { defineConfig } from 'vite'
 
-export default defineConfig({
+const config = {
   test: {
     include: [
       process.env.TEST_DIR || 'ci-visibility/vitest-tests/test-visibility*'
     ]
   }
-})
+}
+
+if (process.env.COVERAGE_PROVIDER) {
+  config.test.coverage = {
+    provider: process.env.COVERAGE_PROVIDER || 'v8',
+    include: ['ci-visibility/vitest-tests/**'],
+    reporter: ['text-summary']
+  }
+}
+
+export default defineConfig(config)

integration-tests/vitest/vitest.spec.js

@@ -13,18 +13,24 @@ const {
   TEST_STATUS,
   TEST_TYPE,
   TEST_IS_RETRY,
-  TEST_CODE_OWNERS
+  TEST_CODE_OWNERS,
+  TEST_CODE_COVERAGE_LINES_PCT
 } = require('../../packages/dd-trace/src/plugins/util/test')
 
-// tested with 1.6.0
 const versions = ['1.6.0', 'latest']
 
+const linePctMatchRegex = /Lines\s+:\s+([\d.]+)%/
+
 versions.forEach((version) => {
   describe(`vitest@${version}`, () => {
-    let sandbox, cwd, receiver, childProcess
+    let sandbox, cwd, receiver, childProcess, testOutput
 
     before(async function () {
-      sandbox = await createSandbox([`vitest@${version}`], true)
+      sandbox = await createSandbox([
+        `vitest@${version}`,
+        `@vitest/coverage-istanbul@${version}`,
+        `@vitest/coverage-v8@${version}`
+      ], true)
       cwd = sandbox.folder
     })
 
@@ -37,6 +43,7 @@ versions.forEach((version) => {
     })
 
     afterEach(async () => {
+      testOutput = ''
       childProcess.kill()
       await receiver.stop()
     })
@@ -233,5 +240,59 @@ versions.forEach((version) => {
         }).catch(done)
       })
     })
+
+    // only works for >=2.0.0
+    if (version === 'latest') {
+      const coverageProviders = ['v8', 'istanbul']
+
+      coverageProviders.forEach((coverageProvider) => {
+        it(`reports code coverage for ${coverageProvider} provider`, (done) => {
+          let codeCoverageExtracted
+          const eventsPromise = receiver
+            .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+
+              const testSession = events.find(event => event.type === 'test_session_end').content
+
+              codeCoverageExtracted = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]
+            })
+
+          childProcess = exec(
+            './node_modules/.bin/vitest run --coverage',
+            {
+              cwd,
+              env: {
+                ...getCiVisAgentlessConfig(receiver.port),
+                NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init',
+                COVERAGE_PROVIDER: coverageProvider,
+                TEST_DIR: 'ci-visibility/vitest-tests/coverage-test*'
+              },
+              stdio: 'inherit'
+            }
+          )
+
+          childProcess.stdout.on('data', (chunk) => {
+            testOutput += chunk.toString()
+          })
+          childProcess.stderr.on('data', (chunk) => {
+            testOutput += chunk.toString()
+          })
+
+          childProcess.on('exit', () => {
+            eventsPromise.then(() => {
+              const linePctMatch = testOutput.match(linePctMatchRegex)
+              const linesPctFromNyc = linePctMatch ? Number(linePctMatch[1]) : null
+
+              assert.equal(
+                linesPctFromNyc,
+                codeCoverageExtracted,
+                'coverage reported by vitest does not match extracted coverage'
+              )
+              done()
+            }).catch(done)
+          })
+        })
+      })
+    }
   })
 })

packages/datadog-instrumentations/src/vitest.js

@@ -121,6 +121,21 @@ function getSortWrapper (sort) {
       this.ctx.config.retry = NUM_FAILED_TEST_RETRIES
     }
 
+    let testCodeCoverageLinesTotal
+
+    if (this.ctx.coverageProvider?.generateCoverage) {
+      shimmer.wrap(this.ctx.coverageProvider, 'generateCoverage', generateCoverage => async function () {
+        const totalCodeCoverage = await generateCoverage.apply(this, arguments)
+
+        try {
+          testCodeCoverageLinesTotal = totalCodeCoverage.getCoverageSummary().lines.pct
+        } catch (e) {
+          // ignore errors
+        }
+        return totalCodeCoverage
+      })
+    }
+
     shimmer.wrap(this.ctx, 'exit', exit => async function () {
       let onFinish
 
@@ -136,8 +151,9 @@ function getSortWrapper (sort) {
       sessionAsyncResource.runInAsyncScope(() => {
         testSessionFinishCh.publish({
           status: getSessionStatus(this.state),
-          onFinish,
-          error
+          testCodeCoverageLinesTotal,
+          error,
+          onFinish
         })
       })
 

packages/datadog-plugin-vitest/src/index.js

@@ -7,7 +7,8 @@ const {
   getTestSuitePath,
   getTestSuiteCommonTags,
   TEST_SOURCE_FILE,
-  TEST_IS_RETRY
+  TEST_IS_RETRY,
+  TEST_CODE_COVERAGE_LINES_PCT
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 
@@ -150,13 +151,17 @@ class VitestPlugin extends CiPlugin {
       }
     })
 
-    this.addSub('ci:vitest:session:finish', ({ status, onFinish, error }) => {
+    this.addSub('ci:vitest:session:finish', ({ status, onFinish, error, testCodeCoverageLinesTotal }) => {
       this.testSessionSpan.setTag(TEST_STATUS, status)
       this.testModuleSpan.setTag(TEST_STATUS, status)
       if (error) {
         this.testModuleSpan.setTag('error', error)
         this.testSessionSpan.setTag('error', error)
       }
+      if (testCodeCoverageLinesTotal) {
+        this.testModuleSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
+        this.testSessionSpan.setTag(TEST_CODE_COVERAGE_LINES_PCT, testCodeCoverageLinesTotal)
+      }
       this.testModuleSpan.finish()
       this.testSessionSpan.finish()
       finishAllTraceSpans(this.testSessionSpan)