[tests-only][full-ci] report tracings of failing tests only (#11571)

* test: save the tracing reports of failed tests only

* test: bump ocis

* test: disable tracing by default

* bump web commit for tests

.drone.env

@@ -1,3 +1,3 @@
 # The version of OCIS to use in pipelines that test against OCIS
-OCIS_COMMITID=91969e6434471b49e64d3c2b97da62d7c94789f0
+OCIS_COMMITID=c83b6cf618350064b2a48e05c04258f1e8626e1b
 OCIS_BRANCH=master

.drone.star

@@ -1829,7 +1829,7 @@ def e2eTestsOnKeycloak(ctx):
                          "BASE_URL_OCIS": "ocis:9200",
                          "HEADLESS": "true",
                          "RETRY": "1",
-                         "REPORT_TRACING": "true",
+                         "REPORT_TRACING": "with-tracing" in ctx.build.title.lower(),
                          "KEYCLOAK": "true",
                          "KEYCLOAK_HOST": "keycloak:8443",
                      },

docker-compose.yml

@@ -98,7 +98,7 @@ services:
       COLLABORA_DOMAIN: host.docker.internal:9980
       FRONTEND_APP_HANDLER_SECURE_VIEW_APP_ADDR: com.owncloud.api.collaboration.Collabora
       # Needed for enabling all roles
-      "GRAPH_AVAILABLE_ROLES": b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6
+      GRAPH_AVAILABLE_ROLES: b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6
     labels:
       traefik.enable: true
       traefik.http.routers.ocis.tls: true

tests/e2e/config.js

@@ -37,6 +37,9 @@ export const config = {
   acceptDownloads: process.env.DOWNLOADS !== 'false',
   browser: process.env.BROWSER ?? 'chrome',
   reportDir: process.env.REPORT_DIR || 'reports/e2e',
+  get tracingReportDir() {
+    return this.reportDir + '/playwright/tracing'
+  },
   reportVideo: process.env.REPORT_VIDEO === 'true',
   reportHar: process.env.REPORT_HAR === 'true',
   reportTracing: process.env.REPORT_TRACING === 'true',

tests/e2e/cucumber/environment/index.ts

@@ -5,10 +5,13 @@ import {
   setWorldConstructor,
   ITestCaseHookParameter,
   AfterAll,
-  After
+  After,
+  Status
 } from '@cucumber/cucumber'
 import pino from 'pino'
 import { Browser, chromium, firefox, webkit } from '@playwright/test'
+import path from 'path'
+import fs from 'fs'
 
 import { config } from '../../config'
 import { api, environment } from '../../support'
@@ -39,7 +42,9 @@ const logger = pino({
     }
   }
 })
+
 setDefaultTimeout(config.debug ? -1 : config.timeout * 1000)
+setWorldConstructor(World)
 
 Before(async function (this: World, { pickle }: ITestCaseHookParameter) {
   this.feature = pickle
@@ -137,6 +142,10 @@ After(async function (this: World, { result, willBeRetried }: ITestCaseHookParam
   keycloakTokenStore.clear()
   removeTempUploadDirectory()
   closeSSEConnections()
+
+  if (config.reportTracing) {
+    filterTracingReports(result.status)
+  }
 })
 
 AfterAll(async () => {
@@ -145,9 +154,35 @@ AfterAll(async () => {
   if (state.browser) {
     await state.browser.close()
   }
+
+  if (config.reportTracing) {
+    // move failed tracing reports
+    const failedDir = path.dirname(config.tracingReportDir) + '/failed'
+
+    if (fs.existsSync(failedDir)) {
+      fs.renameSync(failedDir, config.tracingReportDir)
+    }
+  }
 })
 
-setWorldConstructor(World)
+function filterTracingReports(status: string) {
+  const traceDir = config.tracingReportDir
+  const failedDir = path.dirname(config.tracingReportDir) + '/failed'
+
+  if (status !== Status.PASSED) {
+    if (!fs.existsSync(failedDir)) {
+      fs.mkdirSync(failedDir, { recursive: true })
+    }
+    const reports = fs.readdirSync(traceDir)
+    // collect tracings for failed tests
+    reports.forEach((report) => {
+      fs.renameSync(`${traceDir}/${report}`, `${failedDir}/${report}`)
+    })
+  } else {
+    // clean up the tracing directory
+    fs.rmSync(traceDir, { recursive: true })
+  }
+}
 
 const cleanUpUser = async (adminUser: User) => {
   const requests: Promise<User>[] = []

tests/e2e/cucumber/environment/world.ts

@@ -26,6 +26,7 @@ export class World extends CucumberWorld {
       context: {
         acceptDownloads: config.acceptDownloads,
         reportDir: config.reportDir,
+        tracingReportDir: config.tracingReportDir,
         reportHar: config.reportHar,
         reportTracing: config.reportTracing,
         reportVideo: config.reportVideo,

tests/e2e/support/environment/actor/actor.ts

@@ -60,12 +60,7 @@ export class ActorEnvironment extends EventEmitter implements Actor {
   async close(): Promise<void> {
     if (this.options.context.reportTracing) {
       await this.context?.tracing.stop({
-        path: path.join(
-          this.options.context.reportDir,
-          'playwright',
-          'tracing',
-          `${this.options.namespace}.zip`
-        )
+        path: path.join(this.options.context.tracingReportDir, `${this.options.namespace}.zip`)
       })
     }
 

tests/e2e/support/environment/actor/shared.ts

@@ -6,6 +6,7 @@ export interface ActorsOptions {
   context: {
     acceptDownloads: boolean
     reportDir: string
+    tracingReportDir: string
     reportVideo: boolean
     reportHar: boolean
     reportTracing: boolean