From 540580504d7bc70bfdbdf8b6a22abde5d2464e90 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Mon, 13 Jan 2025 17:43:28 +0000 Subject: [PATCH] Fix flaky playwright tests (#28984) Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- playwright/e2e/crypto/backups-mas.spec.ts | 17 +++++++---- playwright/e2e/login/login-sso.spec.ts | 4 +-- .../e2e/login/soft_logout_oauth.spec.ts | 4 +-- playwright/e2e/login/utils.ts | 5 ++-- .../synapse/legacyOAuthHomeserver.ts | 29 +++++++++++++------ playwright/plugins/oauth_server/index.ts | 13 +++++++-- playwright/services.ts | 4 +++ 7 files changed, 53 insertions(+), 23 deletions(-) diff --git a/playwright/e2e/crypto/backups-mas.spec.ts b/playwright/e2e/crypto/backups-mas.spec.ts index 03c83efb1af..b51f7372558 100644 --- a/playwright/e2e/crypto/backups-mas.spec.ts +++ b/playwright/e2e/crypto/backups-mas.spec.ts @@ -19,19 +19,19 @@ test.use(masHomeserver); test.describe("Encryption state after registration", () => { test.skip(isDendrite, "does not yet support MAS"); - test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => { + test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); await app.settings.openUserSettings("Security & Privacy"); await expect(page.getByText("This session is backing up your keys.")).toBeVisible(); }); - test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => { + test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => { await page.goto("/#/login"); await page.getByRole("button", { name: "Continue" }).click(); - await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!"); + await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!"); await page.getByRole("button", { name: "Add room" }).click(); await page.getByRole("menuitem", { name: "New room" }).click(); @@ -45,8 +45,13 @@ test.describe("Encryption state after registration", () => { test.describe("Key backup reset from elsewhere", () => { test.skip(isDendrite, "does not yet support MAS"); - test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => { - const testUsername = "alice"; + test("Key backup is disabled when reset from elsewhere", async ({ + page, + mailhogClient, + request, + homeserver, + }, testInfo) => { + const testUsername = `alice_${testInfo.testId}`; const testPassword = "Pa$sW0rD!"; // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake diff --git a/playwright/e2e/login/login-sso.spec.ts b/playwright/e2e/login/login-sso.spec.ts index fbe190b9358..22428af5028 100644 --- a/playwright/e2e/login/login-sso.spec.ts +++ b/playwright/e2e/login/login-sso.spec.ts @@ -17,13 +17,13 @@ test.use(legacyOAuthHomeserver); test.describe("SSO login", () => { test.skip(isDendrite, "does not yet support SSO"); - test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => { + test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => { // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to // your firewall settings: Synapse is unable to reach the OIDC server. // // If you are using ufw, try something like: // sudo ufw allow in on docker0 // - await doTokenRegistration(page, homeserver); + await doTokenRegistration(page, homeserver, testInfo); }); }); diff --git a/playwright/e2e/login/soft_logout_oauth.spec.ts b/playwright/e2e/login/soft_logout_oauth.spec.ts index 19b1fc0124c..f6814d0cf4a 100644 --- a/playwright/e2e/login/soft_logout_oauth.spec.ts +++ b/playwright/e2e/login/soft_logout_oauth.spec.ts @@ -26,8 +26,8 @@ test.use({ test.use(legacyOAuthHomeserver); test.describe("Soft logout with SSO user", () => { test.use({ - user: async ({ page, homeserver }, use) => { - const user = await doTokenRegistration(page, homeserver); + user: async ({ page, homeserver }, use, testInfo) => { + const user = await doTokenRegistration(page, homeserver, testInfo); // Eventually, we should end up at the home screen. await expect(page).toHaveURL(/\/#\/home$/); diff --git a/playwright/e2e/login/utils.ts b/playwright/e2e/login/utils.ts index 6e52dd99818..59a5c330f30 100644 --- a/playwright/e2e/login/utils.ts +++ b/playwright/e2e/login/utils.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { Page, expect } from "@playwright/test"; +import { Page, expect, TestInfo } from "@playwright/test"; import { Credentials, HomeserverInstance } from "../../plugins/homeserver"; @@ -15,6 +15,7 @@ import { Credentials, HomeserverInstance } from "../../plugins/homeserver"; export async function doTokenRegistration( page: Page, homeserver: HomeserverInstance, + testInfo: TestInfo, ): Promise { await page.goto("/#/login"); @@ -35,7 +36,7 @@ export async function doTokenRegistration( // Synapse prompts us to pick a user ID await expect(page.getByRole("heading", { name: "Create your account" })).toBeVisible(); - await page.getByRole("textbox", { name: "Username (required)" }).fill("alice"); + await page.getByRole("textbox", { name: "Username (required)" }).fill(`alice_${testInfo.testId}`); // wait for username validation to start, and complete await expect(page.locator("#field-username-output")).toHaveText(""); diff --git a/playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts b/playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts index 246829e422d..2e3d20c9c0d 100644 --- a/playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts +++ b/playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts @@ -6,20 +6,31 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com Please see LICENSE files in the repository root for full details. */ -import { Fixtures } from "@playwright/test"; +import { Fixtures, PlaywrightTestArgs } from "@playwright/test"; import { TestContainers } from "testcontainers"; import { Services } from "../../../services.ts"; import { OAuthServer } from "../../oauth_server"; -export const legacyOAuthHomeserver: Fixtures<{}, Services> = { - _homeserver: [ - async ({ _homeserver: container }, use) => { +export const legacyOAuthHomeserver: Fixtures = { + oAuthServer: [ + // eslint-disable-next-line no-empty-pattern + async ({}, use) => { const server = new OAuthServer(); - const port = server.start(); - + await use(server); + server.stop(); + }, + { scope: "worker" }, + ], + context: async ({ context, oAuthServer }, use, testInfo) => { + oAuthServer.onTestStarted(testInfo); + await use(context); + }, + _homeserver: [ + async ({ oAuthServer, _homeserver: homeserver }, use) => { + const port = oAuthServer.start(); await TestContainers.exposeHostPorts(port); - container.withConfig({ + homeserver.withConfig({ oidc_providers: [ { idp_id: "test", @@ -43,8 +54,8 @@ export const legacyOAuthHomeserver: Fixtures<{}, Services> = { }, ], }); - await use(container); - server.stop(); + + await use(homeserver); }, { scope: "worker" }, ], diff --git a/playwright/plugins/oauth_server/index.ts b/playwright/plugins/oauth_server/index.ts index 3f80dc11ca9..df5ee0f461b 100644 --- a/playwright/plugins/oauth_server/index.ts +++ b/playwright/plugins/oauth_server/index.ts @@ -9,12 +9,21 @@ Please see LICENSE files in the repository root for full details. import http from "http"; import express from "express"; import { AddressInfo } from "net"; +import { TestInfo } from "@playwright/test"; + +import { randB64Bytes } from "../utils/rand.ts"; export class OAuthServer { private server?: http.Server; + private sub?: string; + + public onTestStarted(testInfo: TestInfo): void { + this.sub = testInfo.testId; + } public start(): number { if (this.server) this.stop(); + const token = randB64Bytes(16); const app = express(); @@ -28,7 +37,7 @@ export class OAuthServer { const code = req.body.code; if (code === "valid_auth_code") { res.send({ - access_token: "oauth_access_token", + access_token: token, token_type: "Bearer", expires_in: "3600", }); @@ -43,7 +52,7 @@ export class OAuthServer { // return an OAuth2 user info object res.send({ - sub: "alice", + sub: this.sub, name: "Alice", }); }); diff --git a/playwright/services.ts b/playwright/services.ts index 2444adf3a0d..3abed280e98 100644 --- a/playwright/services.ts +++ b/playwright/services.ts @@ -15,6 +15,7 @@ import { Logger } from "./logger.ts"; import { StartedMatrixAuthenticationServiceContainer } from "./testcontainers/mas.ts"; import { HomeserverContainer, StartedHomeserverContainer } from "./testcontainers/HomeserverContainer.ts"; import { MailhogContainer, StartedMailhogContainer } from "./testcontainers/mailhog.ts"; +import { OAuthServer } from "./plugins/oauth_server"; interface TestFixtures { mailhogClient: mailhog.API; @@ -30,7 +31,10 @@ export interface Services { synapseConfigOptions: SynapseConfigOptions; _homeserver: HomeserverContainer; homeserver: StartedHomeserverContainer; + // Set in masHomeserver only mas?: StartedMatrixAuthenticationServiceContainer; + // Set in legacyOAuthHomeserver only + oAuthServer?: OAuthServer; } export const test = base.extend({