More playwright tests (#330)

* add docs, move scoring functions to scoring.ts, move experiment naming to utils.ts * add initStagehand.ts * break up index.evals.ts and utils into smaller files * export LogLineEval * typing * follow StagehandConfig pattern * choose api key based on model name * Use CI on v2 branch * branch * stagehand.page tests * dont run on BB * prettier * pls dont fail * headless --------- Co-authored-by: Anirudh Kamath <[email protected]>
browserbase · Dec 23, 2024 · 9fb9413 · 9fb9413
1 parent 7b22b83
commit 9fb9413
Show file tree

Hide file tree

Showing 14 changed files with 769 additions and 2 deletions.
diff --git a/evals/deterministic/stagehand.config.ts b/evals/deterministic/stagehand.config.ts
@@ -1,12 +1,12 @@
 import type { ConstructorParams, LogLine } from "../../lib";
 
 const StagehandConfig: ConstructorParams = {
-  env: "BROWSERBASE" /* Environment to run Stagehand in */,
+  env: "LOCAL" /* Environment to run Stagehand in */,
   apiKey: process.env.BROWSERBASE_API_KEY! /* API key for authentication */,
   projectId: process.env.BROWSERBASE_PROJECT_ID! /* Project identifier */,
   verbose: 1 /* Logging verbosity level (0=quiet, 1=normal, 2=verbose) */,
   debugDom: true /* Enable DOM debugging features */,
-  headless: false /* Run browser in headless mode */,
+  headless: true /* Run browser in headless mode */,
   logger: (message: LogLine) =>
     console.log(
       `[stagehand::${message.category}] ${message.message}`,

diff --git a/evals/deterministic/tests/addInitScript.test.ts b/evals/deterministic/tests/addInitScript.test.ts
@@ -0,0 +1,40 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - addInitScript", () => {
+  test("should inject a script before the page loads", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const page = stagehand.page;
+
+    await page.addInitScript(() => {
+      const w = window as typeof window & {
+        __testInitScriptVar?: string;
+      };
+      w.__testInitScriptVar = "Hello from init script!";
+    });
+
+    await page.goto("https://example.com");
+
+    const result = await page.evaluate(() => {
+      const w = window as typeof window & {
+        __testInitScriptVar?: string;
+      };
+      return w.__testInitScriptVar;
+    });
+    expect(result).toBe("Hello from init script!");
+
+    await page.goto("https://www.browserbase.com/");
+    const resultAfterNavigation = await page.evaluate(() => {
+      const w = window as typeof window & {
+        __testInitScriptVar?: string;
+      };
+      return w.__testInitScriptVar;
+    });
+    expect(resultAfterNavigation).toBe("Hello from init script!");
+
+    await stagehand.close();
+  });
+});
diff --git a/evals/deterministic/tests/addRemoveLocatorHandler.test.ts b/evals/deterministic/tests/addRemoveLocatorHandler.test.ts
@@ -0,0 +1,93 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - addLocatorHandler and removeLocatorHandler", () => {
+  // This HTML snippet is reused by both tests.
+  // The "Sign up to the newsletter" overlay appears after 2 seconds.
+  // The "No thanks" button hides it.
+  const overlayHTML = `
+    <html>
+      <body>
+        <button id="cta">Start here</button>
+        <div id="overlay" style="display: none;">
+          <p>Sign up to the newsletter</p>
+          <button id="no-thanks">No thanks</button>
+        </div>
+        <script>
+          // Show the overlay after 2 seconds
+          setTimeout(() => {
+            document.getElementById('overlay').style.display = 'block';
+          }, 2000);
+
+          // Hide the overlay when "No thanks" is clicked
+          document.getElementById('no-thanks').addEventListener('click', () => {
+            document.getElementById('overlay').style.display = 'none';
+          });
+        </script>
+      </body>
+    </html>
+  `;
+
+  test("should use a custom locator handler to dismiss the overlay", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const { page } = stagehand;
+
+    await page.addLocatorHandler(
+      page.getByText("Sign up to the newsletter"),
+      async () => {
+        console.log("Overlay detected. Clicking 'No thanks' to remove it...");
+        await page.getByRole("button", { name: "No thanks" }).click();
+      },
+    );
+
+    await page.goto("https://example.com");
+    await page.setContent(overlayHTML);
+
+    await page.waitForTimeout(5000);
+
+    await page.getByRole("button", { name: "Start here" }).click();
+
+    const isOverlayVisible = await page
+      .getByText("Sign up to the newsletter")
+      .isVisible()
+      .catch(() => false);
+
+    await stagehand.close();
+
+    expect(isOverlayVisible).toBeFalsy();
+  });
+
+  test("should remove a custom locator handler so overlay stays visible", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const { page } = stagehand;
+
+    const locator = page.getByText("Sign up to the newsletter");
+    await page.addLocatorHandler(locator, async () => {
+      console.log("Overlay detected. Clicking 'No thanks' to remove it...");
+      await page.getByRole("button", { name: "No thanks" }).click();
+    });
+
+    await page.removeLocatorHandler(locator);
+    console.log("Locator handler removed — overlay will not be dismissed now.");
+
+    await page.goto("https://example.com");
+    await page.setContent(overlayHTML);
+
+    await page.waitForTimeout(5000);
+
+    await page.getByRole("button", { name: "Start here" }).click();
+
+    const isOverlayVisible = await page
+      .getByText("Sign up to the newsletter")
+      .isVisible()
+      .catch(() => false);
+
+    await stagehand.close();
+    expect(isOverlayVisible).toBe(true);
+  });
+});
diff --git a/evals/deterministic/tests/addTags.test.ts b/evals/deterministic/tests/addTags.test.ts
@@ -0,0 +1,79 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - addScriptTag and addStyleTag", () => {
+  let stagehand: Stagehand;
+
+  test.beforeAll(async () => {
+    stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+  });
+
+  test.afterAll(async () => {
+    await stagehand.close();
+  });
+
+  test("should inject a script tag and have access to the defined function", async () => {
+    const { page } = stagehand;
+
+    await page.setContent(`
+      <html>
+      <body>
+        <h1 id="greeting">Hello, world!</h1>
+      </body>
+      </html>
+    `);
+
+    await page.addScriptTag({
+      content: `
+        window.sayHello = function() {
+          document.getElementById("greeting").textContent = "Hello from injected script!";
+        }
+      `,
+    });
+
+    await page.evaluate(() => {
+      const w = window as typeof window & {
+        sayHello?: () => void;
+      };
+      w.sayHello?.();
+    });
+
+    const text = await page.locator("#greeting").textContent();
+    expect(text).toBe("Hello from injected script!");
+  });
+
+  test("should inject a style tag and apply styles", async () => {
+    const { page } = stagehand;
+
+    await page.setContent(`
+      <html>
+      <body>
+        <div id="styledDiv">Some text</div>
+      </body>
+      </html>
+    `);
+
+    await page.addStyleTag({
+      content: `
+        #styledDiv {
+          color: red;
+          font-weight: bold;
+        }
+      `,
+    });
+
+    const color = await page.evaluate(() => {
+      const el = document.getElementById("styledDiv");
+      return window.getComputedStyle(el!).color;
+    });
+    expect(color).toBe("rgb(255, 0, 0)");
+
+    const fontWeight = await page.evaluate(() => {
+      const el = document.getElementById("styledDiv");
+      return window.getComputedStyle(el!).fontWeight;
+    });
+    expect(["bold", "700"]).toContain(fontWeight);
+  });
+});
diff --git a/evals/deterministic/tests/bringToFront.test.ts b/evals/deterministic/tests/bringToFront.test.ts
@@ -0,0 +1,37 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - bringToFront", () => {
+  test("should bring a background page to the front and allow further actions", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const { page: page1 } = stagehand;
+
+    const page2 = await stagehand.context.newPage();
+    await page2.goto("https://example.com");
+    const page2Title = await page2.title();
+    console.log("Page2 Title:", page2Title);
+
+    await page1.goto("https://www.google.com");
+    const page1TitleBefore = await page1.title();
+    console.log("Page1 Title before:", page1TitleBefore);
+
+    await page1.bringToFront();
+
+    await page1.goto("https://www.browserbase.com");
+    const page1TitleAfter = await page1.title();
+    console.log("Page1 Title after:", page1TitleAfter);
+
+    await page2.bringToFront();
+    const page2URLBefore = page2.url();
+    console.log("Page2 URL before navigation:", page2URLBefore);
+
+    await stagehand.close();
+
+    expect(page1TitleBefore).toContain("Google");
+    expect(page1TitleAfter).toContain("Browserbase");
+    expect(page2Title).toContain("Example Domain");
+  });
+});
diff --git a/evals/deterministic/tests/content.test.ts b/evals/deterministic/tests/content.test.ts
@@ -0,0 +1,18 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - content", () => {
+  test("should retrieve the full HTML content of the page", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const page = stagehand.page;
+    await page.goto("https://example.com");
+    const html = await page.content();
+    expect(html).toContain("<title>Example Domain</title>");
+    expect(html).toContain("<h1>Example Domain</h1>");
+
+    await stagehand.close();
+  });
+});
diff --git a/evals/deterministic/tests/evaluate.test.ts b/evals/deterministic/tests/evaluate.test.ts
@@ -0,0 +1,31 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - JavaScript Evaluation", () => {
+  test("can evaluate JavaScript in the page context", async () => {
+    const stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+
+    const page = stagehand.page;
+
+    await page.goto("https://example.com");
+
+    const sum = await page.evaluate(() => 2 + 2);
+    expect(sum).toBe(4);
+
+    const pageTitle = await page.evaluate(() => document.title);
+    expect(pageTitle).toMatch(/example/i);
+
+    const obj = await page.evaluate(() => {
+      return {
+        message: "Hello from the browser",
+        userAgent: navigator.userAgent,
+      };
+    });
+    expect(obj).toHaveProperty("message", "Hello from the browser");
+    expect(obj.userAgent).toBeDefined();
+
+    await stagehand.close();
+  });
+});
diff --git a/evals/deterministic/tests/expose.test.ts b/evals/deterministic/tests/expose.test.ts
@@ -0,0 +1,63 @@
+import { test, expect } from "@playwright/test";
+import { Stagehand } from "../../../lib";
+import StagehandConfig from "../stagehand.config";
+
+test.describe("StagehandPage - evaluateHandle, exposeBinding, exposeFunction", () => {
+  let stagehand: Stagehand;
+
+  test.beforeAll(async () => {
+    stagehand = new Stagehand(StagehandConfig);
+    await stagehand.init();
+  });
+
+  test.afterAll(async () => {
+    await stagehand.close();
+  });
+
+  test("demonstrates evaluateHandle, exposeBinding, and exposeFunction", async () => {
+    const { page } = stagehand;
+
+    await page.setContent(`
+      <html>
+        <body>
+          <div id="myDiv">Initial Text</div>
+        </body>
+      </html>
+    `);
+
+    const divHandle = await page.evaluateHandle(() => {
+      return document.getElementById("myDiv");
+    });
+    await divHandle.evaluate((div, newText) => {
+      div.textContent = newText;
+    }, "Text updated via evaluateHandle");
+
+    const text = await page.locator("#myDiv").textContent();
+    expect(text).toBe("Text updated via evaluateHandle");
+
+    await page.exposeBinding("myBinding", async (source, arg: string) => {
+      console.log("myBinding called from page with arg:", arg);
+      return `Node responded with: I got your message: "${arg}"`;
+    });
+
+    const responseFromBinding = await page.evaluate(async () => {
+      const w = window as typeof window & {
+        myBinding?: (arg: string) => Promise<string>;
+      };
+      return w.myBinding?.("Hello from the browser");
+    });
+    expect(responseFromBinding).toMatch(/I got your message/);
+
+    await page.exposeFunction("addNumbers", (a: number, b: number) => {
+      return a + b;
+    });
+
+    const sum = await page.evaluate(async () => {
+      const w = window as typeof window & {
+        addNumbers?: (a: number, b: number) => number;
+      };
+      return w.addNumbers?.(3, 7);
+    });
+    expect(sum).toBe(10);
+  });
+});