From e935f10a742b18b3512708901028ed3cdbd8f09a Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Mon, 2 Sep 2024 17:28:19 +0300 Subject: [PATCH 001/164] feat(copilot): introduce `CopilotFacade` type. --- detox-copilot/src/index.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/detox-copilot/src/index.ts b/detox-copilot/src/index.ts index 0231f0d719..eb966e2f6b 100644 --- a/detox-copilot/src/index.ts +++ b/detox-copilot/src/index.ts @@ -1,22 +1,29 @@ import { Copilot } from "@/Copilot"; -export const copilot = { +export type CopilotFacade = { + init: (config: Config) => void; + reset: () => void; + act: (action: string) => Promise; + assert: (assertion: string) => Promise; +}; + +export const copilot: CopilotFacade = { init: (config: Config) => { Copilot.init(config); }, reset: () => { Copilot.getInstance().reset(); }, - act: async (action: string) => { - return await Copilot.getInstance().perform({ + act: (action: string) => { + return Copilot.getInstance().perform({ type: 'action', value: action }); }, - assert: async (assertion: string) => { - return await Copilot.getInstance().perform({ + assert: (assertion: string) => { + return Copilot.getInstance().perform({ type: 'assertion', value: assertion }); } -} +}; From f7b2ca9a023b427d932617621b0439f963643dc8 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Mon, 2 Sep 2024 17:28:44 +0300 Subject: [PATCH 002/164] docs(copilot): update readme file. --- detox-copilot/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detox-copilot/README.md b/detox-copilot/README.md index 1106cc55d6..9a3d3f596c 100644 --- a/detox-copilot/README.md +++ b/detox-copilot/README.md @@ -10,7 +10,7 @@ It provides APIs to perform actions and assertions within your Detox tests while We will provide a high-level overview of the API that Detox Copilot will expose, this is a work in progress and the final API may differ. We will also provide a more extensive documentation once the API is finalized. -- `copilot.init(config)`: Initializes the Copilot with the provided configuration, must be called before using Copilot, e.g. `copilot.init(...)` -- `copilot.reset()`: Resets the Copilot by clearing the previous steps, e.g. `copilot.reset()` +- `init(config)`: Initializes the Copilot with the provided configuration, must be called before using Copilot, e.g. `copilot.init(...)` +- `reset()`: Resets the Copilot by clearing the previous steps, e.g. `copilot.reset()` - `act(prompt)`: Semantic action invocation, e.g. `copilot.act('tap the sign-in button')` - `assert(prompt)`: Semantic assertion invocation, e.g. `copilot.assert('the sign-in button is visible')` From e457fdd6b239c0943efc9a29b49b87b81975e029 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Mon, 2 Sep 2024 17:40:29 +0300 Subject: [PATCH 003/164] refactor(copilot): add type and document the copilot facade. --- detox-copilot/src/CopilotFacade.ts | 34 ++++++++++++++++++++++++++++++ detox-copilot/src/index.ts | 12 ++++------- 2 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 detox-copilot/src/CopilotFacade.ts diff --git a/detox-copilot/src/CopilotFacade.ts b/detox-copilot/src/CopilotFacade.ts new file mode 100644 index 0000000000..39d3ff49d5 --- /dev/null +++ b/detox-copilot/src/CopilotFacade.ts @@ -0,0 +1,34 @@ +/** + * Public API for interacting with the Copilot. + */ +export type CopilotFacade = { + /** + * Initializes the Copilot with the given configuration. + * Must be called before any other Copilot methods. + * @param config The configuration for the Copilot. + */ + init: (config: Config) => void; + + /** + * Resets the Copilot instance. + * Must be called before each test to ensure a clean state (the Copilot uses the operations history as part of + * its context). + */ + reset: () => void; + + /** + * Performs an action in the app. + * @param action The action to perform (in free-form text). + * @example Tap on the login button + * @example Scroll down to the 7th item in the Events list + */ + act: (action: string) => Promise; + + /** + * Asserts a condition in the app. + * @param assertion The assertion to check (in free-form text). + * @example The welcome message should be visible + * @example The welcome message text should be "Hello, world!" + */ + assert: (assertion: string) => Promise; +}; diff --git a/detox-copilot/src/index.ts b/detox-copilot/src/index.ts index eb966e2f6b..7841a12197 100644 --- a/detox-copilot/src/index.ts +++ b/detox-copilot/src/index.ts @@ -1,13 +1,7 @@ import { Copilot } from "@/Copilot"; +import {CopilotFacade} from "@/CopilotFacade"; -export type CopilotFacade = { - init: (config: Config) => void; - reset: () => void; - act: (action: string) => Promise; - assert: (assertion: string) => Promise; -}; - -export const copilot: CopilotFacade = { +const copilot: CopilotFacade = { init: (config: Config) => { Copilot.init(config); }, @@ -27,3 +21,5 @@ export const copilot: CopilotFacade = { }); } }; + +export default copilot; From dc7c1f070fc4eaa391a764b21b20aff608deea29 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 3 Sep 2024 18:15:00 +0300 Subject: [PATCH 004/164] feat(copilot): expose copilot types publicly. --- detox-copilot/package.json | 3 ++ detox-copilot/src/Copilot.test.ts | 1 + detox-copilot/src/Copilot.ts | 1 + detox-copilot/src/CopilotFacade.ts | 34 ------------- .../src/actions/StepPerformer.test.ts | 1 + detox-copilot/src/actions/StepPerformer.ts | 1 + detox-copilot/src/index.ts | 11 ++++- .../src/integration tests/index.test.ts | 13 +++-- .../src/{types/index.ts => types.ts} | 49 ++++++++++++++++--- detox-copilot/src/utils/CodeEvaluator.test.ts | 1 - detox-copilot/src/utils/PromptCreator.test.ts | 1 + detox-copilot/src/utils/PromptCreator.ts | 2 + detox-copilot/src/utils/SnapshotManager.ts | 2 + 13 files changed, 70 insertions(+), 50 deletions(-) delete mode 100644 detox-copilot/src/CopilotFacade.ts rename detox-copilot/src/{types/index.ts => types.ts} (69%) diff --git a/detox-copilot/package.json b/detox-copilot/package.json index a75ea0a95f..e46da38fb4 100644 --- a/detox-copilot/package.json +++ b/detox-copilot/package.json @@ -12,6 +12,9 @@ "license": "MIT", "main": "dist/index.js", "types": "dist/index.d.ts", + "files": [ + "dist" + ], "publishConfig": { "registry": "https://registry.npmjs.org" }, diff --git a/detox-copilot/src/Copilot.test.ts b/detox-copilot/src/Copilot.test.ts index 7044b621d4..bdcac28873 100644 --- a/detox-copilot/src/Copilot.test.ts +++ b/detox-copilot/src/Copilot.test.ts @@ -1,6 +1,7 @@ import { Copilot } from '@/Copilot'; import { StepPerformer } from '@/actions/StepPerformer'; import { CopilotError } from '@/errors/CopilotError'; +import {Config, ExecutionStep} from "@/types"; jest.mock('@/actions/StepPerformer'); diff --git a/detox-copilot/src/Copilot.ts b/detox-copilot/src/Copilot.ts index 3822377d2a..2f70bb8f94 100644 --- a/detox-copilot/src/Copilot.ts +++ b/detox-copilot/src/Copilot.ts @@ -3,6 +3,7 @@ import {PromptCreator} from "@/utils/PromptCreator"; import {CodeEvaluator} from "@/utils/CodeEvaluator"; import {SnapshotManager} from "@/utils/SnapshotManager"; import {StepPerformer} from "@/actions/StepPerformer"; +import {Config, ExecutionStep} from "@/types"; /** * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework. diff --git a/detox-copilot/src/CopilotFacade.ts b/detox-copilot/src/CopilotFacade.ts deleted file mode 100644 index 39d3ff49d5..0000000000 --- a/detox-copilot/src/CopilotFacade.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Public API for interacting with the Copilot. - */ -export type CopilotFacade = { - /** - * Initializes the Copilot with the given configuration. - * Must be called before any other Copilot methods. - * @param config The configuration for the Copilot. - */ - init: (config: Config) => void; - - /** - * Resets the Copilot instance. - * Must be called before each test to ensure a clean state (the Copilot uses the operations history as part of - * its context). - */ - reset: () => void; - - /** - * Performs an action in the app. - * @param action The action to perform (in free-form text). - * @example Tap on the login button - * @example Scroll down to the 7th item in the Events list - */ - act: (action: string) => Promise; - - /** - * Asserts a condition in the app. - * @param assertion The assertion to check (in free-form text). - * @example The welcome message should be visible - * @example The welcome message text should be "Hello, world!" - */ - assert: (assertion: string) => Promise; -}; diff --git a/detox-copilot/src/actions/StepPerformer.test.ts b/detox-copilot/src/actions/StepPerformer.test.ts index 5b89dc4e1b..82003c04a3 100644 --- a/detox-copilot/src/actions/StepPerformer.test.ts +++ b/detox-copilot/src/actions/StepPerformer.test.ts @@ -2,6 +2,7 @@ import { StepPerformer } from '@/actions/StepPerformer'; import { PromptCreator } from '@/utils/PromptCreator'; import { CodeEvaluator } from '@/utils/CodeEvaluator'; import { SnapshotManager } from '@/utils/SnapshotManager'; +import {ExecutionStep, PromptHandler} from "@/types"; jest.mock('@/utils/PromptCreator'); jest.mock('@/utils/CodeEvaluator'); diff --git a/detox-copilot/src/actions/StepPerformer.ts b/detox-copilot/src/actions/StepPerformer.ts index 15958b16c2..91365af642 100644 --- a/detox-copilot/src/actions/StepPerformer.ts +++ b/detox-copilot/src/actions/StepPerformer.ts @@ -1,6 +1,7 @@ import { PromptCreator } from '@/utils/PromptCreator'; import { CodeEvaluator } from '@/utils/CodeEvaluator'; import { SnapshotManager } from '@/utils/SnapshotManager'; +import {ExecutionStep, PromptHandler} from "@/types"; export class StepPerformer { constructor( diff --git a/detox-copilot/src/index.ts b/detox-copilot/src/index.ts index 7841a12197..3ffeb5f5bd 100644 --- a/detox-copilot/src/index.ts +++ b/detox-copilot/src/index.ts @@ -1,5 +1,5 @@ import { Copilot } from "@/Copilot"; -import {CopilotFacade} from "@/CopilotFacade"; +import {CopilotFacade, Config} from "@/types"; const copilot: CopilotFacade = { init: (config: Config) => { @@ -23,3 +23,12 @@ const copilot: CopilotFacade = { }; export default copilot; + +export { + CopilotFacade, + Config, + PromptHandler, + TestingFrameworkDriver, + TestingFrameworkAPI, + TestingFrameworkAPIMethod +} from './types'; diff --git a/detox-copilot/src/integration tests/index.test.ts b/detox-copilot/src/integration tests/index.test.ts index ff03691ebf..de054785cb 100644 --- a/detox-copilot/src/integration tests/index.test.ts +++ b/detox-copilot/src/integration tests/index.test.ts @@ -1,7 +1,6 @@ -import { copilot } from "@/index"; -import { CopilotError } from '@/errors/CopilotError'; +import copilot from "@/index"; import { Copilot } from "@/Copilot"; -import { CodeEvaluationError } from "@/errors/CodeEvaluationError"; +import {PromptHandler, TestingFrameworkDriver} from "@/types"; describe('Integration', () => { let mockFrameworkDriver: jest.Mocked; @@ -37,12 +36,12 @@ describe('Integration', () => { Copilot['instance'] = undefined; }); - it('should throw an error when act is called before initialization', async () => { - await expect(copilot.act('Some action')).rejects.toThrow(); + it('should synchronously throw an error when act is called before initialization', async () => { + expect(() => copilot.act('Some action')).toThrow(); }); - it('should throw an error when expect is called before initialization', async () => { - await expect(copilot.act('Some assertion')).rejects.toThrow(); + it('should synchronously throw an error when expect is called before initialization', async () => { + expect(() => copilot.act('Some assertion')).toThrow(); }); }); diff --git a/detox-copilot/src/types/index.ts b/detox-copilot/src/types.ts similarity index 69% rename from detox-copilot/src/types/index.ts rename to detox-copilot/src/types.ts index 5e7c091112..b68d64d618 100644 --- a/detox-copilot/src/types/index.ts +++ b/detox-copilot/src/types.ts @@ -1,7 +1,42 @@ +/** + * Interface for interacting with the Copilot. + */ +export interface CopilotFacade { + /** + * Initializes the Copilot with the given configuration. + * Must be called before any other Copilot methods. + * @param config The configuration for the Copilot. + */ + init: (config: Config) => void; + + /** + * Resets the Copilot instance. + * Must be called before each test to ensure a clean state (the Copilot uses the operations history as part of + * its context). + */ + reset: () => void; + + /** + * Performs an action in the app. + * @param action The action to perform (in free-form text). + * @example Tap on the login button + * @example Scroll down to the 7th item in the Events list + */ + act: (action: string) => Promise; + + /** + * Asserts a condition in the app. + * @param assertion The assertion to check (in free-form text). + * @example The welcome message should be visible + * @example The welcome message text should be "Hello, world!" + */ + assert: (assertion: string) => Promise; +} + /** * Interface for the testing driver that will be used to interact with the underlying testing framework. */ -interface TestingFrameworkDriver { +export interface TestingFrameworkDriver { /** * Takes a snapshot of the current screen and returns the path to the saved image. * If the driver does not support image, return undefined. @@ -25,7 +60,7 @@ interface TestingFrameworkDriver { * @property actions The available actions API of the testing framework. * @property assertions The available assertions API of the testing framework. */ -type TestingFrameworkAPI = { +export type TestingFrameworkAPI = { matchers: TestingFrameworkAPIMethod[]; actions: TestingFrameworkAPIMethod[]; assertions: TestingFrameworkAPIMethod[]; @@ -49,7 +84,7 @@ type TestingFrameworkAPI = { * ] * }; */ -type TestingFrameworkAPIMethod = { +export type TestingFrameworkAPIMethod = { signature: string; description: string; example: string; @@ -59,7 +94,7 @@ type TestingFrameworkAPIMethod = { /** * Interface for the prompt handler that will be used to interact with the AI service (e.g. OpenAI). */ -interface PromptHandler { +export interface PromptHandler { /** * Sends a prompt to the AI service and returns the response. * @param prompt The prompt to send to the AI service. @@ -77,7 +112,7 @@ interface PromptHandler { /** * Configuration options for Copilot. */ -interface Config { +export interface Config { /** * The testing driver to use for interacting with the underlying testing framework. */ @@ -94,7 +129,7 @@ interface Config { * @property type The type of the step. * @property value The prompt for the step. */ -type ExecutionStep = { +export type ExecutionStep = { type: ExecutionStepType; value: string; } @@ -102,4 +137,4 @@ type ExecutionStep = { /** * Represents the type of step in the test script. */ -type ExecutionStepType = 'action' | 'assertion'; +export type ExecutionStepType = 'action' | 'assertion'; diff --git a/detox-copilot/src/utils/CodeEvaluator.test.ts b/detox-copilot/src/utils/CodeEvaluator.test.ts index d8582a7cca..cb45f0187a 100644 --- a/detox-copilot/src/utils/CodeEvaluator.test.ts +++ b/detox-copilot/src/utils/CodeEvaluator.test.ts @@ -1,5 +1,4 @@ import { CodeEvaluator } from '@/utils/CodeEvaluator'; -import { CodeEvaluationError } from '@/errors/CodeEvaluationError'; describe('CodeEvaluator', () => { let codeEvaluator: CodeEvaluator; diff --git a/detox-copilot/src/utils/PromptCreator.test.ts b/detox-copilot/src/utils/PromptCreator.test.ts index 2a7de93fe9..246a0dfd42 100644 --- a/detox-copilot/src/utils/PromptCreator.test.ts +++ b/detox-copilot/src/utils/PromptCreator.test.ts @@ -1,4 +1,5 @@ import { PromptCreator } from './PromptCreator'; +import {ExecutionStep, TestingFrameworkAPI} from "@/types"; const mockAPI: TestingFrameworkAPI = { actions: [ diff --git a/detox-copilot/src/utils/PromptCreator.ts b/detox-copilot/src/utils/PromptCreator.ts index 5ba136b2c7..f8e473c7c7 100644 --- a/detox-copilot/src/utils/PromptCreator.ts +++ b/detox-copilot/src/utils/PromptCreator.ts @@ -1,3 +1,5 @@ +import {ExecutionStep, ExecutionStepType, TestingFrameworkAPI, TestingFrameworkAPIMethod} from "@/types"; + export class PromptCreator { constructor(private availableAPI: TestingFrameworkAPI) {} diff --git a/detox-copilot/src/utils/SnapshotManager.ts b/detox-copilot/src/utils/SnapshotManager.ts index 8679ebcd53..7cd6534f47 100644 --- a/detox-copilot/src/utils/SnapshotManager.ts +++ b/detox-copilot/src/utils/SnapshotManager.ts @@ -1,3 +1,5 @@ +import {TestingFrameworkDriver} from "@/types"; + export class SnapshotManager { constructor(private driver: TestingFrameworkDriver) {} From 33814a12e5cd7ea6a2299f578f7a7a72f5fd4744 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 3 Sep 2024 20:57:29 +0300 Subject: [PATCH 005/164] refactor(CopilotDriver): rename to `detoxCopilotFrameworkDriver`. --- detox/src/copilot/CopilotDriver.js | 140 ------------------ .../copilot/detoxCopilotFrameworkDriver.js | 85 +++++++++++ 2 files changed, 85 insertions(+), 140 deletions(-) delete mode 100644 detox/src/copilot/CopilotDriver.js create mode 100644 detox/src/copilot/detoxCopilotFrameworkDriver.js diff --git a/detox/src/copilot/CopilotDriver.js b/detox/src/copilot/CopilotDriver.js deleted file mode 100644 index e734aaab33..0000000000 --- a/detox/src/copilot/CopilotDriver.js +++ /dev/null @@ -1,140 +0,0 @@ -const { device } = require('../..'); - -class DetoxDriver { - constructor() { - this.availableAPI = { - matchers: [ - { - signature: 'by.id(id: string)', - description: 'Matches elements by their test ID.', - example: "element(by.id('loginButton'))", - guidelines: ['Always use test-ids (accessibility identifiers) from the UI hierarchy to identify elements.'] - }, - { - signature: 'by.text(text: string)', - description: 'Matches elements by their text.', - example: "element(by.text('Login'))", - guidelines: ['Avoid using text matchers when possible, prefer test-ids.'] - }, - { - signature: 'by.type(type: string)', - description: 'Matches elements by their type.', - example: "element(by.type('RCTTextInput'))", - guidelines: ['Use type matchers as a last resort, prefer test-ids.'] - } - ], - actions: [ - { - signature: 'tap(point?: Point2D)', - description: 'Simulates tap on an element', - example: "await element(by.id('loginButton')).tap();", - guidelines: ['Use element(by.id(\'testID\')) to locate elements.'] - }, - { - signature: 'longPress(point?: Point2D, duration?: number)', - description: 'Simulates long press on an element', - example: "await element(by.id('menuItem')).longPress();", - guidelines: ['If the target element is not accessible, interact with its container or the most relevant parent element.'] - }, - { - signature: 'typeText(text: string)', - description: 'Types text into a text field', - example: "await element(by.id('usernameInput')).typeText('myusername');", - guidelines: ['Typing can only be done on text field elements.'] - }, - { - signature: 'replaceText(text: string)', - description: 'Replaces text in a text field', - example: "await element(by.id('usernameInput')).replaceText('newusername');", - guidelines: ['Use this to replace existing text in a field.'] - }, - { - signature: 'clearText()', - description: 'Clears text from a text field', - example: "await element(by.id('usernameInput')).clearText();", - guidelines: ['Use this to clear existing text from a field.'] - }, - { - signature: 'scrollTo(edge: Direction, startPositionX?: number, startPositionY?: number)', - description: 'Scrolls to an edge', - example: "await element(by.id('scrollView')).scrollTo('bottom');", - guidelines: ['Scrolling must be done only on scroll-view elements.'] - }, - { - signature: 'scrollToIndex(index: Number)', - description: 'Scrolls to a specific index', - example: "await element(by.id('flatList')).scrollToIndex(5);", - guidelines: ['Use this for scrolling to a specific item in a list.'] - }, - { - signature: 'adjustSliderToPosition(newPosition: number)', - description: 'Adjusts slider to a position', - example: "await element(by.id('slider')).adjustSliderToPosition(0.75);", - guidelines: ['The position should be a number between 0 and 1.'] - }, - { - signature: 'setColumnToValue(column: number, value: string)', - description: 'Sets picker view column to a value (iOS only)', - example: "await element(by.id('datePicker')).setColumnToValue(1, '2023');", - guidelines: ['This is only available on iOS.'] - }, - { - signature: 'performAccessibilityAction(actionName: string)', - description: 'Triggers an accessibility action', - example: "await element(by.id('button')).performAccessibilityAction('longpress');", - guidelines: ['Use the provided value from the intent and do not change it.'] - }, - { - signature: 'swipe(direction: Direction, speed?: Speed, percentage?: number)', - description: 'Swipes in the specified direction', - example: "await element(by.id('card')).swipe('left', 'fast');", - guidelines: ['Use this for swiping gestures on elements.'] - }, - { - signature: 'pinch(scale: number, speed?: Speed, angle?: number)', - description: 'Performs a pinch gesture (iOS only)', - example: "await element(by.id('image')).pinch(0.5);", - guidelines: ['This is only available on iOS. Scale < 1 zooms out, scale > 1 zooms in.'] - } - ], - assertions: [ - { - signature: 'toBeVisible()', - description: 'Asserts that the element is visible', - example: "await expect(element(by.id('loginButton'))).toBeVisible();", - guidelines: ['Use this to check if an element is visible on the screen.'] - }, - { - signature: 'toExist()', - description: 'Asserts that the element exists', - example: "await expect(element(by.id('username'))).toExist();", - guidelines: ['Use this to check if an element exists in the hierarchy, even if not visible.'] - }, - { - signature: 'toHaveText(text: string)', - description: 'Asserts that the element has the specified text', - example: "await expect(element(by.id('label'))).toHaveText('Hello, World!');", - guidelines: ['Use this to check the text content of an element.'] - }, - { - signature: 'toHaveValue(value: string)', - description: 'Asserts that the element has the specified value', - example: "await expect(element(by.id('slider'))).toHaveValue('0.5');", - guidelines: ['Use this to check the value of an element.'] - } - ] - }; - } - - async captureSnapshotImage() { - const fileName = `snapshot_${Date.now()}.png`; - await device.takeScreenshot(fileName); - return fileName; - } - - async captureViewHierarchyString() { - return device.generateViewHierarchyXml(); - } -} - -module.exports = DetoxDriver; diff --git a/detox/src/copilot/detoxCopilotFrameworkDriver.js b/detox/src/copilot/detoxCopilotFrameworkDriver.js new file mode 100644 index 0000000000..fe83313078 --- /dev/null +++ b/detox/src/copilot/detoxCopilotFrameworkDriver.js @@ -0,0 +1,85 @@ +const { device } = require('../..'); + +const detoxCopilotFrameworkDriver = { + availableAPI: { + matchers: [ + { + signature: 'by.id(id: string)', + description: 'Matches elements by their test ID.', + example: "element(by.id('loginButton'))", + guidelines: ['Always use test-ids (accessibility identifiers) from the UI hierarchy to identify elements.'] + }, + { + signature: 'by.text(text: string)', + description: 'Matches elements by their text.', + example: "element(by.text('Login'))", + guidelines: ['Avoid using text matchers when possible, prefer test-ids.'] + }, + { + signature: 'by.type(type: string)', + description: 'Matches elements by their type.', + example: "element(by.type('RCTTextInput'))", + guidelines: ['Use type matchers as a last resort, prefer test-ids.'] + } + ], + actions: [ + { + signature: 'tap(point?: Point2D)', + description: 'Simulates tap on an element', + example: "await element(by.id('loginButton')).tap();", + guidelines: ['Use element(by.id(\'testID\')) to locate elements.'] + }, + { + signature: 'longPress(point?: Point2D, duration?: number)', + description: 'Simulates long press on an element', + example: "await element(by.id('menuItem')).longPress();", + guidelines: ['If the target element is not accessible, interact with its container or the most relevant parent element.'] + }, + { + signature: 'typeText(text: string)', + description: 'Types text into a text field', + example: "await element(by.id('usernameInput')).typeText('myusername');", + guidelines: ['Typing can only be done on text field elements.'] + }, + // ... (other actions remain the same) + ], + assertions: [ + { + signature: 'toBeVisible()', + description: 'Asserts that the element is visible', + example: "await expect(element(by.id('loginButton'))).toBeVisible();", + guidelines: ['Use this to check if an element is visible on the screen.'] + }, + { + signature: 'toExist()', + description: 'Asserts that the element exists', + example: "await expect(element(by.id('username'))).toExist();", + guidelines: ['Use this to check if an element exists in the hierarchy, even if not visible.'] + }, + { + signature: 'toHaveText(text: string)', + description: 'Asserts that the element has the specified text', + example: "await expect(element(by.id('label'))).toHaveText('Hello, World!');", + guidelines: ['Use this to check the text content of an element.'] + }, + { + signature: 'toHaveValue(value: string)', + description: 'Asserts that the element has the specified value', + example: "await expect(element(by.id('slider'))).toHaveValue('0.5');", + guidelines: ['Use this to check the value of an element.'] + } + ] + }, + + captureSnapshotImage: async function() { + const fileName = `snapshot_${Date.now()}.png`; + await device.takeScreenshot(fileName); + return fileName; + }, + + captureViewHierarchyString: async function() { + return device.generateViewHierarchyXml(); + } +}; + +module.exports = detoxCopilotFrameworkDriver; From ada2685e91b36d7ed8e9ea1114fab49bba886611 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 3 Sep 2024 20:58:08 +0300 Subject: [PATCH 006/164] feat(DetoxCopilot): initial commit. --- detox/src/copilot/DetoxCopilot.js | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 detox/src/copilot/DetoxCopilot.js diff --git a/detox/src/copilot/DetoxCopilot.js b/detox/src/copilot/DetoxCopilot.js new file mode 100644 index 0000000000..0081a6ffdd --- /dev/null +++ b/detox/src/copilot/DetoxCopilot.js @@ -0,0 +1,37 @@ +const copilot = require('detox-copilot').default; + +const detoxCopilotFrameworkDriver = require('./detoxCopilotFrameworkDriver'); + +class DetoxCopilot { + constructor() { + this.isInitialized = false; + } + + init({ promptHandler }) { + copilot.init({ + frameworkDriver: detoxCopilotFrameworkDriver, + promptHandler: promptHandler + }); + + this.isInitialized = true; + } + + resetIfNeeded() { + if (!this.isInitialized) { + // Copilot is not initialized, nothing to reset + return; + } + + copilot.reset(); + } + + act(action) { + return copilot.act(action); + } + + assert(assertion) { + return copilot.assert(assertion); + } +} + +module.exports = DetoxCopilot; From 0dbc36337b1207a25ed1f651d10ceae871ee2060 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 3 Sep 2024 21:06:53 +0300 Subject: [PATCH 007/164] test(DetoxCopilot): add unit tests. --- detox/src/copilot/DetoxCopilot.test.js | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 detox/src/copilot/DetoxCopilot.test.js diff --git a/detox/src/copilot/DetoxCopilot.test.js b/detox/src/copilot/DetoxCopilot.test.js new file mode 100644 index 0000000000..6be17b12d9 --- /dev/null +++ b/detox/src/copilot/DetoxCopilot.test.js @@ -0,0 +1,71 @@ +const copilot = require('detox-copilot').default; + +const DetoxCopilot = require('./DetoxCopilot'); +const detoxCopilotFrameworkDriver = require('./detoxCopilotFrameworkDriver'); + +jest.mock('detox-copilot', () => ({ + default: { + init: jest.fn(), + reset: jest.fn(), + act: jest.fn(), + assert: jest.fn(), + }, +})); + +jest.mock('./detoxCopilotFrameworkDriver', () => ({})); + +describe('DetoxCopilot', () => { + let detoxCopilot; + const mockPromptHandler = jest.fn(); + + beforeEach(() => { + detoxCopilot = new DetoxCopilot(); + jest.clearAllMocks(); + }); + + describe('init', () => { + it('should initialize copilot with correct parameters', () => { + detoxCopilot.init({ promptHandler: mockPromptHandler }); + + expect(copilot.init).toHaveBeenCalledWith({ + frameworkDriver: detoxCopilotFrameworkDriver, + promptHandler: mockPromptHandler, + }); + expect(detoxCopilot.isInitialized).toBe(true); + }); + }); + + describe('resetIfNeeded', () => { + it('should reset copilot if initialized', () => { + detoxCopilot.isInitialized = true; + detoxCopilot.resetIfNeeded(); + + expect(copilot.reset).toHaveBeenCalled(); + }); + + it('should not reset copilot if not initialized', () => { + detoxCopilot.isInitialized = false; + detoxCopilot.resetIfNeeded(); + + expect(copilot.reset).not.toHaveBeenCalled(); + }); + }); + + describe('act', () => { + it('should call copilot.act with the given action', async () => { + const action = 'test action'; + await detoxCopilot.act(action); + + expect(copilot.act).toHaveBeenCalledWith(action); + }); + }); + + describe('assert', () => { + it('should call copilot.assert with the given assertion', async () => { + const assertion = 'test assertion'; + await detoxCopilot.assert(assertion); + + expect(copilot.assert).toHaveBeenCalledWith(assertion); + }); + }); +}); From c10a67ab89d1aef3379647af8151652517429902 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Tue, 3 Sep 2024 21:18:56 +0300 Subject: [PATCH 008/164] feat(copilot): expose Copilot API from Detox. --- detox/detox.d.ts | 14 ++++++++++++++ detox/src/DetoxWorker.js | 5 +++++ detox/src/copilot/DetoxCopilot.js | 14 +++++++++++++- detox/src/realms/DetoxContext.js | 2 ++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/detox/detox.d.ts b/detox/detox.d.ts index a846c3b9b8..bd6ade3675 100644 --- a/detox/detox.d.ts +++ b/detox/detox.d.ts @@ -9,6 +9,7 @@ // * Dor Ben Baruch import { BunyanDebugStreamOptions } from 'bunyan-debug-stream'; +import { CopilotFacade, PromptHandler } from "detox-copilot"; declare global { namespace Detox { @@ -445,6 +446,8 @@ declare global { readonly system: SystemFacade; + readonly copilot: DetoxCopilotFacade; + readonly DetoxConstants: { userNotificationTriggers: { push: 'push'; @@ -1287,6 +1290,17 @@ declare global { element(systemMatcher: SystemMatcher): IndexableSystemElement; } + interface DetoxCopilotFacade extends Pick { + /** + * Initializes the Copilot with the given prompt handler. + * Must be called before any other Copilot methods. + * @param promptHandler The prompt handler to use. + */ + init: (promptHandler: DetoxCopilotPromptHandler) => void; + } + + interface DetoxCopilotPromptHandler extends PromptHandler {} + interface IndexableSystemElement extends SystemElement { /** * Choose from multiple elements matching the same matcher using index diff --git a/detox/src/DetoxWorker.js b/detox/src/DetoxWorker.js index 7d3f69a32f..cbda13461d 100644 --- a/detox/src/DetoxWorker.js +++ b/detox/src/DetoxWorker.js @@ -2,6 +2,7 @@ const CAF = require('caf'); const _ = require('lodash'); const Client = require('./client/Client'); +const DetoxCopilot = require('./copilot/DetoxCopilot'); const environmentFactory = require('./environmentFactory'); const { DetoxRuntimeErrorComposer } = require('./errors'); const { InvocationManager } = require('./invoke'); @@ -58,6 +59,8 @@ class DetoxWorker { this.by = null; /** @type {Detox.WebFacade} */ this.web = null; + /** @type {Detox.DetoxCopilotFacade} */ + this.copilot = null; this._deviceCookie = null; @@ -121,6 +124,8 @@ class DetoxWorker { runtimeDeviceFactory, } = environmentFactory.createFactories(deviceConfig); + this.copilot = new DetoxCopilot(); + const envValidator = envValidatorFactory.createValidator(); yield envValidator.validate(); diff --git a/detox/src/copilot/DetoxCopilot.js b/detox/src/copilot/DetoxCopilot.js index 0081a6ffdd..4bb87e99f0 100644 --- a/detox/src/copilot/DetoxCopilot.js +++ b/detox/src/copilot/DetoxCopilot.js @@ -2,12 +2,18 @@ const copilot = require('detox-copilot').default; const detoxCopilotFrameworkDriver = require('./detoxCopilotFrameworkDriver'); +/** + * @typedef {Object} Detox.DetoxCopilotFacade + */ class DetoxCopilot { constructor() { this.isInitialized = false; } - init({ promptHandler }) { + /** + * @param {Detox.DetoxCopilotPromptHandler} promptHandler + */ + init(promptHandler) { copilot.init({ frameworkDriver: detoxCopilotFrameworkDriver, promptHandler: promptHandler @@ -25,10 +31,16 @@ class DetoxCopilot { copilot.reset(); } + /** + * @param {String} action + */ act(action) { return copilot.act(action); } + /** + * @param {String} assertion + */ assert(assertion) { return copilot.assert(assertion); } diff --git a/detox/src/realms/DetoxContext.js b/detox/src/realms/DetoxContext.js index bf5111d343..630f40c8b3 100644 --- a/detox/src/realms/DetoxContext.js +++ b/detox/src/realms/DetoxContext.js @@ -81,6 +81,8 @@ class DetoxContext { web = funpermaproxy.callable(() => this[symbols.worker].web); + copilot = funpermaproxy.callable(() => this[symbols.worker].copilot); + get DetoxConstants() { return DetoxConstants; } From b49306ed223f7040563a0ec2672fcec401067a44 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 01:10:31 +0300 Subject: [PATCH 009/164] build(copilot): add tsc-alias to resolve alias paths. --- detox-copilot/package.json | 3 ++- detox-copilot/tsconfig.json | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/detox-copilot/package.json b/detox-copilot/package.json index e46da38fb4..a6412cd930 100644 --- a/detox-copilot/package.json +++ b/detox-copilot/package.json @@ -23,7 +23,7 @@ "url": "git+https://github.com/wix/Detox.git" }, "scripts": { - "build": "tsc", + "build": "tsc && tsc-alias", "test": "jest", "prepublishOnly": "npm run build" }, @@ -34,6 +34,7 @@ "@types/jest": "^29.5.12", "jest": "^29.7.0", "ts-jest": "^29.2.4", + "tsc-alias": "^1.8.10", "typescript": "^5.5.4" } } diff --git a/detox-copilot/tsconfig.json b/detox-copilot/tsconfig.json index 47373bd30c..8dcaecd274 100644 --- a/detox-copilot/tsconfig.json +++ b/detox-copilot/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2018", + "target": "es2022", "module": "commonjs", "declaration": true, "outDir": "./dist", @@ -10,9 +10,9 @@ "forceConsistentCasingInFileNames": true, "rootDir": "./src", "types": ["jest", "node"], - "baseUrl": "./", + "baseUrl": "./src", "paths": { - "@/*": ["src/*"] + "@/*": ["./*"] } }, "include": ["src/**/*"], From ca232096239378972a41627212d5c6716bdd2318 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 14:26:35 +0300 Subject: [PATCH 010/164] test(unit): add copilot constructor tests for DetoxWorker. --- detox/src/DetoxWorker.test.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/detox/src/DetoxWorker.test.js b/detox/src/DetoxWorker.test.js index 9946e7d1a9..f84d41ceec 100644 --- a/detox/src/DetoxWorker.test.js +++ b/detox/src/DetoxWorker.test.js @@ -3,6 +3,7 @@ const testSummaries = require('./artifacts/__mocks__/testSummaries.mock'); const configuration = require('./configuration'); const Deferred = require('./utils/Deferred'); +jest.mock('./copilot/DetoxCopilot'); jest.mock('./utils/logger'); jest.mock('./client/Client'); jest.mock('./utils/AsyncEmitter'); @@ -290,6 +291,29 @@ describe('DetoxWorker', () => { await expect(init).rejects.toThrowError('Mock validation failure'); }); }); + + describe('copilot initialization', () => { + let DetoxCopilot; + + beforeEach(async () => { + DetoxCopilot = require('./copilot/DetoxCopilot'); + + await init(); + }); + + it('should create a new DetoxCopilot instance', () => { + expect(DetoxCopilot).toHaveBeenCalledTimes(1); + }); + + it('should assign the DetoxCopilot instance to the copilot property', () => { + expect(detox.copilot).toBeDefined(); + expect(detox.copilot).toBeInstanceOf(DetoxCopilot); + }); + + it('should not initialize the copilot', () => { + expect(detox.copilot.init).not.toHaveBeenCalled(); + }); + }); }); describe('when DetoxWorker#@onTestStart() is called', () => { From de02672d082dbecb0eda50c6f80dd7be34917a81 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 14:26:54 +0300 Subject: [PATCH 011/164] test(unit): fix DetoxCopilot test. --- detox/src/copilot/DetoxCopilot.js | 12 ------------ detox/src/copilot/DetoxCopilot.test.js | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/detox/src/copilot/DetoxCopilot.js b/detox/src/copilot/DetoxCopilot.js index 4bb87e99f0..7c4fc82966 100644 --- a/detox/src/copilot/DetoxCopilot.js +++ b/detox/src/copilot/DetoxCopilot.js @@ -2,17 +2,11 @@ const copilot = require('detox-copilot').default; const detoxCopilotFrameworkDriver = require('./detoxCopilotFrameworkDriver'); -/** - * @typedef {Object} Detox.DetoxCopilotFacade - */ class DetoxCopilot { constructor() { this.isInitialized = false; } - /** - * @param {Detox.DetoxCopilotPromptHandler} promptHandler - */ init(promptHandler) { copilot.init({ frameworkDriver: detoxCopilotFrameworkDriver, @@ -31,16 +25,10 @@ class DetoxCopilot { copilot.reset(); } - /** - * @param {String} action - */ act(action) { return copilot.act(action); } - /** - * @param {String} assertion - */ assert(assertion) { return copilot.assert(assertion); } diff --git a/detox/src/copilot/DetoxCopilot.test.js b/detox/src/copilot/DetoxCopilot.test.js index 6be17b12d9..5a4f1093aa 100644 --- a/detox/src/copilot/DetoxCopilot.test.js +++ b/detox/src/copilot/DetoxCopilot.test.js @@ -25,7 +25,7 @@ describe('DetoxCopilot', () => { describe('init', () => { it('should initialize copilot with correct parameters', () => { - detoxCopilot.init({ promptHandler: mockPromptHandler }); + detoxCopilot.init(mockPromptHandler); expect(copilot.init).toHaveBeenCalledWith({ frameworkDriver: detoxCopilotFrameworkDriver, From 7e06d2e77def48c6eb682841def0c670df6d01aa Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 14:34:16 +0300 Subject: [PATCH 012/164] feat(DetoxWorker): reset detox copilot if-needed before each test. --- detox/src/DetoxWorker.js | 3 +++ detox/src/DetoxWorker.test.js | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/detox/src/DetoxWorker.js b/detox/src/DetoxWorker.js index cbda13461d..90ba07937f 100644 --- a/detox/src/DetoxWorker.js +++ b/detox/src/DetoxWorker.js @@ -226,6 +226,9 @@ class DetoxWorker { onTestStart = function* (_signal, testSummary) { this._validateTestSummary('beforeEach', testSummary); + // Copilot is reset before each test to ensure a clean state + this.copilot.resetIfNeeded(); + yield this._dumpUnhandledErrorsIfAny({ pendingRequests: false, testName: testSummary.fullName, diff --git a/detox/src/DetoxWorker.test.js b/detox/src/DetoxWorker.test.js index f84d41ceec..2b08f577c3 100644 --- a/detox/src/DetoxWorker.test.js +++ b/detox/src/DetoxWorker.test.js @@ -347,15 +347,18 @@ describe('DetoxWorker', () => { it('should notify artifacts manager about "testStart', () => expect(artifactsManager.onTestStart).toHaveBeenCalledWith(testSummaries.running())); + it('should reset copilot if needed', async () => { + expect(detox.copilot.resetIfNeeded).toHaveBeenCalled(); + }); + it('should not relaunch app', async () => { - await detox.onTestStart(testSummaries.running()); expect(runtimeDevice.launchApp).not.toHaveBeenCalled(); }); it('should not dump pending network requests', async () => { - await detox.onTestStart(testSummaries.running()); expect(client().dumpPendingRequests).not.toHaveBeenCalled(); }); + }); }); From 93f08d043c0653dfb527322aa0f9d523b801a6fd Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 14:34:52 +0300 Subject: [PATCH 013/164] test(unit): remove duplicated unit test. --- detox/src/DetoxWorker.test.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/detox/src/DetoxWorker.test.js b/detox/src/DetoxWorker.test.js index 2b08f577c3..374ef3ae4e 100644 --- a/detox/src/DetoxWorker.test.js +++ b/detox/src/DetoxWorker.test.js @@ -334,13 +334,6 @@ describe('DetoxWorker', () => { })).rejects.toThrowError(/Invalid test summary status/); }); - it('should validate test summary status', async () => { - await expect(detox.onTestStart({ - ...testSummaries.running(), - status: undefined, - })).rejects.toThrowError(/Invalid test summary status/); - }); - describe('with a valid test summary', () => { beforeEach(() => detox.onTestStart(testSummaries.running())); From f4b4138a30aac18706c3419c50dce55c5be0ea70 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 14:43:21 +0300 Subject: [PATCH 014/164] feat(DetoxWorker): always reset copilot before test if needed. --- detox/src/DetoxWorker.js | 4 ++-- detox/src/DetoxWorker.test.js | 30 ++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/detox/src/DetoxWorker.js b/detox/src/DetoxWorker.js index 90ba07937f..b033e0c45f 100644 --- a/detox/src/DetoxWorker.js +++ b/detox/src/DetoxWorker.js @@ -224,11 +224,11 @@ class DetoxWorker { }; onTestStart = function* (_signal, testSummary) { - this._validateTestSummary('beforeEach', testSummary); - // Copilot is reset before each test to ensure a clean state this.copilot.resetIfNeeded(); + this._validateTestSummary('beforeEach', testSummary); + yield this._dumpUnhandledErrorsIfAny({ pendingRequests: false, testName: testSummary.fullName, diff --git a/detox/src/DetoxWorker.test.js b/detox/src/DetoxWorker.test.js index 374ef3ae4e..3ca25916ae 100644 --- a/detox/src/DetoxWorker.test.js +++ b/detox/src/DetoxWorker.test.js @@ -321,17 +321,27 @@ describe('DetoxWorker', () => { detox = await new Detox(detoxContext).init(); }); - it('should validate test summary object', async () => { - await expect(detox.onTestStart('Test')).rejects.toThrowError( - /Invalid test summary was passed/ - ); - }); + describe('with an invalid test summary', () => { + it('should validate test summary object', async () => { + await expect(detox.onTestStart('Test')).rejects.toThrowError( + /Invalid test summary was passed/ + ); + }); + + it('should validate test summary status', async () => { + await expect(detox.onTestStart({ + ...testSummaries.running(), + status: undefined, + })).rejects.toThrowError(/Invalid test summary status/); + }); - it('should validate test summary status', async () => { - await expect(detox.onTestStart({ - ...testSummaries.running(), - status: undefined, - })).rejects.toThrowError(/Invalid test summary status/); + it('should reset copilot if needed', async () => { + try { + await detox.onTestStart('Test'); + } catch {} + + expect(detox.copilot.resetIfNeeded).toHaveBeenCalled(); + }); }); describe('with a valid test summary', () => { From 416bb8ea73d4b095946b91ef4c7f6640eeba0baf Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 17:56:00 +0300 Subject: [PATCH 015/164] fix(DetoxWorker): inject copilot to globals. --- detox/src/DetoxWorker.js | 1 + 1 file changed, 1 insertion(+) diff --git a/detox/src/DetoxWorker.js b/detox/src/DetoxWorker.js index b033e0c45f..bbfdb13e74 100644 --- a/detox/src/DetoxWorker.js +++ b/detox/src/DetoxWorker.js @@ -162,6 +162,7 @@ class DetoxWorker { const injectedGlobals = { ...matchers, device: this.device, + copilot: this.copilot, detox: this, }; From 422ccc011bc2b215b6838d70e3bf6036bba0cea5 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 17:56:13 +0300 Subject: [PATCH 016/164] feat(globals): expose copilot global. --- detox/globals.d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/detox/globals.d.ts b/detox/globals.d.ts index 3da05b8e21..a34fd9bf5a 100644 --- a/detox/globals.d.ts +++ b/detox/globals.d.ts @@ -9,6 +9,7 @@ declare global { const by: Detox.DetoxExportWrapper['by']; const web: Detox.DetoxExportWrapper['web']; const system: Detox.DetoxExportWrapper['system']; + const copilot: Detox.DetoxExportWrapper['copilot']; namespace NodeJS { interface Global { @@ -20,6 +21,7 @@ declare global { by: Detox.DetoxExportWrapper['by']; web: Detox.DetoxExportWrapper['web']; system: Detox.DetoxExportWrapper['system']; + copilot: Detox.DetoxExportWrapper['copilot']; } } } From 02482a2fbb5a82439bcbe337f0f200a987afb587 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 17:56:35 +0300 Subject: [PATCH 017/164] fix(detoxCopilotFrameworkDriver): fix take image snapshot. --- detox/src/copilot/detoxCopilotFrameworkDriver.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/detox/src/copilot/detoxCopilotFrameworkDriver.js b/detox/src/copilot/detoxCopilotFrameworkDriver.js index fe83313078..cd504baecb 100644 --- a/detox/src/copilot/detoxCopilotFrameworkDriver.js +++ b/detox/src/copilot/detoxCopilotFrameworkDriver.js @@ -73,8 +73,7 @@ const detoxCopilotFrameworkDriver = { captureSnapshotImage: async function() { const fileName = `snapshot_${Date.now()}.png`; - await device.takeScreenshot(fileName); - return fileName; + return await device.takeScreenshot(fileName); }, captureViewHierarchyString: async function() { From 835968a88180bfd36a81c247714934989bae96ae Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 17:56:49 +0300 Subject: [PATCH 018/164] feat(PromptHandler): initial commit. --- detox/test/e2e/copilot/PromptHandler.js | 118 ++++++++++++++++++++++++ detox/test/package.json | 2 + 2 files changed, 120 insertions(+) create mode 100644 detox/test/e2e/copilot/PromptHandler.js diff --git a/detox/test/e2e/copilot/PromptHandler.js b/detox/test/e2e/copilot/PromptHandler.js new file mode 100644 index 0000000000..3995dcd2ba --- /dev/null +++ b/detox/test/e2e/copilot/PromptHandler.js @@ -0,0 +1,118 @@ +const axios = require('axios'); +const fs = require('fs').promises; +const path = require('path'); + +require('dotenv').config({ + path: path.resolve(__dirname, '.env') +}); + +class PromptHandler { + constructor() { + this.config = { + appID: process.env.APP_ID, + appSecret: process.env.APP_SECRET, + instanceId: process.env.INSTANCE_ID, + serverlessEndpoint: process.env.SERVERLESS_ENDPOINT, + tokenEndpoint: process.env.TOKEN_ENDPOINT, + uploadUrlEndpoint: process.env.UPLOAD_URL_ENDPOINT, + promptId: process.env.PROMPT_ID, + }; + + const emptyValues = Object.values(this.config).some(value => !value); + if (emptyValues) { + throw new Error(`Missing required environment variables: ${Object.keys(this.config).join(', ')}`); + } + } + + async createToken() { + const response = await axios.post(this.config.tokenEndpoint, { + grant_type: 'client_credentials', + client_id: this.config.appID, + client_secret: this.config.appSecret, + instance_id: this.config.instanceId, + }, { + headers: { 'Content-Type': 'application/json' } + }); + + return response.data.access_token; + } + + async createUploadUrl(token, filename) { + const response = await axios.post(this.config.uploadUrlEndpoint, { + mimeType: 'image/png', + fileName: filename, + namespace: 'NO_NAMESPACE', + }, { + headers: { + Authorization: token, + 'Content-Type': 'application/json', + } + }); + + return response.data.uploadUrl; + } + + async uploadFile(uploadUrl, fileName, fileContent) { + const params = new URLSearchParams({ filename: fileName }); + const urlWithParams = `${uploadUrl}?${params.toString()}`; + + try { + const response = await axios.put(urlWithParams, fileContent, { + headers: { 'Content-Type': 'image/png' } + }); + + return response.data; + } catch (error) { + console.error('Error uploading file', error); + return null; + } + } + + async uploadImage(imagePath) { + const fileContent = await fs.readFile(imagePath); + const token = await this.createToken(); + const uploadUrl = await this.createUploadUrl(token, path.basename(imagePath)); + const response = await this.uploadFile(uploadUrl, path.basename(imagePath), fileContent); + return response.file.url; + } + + async runPrompt(prompt, image) { + if (!image) { + throw new Error('Image is required'); + } + + const imageUrl = await this.uploadImage(image); + + const body = { + promptId: this.config.promptId, + prompt, + image: imageUrl + }; + + try { + const response = await axios.post(this.config.serverlessEndpoint, body, { + headers: { + 'x-wix-model-hub-timeout': '600000', + 'x-time-budget': '600000', + }, + timeout: 600000 + }); + + const generatedText = response.data?.response?.generatedTexts?.[0]; + if (!generatedText) { + throw new Error('Failed to generate text'); + } + + return generatedText; + } catch (error) { + console.error('Error running prompt:', error); + throw error; + } + } + + isSnapshotImageSupported() { + return true; + } +} + +module.exports = PromptHandler; diff --git a/detox/test/package.json b/detox/test/package.json index ef104e94b7..675a800e16 100644 --- a/detox/test/package.json +++ b/detox/test/package.json @@ -59,9 +59,11 @@ "@types/react": "^18.2.45", "@typescript-eslint/eslint-plugin": "^6.16.0", "@typescript-eslint/parser": "^6.16.0", + "axios": "^1.7.7", "cross-env": "^7.0.3", "detox": "^20.26.2", "detox-allure2-adapter": "^1.0.0-alpha.8", + "dotenv": "^16.4.5", "eslint": "^8.56.0", "eslint-plugin-unicorn": "^50.0.1", "execa": "^5.1.1", From ce0598b029c6c5f6c27c693d2b1e8c8dcb4fc93a Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 17:57:03 +0300 Subject: [PATCH 019/164] test(e2e): add copilot sanity tests. --- detox/test/e2e/38.copilot.sanity.test.js | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 detox/test/e2e/38.copilot.sanity.test.js diff --git a/detox/test/e2e/38.copilot.sanity.test.js b/detox/test/e2e/38.copilot.sanity.test.js new file mode 100644 index 0000000000..1a03c37d93 --- /dev/null +++ b/detox/test/e2e/38.copilot.sanity.test.js @@ -0,0 +1,35 @@ +const PromptHandler = require('./copilot/PromptHandler'); + +describe('Copilot Sanity', () => { + beforeAll(async () => { + await device.launchApp({ + delete: true, + newInstance: true, + }); + + console.log('something', copilot); + await copilot.init(new PromptHandler()); + }); + + beforeEach(async () => { + await device.reloadReactNative(); + + await copilot.act('Navigate to sanity'); + }); + + it('should have welcome screen', async () => { + await copilot.assert('Welcome text is displayed'); + await copilot.assert('Say Hello button is visible to the user'); + await copilot.assert('Can see a Say World button'); + }); + + it('should show hello screen after tap', async () => { + await copilot.act('Tap on Say Hello button'); + await copilot.assert('"Hello!!!" text is visible'); + }); + + it('should show world screen after tap', async () => { + await copilot.act('Tap on Say World button'); + await copilot.assert('"World!!!" text is displayed'); + }); +}); From e8fa194996f0f93b0806cc09120954ef17f34d3e Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 18:02:49 +0300 Subject: [PATCH 020/164] test(fix): remove redundant comment. --- detox/test/e2e/38.copilot.sanity.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/detox/test/e2e/38.copilot.sanity.test.js b/detox/test/e2e/38.copilot.sanity.test.js index 1a03c37d93..bd410aeb92 100644 --- a/detox/test/e2e/38.copilot.sanity.test.js +++ b/detox/test/e2e/38.copilot.sanity.test.js @@ -7,7 +7,6 @@ describe('Copilot Sanity', () => { newInstance: true, }); - console.log('something', copilot); await copilot.init(new PromptHandler()); }); From 364f968bc9684bc859c2a998a272c3643c64c198 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 19:11:17 +0300 Subject: [PATCH 021/164] refactor: fix indent. --- detox-copilot/src/actions/StepPerformer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/detox-copilot/src/actions/StepPerformer.ts b/detox-copilot/src/actions/StepPerformer.ts index 91365af642..03d119f117 100644 --- a/detox-copilot/src/actions/StepPerformer.ts +++ b/detox-copilot/src/actions/StepPerformer.ts @@ -10,7 +10,8 @@ export class StepPerformer { private snapshotManager: SnapshotManager, private promptHandler: PromptHandler ) {} - async perform(step: ExecutionStep, previous: ExecutionStep[] = []): Promise { + + async perform(step: ExecutionStep, previous: ExecutionStep[] = []): Promise { const snapshot = await this.snapshotManager.captureSnapshotImage(); const viewHierarchy = await this.snapshotManager.captureViewHierarchyString(); From 5c669cc0dd08c734cc0f9dc0ab92eca5db07d072 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 20:02:59 +0300 Subject: [PATCH 022/164] feat(copilot): capture framework context from config. --- detox-copilot/src/Copilot.test.ts | 1 + detox-copilot/src/Copilot.ts | 8 ++++++- .../src/actions/StepPerformer.test.ts | 21 +++++++++++-------- detox-copilot/src/actions/StepPerformer.ts | 5 +++-- .../src/integration tests/index.test.ts | 1 + detox-copilot/src/types.ts | 6 ++++-- detox-copilot/src/utils/CodeEvaluator.test.ts | 21 ++++++++++++++----- detox-copilot/src/utils/CodeEvaluator.ts | 14 +++++++------ detox-copilot/src/utils/PromptCreator.test.ts | 1 + .../copilot/detoxCopilotFrameworkDriver.js | 8 +++---- 10 files changed, 57 insertions(+), 29 deletions(-) diff --git a/detox-copilot/src/Copilot.test.ts b/detox-copilot/src/Copilot.test.ts index bdcac28873..df3b2e3f75 100644 --- a/detox-copilot/src/Copilot.test.ts +++ b/detox-copilot/src/Copilot.test.ts @@ -14,6 +14,7 @@ describe('Copilot', () => { captureSnapshotImage: jest.fn(), captureViewHierarchyString: jest.fn(), availableAPI: { + context: {}, matchers: [], actions: [], assertions: [] diff --git a/detox-copilot/src/Copilot.ts b/detox-copilot/src/Copilot.ts index 2f70bb8f94..59752c1e37 100644 --- a/detox-copilot/src/Copilot.ts +++ b/detox-copilot/src/Copilot.ts @@ -23,7 +23,13 @@ export class Copilot { this.promptCreator = new PromptCreator(config.frameworkDriver.availableAPI); this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); - this.stepPerformer = new StepPerformer(this.promptCreator, this.codeEvaluator, this.snapshotManager, config.promptHandler); + this.stepPerformer = new StepPerformer( + config.frameworkDriver.availableAPI.context, + this.promptCreator, + this.codeEvaluator, + this.snapshotManager, + config.promptHandler + ); } /** diff --git a/detox-copilot/src/actions/StepPerformer.test.ts b/detox-copilot/src/actions/StepPerformer.test.ts index 82003c04a3..0c4eba56f2 100644 --- a/detox-copilot/src/actions/StepPerformer.test.ts +++ b/detox-copilot/src/actions/StepPerformer.test.ts @@ -10,14 +10,16 @@ jest.mock('@/utils/SnapshotManager'); describe('StepPerformer', () => { let stepPerformer: StepPerformer; + let mockContext: jest.Mocked; let mockPromptCreator: jest.Mocked; let mockCodeEvaluator: jest.Mocked; let mockSnapshotManager: jest.Mocked; let mockPromptHandler: jest.Mocked; beforeEach(() => { - const availableAPI = { matchers: [], actions: [], assertions: [] }; + const availableAPI = { context: {}, matchers: [], actions: [], assertions: [] }; + mockContext = {} as jest.Mocked; mockPromptCreator = new PromptCreator(availableAPI) as jest.Mocked; mockCodeEvaluator = new CodeEvaluator() as jest.Mocked; mockSnapshotManager = new SnapshotManager({} as any) as jest.Mocked; @@ -27,10 +29,11 @@ describe('StepPerformer', () => { } as jest.Mocked; stepPerformer = new StepPerformer( + mockContext, mockPromptCreator, mockCodeEvaluator, mockSnapshotManager, - mockPromptHandler + mockPromptHandler, ); }); @@ -45,12 +48,12 @@ describe('StepPerformer', () => { } const setupMocks = ({ - isSnapshotSupported = true, - snapshotData = 'snapshot_data', - viewHierarchy = '', - promptResult = 'generated code', - codeEvaluationResult = 'success' - }: SetupMockOptions = {}) => { + isSnapshotSupported = true, + snapshotData = 'snapshot_data', + viewHierarchy = '', + promptResult = 'generated code', + codeEvaluationResult = 'success' + }: SetupMockOptions = {}) => { mockPromptHandler.isSnapshotImageSupported.mockReturnValue(isSnapshotSupported); mockSnapshotManager.captureSnapshotImage.mockResolvedValue(snapshotData as string); mockSnapshotManager.captureViewHierarchyString.mockResolvedValue(viewHierarchy); @@ -68,7 +71,7 @@ describe('StepPerformer', () => { expect(result).toBe('success'); expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '', true, []); expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); - expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code'); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); }); it('should perform a step successfully without snapshot image support', async () => { diff --git a/detox-copilot/src/actions/StepPerformer.ts b/detox-copilot/src/actions/StepPerformer.ts index 03d119f117..d742bc79ee 100644 --- a/detox-copilot/src/actions/StepPerformer.ts +++ b/detox-copilot/src/actions/StepPerformer.ts @@ -5,10 +5,11 @@ import {ExecutionStep, PromptHandler} from "@/types"; export class StepPerformer { constructor( + private context: any, private promptCreator: PromptCreator, private codeEvaluator: CodeEvaluator, private snapshotManager: SnapshotManager, - private promptHandler: PromptHandler + private promptHandler: PromptHandler, ) {} async perform(step: ExecutionStep, previous: ExecutionStep[] = []): Promise { @@ -21,6 +22,6 @@ export class StepPerformer { const prompt = this.promptCreator.createPrompt(step, viewHierarchy, isSnapshotImageAttached, previous); const promptResult = await this.promptHandler.runPrompt(prompt, snapshot); - return this.codeEvaluator.evaluate(promptResult); + return this.codeEvaluator.evaluate(promptResult, this.context); } } diff --git a/detox-copilot/src/integration tests/index.test.ts b/detox-copilot/src/integration tests/index.test.ts index de054785cb..b0972a5040 100644 --- a/detox-copilot/src/integration tests/index.test.ts +++ b/detox-copilot/src/integration tests/index.test.ts @@ -13,6 +13,7 @@ describe('Integration', () => { captureSnapshotImage: jest.fn().mockResolvedValue('mock_snapshot'), captureViewHierarchyString: jest.fn().mockResolvedValue(''), availableAPI: { + context: {}, matchers: [], actions: [], assertions: [] diff --git a/detox-copilot/src/types.ts b/detox-copilot/src/types.ts index b68d64d618..7493886743 100644 --- a/detox-copilot/src/types.ts +++ b/detox-copilot/src/types.ts @@ -49,25 +49,27 @@ export interface TestingFrameworkDriver { captureViewHierarchyString: () => Promise; /** - * The available API methods of the testing framework. + * The available API methods docs of the testing framework. */ availableAPI: TestingFrameworkAPI; } /** * Represents the API of the testing framework that can be used by Copilot. + * @property context The available variables of the testing framework (i.e. exposes the matching function, expect, etc.). * @property matchers The available matchers API of the testing framework. * @property actions The available actions API of the testing framework. * @property assertions The available assertions API of the testing framework. */ export type TestingFrameworkAPI = { + context: any; matchers: TestingFrameworkAPIMethod[]; actions: TestingFrameworkAPIMethod[]; assertions: TestingFrameworkAPIMethod[]; } /** - * Represents a method in the API of the testing framework that can be used by Copilot. + * Represents a method docs in the API of the testing framework that can be used by Copilot. * @property signature The method signature of the API. * @property description A description of the API. * @property example An example of how to use the API. diff --git a/detox-copilot/src/utils/CodeEvaluator.test.ts b/detox-copilot/src/utils/CodeEvaluator.test.ts index cb45f0187a..0fa1e91138 100644 --- a/detox-copilot/src/utils/CodeEvaluator.test.ts +++ b/detox-copilot/src/utils/CodeEvaluator.test.ts @@ -9,26 +9,37 @@ describe('CodeEvaluator', () => { it('should evaluate valid code successfully', async () => { const validCode = 'return 2 + 2;'; - await expect(codeEvaluator.evaluate(validCode)).resolves.not.toThrow(); + await expect(codeEvaluator.evaluate(validCode, {})).resolves.not.toThrow(); }); it('should evaluate valid code with comments successfully', async () => { const validCode = 'return 2 + 2; // This is a comment'; - await expect(codeEvaluator.evaluate(validCode)).resolves.not.toThrow(); + await expect(codeEvaluator.evaluate(validCode, {})).resolves.not.toThrow(); + }); + + it('should evaluate valid code with multiple lines successfully', async () => { + const validCode = 'return 2 + 2;\nreturn 3 + 3;'; + await expect(codeEvaluator.evaluate(validCode, {})).resolves.not.toThrow(); + }); + + it('should evaluate valid code with context successfully', async () => { + const contextVariable = 43; + const validCode = 'return contextVariable - 1;'; + await expect(codeEvaluator.evaluate(validCode, { contextVariable })).resolves.toBe(42); }); it('should throw CodeEvaluationError for invalid code', async () => { const invalidCode = 'throw new Error("Test error");'; - await expect(codeEvaluator.evaluate(invalidCode)).rejects.toThrow(new Error('Test error')); + await expect(codeEvaluator.evaluate(invalidCode, {})).rejects.toThrow(new Error('Test error')); }); it('should handle asynchronous code', async () => { const asyncCode = 'await new Promise(resolve => setTimeout(resolve, 100)); return "done";'; - await expect(codeEvaluator.evaluate(asyncCode)).resolves.toBe('done'); + await expect(codeEvaluator.evaluate(asyncCode, {})).resolves.toBe('done'); }); it('should throw CodeEvaluationError with original error message', async () => { const errorCode = 'throw new Error("Custom error message");'; - await expect(codeEvaluator.evaluate(errorCode)).rejects.toThrow(new Error('Custom error message')); + await expect(codeEvaluator.evaluate(errorCode, {})).rejects.toThrow(new Error('Custom error message')); }); }); diff --git a/detox-copilot/src/utils/CodeEvaluator.ts b/detox-copilot/src/utils/CodeEvaluator.ts index dd3d8cd878..480425518e 100644 --- a/detox-copilot/src/utils/CodeEvaluator.ts +++ b/detox-copilot/src/utils/CodeEvaluator.ts @@ -1,19 +1,21 @@ import { CodeEvaluationError } from '@/errors/CodeEvaluationError'; export class CodeEvaluator { - async evaluate(code: string): Promise { - const asyncFunction = this.createAsyncFunction(code); + async evaluate(code: string, context: any): Promise { + const asyncFunction = this.createAsyncFunction(code, context); return await asyncFunction(); } - private createAsyncFunction(code: string): Function { + private createAsyncFunction(code: string, context: any): Function { const codeBlock = this.extractCodeBlock(code); try { - // Wrap the code in an immediately-invoked async function expression (IIFE) - return new Function(`return (async () => { + const contextValues = Object.values(context); + + // Wrap the code in an immediately-invoked async function expression (IIFE), and inject context variables into the function + return new Function(...Object.keys(context), `return (async () => { ${codeBlock} - })();`); + })();`).bind(null, ...contextValues); } catch (error) { const underlyingErrorMessage = (error as Error)?.message; throw new CodeEvaluationError( diff --git a/detox-copilot/src/utils/PromptCreator.test.ts b/detox-copilot/src/utils/PromptCreator.test.ts index 246a0dfd42..888166684c 100644 --- a/detox-copilot/src/utils/PromptCreator.test.ts +++ b/detox-copilot/src/utils/PromptCreator.test.ts @@ -2,6 +2,7 @@ import { PromptCreator } from './PromptCreator'; import {ExecutionStep, TestingFrameworkAPI} from "@/types"; const mockAPI: TestingFrameworkAPI = { + context: {}, actions: [ { signature: 'tap(element: Element)', diff --git a/detox/src/copilot/detoxCopilotFrameworkDriver.js b/detox/src/copilot/detoxCopilotFrameworkDriver.js index cd504baecb..f283d7f562 100644 --- a/detox/src/copilot/detoxCopilotFrameworkDriver.js +++ b/detox/src/copilot/detoxCopilotFrameworkDriver.js @@ -1,7 +1,8 @@ -const { device } = require('../..'); +const detox = require('../..'); const detoxCopilotFrameworkDriver = { availableAPI: { + context: detox, matchers: [ { signature: 'by.id(id: string)', @@ -41,7 +42,6 @@ const detoxCopilotFrameworkDriver = { example: "await element(by.id('usernameInput')).typeText('myusername');", guidelines: ['Typing can only be done on text field elements.'] }, - // ... (other actions remain the same) ], assertions: [ { @@ -73,11 +73,11 @@ const detoxCopilotFrameworkDriver = { captureSnapshotImage: async function() { const fileName = `snapshot_${Date.now()}.png`; - return await device.takeScreenshot(fileName); + return await detox.device.takeScreenshot(fileName); }, captureViewHierarchyString: async function() { - return device.generateViewHierarchyXml(); + return detox.device.generateViewHierarchyXml(); } }; From 42751d384f0174e126a92c7109dcb45c114ac507 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 21:25:07 +0300 Subject: [PATCH 023/164] chore(detox-copilot): bump version to `0.0.1`. --- detox-copilot/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detox-copilot/package.json b/detox-copilot/package.json index a6412cd930..3381ddcf5b 100644 --- a/detox-copilot/package.json +++ b/detox-copilot/package.json @@ -1,6 +1,6 @@ { "name": "detox-copilot", - "version": "0.0.0", + "version": "0.0.1", "description": "A Detox-based plugin that leverages AI to seamlessly invoke UI testing framework operations", "keywords": [ "detox", From 498b75f29e4a15b52fcdbe95cce813c3bcc31c59 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Wed, 4 Sep 2024 21:31:10 +0300 Subject: [PATCH 024/164] refactor(copilot): rename api catalog type. --- detox-copilot/src/index.ts | 4 ++-- detox-copilot/src/types.ts | 16 ++++++++-------- detox-copilot/src/utils/PromptCreator.test.ts | 4 ++-- detox-copilot/src/utils/PromptCreator.ts | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/detox-copilot/src/index.ts b/detox-copilot/src/index.ts index 3ffeb5f5bd..ecb4001170 100644 --- a/detox-copilot/src/index.ts +++ b/detox-copilot/src/index.ts @@ -29,6 +29,6 @@ export { Config, PromptHandler, TestingFrameworkDriver, - TestingFrameworkAPI, - TestingFrameworkAPIMethod + TestingFrameworkAPICatalog, + TestingFrameworkAPICatalogItem } from './types'; diff --git a/detox-copilot/src/types.ts b/detox-copilot/src/types.ts index 7493886743..6bb676f05c 100644 --- a/detox-copilot/src/types.ts +++ b/detox-copilot/src/types.ts @@ -49,23 +49,23 @@ export interface TestingFrameworkDriver { captureViewHierarchyString: () => Promise; /** - * The available API methods docs of the testing framework. + * A descriptive catalog for the available API of the testing framework that can be used by Copilot. */ - availableAPI: TestingFrameworkAPI; + availableAPI: TestingFrameworkAPICatalog; } /** - * Represents the API of the testing framework that can be used by Copilot. + * Represents the available API of the testing framework that can be used by Copilot. * @property context The available variables of the testing framework (i.e. exposes the matching function, expect, etc.). * @property matchers The available matchers API of the testing framework. * @property actions The available actions API of the testing framework. * @property assertions The available assertions API of the testing framework. */ -export type TestingFrameworkAPI = { +export type TestingFrameworkAPICatalog = { context: any; - matchers: TestingFrameworkAPIMethod[]; - actions: TestingFrameworkAPIMethod[]; - assertions: TestingFrameworkAPIMethod[]; + matchers: TestingFrameworkAPICatalogItem[]; + actions: TestingFrameworkAPICatalogItem[]; + assertions: TestingFrameworkAPICatalogItem[]; } /** @@ -86,7 +86,7 @@ export type TestingFrameworkAPI = { * ] * }; */ -export type TestingFrameworkAPIMethod = { +export type TestingFrameworkAPICatalogItem = { signature: string; description: string; example: string; diff --git a/detox-copilot/src/utils/PromptCreator.test.ts b/detox-copilot/src/utils/PromptCreator.test.ts index 888166684c..995b55f52d 100644 --- a/detox-copilot/src/utils/PromptCreator.test.ts +++ b/detox-copilot/src/utils/PromptCreator.test.ts @@ -1,7 +1,7 @@ import { PromptCreator } from './PromptCreator'; -import {ExecutionStep, TestingFrameworkAPI} from "@/types"; +import {ExecutionStep, TestingFrameworkAPICatalog} from "@/types"; -const mockAPI: TestingFrameworkAPI = { +const mockAPI: TestingFrameworkAPICatalog = { context: {}, actions: [ { diff --git a/detox-copilot/src/utils/PromptCreator.ts b/detox-copilot/src/utils/PromptCreator.ts index f8e473c7c7..f8300e679d 100644 --- a/detox-copilot/src/utils/PromptCreator.ts +++ b/detox-copilot/src/utils/PromptCreator.ts @@ -1,7 +1,7 @@ -import {ExecutionStep, ExecutionStepType, TestingFrameworkAPI, TestingFrameworkAPIMethod} from "@/types"; +import {ExecutionStep, ExecutionStepType, TestingFrameworkAPICatalog, TestingFrameworkAPICatalogItem} from "@/types"; export class PromptCreator { - constructor(private availableAPI: TestingFrameworkAPI) {} + constructor(private availableAPI: TestingFrameworkAPICatalog) {} createPrompt( step: ExecutionStep, @@ -82,7 +82,7 @@ export class PromptCreator { ); } - private formatAPIMethod(method: TestingFrameworkAPIMethod): string[] { + private formatAPIMethod(method: TestingFrameworkAPICatalogItem): string[] { const methodInfo = [ `### ${method.signature}`, "", From 19c9413c58eba52c08987703ef026c200a5904e7 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 08:50:13 +0300 Subject: [PATCH 025/164] chore(detox-copilot): remove prepublish script. --- detox-copilot/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/detox-copilot/package.json b/detox-copilot/package.json index 3381ddcf5b..5c568f0732 100644 --- a/detox-copilot/package.json +++ b/detox-copilot/package.json @@ -24,8 +24,7 @@ }, "scripts": { "build": "tsc && tsc-alias", - "test": "jest", - "prepublishOnly": "npm run build" + "test": "jest" }, "bugs": { "url": "https://github.com/wix/Detox/issues" From 51b13cca8fd9fed27c0c801eb30b96dca03db746 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 09:06:09 +0300 Subject: [PATCH 026/164] Revert "chore: remove detox-copilot from lerna packages." This reverts commit 38ba77d6ec5908696b0b8cda6351040762fe3ac4. --- lerna.json | 1 + scripts/ci.release.js | 1 + 2 files changed, 2 insertions(+) diff --git a/lerna.json b/lerna.json index 435ee98564..3e61c806b3 100644 --- a/lerna.json +++ b/lerna.json @@ -4,6 +4,7 @@ "packages": [ "detox", "detox-cli", + "detox-copilot", "detox/test", "examples/demo-native-android", "examples/demo-native-ios", diff --git a/scripts/ci.release.js b/scripts/ci.release.js index 51c6d04929..d6aa474555 100644 --- a/scripts/ci.release.js +++ b/scripts/ci.release.js @@ -43,6 +43,7 @@ email=\${NPM_EMAIL} // Workaround. see https://github.com/lerna/lerna/issues/361 fs.copyFileSync('.npmrc', 'detox/.npmrc'); fs.copyFileSync('.npmrc', 'detox-cli/.npmrc'); + fs.copyFileSync('.npmrc', 'detox-copilot/.npmrc'); } function versionTagAndPublish() { From 59f0a49ee9dc14ecabc1d9ddcb4752581a9c4281 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 09:30:32 +0300 Subject: [PATCH 027/164] feat: add more apis to detox copilot catalog. --- .../copilot/detoxCopilotFrameworkDriver.js | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/detox/src/copilot/detoxCopilotFrameworkDriver.js b/detox/src/copilot/detoxCopilotFrameworkDriver.js index f283d7f562..6a350cd035 100644 --- a/detox/src/copilot/detoxCopilotFrameworkDriver.js +++ b/detox/src/copilot/detoxCopilotFrameworkDriver.js @@ -36,12 +36,84 @@ const detoxCopilotFrameworkDriver = { example: "await element(by.id('menuItem')).longPress();", guidelines: ['If the target element is not accessible, interact with its container or the most relevant parent element.'] }, + { + signature: 'multiTap(times: number)', + description: 'Simulates multiple taps on an element', + example: "await element(by.id('tappable')).multiTap(3);", + guidelines: ['All taps are applied as part of the same gesture.'] + }, { signature: 'typeText(text: string)', description: 'Types text into a text field', example: "await element(by.id('usernameInput')).typeText('myusername');", guidelines: ['Typing can only be done on text field elements.'] }, + { + signature: 'replaceText(text: string)', + description: 'Replaces text in a text field', + example: "await element(by.id('textField')).replaceText('new text');", + guidelines: ['Faster than typeText(), but may not trigger all text input callbacks.'] + }, + { + signature: 'clearText()', + description: 'Clears text from a text field', + example: "await element(by.id('textField')).clearText();", + guidelines: ['Use this to clear text from input fields.'] + }, + { + signature: 'scroll(offset: number, direction: string, startPositionX?: number, startPositionY?: number)', + description: 'Scrolls an element', + example: "await element(by.id('scrollView')).scroll(100, 'down');", + guidelines: ['Specify direction as "up", "down", "left", or "right".'] + }, + { + signature: 'scrollTo(edge: string)', + description: 'Scrolls to an edge of the element', + example: "await element(by.id('scrollView')).scrollTo('bottom');", + guidelines: ['Specify edge as "top", "bottom", "left", or "right".'] + }, + { + signature: 'swipe(direction: string, speed?: string, normalizedOffset?: number)', + description: 'Simulates a swipe on the element', + example: "await element(by.id('scrollView')).swipe('up', 'slow', 0.5);", + guidelines: ['Specify direction as "up", "down", "left", or "right".'] + }, + { + signature: 'setColumnToValue(column: number, value: string)', + description: 'Sets a picker column to a specific value (iOS only)', + example: "await element(by.id('pickerView')).setColumnToValue(1, '6');", + guidelines: ['Use this for picker views on iOS.'] + }, + { + signature: 'setDatePickerDate(dateString: string, dateFormat: string)', + description: 'Sets a date picker to a specific date', + example: "await element(by.id('datePicker')).setDatePickerDate('2023-05-25', 'yyyy-MM-dd');", + guidelines: ['Use ISO8601 format when possible.'] + }, + { + signature: 'performAccessibilityAction(actionName: string)', + description: 'Triggers an accessibility action', + example: "await element(by.id('scrollView')).performAccessibilityAction('activate');", + guidelines: ['Use this to trigger specific accessibility actions.'] + }, + { + signature: 'pinch(scale: number, speed?: string, angle?: number)', + description: 'Simulates a pinch gesture', + example: "await element(by.id('PinchableScrollView')).pinch(1.1);", + guidelines: ['Use scale < 1 to zoom out, > 1 to zoom in.'] + }, + { + signature: 'launchApp(params: object)', + description: 'Launches the app with specified parameters', + example: 'await device.launchApp({newInstance: true});', + guidelines: ['Use this to launch the app with specific configurations.', 'this is part of the device API.'] + }, + { + signature: 'reloadReactNative()', + description: 'Reloads the React Native JS bundle', + example: 'await device.reloadReactNative();', + guidelines: ['Faster than launchApp(), use when you just need to reset React Native logic.', 'this is part of the device API.'] + } ], assertions: [ { From 9094fe9a5a6f825ddf4568e2111166aab85868f3 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 09:57:55 +0300 Subject: [PATCH 028/164] feat(detox-copilot): update detox driver catalog. --- detox/src/copilot/detoxCopilotFrameworkDriver.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detox/src/copilot/detoxCopilotFrameworkDriver.js b/detox/src/copilot/detoxCopilotFrameworkDriver.js index 6a350cd035..69685c35b0 100644 --- a/detox/src/copilot/detoxCopilotFrameworkDriver.js +++ b/detox/src/copilot/detoxCopilotFrameworkDriver.js @@ -106,13 +106,13 @@ const detoxCopilotFrameworkDriver = { signature: 'launchApp(params: object)', description: 'Launches the app with specified parameters', example: 'await device.launchApp({newInstance: true});', - guidelines: ['Use this to launch the app with specific configurations.', 'this is part of the device API.'] + guidelines: ['Use this to launch the app with specific configurations.'] }, { signature: 'reloadReactNative()', description: 'Reloads the React Native JS bundle', example: 'await device.reloadReactNative();', - guidelines: ['Faster than launchApp(), use when you just need to reset React Native logic.', 'this is part of the device API.'] + guidelines: ['Faster than launchApp(), use when you just need to reset React Native state/logic.'] } ], assertions: [ From acdb79613123bbac697450d890b477ec6a4ac23d Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 09:58:06 +0300 Subject: [PATCH 029/164] test(e2e): update copilot sanity tests. --- detox/test/e2e/38.copilot.sanity.test.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/detox/test/e2e/38.copilot.sanity.test.js b/detox/test/e2e/38.copilot.sanity.test.js index bd410aeb92..5bddd3b919 100644 --- a/detox/test/e2e/38.copilot.sanity.test.js +++ b/detox/test/e2e/38.copilot.sanity.test.js @@ -2,17 +2,13 @@ const PromptHandler = require('./copilot/PromptHandler'); describe('Copilot Sanity', () => { beforeAll(async () => { - await device.launchApp({ - delete: true, - newInstance: true, - }); - await copilot.init(new PromptHandler()); + + await copilot.act('Launch the app'); }); beforeEach(async () => { - await device.reloadReactNative(); - + await copilot.act('Reset react native state'); await copilot.act('Navigate to sanity'); }); From 418736a704889a3c36266a470a7f72a6ee37292f Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 10:03:45 +0300 Subject: [PATCH 030/164] chore: update detox-copilot. --- detox/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detox/package.json b/detox/package.json index 3973b176e1..77016862dc 100644 --- a/detox/package.json +++ b/detox/package.json @@ -71,7 +71,7 @@ "caf": "^15.0.1", "chalk": "^4.0.0", "child-process-promise": "^2.2.0", - "detox-copilot": "^0.0.0", + "detox-copilot": "^0.0.1", "execa": "^5.1.1", "find-up": "^5.0.0", "fs-extra": "^11.0.0", From ac0aeed71bbe75842f05dcf033a6f7db55d37651 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 10:11:54 +0300 Subject: [PATCH 031/164] feat(copilot): add code-evaluation log. --- detox-copilot/src/utils/CodeEvaluator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detox-copilot/src/utils/CodeEvaluator.ts b/detox-copilot/src/utils/CodeEvaluator.ts index 480425518e..05b3e7b651 100644 --- a/detox-copilot/src/utils/CodeEvaluator.ts +++ b/detox-copilot/src/utils/CodeEvaluator.ts @@ -9,6 +9,9 @@ export class CodeEvaluator { private createAsyncFunction(code: string, context: any): Function { const codeBlock = this.extractCodeBlock(code); + // todo: this is a temp log for debugging, we'll need to pass a logging mechanism from the framework. + console.log("\x1b[36m%s\x1b[0m", `Copilot evaluating code block: \`${codeBlock}\``); + try { const contextValues = Object.values(context); From 10a73d8564524a5f32e8707083f06361db2fcb2b Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Thu, 5 Sep 2024 10:23:31 +0300 Subject: [PATCH 032/164] feat(copilot): add more debugging logs. --- detox-copilot/src/actions/StepPerformer.ts | 2 ++ detox-copilot/src/utils/CodeEvaluator.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/detox-copilot/src/actions/StepPerformer.ts b/detox-copilot/src/actions/StepPerformer.ts index d742bc79ee..fe23ec07d3 100644 --- a/detox-copilot/src/actions/StepPerformer.ts +++ b/detox-copilot/src/actions/StepPerformer.ts @@ -13,6 +13,8 @@ export class StepPerformer { ) {} async perform(step: ExecutionStep, previous: ExecutionStep[] = []): Promise { + console.log("\x1b[36m%s\x1b[0m", `Running ${step.type} step: ${step.value}`); + const snapshot = await this.snapshotManager.captureSnapshotImage(); const viewHierarchy = await this.snapshotManager.captureViewHierarchyString(); diff --git a/detox-copilot/src/utils/CodeEvaluator.ts b/detox-copilot/src/utils/CodeEvaluator.ts index 05b3e7b651..b787e69b90 100644 --- a/detox-copilot/src/utils/CodeEvaluator.ts +++ b/detox-copilot/src/utils/CodeEvaluator.ts @@ -10,7 +10,7 @@ export class CodeEvaluator { const codeBlock = this.extractCodeBlock(code); // todo: this is a temp log for debugging, we'll need to pass a logging mechanism from the framework. - console.log("\x1b[36m%s\x1b[0m", `Copilot evaluating code block: \`${codeBlock}\``); + console.log("\x1b[36m%s\x1b[0m", `Copilot evaluating code block: \`${codeBlock}\`\n`); try { const contextValues = Object.values(context); From 1b0fb5cc777990d8d547820f5abc610bfa3453a7 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Sun, 15 Sep 2024 09:01:19 +0300 Subject: [PATCH 033/164] test(e2e): add copilot tests. --- detox/test/e2e/39.copilot.actions.test.js | 105 ++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 detox/test/e2e/39.copilot.actions.test.js diff --git a/detox/test/e2e/39.copilot.actions.test.js b/detox/test/e2e/39.copilot.actions.test.js new file mode 100644 index 0000000000..3e57a8d960 --- /dev/null +++ b/detox/test/e2e/39.copilot.actions.test.js @@ -0,0 +1,105 @@ +const PromptHandler = require('./copilot/PromptHandler'); + +describe('Copilot Actions', () => { + beforeAll(async () => { + await copilot.init(new PromptHandler()); + + await copilot.act('Start the application'); + }); + + beforeEach(async () => { + await copilot.act('Restart the React Native environment'); + await copilot.act('Go to the Actions screen'); + }); + + it('should tap on an element', async () => { + await copilot.act('Press the "Tap Me" button'); + await copilot.assert('The text "Tap Working!!!" is shown on the screen'); + }); + + it('should long press on an element', async () => { + await copilot.act('Perform a long press on the "Tap Me" button'); + await copilot.assert('The message "Long Press Working!!!" is displayed'); + }); + + it('should long press with duration on an element', async () => { + await copilot.act('Hold the "Long Press Me 1.5s" button for 1.5 seconds'); + await copilot.assert('Can see "Long Press With Duration Working!!!" on the screen'); + }); + + it('should long press with point', async () => { + await copilot.act('Long press the top left corner of the "Long Press on Top Left" button'); + await copilot.assert('The text "Long Press on Top Left Working!!!" appears'); + }); + + it('should not succeed in long pressing with point outside the target area', async () => { + await copilot.act('Attempt a long press outside the "Long Press on Top Left" button'); + await copilot.assert('The message "Long Press on Top Left Working!!!" is not present'); + }); + + it('should type in an element', async () => { + const typedText = 'Type Working!!!'; + await copilot.act(`Enter "${typedText}" into the text input field`); + await copilot.assert(`The typed text "${typedText}" is visible on the screen`); + }); + + it('should press the backspace key on an element', async () => { + const typedText = 'test'; + await copilot.act(`Input "${typedText}x" in the text field`); + await copilot.act('Hit the backspace key in the text input'); + await copilot.assert(`The text "${typedText}" is shown in the input field`); + }); + + it('should press the return key on an element', async () => { + await copilot.act('Tap the return key on the keyboard for the text input'); + await copilot.assert('The message "Return Working!!!" is visible to the user'); + }); + + it('should clear text in an element', async () => { + await copilot.act('Remove all text from the clearable text input'); + await copilot.assert('The text "Clear Working!!!" appears on the screen'); + }); + + it('should replace text in an element', async () => { + await copilot.act('Substitute the existing text with "replaced_text" in the editable field'); + await copilot.assert('The message "Replace Working!!!" is shown'); + }); + + it('should swipe down until pull to reload is triggered', async () => { + await copilot.act('Drag the scrollable area downwards until the refresh is activated'); + await copilot.assert('The text "PullToReload Working!!!" becomes visible'); + }); + + it('should swipe vertically', async () => { + await copilot.assert('The element with text "Text1" can be seen'); + await copilot.act('Slide the vertical scrollable area upwards'); + await copilot.assert('The "Text1" element is no longer in view'); + await copilot.act('Scroll the vertical area back down'); + await copilot.assert('"Text1" has reappeared on the screen'); + }); + + it('should swipe horizontally', async () => { + await copilot.assert('The "HText1" element is present'); + await copilot.act('Swipe the horizontal scrollable area towards the left'); + await copilot.assert('"HText1" is not in the visible area'); + await copilot.act('Slide the horizontal scroll to the right'); + await copilot.assert('The "HText1" element has come back into view'); + }); + + it('should adjust slider and assert its value', async () => { + await copilot.assert('The slider is set to 25%'); + await copilot.act('Move the slider to the 75% position'); + await copilot.assert('The slider value is approximately 75%, give or take 10%'); + }); + + it('should expect text fields to be focused after tap but not before', async () => { + await copilot.assert('The first text field does not have focus'); + await copilot.assert('Text input 2 is not currently focused'); + await copilot.act('Tap to focus on the first text field'); + await copilot.assert('Text field 1 now has the focus'); + await copilot.assert('The second text input remains unfocused'); + await copilot.act('Touch the second text field to give it focus'); + await copilot.assert('The first text input has lost focus'); + await copilot.assert('Text field 2 is now the active input'); + }); +}); From 358290b4f5515ff6c50f7d347bd7a71113d7e359 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Sun, 15 Sep 2024 09:01:40 +0300 Subject: [PATCH 034/164] feat(copilot): remove id from view hierarchy (Android). --- .../detox/espresso/hierarchy/ViewHierarchyGenerator.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/detox/android/detox/src/full/java/com/wix/detox/espresso/hierarchy/ViewHierarchyGenerator.kt b/detox/android/detox/src/full/java/com/wix/detox/espresso/hierarchy/ViewHierarchyGenerator.kt index 466597e2ea..fc67763211 100644 --- a/detox/android/detox/src/full/java/com/wix/detox/espresso/hierarchy/ViewHierarchyGenerator.kt +++ b/detox/android/detox/src/full/java/com/wix/detox/espresso/hierarchy/ViewHierarchyGenerator.kt @@ -79,14 +79,6 @@ object ViewHierarchyGenerator { "label" to (view.getAccessibilityLabel()?.toString() ?: "") ) - view.id.takeIf { it != View.NO_ID }?.let { - attributes["id"] = try { - view.resources.getResourceName(it) - } catch (e: Exception) { - it.toString() - } - } - val location = IntArray(2).apply { view.getLocationInWindow(this) } attributes["x"] = location[0].toString() attributes["y"] = location[1].toString() From 7432d76ca12ee0aa63090365cafd525afd5aed62 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Sun, 15 Sep 2024 09:56:34 +0300 Subject: [PATCH 035/164] feat(copilot): merge act & assert APIs to perform. --- detox-copilot/src/Copilot.test.ts | 45 ++- detox-copilot/src/Copilot.ts | 12 +- .../src/actions/StepPerformer.test.ts | 71 +++-- detox-copilot/src/actions/StepPerformer.ts | 8 +- detox-copilot/src/index.ts | 13 +- .../src/integration tests/index.test.ts | 84 ++--- detox-copilot/src/types.ts | 48 +-- detox-copilot/src/utils/PromptCreator.test.ts | 94 +++--- detox-copilot/src/utils/PromptCreator.ts | 84 ++--- .../__snapshots__/PromptCreator.test.ts.snap | 214 ++++++++++--- detox/detox.d.ts | 2 +- detox/src/copilot/DetoxCopilot.js | 8 +- .../copilot/detoxCopilotFrameworkDriver.js | 291 +++++++++--------- 13 files changed, 561 insertions(+), 413 deletions(-) diff --git a/detox-copilot/src/Copilot.test.ts b/detox-copilot/src/Copilot.test.ts index df3b2e3f75..182a925db6 100644 --- a/detox-copilot/src/Copilot.test.ts +++ b/detox-copilot/src/Copilot.test.ts @@ -1,7 +1,7 @@ import { Copilot } from '@/Copilot'; import { StepPerformer } from '@/actions/StepPerformer'; import { CopilotError } from '@/errors/CopilotError'; -import {Config, ExecutionStep} from "@/types"; +import { Config } from "@/types"; jest.mock('@/actions/StepPerformer'); @@ -13,11 +13,9 @@ describe('Copilot', () => { frameworkDriver: { captureSnapshotImage: jest.fn(), captureViewHierarchyString: jest.fn(), - availableAPI: { + apiCatalog: { context: {}, - matchers: [], - actions: [], - assertions: [] + categories: [] }, }, promptHandler: { @@ -73,52 +71,53 @@ describe('Copilot', () => { }); }); - describe('execute', () => { - it('should call StepPerformer.perform with the given step', async () => { + describe('perform', () => { + it('should call StepPerformer.perform with the given intent', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); - const step: ExecutionStep = { type: 'action', value: 'tap button' }; + const intent = 'tap button'; - await instance.perform(step); + await instance.perform(intent); - expect(StepPerformer.prototype.perform).toHaveBeenCalledWith(step, []); + expect(StepPerformer.prototype.perform).toHaveBeenCalledWith(intent, []); }); it('should return the result from StepPerformer.perform', async () => { (StepPerformer.prototype.perform as jest.Mock).mockResolvedValue(true); Copilot.init(mockConfig); const instance = Copilot.getInstance(); + const intent = 'tap button'; - const result = await instance.perform({ type: 'action', value: 'tap button' }); + const result = await instance.perform(intent); expect(result).toBe(true); }); - it('should accumulate previous steps', async () => { + it('should accumulate previous intents', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); - const step1: ExecutionStep = { type: 'action', value: 'tap button 1' }; - const step2 : ExecutionStep = { type: 'action', value: 'tap button 2' }; + const intent1 = 'tap button 1'; + const intent2 = 'tap button 2'; - await instance.perform(step1); - await instance.perform(step2); + await instance.perform(intent1); + await instance.perform(intent2); - expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(step2, [step1]); + expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(intent2, [intent1]); }); }); describe('reset', () => { - it('should clear previous steps', async () => { + it('should clear previous intents', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); - const step1: ExecutionStep = { type: 'action', value: 'tap button 1' }; - const step2: ExecutionStep = { type: 'action', value: 'tap button 2' }; + const intent1 = 'tap button 1'; + const intent2 = 'tap button 2'; - await instance.perform(step1); + await instance.perform(intent1); instance.reset(); - await instance.perform(step2); + await instance.perform(intent2); - expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(step2, []); + expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(intent2, []); }); }); }); diff --git a/detox-copilot/src/Copilot.ts b/detox-copilot/src/Copilot.ts index 59752c1e37..6809cf3468 100644 --- a/detox-copilot/src/Copilot.ts +++ b/detox-copilot/src/Copilot.ts @@ -3,7 +3,7 @@ import {PromptCreator} from "@/utils/PromptCreator"; import {CodeEvaluator} from "@/utils/CodeEvaluator"; import {SnapshotManager} from "@/utils/SnapshotManager"; import {StepPerformer} from "@/actions/StepPerformer"; -import {Config, ExecutionStep} from "@/types"; +import {Config} from "@/types"; /** * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework. @@ -16,15 +16,15 @@ export class Copilot { private readonly promptCreator: PromptCreator; private readonly codeEvaluator: CodeEvaluator; private readonly snapshotManager: SnapshotManager; - private previousSteps: ExecutionStep[] = []; + private previousSteps: string[] = []; private stepPerformer: StepPerformer; private constructor(config: Config) { - this.promptCreator = new PromptCreator(config.frameworkDriver.availableAPI); + this.promptCreator = new PromptCreator(config.frameworkDriver.apiCatalog); this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); this.stepPerformer = new StepPerformer( - config.frameworkDriver.availableAPI.context, + config.frameworkDriver.apiCatalog.context, this.promptCreator, this.codeEvaluator, this.snapshotManager, @@ -56,7 +56,7 @@ export class Copilot { * Performs a test step based on the given prompt. * @param step The step describing the operation to perform. */ - async perform(step: ExecutionStep): Promise { + async perform(step: string): Promise { const result = await this.stepPerformer.perform(step, this.previousSteps); this.didPerformStep(step); @@ -71,7 +71,7 @@ export class Copilot { this.previousSteps = []; } - private didPerformStep(step: ExecutionStep): void { + private didPerformStep(step: string): void { this.previousSteps = [...this.previousSteps, step]; } } diff --git a/detox-copilot/src/actions/StepPerformer.test.ts b/detox-copilot/src/actions/StepPerformer.test.ts index 0c4eba56f2..414c5d45a8 100644 --- a/detox-copilot/src/actions/StepPerformer.test.ts +++ b/detox-copilot/src/actions/StepPerformer.test.ts @@ -2,7 +2,7 @@ import { StepPerformer } from '@/actions/StepPerformer'; import { PromptCreator } from '@/utils/PromptCreator'; import { CodeEvaluator } from '@/utils/CodeEvaluator'; import { SnapshotManager } from '@/utils/SnapshotManager'; -import {ExecutionStep, PromptHandler} from "@/types"; +import { PromptHandler, TestingFrameworkAPICatalog } from "@/types"; jest.mock('@/utils/PromptCreator'); jest.mock('@/utils/CodeEvaluator'); @@ -17,10 +17,13 @@ describe('StepPerformer', () => { let mockPromptHandler: jest.Mocked; beforeEach(() => { - const availableAPI = { context: {}, matchers: [], actions: [], assertions: [] }; + const apiCatalog: TestingFrameworkAPICatalog = { + context: {}, + categories: [] + }; mockContext = {} as jest.Mocked; - mockPromptCreator = new PromptCreator(availableAPI) as jest.Mocked; + mockPromptCreator = new PromptCreator(apiCatalog) as jest.Mocked; mockCodeEvaluator = new CodeEvaluator() as jest.Mocked; mockSnapshotManager = new SnapshotManager({} as any) as jest.Mocked; mockPromptHandler = { @@ -37,79 +40,83 @@ describe('StepPerformer', () => { ); }); - const createStep = (value: string): ExecutionStep => ({ type: 'action', value }); - interface SetupMockOptions { isSnapshotSupported?: boolean; snapshotData?: string | null; viewHierarchy?: string; promptResult?: string; - codeEvaluationResult?: string; + codeEvaluationResult?: any; } const setupMocks = ({ - isSnapshotSupported = true, - snapshotData = 'snapshot_data', - viewHierarchy = '', - promptResult = 'generated code', - codeEvaluationResult = 'success' - }: SetupMockOptions = {}) => { + isSnapshotSupported = true, + snapshotData = 'snapshot_data', + viewHierarchy = '', + promptResult = 'generated code', + codeEvaluationResult = 'success' + }: SetupMockOptions = {}) => { mockPromptHandler.isSnapshotImageSupported.mockReturnValue(isSnapshotSupported); - mockSnapshotManager.captureSnapshotImage.mockResolvedValue(snapshotData as string); + mockSnapshotManager.captureSnapshotImage.mockResolvedValue(snapshotData != null ? 'snapshot_data' : undefined); mockSnapshotManager.captureViewHierarchyString.mockResolvedValue(viewHierarchy); mockPromptCreator.createPrompt.mockReturnValue('generated prompt'); mockPromptHandler.runPrompt.mockResolvedValue(promptResult); mockCodeEvaluator.evaluate.mockResolvedValue(codeEvaluationResult); }; - it('should perform a step successfully with snapshot image support', async () => { - const step = createStep('tap button'); + it('should perform an intent successfully with snapshot image support', async () => { + const intent = 'tap button'; setupMocks(); - const result = await stepPerformer.perform(step); + const result = await stepPerformer.perform(intent); expect(result).toBe('success'); - expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '', true, []); + expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(intent, '', true, []); expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); }); - it('should perform a step successfully without snapshot image support', async () => { - const step = createStep('tap button'); + it('should perform an intent successfully without snapshot image support', async () => { + const intent = 'tap button'; setupMocks({ isSnapshotSupported: false }); - const result = await stepPerformer.perform(step); + const result = await stepPerformer.perform(intent); expect(result).toBe('success'); - expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '', false, []); + expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(intent, '', false, []); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); }); - it('should perform a step with null snapshot', async () => { - const step = createStep('tap button'); + it('should perform an intent with undefined snapshot', async () => { + const intent = 'tap button'; setupMocks({ snapshotData: null }); - const result = await stepPerformer.perform(step); + const result = await stepPerformer.perform(intent); expect(result).toBe('success'); - expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', null); + expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(intent, '', false, []); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', undefined); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); }); - it('should perform a step successfully with previous steps', async () => { - const step = createStep('current step'); - const previousSteps = [createStep('previous step')]; + it('should perform an intent successfully with previous intents', async () => { + const intent = 'current intent'; + const previousIntents = ['previous intent']; setupMocks(); - const result = await stepPerformer.perform(step, previousSteps); + const result = await stepPerformer.perform(intent, previousIntents); expect(result).toBe('success'); - expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(step, '', true, previousSteps); + expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith(intent, '', true, previousIntents); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); }); it('should throw an error if code evaluation fails', async () => { - const step = createStep('tap button'); + const intent = 'tap button'; setupMocks(); mockCodeEvaluator.evaluate.mockRejectedValue(new Error('Evaluation failed')); - await expect(stepPerformer.perform(step)).rejects.toThrow('Evaluation failed'); + await expect(stepPerformer.perform(intent)).rejects.toThrow('Evaluation failed'); }); }); diff --git a/detox-copilot/src/actions/StepPerformer.ts b/detox-copilot/src/actions/StepPerformer.ts index fe23ec07d3..ef1420f747 100644 --- a/detox-copilot/src/actions/StepPerformer.ts +++ b/detox-copilot/src/actions/StepPerformer.ts @@ -1,7 +1,7 @@ import { PromptCreator } from '@/utils/PromptCreator'; import { CodeEvaluator } from '@/utils/CodeEvaluator'; import { SnapshotManager } from '@/utils/SnapshotManager'; -import {ExecutionStep, PromptHandler} from "@/types"; +import {PromptHandler} from "@/types"; export class StepPerformer { constructor( @@ -12,14 +12,14 @@ export class StepPerformer { private promptHandler: PromptHandler, ) {} - async perform(step: ExecutionStep, previous: ExecutionStep[] = []): Promise { - console.log("\x1b[36m%s\x1b[0m", `Running ${step.type} step: ${step.value}`); + async perform(step: string, previous: string[] = []): Promise { + console.log("\x1b[36m%s\x1b[0m", `Copilot performing: \"${step}\"`); const snapshot = await this.snapshotManager.captureSnapshotImage(); const viewHierarchy = await this.snapshotManager.captureViewHierarchyString(); const isSnapshotImageAttached = - snapshot !== undefined && this.promptHandler.isSnapshotImageSupported(); + snapshot != null && this.promptHandler.isSnapshotImageSupported(); const prompt = this.promptCreator.createPrompt(step, viewHierarchy, isSnapshotImageAttached, previous); const promptResult = await this.promptHandler.runPrompt(prompt, snapshot); diff --git a/detox-copilot/src/index.ts b/detox-copilot/src/index.ts index ecb4001170..36d8aa8060 100644 --- a/detox-copilot/src/index.ts +++ b/detox-copilot/src/index.ts @@ -8,17 +8,8 @@ const copilot: CopilotFacade = { reset: () => { Copilot.getInstance().reset(); }, - act: (action: string) => { - return Copilot.getInstance().perform({ - type: 'action', - value: action - }); - }, - assert: (assertion: string) => { - return Copilot.getInstance().perform({ - type: 'assertion', - value: assertion - }); + perform: (intent: string) => { + return Copilot.getInstance().perform(intent); } }; diff --git a/detox-copilot/src/integration tests/index.test.ts b/detox-copilot/src/integration tests/index.test.ts index b0972a5040..2870104736 100644 --- a/detox-copilot/src/integration tests/index.test.ts +++ b/detox-copilot/src/integration tests/index.test.ts @@ -1,6 +1,6 @@ import copilot from "@/index"; import { Copilot } from "@/Copilot"; -import {PromptHandler, TestingFrameworkDriver} from "@/types"; +import { PromptHandler, TestingFrameworkDriver } from "@/types"; describe('Integration', () => { let mockFrameworkDriver: jest.Mocked; @@ -12,11 +12,9 @@ describe('Integration', () => { mockFrameworkDriver = { captureSnapshotImage: jest.fn().mockResolvedValue('mock_snapshot'), captureViewHierarchyString: jest.fn().mockResolvedValue(''), - availableAPI: { + apiCatalog: { context: {}, - matchers: [], - actions: [], - assertions: [] + categories: [] } }; @@ -24,36 +22,36 @@ describe('Integration', () => { runPrompt: jest.fn(), isSnapshotImageSupported: jest.fn().mockReturnValue(true) }; + }); - copilot.init({ - frameworkDriver: mockFrameworkDriver, - promptHandler: mockPromptHandler - }); + afterEach(() => { + // Reset Copilot instance after each test to ensure a clean state + Copilot['instance'] = undefined; }); describe('Initialization', () => { - beforeEach(() => { - // Reset Copilot instance before each test - Copilot['instance'] = undefined; - }); - - it('should synchronously throw an error when act is called before initialization', async () => { - expect(() => copilot.act('Some action')).toThrow(); + it('should synchronously throw an error when perform is called before initialization', () => { + expect(() => copilot.perform('Some action')).toThrowError( + 'Copilot has not been initialized. Please call the `init()` method before using it.' + ); }); + }); - it('should synchronously throw an error when expect is called before initialization', async () => { - expect(() => copilot.act('Some assertion')).toThrow(); + describe('perform method', () => { + beforeEach(() => { + copilot.init({ + frameworkDriver: mockFrameworkDriver, + promptHandler: mockPromptHandler + }); }); - }); - describe('act method', () => { it('should successfully perform an action', async () => { mockPromptHandler.runPrompt.mockResolvedValue('// No operation'); - await expect(copilot.act('Tap on the login button')).resolves.not.toThrow(); + await expect(copilot.perform('Tap on the login button')).resolves.not.toThrow(); expect(mockFrameworkDriver.captureSnapshotImage).toHaveBeenCalled(); - expect(mockFrameworkDriver.captureViewHierarchyString).toHaveBeenCalledWith(); + expect(mockFrameworkDriver.captureViewHierarchyString).toHaveBeenCalled(); expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith( expect.stringContaining('Tap on the login button'), @@ -61,57 +59,63 @@ describe('Integration', () => { ); }); - it('should handle errors during action execution', async () => { - mockPromptHandler.runPrompt.mockResolvedValue('throw new Error("Element not found");'); - - await expect(copilot.act('Tap on a non-existent button')).rejects.toThrow(new Error('Element not found')); - }); - }); - - describe('expect method', () => { - it('should successfully perform an expectation', async () => { + it('should successfully perform an assertion', async () => { mockPromptHandler.runPrompt.mockResolvedValue('// No operation'); - await copilot.assert('The welcome message should be visible'); + await expect(copilot.perform('The welcome message should be visible')).resolves.not.toThrow(); expect(mockFrameworkDriver.captureSnapshotImage).toHaveBeenCalled(); expect(mockFrameworkDriver.captureViewHierarchyString).toHaveBeenCalled(); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith( expect.stringContaining('The welcome message should be visible'), 'mock_snapshot' ); }); - it('should handle errors during expectation execution', async () => { + it('should handle errors during action execution', async () => { + mockPromptHandler.runPrompt.mockResolvedValue('throw new Error("Element not found");'); + + await expect(copilot.perform('Tap on a non-existent button')).rejects.toThrow('Element not found'); + }); + + it('should handle errors during assertion execution', async () => { mockPromptHandler.runPrompt.mockResolvedValue('throw new Error("Element not found");'); - await expect(copilot.assert('The welcome message should be visible')).rejects.toThrow(new Error('Element not found')); + await expect(copilot.perform('The welcome message should be visible')).rejects.toThrow('Element not found'); }); it('should handle errors during code evaluation', async () => { mockPromptHandler.runPrompt.mockResolvedValue('foobar'); - await expect(copilot.assert('The welcome message should be visible')).rejects.toThrow(new Error('foobar is not defined')); + await expect(copilot.perform('The welcome message should be visible')).rejects.toThrow(/foobar is not defined/); }); }); describe('error handling', () => { + beforeEach(() => { + copilot.init({ + frameworkDriver: mockFrameworkDriver, + promptHandler: mockPromptHandler + }); + }); + it('should throw error when PromptHandler fails', async () => { mockPromptHandler.runPrompt.mockRejectedValue(new Error('API error')); - await expect(copilot.act('Perform action')).rejects.toThrow(/API error/); + await expect(copilot.perform('Perform action')).rejects.toThrow('API error'); }); it('should throw error when captureSnapshotImage() fails', async () => { - mockFrameworkDriver.captureSnapshotImage.mockRejectedValue(new Error('API error')); + mockFrameworkDriver.captureSnapshotImage.mockRejectedValue(new Error('Snapshot error')); - await expect(copilot.act('Perform action')).rejects.toThrow(/API error/); + await expect(copilot.perform('Perform action')).rejects.toThrow('Snapshot error'); }); it('should throw error when captureViewHierarchyString() fails', async () => { - mockFrameworkDriver.captureViewHierarchyString.mockRejectedValue(new Error('API error')); + mockFrameworkDriver.captureViewHierarchyString.mockRejectedValue(new Error('Hierarchy error')); - await expect(copilot.act('Perform action')).rejects.toThrow(/API error/); + await expect(copilot.perform('Perform action')).rejects.toThrow('Hierarchy error'); }); }); }); diff --git a/detox-copilot/src/types.ts b/detox-copilot/src/types.ts index 6bb676f05c..514186842c 100644 --- a/detox-copilot/src/types.ts +++ b/detox-copilot/src/types.ts @@ -17,20 +17,13 @@ export interface CopilotFacade { reset: () => void; /** - * Performs an action in the app. - * @param action The action to perform (in free-form text). + * Performs a testing operation in the app based on the given `intent`. * @example Tap on the login button * @example Scroll down to the 7th item in the Events list - */ - act: (action: string) => Promise; - - /** - * Asserts a condition in the app. - * @param assertion The assertion to check (in free-form text). * @example The welcome message should be visible * @example The welcome message text should be "Hello, world!" */ - assert: (assertion: string) => Promise; + perform: (intent: string) => Promise; } /** @@ -49,23 +42,29 @@ export interface TestingFrameworkDriver { captureViewHierarchyString: () => Promise; /** - * A descriptive catalog for the available API of the testing framework that can be used by Copilot. + * The available API methods of the testing framework. */ - availableAPI: TestingFrameworkAPICatalog; + apiCatalog: TestingFrameworkAPICatalog; } /** * Represents the available API of the testing framework that can be used by Copilot. * @property context The available variables of the testing framework (i.e. exposes the matching function, expect, etc.). - * @property matchers The available matchers API of the testing framework. - * @property actions The available actions API of the testing framework. - * @property assertions The available assertions API of the testing framework. + * @property categories The available categories of the testing framework API. */ export type TestingFrameworkAPICatalog = { context: any; - matchers: TestingFrameworkAPICatalogItem[]; - actions: TestingFrameworkAPICatalogItem[]; - assertions: TestingFrameworkAPICatalogItem[]; + categories: TestingFrameworkAPICatalogCategory[]; +} + +/** + * Represents a category of the API of the testing framework that can be used by Copilot. + * @property title The title of the category. + * @property items The items in the category. + */ +export type TestingFrameworkAPICatalogCategory = { + title: string; + items: TestingFrameworkAPICatalogItem[]; } /** @@ -125,18 +124,3 @@ export interface Config { */ promptHandler: PromptHandler; } - -/** - * Represents a step in the test script. - * @property type The type of the step. - * @property value The prompt for the step. - */ -export type ExecutionStep = { - type: ExecutionStepType; - value: string; -} - -/** - * Represents the type of step in the test script. - */ -export type ExecutionStepType = 'action' | 'assertion'; diff --git a/detox-copilot/src/utils/PromptCreator.test.ts b/detox-copilot/src/utils/PromptCreator.test.ts index 995b55f52d..64e975d35f 100644 --- a/detox-copilot/src/utils/PromptCreator.test.ts +++ b/detox-copilot/src/utils/PromptCreator.test.ts @@ -1,77 +1,77 @@ import { PromptCreator } from './PromptCreator'; -import {ExecutionStep, TestingFrameworkAPICatalog} from "@/types"; +import { TestingFrameworkAPICatalog, TestingFrameworkAPICatalogCategory, TestingFrameworkAPICatalogItem } from "@/types"; const mockAPI: TestingFrameworkAPICatalog = { context: {}, - actions: [ + categories: [ { - signature: 'tap(element: Element)', - description: 'Taps on the specified element.', - example: 'await element(by.id("button")).tap();', - guidelines: ['Ensure the element is tappable before using this method.'] + title: 'Actions', + items: [ + { + signature: 'tap(element: Element)', + description: 'Taps on the specified element.', + example: 'await element(by.id("button")).tap();', + guidelines: ['Ensure the element is tappable before using this method.'] + }, + { + signature: 'typeText(element: Element, text: string)', + description: 'Types the specified text into the element.', + example: 'await element(by.id("input")).typeText("Hello, World!");', + guidelines: ['Use this method only on text input elements.'] + } + ] }, { - signature: 'typeText(element: Element, text: string)', - description: 'Types the specified text into the element.', - example: 'await element(by.id("input")).typeText("Hello, World!");', - guidelines: ['Use this method only on text input elements.'] - } - ], - assertions: [ - { - signature: 'toBeVisible()', - description: 'Asserts that the element is visible on the screen.', - example: 'await expect(element(by.id("title"))).toBeVisible();', - guidelines: ['Consider scroll position when using this assertion.'] - } - ], - matchers: [ + title: 'Assertions', + items: [ + { + signature: 'toBeVisible()', + description: 'Asserts that the element is visible on the screen.', + example: 'await expect(element(by.id("title"))).toBeVisible();', + guidelines: ['Consider scroll position when using this assertion.'] + } + ] + }, { - signature: 'by.id(id: string)', - description: 'Matches elements by their ID attribute.', - example: 'element(by.id("uniqueId"))', - guidelines: ['Use unique IDs for elements to avoid conflicts, combine with atIndex() if necessary.'] + title: 'Matchers', + items: [ + { + signature: 'by.id(id: string)', + description: 'Matches elements by their ID attribute.', + example: 'element(by.id("uniqueId"))', + guidelines: ['Use unique IDs for elements to avoid conflicts, combine with atIndex() if necessary.'] + } + ] } ] }; -describe('prompt creation', () => { +describe('PromptCreator', () => { let promptCreator: PromptCreator; beforeEach(() => { promptCreator = new PromptCreator(mockAPI); }); - it('should create an action step prompt correctly', () => { - const step: ExecutionStep = { - type: 'action', - value: 'tap button' - }; - + it('should create a prompt for an intent correctly', () => { + const intent = 'tap button'; const viewHierarchy = '