Merge branch 'main' into kevin

.github/workflows/ghcr_runtime.yml

@@ -98,7 +98,7 @@ jobs:
           outputs: type=docker,dest=/tmp/runtime-${{ matrix.base_image.tag }}.tar
           context: containers/runtime
       - name: Upload runtime image for fork
-        if: github.event.pull_request.head.repo.fork != true
+        if: github.event.pull_request.head.repo.fork
         uses: actions/upload-artifact@v4
         with:
           name: runtime-${{ matrix.base_image.tag }}

pyproject.toml

@@ -27,7 +27,7 @@ uvicorn = "*"
 types-toml = "*"
 numpy = "*"
 json-repair = "*"
-browsergym = "0.3.4" # integrate browsergym as the browsing interface
+browsergym = "0.4.3" # integrate browsergym as the browsing interface
 html2text = "*"
 e2b = "^0.17.1"
 pexpect = "*"

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_001.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_002.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------

tests/integration/mock/eventstream_runtime/BrowsingAgent/test_browse_internet/prompt_003.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_002.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_003.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------

tests/integration/mock/eventstream_runtime/CodeActAgent/test_browse_internet/prompt_004.log

@@ -36,13 +36,13 @@ fill(bid: str, value: str)
 
 select_option(bid: str, options: str | list[str])
     Examples:
-        select_option('48', 'blue')
+        select_option('a48', 'blue')
 
-        select_option('48', ['red', 'green', 'blue'])
+        select_option('c48', ['red', 'green', 'blue'])
 
 click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'Meta', 'Shift']] = [])
     Examples:
-        click('51')
+        click('a51')
 
         click('b22', button='right')
 
@@ -98,13 +98,11 @@ goto(url: str)
     Examples:
         goto('http://www.example.com')
 
-Multiple actions can be provided at once. Example:
+Multiple actions can be provided at once, but will be executed sequentially without any feedback from the page.
+More than 2-3 actions usually leads to failure or unexpected behavior. Example:
 fill('a12', 'example with "quotes"')
-click('51')
+click('a51')
 click('48', button='middle', modifiers=['Shift'])
-Multiple actions are meant to be executed sequentially without any feedback from the page.
-Don't execute multiple actions at once if you need feedback from the page.
-
 
 
 ----------