[WIP] Release dataset

crab-benchmark-v0/README.md

@@ -11,4 +11,4 @@ Our benchmark contains two important parts: **Environments** and **Tasks**.
 Since our Ubuntu environment is built upon KVM, setting it up locally requires you an experienced Linux user to deal with many small and miscellaneous issues. Therefore, we provide two environment setup methods:
 
 * [Local setup](./docs/environment_local_setup.md) provides you a step-by-step guideline to build environments on a Linux Machine with **at least one monitor and 32G memory**, but it doesn't cover details like how to install KVM on your machine because they are various on different Linux distros.
-* For those who want a quicker setup, we also provide a setup through [Google Clould Platform](./docs/environment_gcp_setup.md). Specifically, we publish a disk image contains all required softwares and configurations on google cloud, you can use your own google account to create a cloud computer through this disk image and use [google remote desktop](https://remotedesktop.google.com/access/) to connect to it. This method doesn't have any hardware limitations and when you set it up you can run the experiment immediately. As a tradeoff, the cloud computer that meets the minimum hardware requirement costs around $0.4 per hour (depend on the machine zone) .
+* For those who want a quicker setup, we also provide a setup through [Google Clould Platform](./docs/environment_gcp_setup.md). Specifically, we publish a disk image contains all required softwares and configurations on google cloud, you can use your own google account to create a cloud computer through this disk image and use [google remote desktop](https://remotedesktop.google.com/access/) to connect to it. This method doesn't have any hardware limitations and when you set it up you can run the experiment immediately. As a tradeoff, the cloud computer that meets the minimum hardware requirement costs around $0.4 per hour (depend on the machine zone).

crab-benchmark-v0/android_env.py

@@ -0,0 +1,25 @@
+from crab import EnvironmentConfig
+from crab.actions.android_actions import (
+    key_press,
+    open_app_drawer,
+    screenshot,
+    setup,
+    swipe,
+    tap,
+    write_text,
+)
+
+ANDROID_ENV = EnvironmentConfig(
+    name="android",
+    action_space=[tap, key_press, write_text, swipe, open_app_drawer],
+    observation_space=[screenshot],
+    description="""A Google Pixel smartphone runs on the Android operating system. \
+        The interface displays a current screenshot at each step and primarily \
+        supports interaction through tapping and typing. This device offers a suite \
+        of standard applications including Phone, Photos, Camera, Chrome, and \
+        Calendar, among others. Access the app drawer to view all installed \
+        applications on the device. The Google account is pre-logged in, synchronized \
+        with the same account used in the Ubuntu environment.""",
+    extra_attributes={"device": None},
+    reset=setup,
+)

crab-benchmark-v0/dataset/android/22b04776-8eec-4303-b3f6-9c981f7f29b8.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, Using \"Setting\" app, rename the device name of bluetooth as \"Sydney\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548an",
+            "attribute": {
+                "content": "Sydney"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "22b04776-8eec-4303-b3f6-9c981f7f29b8"
+}

crab-benchmark-v0/dataset/android/346caf7c-dc74-4c38-962a-aaffb638e0c7.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, Using \"Calendar\" app, add a new task with text \"meeting\" in date \"June 5th 2024\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ac",
+            "attribute": {
+                "content": "meeting",
+                "date": "05 June 2024"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "346caf7c-dc74-4c38-962a-aaffb638e0c7"
+}

crab-benchmark-v0/dataset/android/379b9c58-5125-41b3-9cc6-ea925c8b094d.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, Using Google Map app, Find the city name of corresponding post code \"560049\" in the country \"India\".",
+    "tasks": [
+        {
+            "task": "51b2463c-9904-4a32-81ba-507bfb89d61f",
+            "attribute": {
+                "country": "India",
+                "number": "560049"
+            },
+            "output": "Bengaluru"
+        }
+    ],
+    "adjlist": "0",
+    "id": "379b9c58-5125-41b3-9cc6-ea925c8b094d"
+}

crab-benchmark-v0/dataset/android/46d7ccdb-d2e4-4b8a-bead-f2641b5ac23c.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, Using \"Contacts\" app, add a contact with a mail \"{mail}\" with a name \"{name}\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ag",
+            "attribute": {
+                "mail": "[email protected]",
+                "name": "John Haruhimiya"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "46d7ccdb-d2e4-4b8a-bead-f2641b5ac23c"
+}

crab-benchmark-v0/dataset/android/6d9f6395-de79-4ad0-8a2a-2d674f93f293.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, Using \"Clock\" app, set the time of \"London\" in the clock, check the time gap between the city and current city.",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ah",
+            "attribute": {
+                "place_name": "London"
+            },
+            "output": "7 hours behind"
+        }
+    ],
+    "adjlist": "0",
+    "test_finished":"1",
+    "id": "6d9f6395-de79-4ad0-8a2a-2d674f93f293"
+}

crab-benchmark-v0/dataset/android/73f78fc3-1ca5-442d-801f-bc175a0bfb89.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, using \"Google Map\" App, find the distance of the shortest route from \"Southern University of Science and Technology\" to \"Lianhuashan Park\"",
+    "tasks": [
+        {
+            "task": "1a1b72d7-78c9-4027-8278-86083ae01045",
+            "attribute": {
+                "place_name_1": "Southern University of Science and Technology",
+                "place_name_2": "Lianhuashan Park"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "73f78fc3-1ca5-442d-801f-bc175a0bfb89"
+}

crab-benchmark-v0/dataset/android/764838cc-9359-4130-9bb2-4a75900b2d89.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, call \"123456789\".",
+    "tasks": [
+        {
+            "task": "955d8773-dd7a-4072-b87c-7e546be7de4e",
+            "attribute": {
+                "number": "123456789"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "764838cc-9359-4130-9bb2-4a75900b2d89"
+}

crab-benchmark-v0/dataset/android/77289141-e52b-48c8-b3a7-1b29520f3e1e.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, Using \"Contacts\" app, find out the mail of contact named \"John Haruhimiya\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ap",
+            "attribute": {
+                "name": "John Haruhimiya"
+            },
+            "output": "[email protected]"  
+        }
+    ],
+    "adjlist": "0",
+    "id": "77289141-e52b-48c8-b3a7-1b29520f3e1e"
+} 

crab-benchmark-v0/dataset/android/b077299d-1acb-40f5-89f3-cc08044345bf.json

@@ -0,0 +1,14 @@
+{
+    "description": "Using \"Tasks\" app, add a new task with text \"Watch camel tutorial video\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548af",
+            "attribute": {
+                "content": "Watch camel tutorial video"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "b077299d-1acb-40f5-89f3-cc08044345bf"
+}

crab-benchmark-v0/dataset/android/c1b1cfeb-40e7-49a8-a3f5-b8c8ba723601.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, Using \"Google Drive\" app, create a new folder named \"Journey\".",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ar",
+            "attribute": {
+                "content": "Journey"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "c1b1cfeb-40e7-49a8-a3f5-b8c8ba723601"
+}

crab-benchmark-v0/dataset/android/c85f03c9-83c4-417b-93d9-0d7b41022525.json

@@ -0,0 +1,14 @@
+{
+    "description": "In android system, use the calendar app, find the title of an event in the date \"15 June, 2024\".",
+    "tasks": [
+        {
+            "task": "2394b768-2ca7-45e9-b41e-2aa4e9573192",
+            "attribute": {
+                "date": "15 June 2024"
+            },
+            "output": "EMNLP ddl"
+        }
+    ],
+    "adjlist": "0",
+    "id": "c85f03c9-83c4-417b-93d9-0d7b41022525"
+}

crab-benchmark-v0/dataset/android/d2d456bb-c7d1-46af-8263-78d8509fb320.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, using \"Gmail\" App, send \"[email protected]\" a message \"Hello, nice to meet you!\"",
+    "tasks": [
+        {
+            "task": "0090f116-e02b-4562-a20d-b5df38be963a",
+            "attribute": {
+                "content": "Hello, nice to meet you!",
+                "mail": "[email protected]"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "d2d456bb-c7d1-46af-8263-78d8509fb320"
+}

crab-benchmark-v0/dataset/android/d4e0f2b3-d0ff-4efd-856f-9f5e598cfd05.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, Using \"Google Map\" app, Find the address of \"University of Oxford\"",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548aw",
+            "attribute": {
+                "content": "University of Oxford"
+            },
+            "output": "Wellington Square, Oxford OX1 2JD, United Kingdom"
+        }
+    ],
+    "adjlist": "0",
+    "id": "d4e0f2b3-d0ff-4efd-856f-9f5e598cfd05"
+}

crab-benchmark-v0/dataset/android/de843952-df8f-4a26-bae9-d0a32ed9a7f5.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, Using \"Files\" app, find the create date of \"Downloads/meow.jpg\" in the sdk system.",
+    "tasks": [
+        {
+            "task": "a3d11574-2acf-4b26-a569-a5dbc9d548ak",
+            "attribute": {
+                "file_path": "Download/meow.jpg.webp"
+            },
+            "output": "May 28"
+        }
+    ],
+    "adjlist": "0",
+    "id": "de843952-df8f-4a26-bae9-d0a32ed9a7f5"
+}

crab-benchmark-v0/dataset/android/e20fd121-b981-42da-94de-efcd66889c11.json

@@ -0,0 +1,15 @@
+{
+    "description": "In Android, using \"Messages\", send \"The meeting starts from 10am today\" to \"123456789\".",
+    "tasks": [
+        {
+            "task": "caa29623-1811-402d-963a-19f7eecc63d8",
+            "attribute": {
+                "content": "The meeting starts from 10am today",
+                "number": "123456789"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "e20fd121-b981-42da-94de-efcd66889c11"
+}

crab-benchmark-v0/dataset/android/fc642cb6-5321-4966-afbf-fb3348bb69ee.json

@@ -0,0 +1,14 @@
+{
+    "description": "In Android, using \"Keep Notes\" App, record \"Camel is the best agent framework in the world!\" in a new note without title.",
+    "tasks": [
+        {
+            "task": "eb92a1e6-4c86-4d56-baac-95fc8397732e",
+            "attribute": {
+                "content": "Camel is the best agent framework in the world!"
+            },
+            "output": null
+        }
+    ],
+    "adjlist": "0",
+    "id": "fc642cb6-5321-4966-afbf-fb3348bb69ee"
+}