Create a single set of example scripts that can run on any executor by specifying external config.

Author: tomwhite

examples/add-asarray.py

@@ -0,0 +1,14 @@
+import cubed.array_api as xp
+
+if __name__ == "__main__":
+    a = xp.asarray(
+        [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]],
+        chunks=(2, 2),
+    )
+    b = xp.asarray(
+        [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]],
+        chunks=(2, 2),
+    )
+    c = xp.add(a, b)
+    res = c.compute()
+    print(res)

examples/add-random.py

@@ -0,0 +1,27 @@
+import logging
+
+import cubed
+import cubed.array_api as xp
+import cubed.random
+from cubed.extensions.history import HistoryCallback
+from cubed.extensions.rich import RichProgressBar
+from cubed.extensions.timeline import TimelineVisualizationCallback
+
+# suppress harmless connection pool warnings
+logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
+
+if __name__ == "__main__":
+    # 200MB chunks
+    a = cubed.random.random((50000, 50000), chunks=(5000, 5000))
+    b = cubed.random.random((50000, 50000), chunks=(5000, 5000))
+    c = xp.add(a, b)
+
+    progress = RichProgressBar()
+    hist = HistoryCallback()
+    timeline_viz = TimelineVisualizationCallback()
+    # use store=None to write to temporary zarr
+    cubed.to_zarr(
+        c,
+        store=None,
+        callbacks=[progress, hist, timeline_viz],
+    )

examples/lithops/aws-lambda/cubed.yaml

@@ -0,0 +1,7 @@
+spec:
+  work_dir: "s3://cubed-$USER-temp"
+  allowed_mem: "2GB"
+  executor_name: "lithops"
+  executor_options:
+    runtime: "cubed-runtime-dev"
+    runtime_memory: 2000

examples/lithops/gcf/cubed.yaml

@@ -0,0 +1,7 @@
+spec:
+  work_dir: "gs://cubed-$USER-temp"
+  allowed_mem: "2GB"
+  executor_name: "lithops"
+  executor_options:
+    runtime: "cubed-runtime-dev"
+    runtime_memory: 2048

examples/matmul-random.py

@@ -0,0 +1,29 @@
+import logging
+
+import cubed
+import cubed.array_api as xp
+import cubed.random
+from cubed.extensions.history import HistoryCallback
+from cubed.extensions.rich import RichProgressBar
+from cubed.extensions.timeline import TimelineVisualizationCallback
+
+# suppress harmless connection pool warnings
+logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
+
+if __name__ == "__main__":
+    # 200MB chunks
+    a = cubed.random.random((50000, 50000), chunks=(5000, 5000))
+    b = cubed.random.random((50000, 50000), chunks=(5000, 5000))
+    c = xp.astype(a, xp.float32)
+    d = xp.astype(b, xp.float32)
+    e = xp.matmul(c, d)
+
+    progress = RichProgressBar()
+    hist = HistoryCallback()
+    timeline_viz = TimelineVisualizationCallback()
+    # use store=None to write to temporary zarr
+    cubed.to_zarr(
+        e,
+        store=None,
+        callbacks=[progress, hist, timeline_viz],
+    )

examples/modal/aws/cubed.yaml

@@ -0,0 +1,6 @@
+spec:
+  work_dir: "s3://cubed-$USER-temp"
+  allowed_mem: "2GB"
+  executor_name: "modal"
+  executor_options:
+    cloud: "aws"

examples/modal/gcp/cubed.yaml

@@ -0,0 +1,6 @@
+spec:
+  work_dir: "gs://cubed-$USER-temp"
+  allowed_mem: "2GB"
+  executor_name: "modal"
+  executor_options:
+    cloud: "gcp"