Merge remote-tracking branch 'upstream/master' into thomasjpfan/comet…

…-ml-init

Signed-off-by: Thomas J. Fan <[email protected]>

examples/productionizing/productionizing/reference_task.py

@@ -1,6 +1,6 @@
-from typing import List
+from typing import List, NamedTuple
 
-from flytekit import reference_task, workflow
+from flytekit import StructuredDataset, reference_task, workflow
 from flytekit.types.file import FlyteFile
 
 # A `flytekit.reference_task` references the Flyte tasks that have already been defined, serialized, and registered.
@@ -26,8 +26,22 @@ def normalize_columns(
     ...
 
 
+outputs = NamedTuple("Outputs", results=StructuredDataset)
+
+
+@reference_task(
+    project="flytesnacks",
+    domain="development",
+    name="bigquery",
+    version="{{ registration.version }}",
+)
+def bigquery_task(version: int) -> outputs:
+    ...
+
+
 @workflow
 def wf() -> FlyteFile:
+    bigquery_task(version=1)
     return normalize_columns(
         csv_url="https://people.sc.fsu.edu/~jburkardt/data/csv/biostats.csv",
         column_names=["Name", "Sex", "Age", "Heights (in)", "Weight (lbs)"],

examples/ray_plugin/ray_plugin/ray_example.py

@@ -55,13 +55,18 @@ def f(x):
 # - `replicas`: Specifies the desired number of replicas for the worker group. The default is 1.
 # - `group_name`: A RayCluster can host multiple worker groups, each differentiated by its name.
 # - `runtime_env`: The [runtime environment](https://docs.ray.io/en/latest/ray-core/handling-dependencies.html#runtime-environments)
+# - `enable_autoscaling`: Enable [Ray Autoscaler](https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/configuring-autoscaling.html)
+# - `ttl_seconds_after_finished`: Shutdown Ray cluster after 1 hour of inactivity.
 #   definition outlines the necessary dependencies for your Ray application to function.
 #   This environment is dynamically installed on the cluster at runtime.
 # %%
 ray_config = RayJobConfig(
     head_node_config=HeadNodeConfig(ray_start_params={"log-color": "True"}),
     worker_node_config=[WorkerNodeConfig(group_name="ray-group", replicas=1)],
     runtime_env={"pip": ["numpy", "pandas"]},  # or runtime_env="./requirements.txt"
+    enable_autoscaling=True,
+    shutdown_after_job_finishes=True,
+    ttl_seconds_after_finished=3600,
 )