[@card] Realtime cards (2/N)

metaflow/cards.py

@@ -6,6 +6,8 @@
     Image,
     Error,
     Markdown,
+    VegaChart,
+    ProgressBar,
 )
 from metaflow.plugins.cards.card_modules.basic import (
     DefaultCard,

metaflow/plugins/cards/card_cli.py

@@ -1,6 +1,11 @@
 from metaflow.client import Task
 from metaflow import JSONType, namespace
-from metaflow.exception import CommandException
+from metaflow.util import resolve_identity
+from metaflow.exception import (
+    CommandException,
+    MetaflowNotFound,
+    MetaflowNamespaceMismatch,
+)
 import webbrowser
 import re
 from metaflow._vendor import click
@@ -945,3 +950,125 @@ def list(
         show_list_as_json=as_json,
         file=file,
     )
+
+
+@card.command(help="Run local card viewer server")
+@click.option(
+    "--run-id",
+    default=None,
+    show_default=True,
+    type=str,
+    help="Run ID of the flow",
+)
+@click.option(
+    "--port",
+    default=8324,
+    show_default=True,
+    type=int,
+    help="Port on which Metaflow card viewer server will run",
+)
+@click.option(
+    "--namespace",
+    "user_namespace",
+    default=None,
+    show_default=True,
+    type=str,
+    help="Namespace of the flow",
+)
+@click.option(
+    "--poll-interval",
+    default=5,
+    show_default=True,
+    type=int,
+    help="Polling interval of the card viewer server.",
+)
+@click.option(
+    "--max-cards",
+    default=30,
+    show_default=True,
+    type=int,
+    help="Maximum number of cards to be shown at any time by the card viewer server",
+)
+@click.pass_context
+def server(ctx, run_id, port, user_namespace, poll_interval, max_cards):
+    from .card_server import create_card_server, CardServerOptions
+
+    user_namespace = resolve_identity() if user_namespace is None else user_namespace
+    run, follow_new_runs, _status_message = _get_run_object(
+        ctx.obj, run_id, user_namespace
+    )
+    if _status_message is not None:
+        ctx.obj.echo(_status_message, fg="red")
+    options = CardServerOptions(
+        flow_name=ctx.obj.flow.name,
+        run_object=run,
+        only_running=False,
+        follow_resumed=False,
+        flow_datastore=ctx.obj.flow_datastore,
+        max_cards=max_cards,
+        follow_new_runs=follow_new_runs,
+        poll_interval=poll_interval,
+    )
+    create_card_server(options, port, ctx.obj)
+
+
+def _get_run_from_cli_set_runid(obj, run_id):
+    # This run-id will be set from the command line args.
+    # So if we hit a MetaflowNotFound exception / Namespace mismatch then
+    # we should raise an exception
+    from metaflow import Run
+
+    flow_name = obj.flow.name
+    if len(run_id.split("/")) > 1:
+        raise CommandException(
+            "run_id should NOT be of the form: `<flowname>/<runid>`. Please provide only run-id"
+        )
+    try:
+        pathspec = "%s/%s" % (flow_name, run_id)
+        # Since we are looking at all namespaces,
+        # we will not
+        namespace(None)
+        return Run(pathspec)
+    except MetaflowNotFound:
+        raise CommandException("No run (%s) found for *%s*." % (run_id, flow_name))
+
+
+def _get_run_object(obj, run_id, user_namespace):
+    from metaflow import Flow
+
+    follow_new_runs = True
+    flow_name = obj.flow.name
+
+    if run_id is not None:
+        follow_new_runs = False
+        run = _get_run_from_cli_set_runid(obj, run_id)
+        obj.echo("Using run-id %s" % run.pathspec, fg="blue", bold=False)
+        return run, follow_new_runs, None
+
+    _msg = "Searching for runs in namespace: %s" % user_namespace
+    obj.echo(_msg, fg="blue", bold=False)
+
+    try:
+        namespace(user_namespace)
+        flow = Flow(pathspec=flow_name)
+        run = flow.latest_run
+    except MetaflowNotFound:
+        # When we have no runs found for the Flow, we need to ensure that
+        # if the `follow_new_runs` is set to True; If `follow_new_runs` is set to True then
+        # we don't raise the Exception and instead we return None and let the
+        # background Thread wait on the Retrieving the run object.
+        _status_msg = "No run found for *%s*." % flow_name
+        return None, follow_new_runs, _status_msg
+
+    except MetaflowNamespaceMismatch:
+        _status_msg = (
+            "No run found for *%s* in namespace *%s*. You can switch the namespace using --namespace"
+            % (
+                flow_name,
+                user_namespace,
+            )
+        )
+        return None, follow_new_runs, _status_msg
+
+    obj.echo("Using run-id %s" % run.pathspec, fg="blue", bold=False)
+    return run, follow_new_runs, None

metaflow/plugins/cards/card_creator.py

@@ -57,8 +57,11 @@ def create(
         logger=None,
         mode="render",
         final=False,
+        sync=False,
     ):
-        # warning_message("calling proc for uuid %s" % self._card_uuid, self._logger)
+        # Setting `final` will affect the Reload token set during the card refresh
+        # data creation along with synchronous execution of subprocess.
+        # Setting `sync` will only cause synchronous execution of subprocess.
         if mode != "render" and not runtime_card:
             # silently ignore runtime updates for cards that don't support them
             return
@@ -68,7 +71,6 @@ def create(
             component_strings = []
         else:
             component_strings = current.card._serialize_components(card_uuid)
-
         data = current.card._get_latest_data(card_uuid, final=final, mode=mode)
         runspec = "/".join([current.run_id, current.step_name, current.task_id])
         self._run_cards_subprocess(
@@ -82,6 +84,7 @@ def create(
             logger,
             data,
             final=final,
+            sync=sync,
         )
 
     def _run_cards_subprocess(
@@ -96,9 +99,10 @@ def _run_cards_subprocess(
         logger,
         data=None,
         final=False,
+        sync=False,
     ):
         components_file = data_file = None
-        wait = final
+        wait = final or sync
 
         if len(component_strings) > 0:
             # note that we can't delete temporary files here when calling the subprocess

metaflow/plugins/cards/card_modules/base.html

@@ -18,16 +18,28 @@
 <body>
   <div class="card_app"></div>
   <script>
+    var METAFLOW_RELOAD_TOKEN = "[METAFLOW_RELOAD_TOKEN]"
     var mfCardDataId = "{{{card_data_id}}}";
 
     if (!window.__MF_DATA__) {
       window.__MF_DATA__ = {};
     }
     window.__MF_DATA__["{{{card_data_id}}}"] = "{{{task_data}}}"
+
   </script>
   <script>
     {{{javascript}}}
   </script>
+  {{#RENDER_COMPLETE}}
+  <script>
+    window.metaflow_card_update = undefined;
+  </script>
+  {{/RENDER_COMPLETE}}
+  {{^RENDER_COMPLETE}}
+  <script> 
+  // This Card was Designed to be Realtime Updatable
+  </script>
+  {{/RENDER_COMPLETE}}
 </body>
 
 </html>