[current.card.refresh] refactor cards to support runtime updates

metaflow/plugins/cards/card_cli.py

@@ -6,7 +6,9 @@
 from metaflow._vendor import click
 import os
 import json
+import uuid
 import signal
+import inspect
 import random
 from contextlib import contextmanager
 from functools import wraps
@@ -375,14 +377,20 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-def render_card(mf_card, task, timeout_value=None):
-    rendered_info = None
+def update_card(mf_card, mode, task, data, timeout_value=None):
+    def _call():
+        # compatibility with old render()-method that doesn't accept the data arg
+        new_render = "data" in inspect.getfullargspec(mf_card.render).args
+        if mode == "render" and not new_render:
+            return mf_card.render(task)
+        else:
+            return getattr(mf_card, mode)(task, data=data)
+
     if timeout_value is None or timeout_value < 0:
-        rendered_info = mf_card.render(task)
+        return _call()
     else:
         with timeout(timeout_value):
-            rendered_info = mf_card.render(task)
-    return rendered_info
+            return _call()
 
 
 @card.command(help="create a HTML card")
@@ -414,29 +422,61 @@ def render_card(mf_card, task, timeout_value=None):
     is_flag=True,
     help="Upon failing to render a card, render a card holding the stack trace",
 )
+@click.option(
+    "--id",
+    default=None,
+    show_default=True,
+    type=str,
+    help="ID of the card",
+)
 @click.option(
     "--component-file",
     default=None,
     show_default=True,
     type=str,
-    help="JSON File with Pre-rendered components.(internal)",
+    help="JSON File with Pre-rendered components. (internal)",
 )
 @click.option(
-    "--id",
+    "--mode",
+    default="render",
+    show_default=True,
+    type=str,
+    help="Rendering mode. (internal)",
+)
+@click.option(
+    "--data-file",
     default=None,
     show_default=True,
     type=str,
-    help="ID of the card",
+    help="JSON file containing data to be updated. (internal)",
+)
+@click.option(
+    "--card-uuid",
+    default=None,
+    show_default=True,
+    type=str,
+    help="Card UUID. (internal)",
+)
+@click.option(
+    "--delete-input-files",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Delete data-file and compontent-file after reading. (internal)",
 )
 @click.pass_context
 def create(
     ctx,
     pathspec,
+    mode=None,
     type=None,
     options=None,
     timeout=None,
     component_file=None,
+    data_file=None,
     render_error_card=False,
+    card_uuid=None,
+    delete_input_files=None,
     id=None,
 ):
     card_id = id
@@ -452,11 +492,26 @@ def create(
 
     graph_dict, _ = ctx.obj.graph.output_steps()
 
+    if card_uuid is None:
+        card_uuid = str(uuid.uuid4()).replace("-", "")
+
     # Components are rendered in a Step and added via `current.card.append` are added here.
     component_arr = []
     if component_file is not None:
         with open(component_file, "r") as f:
             component_arr = json.load(f)
+        # data is passed in as temporary files which can be deleted after use
+        if delete_input_files:
+            os.remove(component_file)
+
+    # Load data to be refreshed for runtime cards
+    data = {}
+    if data_file is not None:
+        with open(data_file, "r") as f:
+            data = json.load(f)
+        # data is passed in as temporary files which can be deleted after use
+        if delete_input_files:
+            os.remove(data_file)
 
     task = Task(full_pathspec)
     from metaflow.plugins import CARDS
@@ -500,18 +555,20 @@ def create(
 
         if mf_card:
             try:
-                rendered_info = render_card(mf_card, task, timeout_value=timeout)
+                rendered_info = update_card(
+                    mf_card, mode, task, data, timeout_value=timeout
+                )
             except:
                 if render_error_card:
                     error_stack_trace = str(UnrenderableCardException(type, options))
                 else:
                     raise UnrenderableCardException(type, options)
         #
 
-    if error_stack_trace is not None:
+    if error_stack_trace is not None and mode != "refresh":
         rendered_info = error_card().render(task, stack_trace=error_stack_trace)
 
-    if rendered_info is None and render_error_card:
+    if rendered_info is None and render_error_card and mode != "refresh":
         rendered_info = error_card().render(
             task, stack_trace="No information rendered From card of type %s" % type
         )
@@ -532,12 +589,20 @@ def create(
         card_id = None
 
     if rendered_info is not None:
-        card_info = card_datastore.save_card(save_type, rendered_info, card_id=card_id)
-        ctx.obj.echo(
-            "Card created with type: %s and hash: %s"
-            % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]),
-            fg="green",
-        )
+        if mode == "refresh":
+            card_datastore.save_data(
+                card_uuid, save_type, rendered_info, card_id=card_id
+            )
+            ctx.obj.echo("Data updated", fg="green")
+        else:
+            card_info = card_datastore.save_card(
+                card_uuid, save_type, rendered_info, card_id=card_id
+            )
+            ctx.obj.echo(
+                "Card created with type: %s and hash: %s"
+                % (card_info.type, card_info.hash[:NUM_SHORT_HASH_CHARS]),
+                fg="green",
+            )
 
 
 @card.command()
@@ -655,7 +720,6 @@ def list(
     as_json=False,
     file=None,
 ):
-
     card_id = id
     if pathspec is None:
         list_many_cards(

metaflow/plugins/cards/card_datastore.py

@@ -6,6 +6,7 @@
 from hashlib import sha1
 from io import BytesIO
 import os
+import json
 import shutil
 
 from metaflow.plugins.datastores.local_storage import LocalStorage
@@ -88,15 +89,15 @@ def __init__(self, flow_datastore, pathspec=None):
         self._temp_card_save_path = self._get_write_path(base_pth=TEMP_DIR_NAME)
 
     @classmethod
-    def get_card_location(cls, base_path, card_name, card_html, card_id=None):
-        chash = sha1(bytes(card_html, "utf-8")).hexdigest()
+    def get_card_location(cls, base_path, card_name, uuid, card_id=None, suffix="html"):
+        chash = uuid
         if card_id is None:
-            card_file_name = "%s-%s.html" % (card_name, chash)
+            card_file_name = "%s-%s.%s" % (card_name, chash, suffix)
         else:
-            card_file_name = "%s-%s-%s.html" % (card_name, card_id, chash)
+            card_file_name = "%s-%s-%s.%s" % (card_name, card_id, chash, suffix)
         return os.path.join(base_path, card_file_name)
 
-    def _make_path(self, base_pth, pathspec=None, with_steps=False):
+    def _make_path(self, base_pth, pathspec=None, with_steps=False, suffix="cards"):
         sysroot = base_pth
         if pathspec is not None:
             # since most cards are at a task level there will always be 4 non-none values returned
@@ -121,7 +122,7 @@ def _make_path(self, base_pth, pathspec=None, with_steps=False):
                 step_name,
                 "tasks",
                 task_id,
-                "cards",
+                suffix,
             ]
         else:
             pth_arr = [
@@ -131,14 +132,16 @@ def _make_path(self, base_pth, pathspec=None, with_steps=False):
                 run_id,
                 "tasks",
                 task_id,
-                "cards",
+                suffix,
             ]
         if sysroot == "" or sysroot is None:
             pth_arr.pop(0)
         return os.path.join(*pth_arr)
 
-    def _get_write_path(self, base_pth=""):
-        return self._make_path(base_pth, pathspec=self._pathspec, with_steps=True)
+    def _get_write_path(self, base_pth="", suffix="cards"):
+        return self._make_path(
+            base_pth, pathspec=self._pathspec, with_steps=True, suffix=suffix
+        )
 
     def _get_read_path(self, base_pth="", with_steps=False):
         return self._make_path(base_pth, pathspec=self._pathspec, with_steps=with_steps)
@@ -173,7 +176,20 @@ def card_info_from_path(path):
         card_hash = card_hash.split(".html")[0]
         return CardInfo(card_type, card_hash, card_id, card_file_name)
 
-    def save_card(self, card_type, card_html, card_id=None, overwrite=True):
+    def save_data(self, uuid, card_type, json_data, card_id=None):
+        card_file_name = card_type
+        loc = self.get_card_location(
+            self._get_write_path(suffix="runtime"),
+            card_file_name,
+            uuid,
+            card_id=card_id,
+            suffix="data.json",
+        )
+        self._backend.save_bytes(
+            [(loc, BytesIO(json.dumps(json_data).encode("utf-8")))], overwrite=True
+        )
+
+    def save_card(self, uuid, card_type, card_html, card_id=None, overwrite=True):
         card_file_name = card_type
         # TEMPORARY_WORKAROUND: FIXME (LATER) : Fix the duplication of below block in a few months.
         # Check file blame to understand the age of this temporary workaround.
@@ -193,7 +209,7 @@ def save_card(self, card_type, card_html, card_id=None, overwrite=True):
         # It will also easily end up breaking the metaflow-ui (which maybe using a client from an older version).
         # Hence, we are writing cards to both paths so that we can introduce breaking changes later in the future.
         card_path_with_steps = self.get_card_location(
-            self._get_write_path(), card_file_name, card_html, card_id=card_id
+            self._get_write_path(), card_file_name, uuid, card_id=card_id
         )
         if SKIP_CARD_DUALWRITE:
             self._backend.save_bytes(
@@ -204,7 +220,7 @@ def save_card(self, card_type, card_html, card_id=None, overwrite=True):
             card_path_without_steps = self.get_card_location(
                 self._get_read_path(with_steps=False),
                 card_file_name,
-                card_html,
+                uuid,
                 card_id=card_id,
             )
             for cp in [card_path_with_steps, card_path_without_steps]: