From 01182b434d407106df83868c52f8a94e439687a0 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 1 Oct 2024 21:05:54 +0800 Subject: [PATCH 01/17] [Flyte Decl] Streaming Decks Signed-off-by: Future-Outlier --- flytekit/bin/entrypoint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 36c5994421..65ff8f0a21 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -103,6 +103,10 @@ def _dispatch_execute( input_proto = utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto) + # TEST DECK + if task_def is not None and not getattr(task_def, "disable_deck", True): + _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) + # Step2 # Invoke task - dispatch_execute outputs = task_def.dispatch_execute(ctx, idl_input_literals) From 9616fc319773e21e922b662ea160f307f661f85e Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 1 Oct 2024 21:25:38 +0800 Subject: [PATCH 02/17] print Signed-off-by: Future-Outlier --- flytekit/bin/entrypoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 65ff8f0a21..58e7c7ccf2 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -106,6 +106,7 @@ def _dispatch_execute( # TEST DECK if task_def is not None and not getattr(task_def, "disable_deck", True): _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) + print("@@@ OUTPUT DECK @@@") # Step2 # Invoke task - dispatch_execute From 98ae7c7f1a34cf201e66263df95479bbcf6091f9 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Tue, 1 Oct 2024 23:17:24 +0800 Subject: [PATCH 03/17] sleep more Signed-off-by: Future-Outlier --- flytekit/bin/entrypoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 58e7c7ccf2..1d90f2b9eb 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -107,6 +107,8 @@ def _dispatch_execute( if task_def is not None and not getattr(task_def, "disable_deck", True): _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) print("@@@ OUTPUT DECK @@@") + import time + time.sleep(15) # Step2 # Invoke task - dispatch_execute From 4e92bb0ad1cb55c9b6288533525359b0182d0e72 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 13:15:07 +0800 Subject: [PATCH 04/17] add dummy deck Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 025306d47b..0ac28a8bc6 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -11,7 +11,40 @@ OUTPUT_DIR_JUPYTER_PREFIX = "jupyter" DECK_FILE_NAME = "deck.html" - +DUMMY_DECK = """ + + + + + + Flytekit Status + + + +
+

Flyte decks have not been created yet.

+
+ + +""" class DeckField(str, enum.Enum): """ @@ -164,6 +197,9 @@ def _get_deck( # The renderer must ensure that the HTML is safe. body_htmls.append(f"
{value}
") + if len(nav_htmls) == 0 and len(body_htmls) == 0: + body_htmls.append(DUMMY_DECK) + body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): try: From 4df18b5cc9bd3a8b5d222ee155ebe319665ffef3 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 13:26:22 +0800 Subject: [PATCH 05/17] nit Signed-off-by: Future-Outlier --- flytekit/bin/entrypoint.py | 2 -- flytekit/deck/deck.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 1d90f2b9eb..58e7c7ccf2 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -107,8 +107,6 @@ def _dispatch_execute( if task_def is not None and not getattr(task_def, "disable_deck", True): _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) print("@@@ OUTPUT DECK @@@") - import time - time.sleep(15) # Step2 # Invoke task - dispatch_execute diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 0ac28a8bc6..35a8ad7428 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -46,6 +46,7 @@ """ + class DeckField(str, enum.Enum): """ DeckField is used to specify the fields that will be rendered in the deck. @@ -197,8 +198,8 @@ def _get_deck( # The renderer must ensure that the HTML is safe. body_htmls.append(f"
{value}
") - if len(nav_htmls) == 0 and len(body_htmls) == 0: - body_htmls.append(DUMMY_DECK) + # if len(nav_htmls) == 0 and len(body_htmls) == 0: + # body_htmls.append(DUMMY_DECK) body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): From ebb4d4e453763a37536c0d9cb99f3dbfa47e7843 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 13:29:47 +0800 Subject: [PATCH 06/17] dummy deck Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 35a8ad7428..d4894285fe 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -200,6 +200,8 @@ def _get_deck( # if len(nav_htmls) == 0 and len(body_htmls) == 0: # body_htmls.append(DUMMY_DECK) + nav_htmls = [] + body_htmls = [] body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): From 99522d9313204cec543b4df8a109d394e848914d Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 14:12:06 +0800 Subject: [PATCH 07/17] update Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index d4894285fe..ca3315d069 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -200,9 +200,9 @@ def _get_deck( # if len(nav_htmls) == 0 and len(body_htmls) == 0: # body_htmls.append(DUMMY_DECK) - nav_htmls = [] - body_htmls = [] - body_htmls.append(DUMMY_DECK) + nav_htmls = [DUMMY_DECK] + body_htmls = [DUMMY_DECK] + # body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): try: From c19d67daecf1445c4fd23b5ac68157baeef39043 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 14:14:01 +0800 Subject: [PATCH 08/17] nit Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index ca3315d069..25fd19253a 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -200,8 +200,7 @@ def _get_deck( # if len(nav_htmls) == 0 and len(body_htmls) == 0: # body_htmls.append(DUMMY_DECK) - nav_htmls = [DUMMY_DECK] - body_htmls = [DUMMY_DECK] + nav_htmls.append(DUMMY_DECK) # body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): From 67cd829640abfe6660a01b72286fa7e2984b5c99 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 14:20:38 +0800 Subject: [PATCH 09/17] test Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 25fd19253a..06ed4fcf1a 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -11,7 +11,7 @@ OUTPUT_DIR_JUPYTER_PREFIX = "jupyter" DECK_FILE_NAME = "deck.html" -DUMMY_DECK = """ +DUMMY_DECK_HTML = """ @@ -46,7 +46,6 @@ """ - class DeckField(str, enum.Enum): """ DeckField is used to specify the fields that will be rendered in the deck. @@ -106,6 +105,7 @@ def __init__(self, name: str, html: Optional[str] = "", auto_add_to_deck: bool = self._html = html if auto_add_to_deck: FlyteContextManager.current_context().user_space_params.decks.append(self) + FlyteContextManager.current_context().user_space_params.decks.append(DummyDeck) def append(self, html: str) -> "Deck": assert isinstance(html, str) @@ -120,6 +120,15 @@ def name(self) -> str: def html(self) -> str: return self._html +class DummyDeck(Deck): + """ + The DummyDeck class is designed + """ + def __init__(self): + name = "dummy_deck" + html = DUMMY_DECK_HTML + super().__init__(name, html) + class TimeLineDeck(Deck): """ @@ -200,7 +209,7 @@ def _get_deck( # if len(nav_htmls) == 0 and len(body_htmls) == 0: # body_htmls.append(DUMMY_DECK) - nav_htmls.append(DUMMY_DECK) + # nav_htmls.append(DUMMY_DECK) # body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): From 06da3df4a18fdf8a4208dea2d043504c8f137b2b Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 14:24:43 +0800 Subject: [PATCH 10/17] return html Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 06ed4fcf1a..1e97ff48f2 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -129,6 +129,9 @@ def __init__(self): html = DUMMY_DECK_HTML super().__init__(name, html) + @property + def html(self) -> str: + return self.html class TimeLineDeck(Deck): """ From 6d99d69259c440fb991be6e559216a22ed576bb1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:12:26 +0800 Subject: [PATCH 11/17] Change Deck Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 1e97ff48f2..06ed4fcf1a 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -129,9 +129,6 @@ def __init__(self): html = DUMMY_DECK_HTML super().__init__(name, html) - @property - def html(self) -> str: - return self.html class TimeLineDeck(Deck): """ From b805cd7f52063ab2ba67a5807e42098b1c212bf0 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:18:25 +0800 Subject: [PATCH 12/17] fix Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 06ed4fcf1a..52ac9f6280 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -105,7 +105,7 @@ def __init__(self, name: str, html: Optional[str] = "", auto_add_to_deck: bool = self._html = html if auto_add_to_deck: FlyteContextManager.current_context().user_space_params.decks.append(self) - FlyteContextManager.current_context().user_space_params.decks.append(DummyDeck) + FlyteContextManager.current_context().user_space_params.decks.append(DummyDeck()) def append(self, html: str) -> "Deck": assert isinstance(html, str) @@ -129,6 +129,10 @@ def __init__(self): html = DUMMY_DECK_HTML super().__init__(name, html) + @property + def html(self) -> str: + return self._html + class TimeLineDeck(Deck): """ From 4c97758ab263a8a564b846c1307e1026cab45cce Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:21:59 +0800 Subject: [PATCH 13/17] fix recursion error Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 52ac9f6280..b604ff1723 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -105,7 +105,6 @@ def __init__(self, name: str, html: Optional[str] = "", auto_add_to_deck: bool = self._html = html if auto_add_to_deck: FlyteContextManager.current_context().user_space_params.decks.append(self) - FlyteContextManager.current_context().user_space_params.decks.append(DummyDeck()) def append(self, html: str) -> "Deck": assert isinstance(html, str) @@ -201,6 +200,9 @@ def _get_deck( Get flyte deck html string If ignore_jupyter is set to True, then it will return a str even in a jupyter environment. """ + if not new_user_params.decks: + new_user_params.decks.append(DummyDeck()) + deck_map = {deck.name: deck.html for deck in new_user_params.decks} nav_htmls = [] body_htmls = [] From 7b3574ad4b3905412fa3e02a3cafffe96fc457de Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:28:08 +0800 Subject: [PATCH 14/17] remove redundant code Signed-off-by: Future-Outlier --- flytekit/bin/entrypoint.py | 8 ++++---- flytekit/deck/deck.py | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/flytekit/bin/entrypoint.py b/flytekit/bin/entrypoint.py index 58e7c7ccf2..5537d7555f 100644 --- a/flytekit/bin/entrypoint.py +++ b/flytekit/bin/entrypoint.py @@ -103,10 +103,10 @@ def _dispatch_execute( input_proto = utils.load_proto_from_file(_literals_pb2.LiteralMap, local_inputs_file) idl_input_literals = _literal_models.LiteralMap.from_flyte_idl(input_proto) - # TEST DECK - if task_def is not None and not getattr(task_def, "disable_deck", True): - _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) - print("@@@ OUTPUT DECK @@@") + # # TEST DECK + # if task_def is not None and not getattr(task_def, "disable_deck", True): + # _output_deck(task_def.name.split(".")[-1], ctx.user_space_params) + # print("@@@ OUTPUT DECK @@@") # Step2 # Invoke task - dispatch_execute diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index b604ff1723..4c633fdeab 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -213,10 +213,6 @@ def _get_deck( # The renderer must ensure that the HTML is safe. body_htmls.append(f"
{value}
") - # if len(nav_htmls) == 0 and len(body_htmls) == 0: - # body_htmls.append(DUMMY_DECK) - # nav_htmls.append(DUMMY_DECK) - # body_htmls.append(DUMMY_DECK) raw_html = get_deck_template().substitute(NAV_HTML="".join(nav_htmls), BODY_HTML="".join(body_htmls)) if not ignore_jupyter and ipython_check(): try: From 9b60564fe56f923649558aca2bb67289e24695e2 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:38:10 +0800 Subject: [PATCH 15/17] add dummy deck to deck init Signed-off-by: Future-Outlier --- flytekit/deck/__init__.py | 2 +- flytekit/deck/deck.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/deck/__init__.py b/flytekit/deck/__init__.py index 58da56cf64..5481493e8f 100644 --- a/flytekit/deck/__init__.py +++ b/flytekit/deck/__init__.py @@ -18,5 +18,5 @@ SourceCodeRenderer """ -from .deck import Deck, DeckField +from .deck import Deck, DeckField, DummyDeck from .renderer import MarkdownRenderer, SourceCodeRenderer, TopFrameRenderer diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index 4c633fdeab..a552ef9e32 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -200,8 +200,8 @@ def _get_deck( Get flyte deck html string If ignore_jupyter is set to True, then it will return a str even in a jupyter environment. """ - if not new_user_params.decks: - new_user_params.decks.append(DummyDeck()) + # if not new_user_params.decks: + # new_user_params.decks.append(DummyDeck()) deck_map = {deck.name: deck.html for deck in new_user_params.decks} nav_htmls = [] From 18c994ff4b09d4c4014ea90352ee5fc678f705a1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Wed, 2 Oct 2024 23:56:04 +0800 Subject: [PATCH 16/17] Better Dummy Deck Logic Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index a552ef9e32..e5d069490c 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -200,10 +200,15 @@ def _get_deck( Get flyte deck html string If ignore_jupyter is set to True, then it will return a str even in a jupyter environment. """ - # if not new_user_params.decks: - # new_user_params.decks.append(DummyDeck()) - deck_map = {deck.name: deck.html for deck in new_user_params.decks} + deck_map = { + deck.name: deck.html for deck in new_user_params.decks if deck.name != "dummy_deck" + } + + # If deck_map is empty after filtering, add DummyDeck + if not deck_map: + deck_map = {"dummy_deck": DUMMY_DECK_HTML} + nav_htmls = [] body_htmls = [] From aabcbbbbceee1c87f69e55c25bc7ec07045d6058 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Mon, 7 Oct 2024 14:21:23 +0800 Subject: [PATCH 17/17] Deck Publish Signed-off-by: Future-Outlier --- flytekit/deck/deck.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/flytekit/deck/deck.py b/flytekit/deck/deck.py index e5d069490c..1db12347d6 100644 --- a/flytekit/deck/deck.py +++ b/flytekit/deck/deck.py @@ -46,6 +46,7 @@ """ + class DeckField(str, enum.Enum): """ DeckField is used to specify the fields that will be rendered in the deck. @@ -119,10 +120,18 @@ def name(self) -> str: def html(self) -> str: return self._html + @classmethod + def publish(cls): + task_name = FlyteContextManager.current_context().user_space_params.task_id.name + new_user_params = FlyteContextManager.current_context().user_space_params + _output_deck(task_name, new_user_params) + + class DummyDeck(Deck): """ The DummyDeck class is designed """ + def __init__(self): name = "dummy_deck" html = DUMMY_DECK_HTML @@ -201,9 +210,7 @@ def _get_deck( If ignore_jupyter is set to True, then it will return a str even in a jupyter environment. """ - deck_map = { - deck.name: deck.html for deck in new_user_params.decks if deck.name != "dummy_deck" - } + deck_map = {deck.name: deck.html for deck in new_user_params.decks if deck.name != "dummy_deck"} # If deck_map is empty after filtering, add DummyDeck if not deck_map: