From 4e50eaacc23fb72692b88ff16678ead7281ce4c2 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 12:46:47 -0700 Subject: [PATCH 01/12] testing --- .github/workflows/reusable_tutorials.yml | 26 ++++++++++++------------ scripts/make_tutorials.py | 14 ++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/reusable_tutorials.yml b/.github/workflows/reusable_tutorials.yml index bf758bd954f..5af913bdff8 100644 --- a/.github/workflows/reusable_tutorials.yml +++ b/.github/workflows/reusable_tutorials.yml @@ -30,20 +30,20 @@ jobs: with: python-version: "3.9" - - if: ${{ inputs.pinned_botorch }} - name: Install dependencies with pinned BoTorch - run: | - pip install -e ".[tutorial]" + # - if: ${{ inputs.pinned_botorch }} + # name: Install dependencies with pinned BoTorch + # run: | + # pip install -e ".[tutorial]" - - if: ${{ !inputs.pinned_botorch }} - name: Install dependencies with latest BoTorch - env: - ALLOW_BOTORCH_LATEST: true - ALLOW_LATEST_GPYTORCH_LINOP: true - run: | - pip install git+https://github.com/cornellius-gp/gpytorch.git - pip install git+https://github.com/pytorch/botorch.git - pip install -e ".[tutorial]" + # - if: ${{ !inputs.pinned_botorch }} + # name: Install dependencies with latest BoTorch + # env: + # ALLOW_BOTORCH_LATEST: true + # ALLOW_LATEST_GPYTORCH_LINOP: true + # run: | + # pip install git+https://github.com/cornellius-gp/gpytorch.git + # pip install git+https://github.com/pytorch/botorch.git + # pip install -e ".[tutorial]" - if: ${{ inputs.smoke_test }} name: Build tutorials with smoke test diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 6adba33c379..3e46785023a 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -268,10 +268,10 @@ def gen_tutorials( "to specify --include-ignored.", ) args = parser.parse_args() - gen_tutorials( - args.repo_dir, - args.exec_tutorials, - args.kernel_name, - smoke_test=args.smoke, - name=args.name, - ) + # gen_tutorials( + # args.repo_dir, + # args.exec_tutorials, + # args.kernel_name, + # smoke_test=args.smoke, + # name=args.name, + # ) From ef54169dd6ee530033c1df6c0d0f0f9b61269984 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 12:48:10 -0700 Subject: [PATCH 02/12] testing --- scripts/make_tutorials.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 3e46785023a..7cd7b9f0401 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -11,10 +11,10 @@ import time from typing import Dict, Optional -import nbformat -from bs4 import BeautifulSoup -from nbconvert import HTMLExporter, ScriptExporter -from nbconvert.preprocessors import ExecutePreprocessor +# import nbformat +# from bs4 import BeautifulSoup +# from nbconvert import HTMLExporter, ScriptExporter +# from nbconvert.preprocessors import ExecutePreprocessor TEMPLATE = """const CWD = process.cwd(); From e07d2e24e7483ac0fb22d41587e0f0610b982f70 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 12:49:35 -0700 Subject: [PATCH 03/12] testing --- .github/workflows/reusable_tutorials.yml | 26 ++++++++++++------------ scripts/make_tutorials.py | 24 ++++++++++++---------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/reusable_tutorials.yml b/.github/workflows/reusable_tutorials.yml index 5af913bdff8..bf758bd954f 100644 --- a/.github/workflows/reusable_tutorials.yml +++ b/.github/workflows/reusable_tutorials.yml @@ -30,20 +30,20 @@ jobs: with: python-version: "3.9" - # - if: ${{ inputs.pinned_botorch }} - # name: Install dependencies with pinned BoTorch - # run: | - # pip install -e ".[tutorial]" + - if: ${{ inputs.pinned_botorch }} + name: Install dependencies with pinned BoTorch + run: | + pip install -e ".[tutorial]" - # - if: ${{ !inputs.pinned_botorch }} - # name: Install dependencies with latest BoTorch - # env: - # ALLOW_BOTORCH_LATEST: true - # ALLOW_LATEST_GPYTORCH_LINOP: true - # run: | - # pip install git+https://github.com/cornellius-gp/gpytorch.git - # pip install git+https://github.com/pytorch/botorch.git - # pip install -e ".[tutorial]" + - if: ${{ !inputs.pinned_botorch }} + name: Install dependencies with latest BoTorch + env: + ALLOW_BOTORCH_LATEST: true + ALLOW_LATEST_GPYTORCH_LINOP: true + run: | + pip install git+https://github.com/cornellius-gp/gpytorch.git + pip install git+https://github.com/pytorch/botorch.git + pip install -e ".[tutorial]" - if: ${{ inputs.smoke_test }} name: Build tutorials with smoke test diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 7cd7b9f0401..3175f1ef0ce 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -11,10 +11,10 @@ import time from typing import Dict, Optional -# import nbformat -# from bs4 import BeautifulSoup -# from nbconvert import HTMLExporter, ScriptExporter -# from nbconvert.preprocessors import ExecutePreprocessor +import nbformat +from bs4 import BeautifulSoup +from nbconvert import HTMLExporter, ScriptExporter +from nbconvert.preprocessors import ExecutePreprocessor TEMPLATE = """const CWD = process.cwd(); @@ -116,6 +116,8 @@ def gen_tutorials( Also create ipynb and py versions of tutorial in Docusaurus site for download. """ + print("In gen_tutorials") + return has_errors = False with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile: @@ -268,10 +270,10 @@ def gen_tutorials( "to specify --include-ignored.", ) args = parser.parse_args() - # gen_tutorials( - # args.repo_dir, - # args.exec_tutorials, - # args.kernel_name, - # smoke_test=args.smoke, - # name=args.name, - # ) + gen_tutorials( + args.repo_dir, + args.exec_tutorials, + args.kernel_name, + smoke_test=args.smoke, + name=args.name, + ) From e6ca45c1b30b5a24e5aeec3ea7a5cc4ba6fb9975 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 12:56:04 -0700 Subject: [PATCH 04/12] testing --- scripts/make_tutorials.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 3175f1ef0ce..eda9f027331 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -117,12 +117,11 @@ def gen_tutorials( download. """ print("In gen_tutorials") - return has_errors = False with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile: tutorial_config = json.loads(infile.read()) - + print("past json loads") # flatten config dict tutorial_configs = [ config for category in tutorial_config.values() for config in category @@ -132,13 +131,14 @@ def gen_tutorials( tutorial_configs = [d for d in tutorial_configs if d["id"] == name] if len(tutorial_configs) == 0: raise RuntimeError(f"No tutorial found with name {name}.") - + print("making dirs") # prepare paths for converted tutorials & files os.makedirs(os.path.join(repo_dir, "website", "_tutorials"), exist_ok=True) os.makedirs(os.path.join(repo_dir, "website", "static", "files"), exist_ok=True) if smoke_test: os.environ["SMOKE_TEST"] = str(smoke_test) + return for config in tutorial_configs: tid = config["id"] t_dir = config.get("dir") From ac8b25ffbd7a25694db25a456ae7a0f962fded87 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 13:04:15 -0700 Subject: [PATCH 05/12] testing --- scripts/make_tutorials.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index eda9f027331..92365fc939b 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -138,8 +138,8 @@ def gen_tutorials( if smoke_test: os.environ["SMOKE_TEST"] = str(smoke_test) - return for config in tutorial_configs: + print("entering loop") tid = config["id"] t_dir = config.get("dir") exec_on_build = config.get("exec_on_build", True) @@ -147,16 +147,18 @@ def gen_tutorials( print("Generating {} tutorial".format(tid)) paths = _get_paths(repo_dir=repo_dir, t_dir=t_dir, tid=tid) + print("got paths") # load notebook with open(paths["tutorial_path"], "r") as infile: nb_str = infile.read() nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) + print("loaded notebook") # track total exec time (non-None if exec_on_build=True) total_time = None - if exec_tutorials and exec_on_build: + if exec_tutorials and exec_on_build and False: print("Executing tutorial {}".format(tid)) kwargs = {"kernel_name": kernel_name} if kernel_name is not None else {} # 2.5 hours, in seconds; 1 hour if smoke test mode @@ -184,12 +186,14 @@ def gen_tutorials( # convert notebook to HTML exporter = HTMLExporter(template_name="classic") html, _ = exporter.from_notebook_node(nb) + print("converted to html") # pull out html div for notebook soup = BeautifulSoup(html, "html.parser") nb_meat = soup.find("div", {"id": "notebook-container"}) del nb_meat.attrs["id"] nb_meat.attrs["class"] = ["notebook"] + print("made into soup") # when output html, iframe it (useful for Ax reports) for html_div in nb_meat.findAll("div", {"class": "output_html"}): @@ -201,12 +205,14 @@ def gen_tutorials( # replace `#` in CSS iframe.attrs["src"] = iframe.attrs["src"].replace("#", "%23") html_div.contents = [iframe] + print("made it into iframe") html_out = MOCK_JS_REQUIRES + str(nb_meat) # generate HTML file with open(paths["html_path"], "w") as html_outfile: html_outfile.write(html_out) + print("wrote to html outfile") # generate JS file t_dir_js = t_dir if t_dir else "" @@ -217,6 +223,7 @@ def gen_tutorials( ) with open(paths["js_path"], "w") as js_outfile: js_outfile.write(script) + print("generated js file") # output tutorial in both ipynb & py form nbformat.write(nb, paths["ipynb_path"]) @@ -224,6 +231,7 @@ def gen_tutorials( script, _ = exporter.from_notebook_node(nb) with open(paths["py_path"], "w") as py_outfile: py_outfile.write(script) + print("wrote to py outfile") # create .tar archive (if necessary) if t_dir is not None: @@ -232,6 +240,7 @@ def gen_tutorials( paths["tutorial_dir"], arcname=os.path.basename(paths["tutorial_dir"]), ) + print("added to tar") if has_errors: raise Exception("There are errors in tutorials, will not continue to publish") From 286fc0a040047474697cf1bd6bc340769129f8e7 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 13:10:22 -0700 Subject: [PATCH 06/12] testing --- scripts/make_tutorials.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 92365fc939b..85417402b09 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -158,17 +158,19 @@ def gen_tutorials( # track total exec time (non-None if exec_on_build=True) total_time = None - if exec_tutorials and exec_on_build and False: + if exec_tutorials and exec_on_build: print("Executing tutorial {}".format(tid)) kwargs = {"kernel_name": kernel_name} if kernel_name is not None else {} # 2.5 hours, in seconds; 1 hour if smoke test mode timeout = int(60 * 60) if smoke_test else int(60 * 60 * 2.5) + print("initializing execute processor") ep = ExecutePreprocessor(timeout=timeout, **kwargs) start_time = time.time() # try / catch failures for now # will re-raise at the end try: + print("In executing tutorial") # execute notebook, using `tutorial_dir` as working directory ep.preprocess(nb, {"metadata": {"path": paths["tutorial_dir"]}}) total_time = time.time() - start_time @@ -178,6 +180,7 @@ def gen_tutorials( ) ) except Exception as exc: + print("CAUGHT AN EXCEPTION") has_errors = True print("Couldn't execute tutorial {}!".format(tid)) print(exc) From f2939fd990980c9090151f742a6f767bfb92b5f8 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 13:25:05 -0700 Subject: [PATCH 07/12] testing --- scripts/make_tutorials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 85417402b09..23ef9026790 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -172,7 +172,7 @@ def gen_tutorials( try: print("In executing tutorial") # execute notebook, using `tutorial_dir` as working directory - ep.preprocess(nb, {"metadata": {"path": paths["tutorial_dir"]}}) + # ep.preprocess(nb, {"metadata": {"path": paths["tutorial_dir"]}}) total_time = time.time() - start_time print( "Done executing tutorial {}. Took {:.2f} seconds.".format( From ca16d94b14d50dbefbdbe86c3c68da67a62d5188 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 14:03:39 -0700 Subject: [PATCH 08/12] testing --- scripts/make_tutorials.py | 49 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 23ef9026790..5a45c96f2c8 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -7,10 +7,11 @@ import argparse import json import os +import subprocess import tarfile import time +from pathlib import Path from typing import Dict, Optional - import nbformat from bs4 import BeautifulSoup from nbconvert import HTMLExporter, ScriptExporter @@ -104,6 +105,21 @@ def _get_paths(repo_dir: str, t_dir: Optional[str], tid: str) -> Dict[str, str]: return paths +def run_script( + tutorial: Path, timeout_minutes: int, env: Optional[Dict[str, str]] = None +) -> None: + if env is not None: + env = {**os.environ, **env} + run_out = subprocess.run( + ["papermill", tutorial, "|"], + capture_output=True, + text=True, + env=env, + # timeout=timeout_minutes * 60, # TODO: utilize time out. + ) + return run_out + + def gen_tutorials( repo_dir: str, exec_tutorials: bool, @@ -116,12 +132,10 @@ def gen_tutorials( Also create ipynb and py versions of tutorial in Docusaurus site for download. """ - print("In gen_tutorials") has_errors = False with open(os.path.join(repo_dir, "website", "tutorials.json"), "r") as infile: tutorial_config = json.loads(infile.read()) - print("past json loads") # flatten config dict tutorial_configs = [ config for category in tutorial_config.values() for config in category @@ -131,7 +145,6 @@ def gen_tutorials( tutorial_configs = [d for d in tutorial_configs if d["id"] == name] if len(tutorial_configs) == 0: raise RuntimeError(f"No tutorial found with name {name}.") - print("making dirs") # prepare paths for converted tutorials & files os.makedirs(os.path.join(repo_dir, "website", "_tutorials"), exist_ok=True) os.makedirs(os.path.join(repo_dir, "website", "static", "files"), exist_ok=True) @@ -139,40 +152,30 @@ def gen_tutorials( os.environ["SMOKE_TEST"] = str(smoke_test) for config in tutorial_configs: - print("entering loop") tid = config["id"] t_dir = config.get("dir") exec_on_build = config.get("exec_on_build", True) - print("Generating {} tutorial".format(tid)) - paths = _get_paths(repo_dir=repo_dir, t_dir=t_dir, tid=tid) - print("got paths") # load notebook with open(paths["tutorial_path"], "r") as infile: nb_str = infile.read() nb = nbformat.reads(nb_str, nbformat.NO_CONVERT) - print("loaded notebook") - - # track total exec time (non-None if exec_on_build=True) - total_time = None if exec_tutorials and exec_on_build: + tutorial_path = Path(paths["tutorial_path"]) print("Executing tutorial {}".format(tid)) kwargs = {"kernel_name": kernel_name} if kernel_name is not None else {} # 2.5 hours, in seconds; 1 hour if smoke test mode - timeout = int(60 * 60) if smoke_test else int(60 * 60 * 2.5) - print("initializing execute processor") - ep = ExecutePreprocessor(timeout=timeout, **kwargs) + timeout = int(60 * 60) if smoke_test else int(60 * 60 * 2.5) # NOTE: unused! start_time = time.time() # try / catch failures for now # will re-raise at the end try: - print("In executing tutorial") - # execute notebook, using `tutorial_dir` as working directory - # ep.preprocess(nb, {"metadata": {"path": paths["tutorial_dir"]}}) + # Execute notebook. TODO: use timeout in the future. + run_script(tutorial=tutorial_path, timeout_minutes=None) total_time = time.time() - start_time print( "Done executing tutorial {}. Took {:.2f} seconds.".format( @@ -180,23 +183,22 @@ def gen_tutorials( ) ) except Exception as exc: - print("CAUGHT AN EXCEPTION") has_errors = True print("Couldn't execute tutorial {}!".format(tid)) print(exc) total_time = None + else: + total_time = None # convert notebook to HTML exporter = HTMLExporter(template_name="classic") html, _ = exporter.from_notebook_node(nb) - print("converted to html") # pull out html div for notebook soup = BeautifulSoup(html, "html.parser") nb_meat = soup.find("div", {"id": "notebook-container"}) del nb_meat.attrs["id"] nb_meat.attrs["class"] = ["notebook"] - print("made into soup") # when output html, iframe it (useful for Ax reports) for html_div in nb_meat.findAll("div", {"class": "output_html"}): @@ -208,14 +210,12 @@ def gen_tutorials( # replace `#` in CSS iframe.attrs["src"] = iframe.attrs["src"].replace("#", "%23") html_div.contents = [iframe] - print("made it into iframe") html_out = MOCK_JS_REQUIRES + str(nb_meat) # generate HTML file with open(paths["html_path"], "w") as html_outfile: html_outfile.write(html_out) - print("wrote to html outfile") # generate JS file t_dir_js = t_dir if t_dir else "" @@ -226,7 +226,6 @@ def gen_tutorials( ) with open(paths["js_path"], "w") as js_outfile: js_outfile.write(script) - print("generated js file") # output tutorial in both ipynb & py form nbformat.write(nb, paths["ipynb_path"]) @@ -234,7 +233,6 @@ def gen_tutorials( script, _ = exporter.from_notebook_node(nb) with open(paths["py_path"], "w") as py_outfile: py_outfile.write(script) - print("wrote to py outfile") # create .tar archive (if necessary) if t_dir is not None: @@ -243,7 +241,6 @@ def gen_tutorials( paths["tutorial_dir"], arcname=os.path.basename(paths["tutorial_dir"]), ) - print("added to tar") if has_errors: raise Exception("There are errors in tutorials, will not continue to publish") From 8f239afea09c589d79f4e68d64eeb039e9d2b589 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 14:09:09 -0700 Subject: [PATCH 09/12] testing --- scripts/make_tutorials.py | 1 - setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 5a45c96f2c8..9be16be2fbe 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -15,7 +15,6 @@ import nbformat from bs4 import BeautifulSoup from nbconvert import HTMLExporter, ScriptExporter -from nbconvert.preprocessors import ExecutePreprocessor TEMPLATE = """const CWD = process.cwd(); diff --git a/setup.py b/setup.py index 47e659b3952..cc7601742ce 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ "matplotlib", # Required for building Multi-objective tutorial notebook. "pyro-ppl", # Required for to call run_inference. "pytorch-lightning", # For the early stopping tutorial. + "papermill", # For executing the tutorials. ] From b8f38f997b9b60f46ccbf5f614baf8754d32a2c5 Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 15:19:12 -0700 Subject: [PATCH 10/12] lint --- scripts/make_tutorials.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 9be16be2fbe..6cf0e857c13 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -165,9 +165,6 @@ def gen_tutorials( if exec_tutorials and exec_on_build: tutorial_path = Path(paths["tutorial_path"]) print("Executing tutorial {}".format(tid)) - kwargs = {"kernel_name": kernel_name} if kernel_name is not None else {} - # 2.5 hours, in seconds; 1 hour if smoke test mode - timeout = int(60 * 60) if smoke_test else int(60 * 60 * 2.5) # NOTE: unused! start_time = time.time() # try / catch failures for now From 6d843c353e5432ef38f80cb570a14073cb9b035a Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 15:22:23 -0700 Subject: [PATCH 11/12] lint --- scripts/make_tutorials.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index 6cf0e857c13..a6917df37ca 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -12,6 +12,7 @@ import time from pathlib import Path from typing import Dict, Optional + import nbformat from bs4 import BeautifulSoup from nbconvert import HTMLExporter, ScriptExporter From b1c6993dc74149a8d148b95c5552841dd862cc7f Mon Sep 17 00:00:00 2001 From: Sait Cakmak Date: Fri, 8 Sep 2023 15:43:01 -0700 Subject: [PATCH 12/12] add timeouts --- scripts/make_tutorials.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/make_tutorials.py b/scripts/make_tutorials.py index a6917df37ca..2eefad9229d 100644 --- a/scripts/make_tutorials.py +++ b/scripts/make_tutorials.py @@ -115,7 +115,7 @@ def run_script( capture_output=True, text=True, env=env, - # timeout=timeout_minutes * 60, # TODO: utilize time out. + timeout=timeout_minutes * 60, ) return run_out @@ -171,8 +171,10 @@ def gen_tutorials( # try / catch failures for now # will re-raise at the end try: - # Execute notebook. TODO: use timeout in the future. - run_script(tutorial=tutorial_path, timeout_minutes=None) + # Execute notebook. + # TODO: [T163244135] Speed up tutorials and reduce timeout limits. + timeout_minutes = 15 if smoke_test else 150 + run_script(tutorial=tutorial_path, timeout_minutes=timeout_minutes) total_time = time.time() - start_time print( "Done executing tutorial {}. Took {:.2f} seconds.".format(