From 59b54139d383e439d2ac6e2931281e4886a06390 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 13:39:58 +0100 Subject: [PATCH 01/56] rm production website config #180 --- .../benchmarking/report/config/config.json | 128 ------------------ 1 file changed, 128 deletions(-) delete mode 100644 src/feelpp/benchmarking/report/config/config.json diff --git a/src/feelpp/benchmarking/report/config/config.json b/src/feelpp/benchmarking/report/config/config.json deleted file mode 100644 index 67d6359f..00000000 --- a/src/feelpp/benchmarking/report/config/config.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "execution_mapping": { - "feelpp_toolbox_heat": { - "gaya": { - "thermal_bridges_case_2":{"platform":"girder","path":"673da5c0b0e95728eb010c1b"}, - "thermal_bridges_case_3":{"platform":"girder","path":"673da5c7b0e95728eb010c1c"} - } - }, - "feelpp_toolbox_thermoelectric": { - "gaya": { - "HL_31":{"platform":"girder","path":"673da5e2b0e95728eb010c1e"} - } - }, - "feelpp_toolbox_heatfluid": { - "gaya": { - "eye":{"platform":"girder","path":"673da5d4b0e95728eb010c1d"} - } - }, - "feelpp_kub_cem": { - "gaya": { - "poznan":{"platform":"girder","path":"673da5a6b0e95728eb010c1a"} - } - } - }, - "machines": { - "gaya": { - "display_name": "Gaya", - "description": "Strasbourg, France" - }, - "meluxina": { - "display_name": "MeluXina", - "description": "Bissen, Luxembourg" - }, - "discoverer": { - "display_name": "Discoverer", - "description": "Sofia, Bulgaria" - }, - "lumi": { - "display_name": "Lumi", - "description": "Kajaani, Finland" - }, - "karolina": { - "display_name": "Karolina", - "description": "Ostrava, Czech Republic" - } - }, - "use_cases": { - "busbar2d": { - "display_name": "Busbar 2D", - "description": "Busbar 2D simulation" - }, - "busbar3d": { - "display_name": "Busbar 3D", - "description": "Busbar 3D simulation" - }, - "thermal_bridges_case_1": { - "display_name": "Thermal Bridges Case 1", - "description": "Thermal bridges case 1 simulation" - }, - "thermal_bridges_case_2": { - "display_name": "Thermal Bridges Case 2", - "description": "Thermal bridges case 2 simulation" - }, - "thermal_bridges_case_3": { - "display_name": "Thermal Bridges Case 3", - "description": "Thermal bridges case 3 simulation" - }, - "thermal_bridges_case_4": { - "display_name": "Thermal Bridges Case 4", - "description": "Thermal bridges case 4 simulation" - }, - "laminar2d": { - "display_name": "Laminar 2D", - "description": "Laminar 2D simulation" - }, - "laminar3d": { - "display_name": "Laminar 3D", - "description": "Laminar 3D simulation" - }, - "quarter_turn_3d": { - "display_name": "Quarter Turn 3D", - "description": "Quarter Turn 3D simulation" - }, - "HL_31": { - "display_name": "HL-31", - "description": "HL-31" - }, - "poznan": { - "display_name":"Poznan", - "description": "Poznan, Poland" - }, - "eye": { - "display_name":"Eye", - "description":"steady aqueous humor (AH) flow in the posterior and anterior chambers of the human eyeball, coupled with the overall heat transfer" - }, - "default": { - "display_name": "Default", - "description": "Unkown test case" - } - }, - "applications": { - "feelpp_toolbox_electric": { - "display_name": "Feel++ electric toolbox", - "description": "Feel++ electric toolbox", - "main_variables": [ "Constructor_init", "PostProcessing_exportResults", "Solve_solve" ] - }, - "feelpp_toolbox_heat": { - "display_name": "Feel++ heat toolbox", - "description": "Feel++ heat toolbox", - "main_variables": [ "Constructor_init", "PostProcessing_exportResults", "Solve_solve" ] - }, - "feelpp_toolbox_heatfluid": { - "display_name": "Feel++ Heat Fluid toolbox", - "description": "Feel++ Heat Fluid toolbox", - "main_variables": [ "Constructor_init", "PostProcessing_exportResults", "Solve_solve" ] - }, - "feelpp_toolbox_thermoelectric": { - "display_name": "Feel++ Thermo-electric toolbox", - "description": "Feel++ Thermo-electric toolbox", - "main_variables": [ "Constructor_init", "PostProcessing_exportResults", "Solve_solve" ] - }, - "feelpp_kub_cem":{ - "display_name": "KUB", - "description": "Ktirio Urban Buildings", - "main_variables": [ "constructor", "updateForUse", "execute", "simulate", "postprocess" ] - } - } -} \ No newline at end of file From 412467d8b6fa00ed2e8a09bd6bb5b5266e597356 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 13:45:00 +0100 Subject: [PATCH 02/56] use remote website config #180 --- .github/workflows/ci.yml | 7 ++++++- netlify-build.sh | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aaa1ff5..0d9f1f5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,10 +99,15 @@ jobs: pip3 install -I -r requirements.txt pip3 install dist/*.whl tree .venv + - name: Download website config + run: | + source .venv/bin/activate + config_id=6752f0734c9ccbdde21a48ca + girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json - name: Render reports run: | source .venv/bin/activate - render-benchmarks + render-benchmarks --config_file=./tmp/website_config.json env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Build Antora Site diff --git a/netlify-build.sh b/netlify-build.sh index d18e6285..416329ba 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -8,5 +8,7 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i -render-benchmarks +config_id=6752f0734c9ccbdde21a48ca +girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json +render-benchmarks --config_file=./tmp/website_config.json npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From ebc320711f3aae343b60a3db7a89f678867705d1 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 13:45:38 +0100 Subject: [PATCH 03/56] ignore tmp --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a9ec23ce..dc1b6586 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,7 @@ docs/modules/ROOT/pages/ reports/ #Test Outputs -tests/data/outputs/ \ No newline at end of file +tests/data/outputs/ + +tmp/ +tmp.ipynb \ No newline at end of file From 4ca1e5a4e5bd4e4019661b1fc894905ba0e4b3f9 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 14:06:34 +0100 Subject: [PATCH 04/56] add date metadata #181 --- docs/modules/ROOT/pages/index.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index ad01ded3..95e40e99 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -3,4 +3,5 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/] = Benchmarking :page-layout: toolboxes -:page-tags: catalog, catalog-index \ No newline at end of file +:page-tags: catalog, catalog-index +:docdatetime: 2024-12-06T13:56:01 \ No newline at end of file From d7c9c08bee900246cb54d78227af2b2d964a42c1 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 14:06:49 +0100 Subject: [PATCH 05/56] start deployment logic #181 --- .github/workflows/benchmark.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 25810b98..f301cec6 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,10 +18,6 @@ on: description: 'Plots related configuration' required: True default: 675053424c9ccbdde21a470a - girder_folder_id: - description: 'ID of the folder to upload to' - required: True - default: 67504ecd4c9ccbdde21a4704 jobs: @@ -166,12 +162,21 @@ jobs: source .venv/bin/activate new_foldername=./tmp/$(date +"%Y_%m_%dT%H_%M_%S") mv ./tmp/results/ $new_foldername - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - girder_upload_id=${{ github.event.inputs.girder_folder_id }}; - elif [[ "${{ github.event_name}}" == "repository_dispatch" ]]; then - girder_upload_id=${{ github.event.client_payload.girder_folder_id }}; - fi - girder-upload --directory $new_foldername --girder_id $girder_upload_id + girder_staging_id=6752b5194c9ccbdde21a48b8 + girder-upload --directory $new_foldername --girder_id $girder_staging_id rm -r $new_foldername env: - GIRDER_API_KEY: ${{ secrets.GIRDER }} \ No newline at end of file + GIRDER_API_KEY: ${{ secrets.GIRDER }} + - name: Update Index Date + run: | + sed -i "s/^:docdatetime: .*/:docdatetime: $(date +'%Y-%m-%dT%H:%M:%S')/" "docs/modules/ROOT/pages/index.adoc" + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + title: "Add benchmark for ${{ needs.factory.outputs.executable_name }} - ${{ needs.factory.outputs.use_case }}" + body: | + Auto-generated by [create-pull-request][1] + [1]: https://github.com/peter-evans/create-pull-request + reviewers: JavierCladellas + env: + GITHUB_TOKEN: ${{ secrets.CR_PAT }} \ No newline at end of file From 826f88ed0e157118a5f10c116dcb4e7832d88e89 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 14:15:28 +0100 Subject: [PATCH 06/56] add some body text [ci skip] --- .github/workflows/benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f301cec6..f9fb178f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -175,6 +175,7 @@ jobs: with: title: "Add benchmark for ${{ needs.factory.outputs.executable_name }} - ${{ needs.factory.outputs.use_case }}" body: | + Generating reports from staging directory. Auto-generated by [create-pull-request][1] [1]: https://github.com/peter-evans/create-pull-request reviewers: JavierCladellas From db0d9044adfc8c29e099d7c1fc34be7a74d6877f Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 14:49:21 +0100 Subject: [PATCH 07/56] bad name --- .github/workflows/benchmark.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f9fb178f..c6a18ef9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,7 +27,7 @@ jobs: if: "!contains(github.event.head_commit.message, 'code skip')" steps: - uses: actions/checkout@v4 - - name: Build Ktirio Cases wheel + - name: Build wheel run: | npm install npx downdoc README.adoc @@ -179,5 +179,6 @@ jobs: Auto-generated by [create-pull-request][1] [1]: https://github.com/peter-evans/create-pull-request reviewers: JavierCladellas + labels: new-benchmark env: GITHUB_TOKEN: ${{ secrets.CR_PAT }} \ No newline at end of file From b9b7ff83c9faac8a59fb8e61553afb71888200c5 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 15:08:54 +0100 Subject: [PATCH 08/56] add dir option to girder-download --- src/feelpp/benchmarking/scripts/girder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index 41e2f1be..9df7aa27 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -6,12 +6,16 @@ def download_cli(): parser = ArgumentParser() parser.add_argument("--girder_id", '-gid', required = True, type=str, help="Girder file id to download.") + parser.add_argument("--directory","-d", action='store_true') parser.add_argument("--output_dir", '-o', required = False, type=str, help="Path to the directory to download files to.", default="./tmp/") parser.add_argument("--filename", '-fn', required = False, type=str, default=None, help="Name to give to the file after downloading ") args = parser.parse_args() girder_hanlder = GirderHandler(args.output_dir) - girder_hanlder.downloadFile(args.girder_id,name=args.filename) + if args.directory: + girder_hanlder.downloadFolder(args.girder_id,output_dir=".") + else: + girder_hanlder.downloadFile(args.girder_id,name=args.filename) return 0 From 62499e823245e7bc6ee2cafc662f655d58090838 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 15:27:27 +0100 Subject: [PATCH 09/56] add merge json config script #181 --- pyproject.toml | 1 + src/feelpp/benchmarking/scripts/misc.py | 41 +++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 src/feelpp/benchmarking/scripts/misc.py diff --git a/pyproject.toml b/pyproject.toml index 4b7bad5e..0ebff188 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ execute-benchmark = "feelpp.benchmarking.reframe.__main__:main_cli" girder-download = "feelpp.benchmarking.scripts.girder:download_cli" girder-upload = "feelpp.benchmarking.scripts.girder:upload_cli" hpc-dispatch = "feelpp.benchmarking.scripts.hpcSystems:hpcSystemDispatcher_cli" +merge-json-configs = "feelpp.benchmarking.scripts.misc:jsonConfigMerge_cli" [tool.pytest.ini_options] minversion = "6.0" diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py new file mode 100644 index 00000000..746b8af4 --- /dev/null +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -0,0 +1,41 @@ +from argparse import ArgumentParser +import glob, json, copy + +def mergeDicts(dict1, dict2): + """Recursively merges two dictionaries.""" + merged = copy.deepcopy(dict1) # Make a deep copy of the first dictionary + for key, value in dict2.items(): + if key in merged: + if isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = mergeDicts(merged[key], value) + elif isinstance(merged[key], list) and isinstance(value, list): + merged[key] = list(set(merged[key] + value)) + else: + merged[key] = value + else: + merged[key] = value + return merged + + + + +def jsonConfigMerge_cli(): + """ Merge multiple json files into a single one""" + parser = ArgumentParser() + parser.add_argument("--file_pattern", '-fp', required = True, type=str, help="File glob pattern of the json files to merge") + parser.add_argument("--output_file_path","-o",required=True,type=str,help="Path of the resulting file to be saved to") + args = parser.parse_args() + + master_config = {} + for filename in glob.glob(args.file_pattern,recursive=True): + with open(filename,"r") as f: + current_config = json.load(f) + master_config = mergeDicts(master_config,current_config) + + with open(args.output_file_path,"w") as f: + f.write(json.dumps(master_config)) + + return 0 + +if __name__=="__main__": + jsonConfigMerge_cli() \ No newline at end of file From 30ddced777d7bb62c5b38e9bf8720f1e056a2d50 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 15:27:38 +0100 Subject: [PATCH 10/56] rm main --- src/feelpp/benchmarking/scripts/misc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index 746b8af4..5a0bf0d8 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -35,7 +35,4 @@ def jsonConfigMerge_cli(): with open(args.output_file_path,"w") as f: f.write(json.dumps(master_config)) - return 0 - -if __name__=="__main__": - jsonConfigMerge_cli() \ No newline at end of file + return 0 \ No newline at end of file From 440e3f8d75c5adffda6ed6d30799010351ee2cd5 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 15:29:03 +0100 Subject: [PATCH 11/56] help --- src/feelpp/benchmarking/scripts/girder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index 9df7aa27..069ecf5e 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -6,7 +6,7 @@ def download_cli(): parser = ArgumentParser() parser.add_argument("--girder_id", '-gid', required = True, type=str, help="Girder file id to download.") - parser.add_argument("--directory","-d", action='store_true') + parser.add_argument("--directory","-d", action='store_true', help="Set this flag if the target is a directory") parser.add_argument("--output_dir", '-o', required = False, type=str, help="Path to the directory to download files to.", default="./tmp/") parser.add_argument("--filename", '-fn', required = False, type=str, default=None, help="Name to give to the file after downloading ") args = parser.parse_args() From a6d294dcd9420ab2647d4b9440ca26b7a2d92d7a Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:09:05 +0100 Subject: [PATCH 12/56] add update config option #181 --- src/feelpp/benchmarking/scripts/misc.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index 5a0bf0d8..689976e7 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -1,5 +1,5 @@ from argparse import ArgumentParser -import glob, json, copy +import glob, json, copy, os def mergeDicts(dict1, dict2): """Recursively merges two dictionaries.""" @@ -24,12 +24,23 @@ def jsonConfigMerge_cli(): parser = ArgumentParser() parser.add_argument("--file_pattern", '-fp', required = True, type=str, help="File glob pattern of the json files to merge") parser.add_argument("--output_file_path","-o",required=True,type=str,help="Path of the resulting file to be saved to") + parser.add_argument("--update_paths", "-u", action="store_true", help="Whether to update the config file paths") args = parser.parse_args() master_config = {} for filename in glob.glob(args.file_pattern,recursive=True): with open(filename,"r") as f: current_config = json.load(f) + + if args.update_paths: + file_dirpath = os.path.dirname(os.path.relpath(filename,".")) + report_paths = glob.glob(os.path.join(file_dirpath,"**","reframe_report.json"),recursive=True) + for report in report_paths: + app,use_case,machine = report.split("/")[-5:-2] + + current_config["execution_mapping"][app][machine][use_case]["path"] = os.path.join(file_dirpath,app,use_case,machine) + current_config["execution_mapping"][app][machine][use_case]["path"]["platform"] = "local" + master_config = mergeDicts(master_config,current_config) with open(args.output_file_path,"w") as f: From f65148b95aa2b982102050a66af0076b5dca0053 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:09:23 +0100 Subject: [PATCH 13/56] add first staging workflow #181 --- .github/workflows/staging_preview.yml | 41 +++++++++++++++++++++++++++ netlify-stage-build.sh | 12 ++++++++ 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/staging_preview.yml create mode 100644 netlify-stage-build.sh diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml new file mode 100644 index 00000000..1ac4c7c4 --- /dev/null +++ b/.github/workflows/staging_preview.yml @@ -0,0 +1,41 @@ +name: Preview Staging Website + +on: + pull_request: + types: + - labeled + - opened +jobs: + preview-staging: + runs-on: self-ubuntu-22.04 + name: Preview stage benchmarks + if: contains(github.event.pull_request.labels.*.name,'new-benchmark') + steps: + - uses: actions/checkout@v4 + - name: Build wheel + run: | + npm install + npx downdoc README.adoc + pip3 wheel --no-deps --wheel-dir dist . + env: + CXX: clang++ + - name: Create python environment + run: | + python3 -m venv .venv + source .venv/bin/activate + pip3 install -r requirements.txt + - name: Download stage benchmarks + run: | + source .venv/bin/activate + girder-download -gid 6752b5194c9ccbdde21a48b8 -o ./tmp/ -d + - name: Merge Website Configs + run: | + source .venv/bin/activate + merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u + - name: Trigger Preview + uses: netlify/actions/build@master + env: + NETLIFY_CMD: "./netlify-stage-build.sh" + NETLIFY_DIR: public + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + diff --git a/netlify-stage-build.sh b/netlify-stage-build.sh new file mode 100644 index 00000000..d86a9f24 --- /dev/null +++ b/netlify-stage-build.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +python3 -m venv .venv +source .venv/bin/activate + +pip3 install --upgrade pip +pip3 install -r requirements.txt +pip3 wheel --no-deps --wheel-dir dist . +pip3 install dist/*.whl +npm i +render-benchmarks --config_file=./tmp/website_config.json +npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From 5236fc3c1a7e888a7a07b41bde2a92dd2b5e9730 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:13:57 +0100 Subject: [PATCH 14/56] add girder api key to env #181 --- .github/workflows/staging_preview.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml index 1ac4c7c4..6830b3ae 100644 --- a/.github/workflows/staging_preview.yml +++ b/.github/workflows/staging_preview.yml @@ -28,6 +28,8 @@ jobs: run: | source .venv/bin/activate girder-download -gid 6752b5194c9ccbdde21a48b8 -o ./tmp/ -d + env: + GIRDER_API_KEY: ${{secrets.GIRDER}} - name: Merge Website Configs run: | source .venv/bin/activate From 5ac7eecc62f17c9033581e6f18dc53d36c87dcf0 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:14:07 +0100 Subject: [PATCH 15/56] add push event for debug --- .github/workflows/staging_preview.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml index 6830b3ae..1179b230 100644 --- a/.github/workflows/staging_preview.yml +++ b/.github/workflows/staging_preview.yml @@ -5,6 +5,9 @@ on: types: - labeled - opened + push: + branches: + - '181-handle-report-deployment' #TODO: Remove (DEBUG) jobs: preview-staging: runs-on: self-ubuntu-22.04 From a72a0b9a856471c844d95980157bad295d78bb92 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:15:13 +0100 Subject: [PATCH 16/56] debug [ci skip] --- .github/workflows/staging_preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml index 1179b230..046c383c 100644 --- a/.github/workflows/staging_preview.yml +++ b/.github/workflows/staging_preview.yml @@ -12,7 +12,7 @@ jobs: preview-staging: runs-on: self-ubuntu-22.04 name: Preview stage benchmarks - if: contains(github.event.pull_request.labels.*.name,'new-benchmark') + if: 1 || contains(github.event.pull_request.labels.*.name,'new-benchmark') steps: - uses: actions/checkout@v4 - name: Build wheel From 63ff1d58f48fe1026c9b0f3b590a0e2b86a74ab5 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:16:22 +0100 Subject: [PATCH 17/56] fix debug --- .github/workflows/staging_preview.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml index 046c383c..58546959 100644 --- a/.github/workflows/staging_preview.yml +++ b/.github/workflows/staging_preview.yml @@ -12,7 +12,8 @@ jobs: preview-staging: runs-on: self-ubuntu-22.04 name: Preview stage benchmarks - if: 1 || contains(github.event.pull_request.labels.*.name,'new-benchmark') + #if + #contains(github.event.pull_request.labels.*.name,'new-benchmark') steps: - uses: actions/checkout@v4 - name: Build wheel From 5135cdf7178957d30888fb140ef37b91b004f188 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:18:27 +0100 Subject: [PATCH 18/56] bug fix --- src/feelpp/benchmarking/scripts/misc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index 689976e7..a67bdcc2 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -39,7 +39,6 @@ def jsonConfigMerge_cli(): app,use_case,machine = report.split("/")[-5:-2] current_config["execution_mapping"][app][machine][use_case]["path"] = os.path.join(file_dirpath,app,use_case,machine) - current_config["execution_mapping"][app][machine][use_case]["path"]["platform"] = "local" master_config = mergeDicts(master_config,current_config) From e30e9ba1f86e27abcb99fc2695dc661a7b90b9fc Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:39:53 +0100 Subject: [PATCH 19/56] rm netlify rebuild [ci skip] #181 Need to think of a different strat :/ --- .github/workflows/staging_preview.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml index 58546959..c76a6eb9 100644 --- a/.github/workflows/staging_preview.yml +++ b/.github/workflows/staging_preview.yml @@ -38,10 +38,9 @@ jobs: run: | source .venv/bin/activate merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u - - name: Trigger Preview - uses: netlify/actions/build@master - env: - NETLIFY_CMD: "./netlify-stage-build.sh" - NETLIFY_DIR: public - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - + - name: Render Benchmarks + run: | + source .venv/bin/activate + render-benchmarks --config_file=./tmp/website_config.json + - name: Deploy Preview + run: echo "HOW???" \ No newline at end of file From f7e130f890f275ba8d8e8d564273e5fd03676d63 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Fri, 6 Dec 2024 17:40:44 +0100 Subject: [PATCH 20/56] rm netlify stage [ci skip] --- netlify-stage-build.sh | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 netlify-stage-build.sh diff --git a/netlify-stage-build.sh b/netlify-stage-build.sh deleted file mode 100644 index d86a9f24..00000000 --- a/netlify-stage-build.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -python3 -m venv .venv -source .venv/bin/activate - -pip3 install --upgrade pip -pip3 install -r requirements.txt -pip3 wheel --no-deps --wheel-dir dist . -pip3 install dist/*.whl -npm i -render-benchmarks --config_file=./tmp/website_config.json -npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From 01365a37c705f69c00e92e4ad406467839bac961 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:11:27 +0100 Subject: [PATCH 21/56] conditional preview #181 --- .github/workflows/benchmark.yml | 1 + netlify-build.sh | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c6a18ef9..fb5e802d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -180,5 +180,6 @@ jobs: [1]: https://github.com/peter-evans/create-pull-request reviewers: JavierCladellas labels: new-benchmark + branch: new-benchmark-{{needs.factory.outputs.executable_name}} env: GITHUB_TOKEN: ${{ secrets.CR_PAT }} \ No newline at end of file diff --git a/netlify-build.sh b/netlify-build.sh index 416329ba..7709fd63 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -8,7 +8,9 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i -config_id=6752f0734c9ccbdde21a48ca -girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json +if [[ $BRANCH != *"new-benchmark"*]]; then + config_id=6752f0734c9ccbdde21a48ca + girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json +fi render-benchmarks --config_file=./tmp/website_config.json npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From a148c7db5807be5d9041c28f667c937b76b472cb Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:15:05 +0100 Subject: [PATCH 22/56] fix error #181 --- netlify-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netlify-build.sh b/netlify-build.sh index 7709fd63..7f0c6828 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -8,7 +8,7 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i -if [[ $BRANCH != *"new-benchmark"*]]; then +if [[ $BRANCH != *"new-benchmark"* ]]; then config_id=6752f0734c9ccbdde21a48ca girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json fi From ac360847b8e42983704257eddec89df77fa8ed2a Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:27:58 +0100 Subject: [PATCH 23/56] add missing girder key to download website config step #181 --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d9f1f5a..9bb8f4f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,8 @@ jobs: source .venv/bin/activate config_id=6752f0734c9ccbdde21a48ca girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json + env: + GIRDER_API_KEY: ${{secrets.GIRDER}} - name: Render reports run: | source .venv/bin/activate From 930ff7171a44e8bc0f80fbe1f9953bdb4b049a67 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:36:57 +0100 Subject: [PATCH 24/56] rm staging preview workflow (handled now by netlify) #181 --- .github/workflows/staging_preview.yml | 46 --------------------------- netlify-build.sh | 8 +++-- 2 files changed, 6 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/staging_preview.yml diff --git a/.github/workflows/staging_preview.yml b/.github/workflows/staging_preview.yml deleted file mode 100644 index c76a6eb9..00000000 --- a/.github/workflows/staging_preview.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Preview Staging Website - -on: - pull_request: - types: - - labeled - - opened - push: - branches: - - '181-handle-report-deployment' #TODO: Remove (DEBUG) -jobs: - preview-staging: - runs-on: self-ubuntu-22.04 - name: Preview stage benchmarks - #if - #contains(github.event.pull_request.labels.*.name,'new-benchmark') - steps: - - uses: actions/checkout@v4 - - name: Build wheel - run: | - npm install - npx downdoc README.adoc - pip3 wheel --no-deps --wheel-dir dist . - env: - CXX: clang++ - - name: Create python environment - run: | - python3 -m venv .venv - source .venv/bin/activate - pip3 install -r requirements.txt - - name: Download stage benchmarks - run: | - source .venv/bin/activate - girder-download -gid 6752b5194c9ccbdde21a48b8 -o ./tmp/ -d - env: - GIRDER_API_KEY: ${{secrets.GIRDER}} - - name: Merge Website Configs - run: | - source .venv/bin/activate - merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u - - name: Render Benchmarks - run: | - source .venv/bin/activate - render-benchmarks --config_file=./tmp/website_config.json - - name: Deploy Preview - run: echo "HOW???" \ No newline at end of file diff --git a/netlify-build.sh b/netlify-build.sh index 7f0c6828..3b85a5a7 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -8,9 +8,13 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i -if [[ $BRANCH != *"new-benchmark"* ]]; then +if [[ $BRANCH == *"new-benchmark"* ]]; then config_id=6752f0734c9ccbdde21a48ca - girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json + girder-download -gid $config_id -o ./tmp/ -d + merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u +else + config_id=6752f0734c9ccbdde21a48ca + girder-download -gid $config_id -o ./tmp/ -fn website_config.json fi render-benchmarks --config_file=./tmp/website_config.json npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From 9b4e4290ae45c13c8dba72c1eba6556370777e46 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:39:46 +0100 Subject: [PATCH 25/56] typo fix #181 --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index fb5e802d..ad0d3607 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -180,6 +180,6 @@ jobs: [1]: https://github.com/peter-evans/create-pull-request reviewers: JavierCladellas labels: new-benchmark - branch: new-benchmark-{{needs.factory.outputs.executable_name}} + branch: new-benchmark-${{needs.factory.outputs.executable_name}} env: GITHUB_TOKEN: ${{ secrets.CR_PAT }} \ No newline at end of file From 41eb4edcc9a5b1f6e21f8f19a49943e1dad6ede7 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:45:16 +0100 Subject: [PATCH 26/56] add echos and fix id #181 [ci skip] --- netlify-build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netlify-build.sh b/netlify-build.sh index 3b85a5a7..aea70b1b 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -9,10 +9,12 @@ pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i if [[ $BRANCH == *"new-benchmark"* ]]; then - config_id=6752f0734c9ccbdde21a48ca + echo "Downloading Staging benchmarks" + config_id=6752b5194c9ccbdde21a48b8 girder-download -gid $config_id -o ./tmp/ -d merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u else + echo "Downloading Production benchmarks" config_id=6752f0734c9ccbdde21a48ca girder-download -gid $config_id -o ./tmp/ -fn website_config.json fi From d17e89ea653a42832305cc8fc4b5a0e13a371bc0 Mon Sep 17 00:00:00 2001 From: JavierCladellas <82624449+JavierCladellas@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:48:31 +0100 Subject: [PATCH 27/56] [create-pull-request] automated change --- docs/modules/ROOT/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 95e40e99..da23ccb8 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -4,4 +4,4 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/] = Benchmarking :page-layout: toolboxes :page-tags: catalog, catalog-index -:docdatetime: 2024-12-06T13:56:01 \ No newline at end of file +:docdatetime: 2024-12-09T10:48:30 \ No newline at end of file From c5c5c9c7b97471bf64e32b3337a7d95e6ea03f45 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:52:48 +0100 Subject: [PATCH 28/56] debug netlify #181 --- netlify-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/netlify-build.sh b/netlify-build.sh index aea70b1b..dc72195e 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -10,6 +10,7 @@ pip3 install dist/*.whl npm i if [[ $BRANCH == *"new-benchmark"* ]]; then echo "Downloading Staging benchmarks" + echo $BRANCH config_id=6752b5194c9ccbdde21a48b8 girder-download -gid $config_id -o ./tmp/ -d merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u From 8760823034196323df8f1c3675f6ac32e3e0ec8e Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 10:56:09 +0100 Subject: [PATCH 29/56] more netlify debug #181 --- netlify-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netlify-build.sh b/netlify-build.sh index dc72195e..0dc262d9 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +echo $BRANCH python3 -m venv .venv source .venv/bin/activate @@ -10,7 +11,6 @@ pip3 install dist/*.whl npm i if [[ $BRANCH == *"new-benchmark"* ]]; then echo "Downloading Staging benchmarks" - echo $BRANCH config_id=6752b5194c9ccbdde21a48b8 girder-download -gid $config_id -o ./tmp/ -d merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u From 62666a53e5e1118f2c220df08ee704992a4b6001 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 11:01:29 +0100 Subject: [PATCH 30/56] more echoing #181 --- netlify-build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netlify-build.sh b/netlify-build.sh index 0dc262d9..043c9ebd 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +echo "BRANCH" echo $BRANCH +echo "HEAD" +echo $HEAD + python3 -m venv .venv source .venv/bin/activate From 0ed3b795e4ca84b1c218c03eddc99c02e03e5ec6 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 11:03:35 +0100 Subject: [PATCH 31/56] branch -> head #181 --- netlify-build.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/netlify-build.sh b/netlify-build.sh index 043c9ebd..4b6bff38 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -1,9 +1,4 @@ #!/usr/bin/env bash -echo "BRANCH" -echo $BRANCH -echo "HEAD" -echo $HEAD - python3 -m venv .venv source .venv/bin/activate @@ -13,7 +8,7 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i -if [[ $BRANCH == *"new-benchmark"* ]]; then +if [[ $HEAD == *"new-benchmark"* ]]; then echo "Downloading Staging benchmarks" config_id=6752b5194c9ccbdde21a48b8 girder-download -gid $config_id -o ./tmp/ -d From 0cfc2974bee35148751aa41555515e060fc887d7 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 13:25:48 +0100 Subject: [PATCH 32/56] add reusable upload to girder wrapped cli --- src/feelpp/benchmarking/report/config/handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/feelpp/benchmarking/report/config/handlers.py b/src/feelpp/benchmarking/report/config/handlers.py index 08225c80..2343f5da 100644 --- a/src/feelpp/benchmarking/report/config/handlers.py +++ b/src/feelpp/benchmarking/report/config/handlers.py @@ -60,7 +60,7 @@ def downloadFile(self, file_id, output_dir="",name=None): self.client.downloadFile(fileId=file_id,path=filepath) - def upload(self, file_pattern, parent_id,leaf_folder_as_items=True): + def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existing=True): """ Upload a local file to an existing folder/item in Girder Args: file_pattern: The file pattern of the files to upload @@ -68,7 +68,7 @@ def upload(self, file_pattern, parent_id,leaf_folder_as_items=True): parent_type (str) : "folder" or "item". Defaults to "folder" leaf_folder_as_items (bool): Whether to upload leaf folders as items or files """ - self.client.upload(filePattern=file_pattern, parentId=parent_id, parentType="folder",leafFoldersAsItems=leaf_folder_as_items) + self.client.upload(filePattern=file_pattern, parentId=parent_id, parentType="folder",leafFoldersAsItems=leaf_folder_as_items, reuseExisting=reuse_existing) class ConfigHandler: From 882cd4cb9a74197e85fb7cc84c8f43b62c95f173 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 13:26:22 +0100 Subject: [PATCH 33/56] rename option (directory -> item) --- .github/workflows/benchmark.yml | 2 +- src/feelpp/benchmarking/scripts/girder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ad0d3607..406eb730 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -163,7 +163,7 @@ jobs: new_foldername=./tmp/$(date +"%Y_%m_%dT%H_%M_%S") mv ./tmp/results/ $new_foldername girder_staging_id=6752b5194c9ccbdde21a48b8 - girder-upload --directory $new_foldername --girder_id $girder_staging_id + girder-upload --item $new_foldername --girder_id $girder_staging_id rm -r $new_foldername env: GIRDER_API_KEY: ${{ secrets.GIRDER }} diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index 069ecf5e..b66ad469 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -21,7 +21,7 @@ def download_cli(): def upload_cli(): parser = ArgumentParser() - parser.add_argument("--directory", required=True, help="Path of the directory to upload as an item") + parser.add_argument("--item", required=True, help="Path of the directory or file to upload as an item") parser.add_argument("--girder_id", required=True, help="The Girder folder ID to which to upload the item") args = parser.parse_args() From ea428164e52c0832ad95f06d57b373d1c49d0d21 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 13:43:14 +0100 Subject: [PATCH 34/56] fix bug in configMerge --- src/feelpp/benchmarking/scripts/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index a67bdcc2..026a816f 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -40,7 +40,7 @@ def jsonConfigMerge_cli(): current_config["execution_mapping"][app][machine][use_case]["path"] = os.path.join(file_dirpath,app,use_case,machine) - master_config = mergeDicts(master_config,current_config) + master_config = mergeDicts(master_config,current_config) with open(args.output_file_path,"w") as f: f.write(json.dumps(master_config)) From 221828c521566684b2650666a752e15711ab5108 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Mon, 9 Dec 2024 13:55:20 +0100 Subject: [PATCH 35/56] imrpove json merge --- src/feelpp/benchmarking/scripts/misc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index 026a816f..e43fbe81 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -6,6 +6,8 @@ def mergeDicts(dict1, dict2): merged = copy.deepcopy(dict1) # Make a deep copy of the first dictionary for key, value in dict2.items(): if key in merged: + if not value: continue + if isinstance(merged[key], dict) and isinstance(value, dict): merged[key] = mergeDicts(merged[key], value) elif isinstance(merged[key], list) and isinstance(value, list): From 154a4167a89b577b9808347f343ddd6595106bff Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 09:27:46 +0100 Subject: [PATCH 36/56] fix arg name --- src/feelpp/benchmarking/scripts/girder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index b66ad469..9598f197 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -26,6 +26,6 @@ def upload_cli(): args = parser.parse_args() girder_handler = GirderHandler(download_base_dir=None) - girder_handler.upload( args.directory, args.girder_id ) + girder_handler.upload( args.item, args.girder_id ) return 0 \ No newline at end of file From 485afead504b600c7e3802d8ebb6b1ad8f3fb2db Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 09:34:29 +0100 Subject: [PATCH 37/56] return item id on upload #181 --- src/feelpp/benchmarking/report/config/handlers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/report/config/handlers.py b/src/feelpp/benchmarking/report/config/handlers.py index 2343f5da..8062bc13 100644 --- a/src/feelpp/benchmarking/report/config/handlers.py +++ b/src/feelpp/benchmarking/report/config/handlers.py @@ -60,7 +60,7 @@ def downloadFile(self, file_id, output_dir="",name=None): self.client.downloadFile(fileId=file_id,path=filepath) - def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existing=True): + def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existing=True, return_id=False): """ Upload a local file to an existing folder/item in Girder Args: file_pattern: The file pattern of the files to upload @@ -70,6 +70,11 @@ def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existin """ self.client.upload(filePattern=file_pattern, parentId=parent_id, parentType="folder",leafFoldersAsItems=leaf_folder_as_items, reuseExisting=reuse_existing) + if return_id: + items = list(self.client.listItem(folderId=parent_id, name=os.path.basename(file_pattern))) + assert len(items) >0, f"File not Found in Girder with the name {os.path.basename(file_pattern)}" + assert len(items) <=1, f"More than one file found with the same name {os.path.basename(file_pattern)}" + return items[0]["_id"] class ConfigHandler: def __init__(self, config_filepath): From d3ef1dba1c330cfc515f9d6084f10d5198e30933 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 10:15:10 +0100 Subject: [PATCH 38/56] return id to find folder #181 --- src/feelpp/benchmarking/report/config/handlers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/report/config/handlers.py b/src/feelpp/benchmarking/report/config/handlers.py index 8062bc13..ca33b4b5 100644 --- a/src/feelpp/benchmarking/report/config/handlers.py +++ b/src/feelpp/benchmarking/report/config/handlers.py @@ -72,8 +72,10 @@ def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existin if return_id: items = list(self.client.listItem(folderId=parent_id, name=os.path.basename(file_pattern))) - assert len(items) >0, f"File not Found in Girder with the name {os.path.basename(file_pattern)}" + if len(items) == 0: + items = list(self.client.listFolder(parentId=parent_id, name=os.path.basename(file_pattern))) assert len(items) <=1, f"More than one file found with the same name {os.path.basename(file_pattern)}" + assert len(items) >0, f"File not Found in Girder with the name {os.path.basename(file_pattern)}" return items[0]["_id"] class ConfigHandler: From 3d0c030fa7108c476dc898c6dc5f5a871ba3600f Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 10:16:33 +0100 Subject: [PATCH 39/56] add script to handle stage configuration parsing #181 --- pyproject.toml | 1 + src/feelpp/benchmarking/scripts/misc.py | 55 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0ebff188..eb6f83c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ girder-download = "feelpp.benchmarking.scripts.girder:download_cli" girder-upload = "feelpp.benchmarking.scripts.girder:upload_cli" hpc-dispatch = "feelpp.benchmarking.scripts.hpcSystems:hpcSystemDispatcher_cli" merge-json-configs = "feelpp.benchmarking.scripts.misc:jsonConfigMerge_cli" +update-stage-config = "feelpp.benchmarking.scripts.misc:upldateStageConfig_cli" [tool.pytest.ini_options] minversion = "6.0" diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index e43fbe81..5f9ab4eb 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -1,5 +1,6 @@ from argparse import ArgumentParser import glob, json, copy, os +from feelpp.benchmarking.report.config.handlers import GirderHandler def mergeDicts(dict1, dict2): """Recursively merges two dictionaries.""" @@ -47,4 +48,58 @@ def jsonConfigMerge_cli(): with open(args.output_file_path,"w") as f: f.write(json.dumps(master_config)) + return 0 + + +def upldateStageConfig_cli(): + """Upload new staged benchmarks to production and update production config""" + parser = ArgumentParser() + parser.add_argument("--stage_config", '-stag', required = True, type=str, help="File path of the stage configuration file") + parser.add_argument("--production_config","-prod",required=True,type=str,help="FIle path of the production configuration file") + parser.add_argument("--production_girder_id", "-prodid", required=True,type=str, help="Girder id of the production folder") + args = parser.parse_args() + + with open(args.stage_config,"r") as f: + stage_cfg = json.load(f) + with open(args.production_config,"r") as f: + prod_cfg = json.load(f) + + girder_client = GirderHandler(None) + + for app_name, machines in stage_cfg["execution_mapping"].items(): + for machine_name, use_cases in machines.items(): + for use_case_name, info in use_cases.items(): + stage_platform = info["platform"] + stage_path = info["path"] + + if ( + app_name in prod_cfg["execution_mapping"] and + machine_name in prod_cfg["execution_mapping"][app_name] and + use_case_name in prod_cfg["execution_mapping"][app_name][machine_name] + ): + prod_info = prod_cfg["execution_mapping"][app_name][machine_name][use_case_name] + prod_platorm = prod_info["platform"] + prod_path = prod_info["path"] + + if stage_platform == "local": + for report_item_basename in os.listdir(report_item_basename): + girder_client.upload( os.path.join(stage_path,report_item_basename), prod_path, reuse_existing=False) + stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["path"] = prod_path + stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["platform"] = prod_platorm + else: + raise ValueError("Unexpected: non local path but benchmark is staged") + + else: + print(f"{app_name}-{machine_name}-{use_case_name} not found in production... Adding it.") + + tmp_location = os.path.join("./tmp",f"{app_name}_{use_case_name}_{machine_name}") + os.rename(stage_path,tmp_location) + uploaded_id = girder_client.upload( tmp_location, args.production_girder_id, return_id=True ) + + stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["path"] = uploaded_id + stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["platform"] = "girder" + + with open(args.stage_config,"w") as f: + json.dump(stage_cfg,f) + return 0 \ No newline at end of file From e71d2c4b488892949638b7100d449c53eff354e2 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 10:25:27 +0100 Subject: [PATCH 40/56] try fix girder upload of benchmarks in wf #181 --- .github/workflows/benchmark.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 406eb730..7aa16dbf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -160,11 +160,8 @@ jobs: - name: Upload to girder run: | source .venv/bin/activate - new_foldername=./tmp/$(date +"%Y_%m_%dT%H_%M_%S") - mv ./tmp/results/ $new_foldername girder_staging_id=6752b5194c9ccbdde21a48b8 - girder-upload --item $new_foldername --girder_id $girder_staging_id - rm -r $new_foldername + girder-upload --item ./tmp/results/ --girder_id $girder_staging_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Update Index Date From 82ba95c0823f1c3507968ace07b9eda7f11faa86 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 10:39:14 +0100 Subject: [PATCH 41/56] fix folder name in girder #181 [ci skip] --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7aa16dbf..b13b371e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -161,7 +161,7 @@ jobs: run: | source .venv/bin/activate girder_staging_id=6752b5194c9ccbdde21a48b8 - girder-upload --item ./tmp/results/ --girder_id $girder_staging_id + girder-upload --item "./tmp/results/*" --girder_id $girder_staging_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Update Index Date From 0b47face999c5ebdcf9b86b0c46155c58e777ffd Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:38:59 +0100 Subject: [PATCH 42/56] add move cli #181 --- src/feelpp/benchmarking/scripts/girder.py | 28 +++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index 9598f197..767bd45d 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -1,6 +1,6 @@ from feelpp.benchmarking.report.config.handlers import GirderHandler from argparse import ArgumentParser - +import requests, os def download_cli(): @@ -28,4 +28,28 @@ def upload_cli(): girder_handler = GirderHandler(download_base_dir=None) girder_handler.upload( args.item, args.girder_id ) - return 0 \ No newline at end of file + return 0 + +def move_cli(): + """Move all folder contents to another one""" + parser = ArgumentParser() + parser.add_argument("--old_id","-oid", required=True, help="Id of the folder to move the items from") + parser.add_argument("--new_id","-nid", required=True, help="Id of the folder to move the items to") + args = parser.parse_args() + + girder_handler = GirderHandler(download_base_dir=None) + children = girder_handler.listChildren(args.old_id) + + token = requests.post( + f"{girder_handler.base_url}/api_key/token", + params={"key":os.environ["GIRDER_API_KEY"]} + ) + + for child in children: + response = requests.put( + f"{girder_handler.base_url}/{child['_modelType']}/{child['_id']}", + params={"parentId":args.new_id,"parentType":"folder"}, + headers={"Girder-Token":token.json()["authToken"]["token"]} + ) + + print(response.text) From 1685903a445f36fb99f849452bd681dabefe475d Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:39:07 +0100 Subject: [PATCH 43/56] add move cli script #181 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index eb6f83c2..36d5620f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ render-benchmarks = "feelpp.benchmarking.report.__main__:main_cli" execute-benchmark = "feelpp.benchmarking.reframe.__main__:main_cli" girder-download = "feelpp.benchmarking.scripts.girder:download_cli" girder-upload = "feelpp.benchmarking.scripts.girder:upload_cli" +girder-move = "feelpp.benchmarking.scripts.girder:move_cli" hpc-dispatch = "feelpp.benchmarking.scripts.hpcSystems:hpcSystemDispatcher_cli" merge-json-configs = "feelpp.benchmarking.scripts.misc:jsonConfigMerge_cli" update-stage-config = "feelpp.benchmarking.scripts.misc:upldateStageConfig_cli" From 7120ea97ea38b3290b370ec26fb1321f8ddcc024 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:39:22 +0100 Subject: [PATCH 44/56] factor id fetching #181 --- .../benchmarking/report/config/handlers.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/feelpp/benchmarking/report/config/handlers.py b/src/feelpp/benchmarking/report/config/handlers.py index ca33b4b5..0f71cd23 100644 --- a/src/feelpp/benchmarking/report/config/handlers.py +++ b/src/feelpp/benchmarking/report/config/handlers.py @@ -60,6 +60,18 @@ def downloadFile(self, file_id, output_dir="",name=None): self.client.downloadFile(fileId=file_id,path=filepath) + def listChildren(self,parent_id, children_name = None): + items = list(self.client.listItem(folderId=parent_id, name=children_name)) + if len(items) == 0: + items = list(self.client.listFolder(parentId=parent_id, name=children_name)) + + if children_name: + assert len(items) <=1, f"More than one file found with the same name {children_name}" + assert len(items) >0, f"File not Found in Girder with the name {children_name}" + return items[0] + + return items + def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existing=True, return_id=False): """ Upload a local file to an existing folder/item in Girder Args: @@ -71,12 +83,8 @@ def upload(self, file_pattern, parent_id,leaf_folder_as_items=True,reuse_existin self.client.upload(filePattern=file_pattern, parentId=parent_id, parentType="folder",leafFoldersAsItems=leaf_folder_as_items, reuseExisting=reuse_existing) if return_id: - items = list(self.client.listItem(folderId=parent_id, name=os.path.basename(file_pattern))) - if len(items) == 0: - items = list(self.client.listFolder(parentId=parent_id, name=os.path.basename(file_pattern))) - assert len(items) <=1, f"More than one file found with the same name {os.path.basename(file_pattern)}" - assert len(items) >0, f"File not Found in Girder with the name {os.path.basename(file_pattern)}" - return items[0]["_id"] + item = self.listChildren(parent_id,os.path.basename(file_pattern)) + return item["_id"] class ConfigHandler: def __init__(self, config_filepath): From d3c40803d2947a301263dfedfe3e55022be995a9 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:41:12 +0100 Subject: [PATCH 45/56] rm print #181 --- src/feelpp/benchmarking/scripts/girder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/girder.py b/src/feelpp/benchmarking/scripts/girder.py index 767bd45d..b03efe2d 100644 --- a/src/feelpp/benchmarking/scripts/girder.py +++ b/src/feelpp/benchmarking/scripts/girder.py @@ -52,4 +52,4 @@ def move_cli(): headers={"Girder-Token":token.json()["authToken"]["token"]} ) - print(response.text) + return 0 From 1ffa0e13bd46ca97e4063627383ab1ea97968cd6 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:48:24 +0100 Subject: [PATCH 46/56] write deploy workflow #181 [ci skip] --- .github/workflows/benchmark_deploy.yml | 108 +++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 .github/workflows/benchmark_deploy.yml diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml new file mode 100644 index 00000000..16a0cfaf --- /dev/null +++ b/.github/workflows/benchmark_deploy.yml @@ -0,0 +1,108 @@ +name: Benchmark Deploy + +on: + pull_request: + types: [closed] + +jobs: + build_wheel: + runs-on: self-ubuntu-22.04 + name: Build wheel package + if: contains(github.event.pull_request.labels, 'new-benchmark') + steps: + - uses: actions/checkout@v4 + - name: Build wheel + run: | + npm install + npx downdoc README.adoc + pip3 wheel --no-deps --wheel-dir dist . + env: + CXX: clang++ + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: wheel-artifacts + path: dist/*.whl + + accepted: + if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels, 'new-benchmark') + runs-on: self-ubuntu-22.04 + name: Move to production + needs: build_wheel + steps: + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: wheel-artifacts + path: dist + + - name: Create Virtual Environment + run: | + python3 -m venv .venv + source .venv/bin/activate + pip3 install -r requirements.txt + - name: Download stage reports + run: | + source .venv/bin/activate + config_id=6752b5194c9ccbdde21a48b8 + girder-download -gid $config_id -o ./tmp/ -d + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} + - name: Download Production config + run: | + source .venv/bin/activate + config_id=6752f0734c9ccbdde21a48ca + girder-download -gid $config_id -o ./configs_tmp/ -fn production_config.json + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} + - name: Merge stage configs + run: | + source .venv/bin/activate + merge-json-configs -fp ./tmp/**/website_config.json -o ./configs_tmp/stage_config.json -u + - name: Update Stage Config + run: | + source .venv/bin/activate + production_folder_id=6752efed4c9ccbdde21a48c7 + update-stage-config -stag ./configs_tmp/stage_config.json -prod ./configs_tmp/production_config.json -prodid $production_folder_id + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} + - name: Merge stage and prod configs + run: | + source .venv/bin/activate + merge-json-configs -fp ./configs_tmp/**/*.json -o ./configs_tmp/website_config.json + - name: Upload New Website config + run: | + source .venv/bin/activate + girder-upload --item ./configs_tmp/website_config.json --girder_id 6752efed4c9ccbdde21a48c7 + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} + - name: Cleanup stage folder + run: | + echo "Moving all folders from staging/ to merged/" + source .venv/bin/activate + staging_id=6752b5194c9ccbdde21a48b8 + merged_id=67580d534c9ccbdde21a4a75 + girder-move -oid $staging_id -nid $merged_id + denied: + if: github.event.pull_request.merged == false && contains(github.event.pull_request.labels, 'new-benchmark') + runs-on: self-ubuntu-22.04 + name: Move to denied + needs: build_wheel + steps: + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: wheel-artifacts + path: dist + - name: Create Virtual Environment + run: | + python3 -m venv .venv + source .venv/bin/activate + pip3 install -r requirements.txt + - name: Cleanup stage folder + run: | + echo "Moving all folders from staging/ to denied/" + source .venv/bin/activate + staging_id=6752b5194c9ccbdde21a48b8 + denied_id=6752b66d4c9ccbdde21a48b9 + girder-move -oid $staging_id -nid $denied_id \ No newline at end of file From 04fc7431c0f933efde30e099d76b352ed8d3a758 Mon Sep 17 00:00:00 2001 From: JavierCladellas <82624449+JavierCladellas@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:52:10 +0100 Subject: [PATCH 47/56] [create-pull-request] automated change --- docs/modules/ROOT/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index da23ccb8..736ae7fe 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -4,4 +4,4 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/] = Benchmarking :page-layout: toolboxes :page-tags: catalog, catalog-index -:docdatetime: 2024-12-09T10:48:30 \ No newline at end of file +:docdatetime: 2024-12-10T11:52:09 \ No newline at end of file From 63c85e25b5c4bc125064c4874c7b140020502997 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 11:56:03 +0100 Subject: [PATCH 48/56] bug fix [ci skip] --- src/feelpp/benchmarking/scripts/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/feelpp/benchmarking/scripts/misc.py b/src/feelpp/benchmarking/scripts/misc.py index 5f9ab4eb..3c9340d6 100644 --- a/src/feelpp/benchmarking/scripts/misc.py +++ b/src/feelpp/benchmarking/scripts/misc.py @@ -82,7 +82,7 @@ def upldateStageConfig_cli(): prod_path = prod_info["path"] if stage_platform == "local": - for report_item_basename in os.listdir(report_item_basename): + for report_item_basename in os.listdir(stage_path): girder_client.upload( os.path.join(stage_path,report_item_basename), prod_path, reuse_existing=False) stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["path"] = prod_path stage_cfg["execution_mapping"][app_name][machine_name][use_case_name]["platform"] = prod_platorm From cacfba52b867711f3d2212f4aea41b7a8958eb56 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 12:29:50 +0100 Subject: [PATCH 49/56] factor girder ids #181 --- .github/workflows/benchmark.yml | 4 ++-- .github/workflows/benchmark_deploy.yml | 22 ++++++++++------------ .github/workflows/ci.yml | 4 ++-- configs_tmp/production_config.json | 1 + configs_tmp/stage_config.json | 1 + configs_tmp/website_config.json | 1 + girder_deploy_config.sh | 7 +++++++ netlify-build.sh | 7 +++---- 8 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 configs_tmp/production_config.json create mode 100644 configs_tmp/stage_config.json create mode 100644 configs_tmp/website_config.json create mode 100644 girder_deploy_config.sh diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b13b371e..2baee8dd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -160,8 +160,8 @@ jobs: - name: Upload to girder run: | source .venv/bin/activate - girder_staging_id=6752b5194c9ccbdde21a48b8 - girder-upload --item "./tmp/results/*" --girder_id $girder_staging_id + source ./girder_deploy_config.sh + girder-upload --item "./tmp/results/*" --girder_id $staging_folder_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Update Index Date diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml index 16a0cfaf..ac0bba79 100644 --- a/.github/workflows/benchmark_deploy.yml +++ b/.github/workflows/benchmark_deploy.yml @@ -44,15 +44,15 @@ jobs: - name: Download stage reports run: | source .venv/bin/activate - config_id=6752b5194c9ccbdde21a48b8 - girder-download -gid $config_id -o ./tmp/ -d + source ./girder_deploy_config.sh + girder-download -gid $staging_folder_id -o ./tmp/ -d env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Download Production config run: | source .venv/bin/activate - config_id=6752f0734c9ccbdde21a48ca - girder-download -gid $config_id -o ./configs_tmp/ -fn production_config.json + source ./girder_deploy_config.sh + girder-download -gid $production_website_config_id -o ./configs_tmp/ -fn production_config.json env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Merge stage configs @@ -62,7 +62,7 @@ jobs: - name: Update Stage Config run: | source .venv/bin/activate - production_folder_id=6752efed4c9ccbdde21a48c7 + source ./girder_deploy_config.sh update-stage-config -stag ./configs_tmp/stage_config.json -prod ./configs_tmp/production_config.json -prodid $production_folder_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} @@ -73,16 +73,15 @@ jobs: - name: Upload New Website config run: | source .venv/bin/activate - girder-upload --item ./configs_tmp/website_config.json --girder_id 6752efed4c9ccbdde21a48c7 + girder-upload --item ./configs_tmp/website_config.json --girder_id $production_folder_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} - name: Cleanup stage folder run: | echo "Moving all folders from staging/ to merged/" source .venv/bin/activate - staging_id=6752b5194c9ccbdde21a48b8 - merged_id=67580d534c9ccbdde21a4a75 - girder-move -oid $staging_id -nid $merged_id + source ./girder_deploy_config.sh + girder-move -oid $staging_folder_id -nid $accepted_folder_id denied: if: github.event.pull_request.merged == false && contains(github.event.pull_request.labels, 'new-benchmark') runs-on: self-ubuntu-22.04 @@ -103,6 +102,5 @@ jobs: run: | echo "Moving all folders from staging/ to denied/" source .venv/bin/activate - staging_id=6752b5194c9ccbdde21a48b8 - denied_id=6752b66d4c9ccbdde21a48b9 - girder-move -oid $staging_id -nid $denied_id \ No newline at end of file + source ./girder_deploy_config.sh + girder-move -oid $staging_folder_id -nid $denied_folder_id \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bb8f4f1..139de6fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,8 +102,8 @@ jobs: - name: Download website config run: | source .venv/bin/activate - config_id=6752f0734c9ccbdde21a48ca - girder-download --girder_id=$config_id --output_dir=./tmp/ --filename=website_config.json + source ./girder_deploy_config.sh + girder-download --girder_id=$production_website_config_id --output_dir=./tmp/ --filename=website_config.json env: GIRDER_API_KEY: ${{secrets.GIRDER}} - name: Render reports diff --git a/configs_tmp/production_config.json b/configs_tmp/production_config.json new file mode 100644 index 00000000..845e7fde --- /dev/null +++ b/configs_tmp/production_config.json @@ -0,0 +1 @@ +{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}, "thermal_bridges_case_2": {"platform": "girder", "path": "673da5c0b0e95728eb010c1b"}, "thermal_bridges_case_3": {"platform": "girder", "path": "673da5c7b0e95728eb010c1c"}}, "discoverer": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49ab"}}}, "feelpp_toolbox_thermoelectric": {"gaya": {"HL_31": {"platform": "girder", "path": "673da5e2b0e95728eb010c1e"}}}, "feelpp_toolbox_heatfluid": {"gaya": {"eye": {"platform": "girder", "path": "673da5d4b0e95728eb010c1d"}}}, "feelpp_kub_cem": {"gaya": {"poznan": {"platform": "girder", "path": "673da5a6b0e95728eb010c1a"}}}}, "machines": {"gaya": {"display_name": "Gaya", "description": "Strasbourg, France"}, "discoverer": {"display_name": "Discoverer", "description": "Sofia, Bulgaria"}, "meluxina": {"display_name": "MeluXina", "description": "Bissen, Luxembourg"}, "lumi": {"display_name": "Lumi", "description": "Kajaani, Finland"}, "karolina": {"display_name": "Karolina", "description": "Ostrava, Czech Republic"}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}, "busbar2d": {"display_name": "Busbar 2D", "description": "Busbar 2D simulation"}, "busbar3d": {"display_name": "Busbar 3D", "description": "Busbar 3D simulation"}, "thermal_bridges_case_1": {"display_name": "Thermal Bridges Case 1", "description": "Thermal bridges case 1 simulation"}, "thermal_bridges_case_2": {"display_name": "Thermal Bridges Case 2", "description": "Thermal bridges case 2 simulation"}, "thermal_bridges_case_3": {"display_name": "Thermal Bridges Case 3", "description": "Thermal bridges case 3 simulation"}, "thermal_bridges_case_4": {"display_name": "Thermal Bridges Case 4", "description": "Thermal bridges case 4 simulation"}, "laminar2d": {"display_name": "Laminar 2D", "description": "Laminar 2D simulation"}, "laminar3d": {"display_name": "Laminar 3D", "description": "Laminar 3D simulation"}, "quarter_turn_3d": {"display_name": "Quarter Turn 3D", "description": "Quarter Turn 3D simulation"}, "HL_31": {"display_name": "HL-31", "description": "HL-31"}, "poznan": {"display_name": "Poznan", "description": "Poznan, Poland"}, "eye": {"display_name": "Eye", "description": "steady aqueous humor (AH) flow in the posterior and anterior chambers of the human eyeball, coupled with the overall heat transfer"}, "default": {"display_name": "Default", "description": "Unkown test case"}}, "applications": {"feelpp_toolbox_heat": {"display_name": "Feel++ heat toolbox", "description": "Feel++ heat toolbox", "main_variables": ["Solve_solve", "PostProcessing_exportResults", "Constructor_init"]}, "feelpp_toolbox_electric": {"display_name": "Feel++ electric toolbox", "description": "Feel++ electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_heatfluid": {"display_name": "Feel++ Heat Fluid toolbox", "description": "Feel++ Heat Fluid toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_thermoelectric": {"display_name": "Feel++ Thermo-electric toolbox", "description": "Feel++ Thermo-electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_kub_cem": {"display_name": "KUB", "description": "Ktirio Urban Buildings", "main_variables": ["constructor", "updateForUse", "execute", "simulate", "postprocess"]}}} \ No newline at end of file diff --git a/configs_tmp/stage_config.json b/configs_tmp/stage_config.json new file mode 100644 index 00000000..be1d113a --- /dev/null +++ b/configs_tmp/stage_config.json @@ -0,0 +1 @@ +{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}}}}, "machines": {"gaya": {"display_name": "gaya", "description": ""}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}}, "applications": {"feelpp_toolbox_heat": {"display_name": "feelpp_toolbox_heat", "description": "", "main_variables": []}}} \ No newline at end of file diff --git a/configs_tmp/website_config.json b/configs_tmp/website_config.json new file mode 100644 index 00000000..24bee281 --- /dev/null +++ b/configs_tmp/website_config.json @@ -0,0 +1 @@ +{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}, "thermal_bridges_case_2": {"platform": "girder", "path": "673da5c0b0e95728eb010c1b"}, "thermal_bridges_case_3": {"platform": "girder", "path": "673da5c7b0e95728eb010c1c"}}, "discoverer": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49ab"}}}, "feelpp_toolbox_thermoelectric": {"gaya": {"HL_31": {"platform": "girder", "path": "673da5e2b0e95728eb010c1e"}}}, "feelpp_toolbox_heatfluid": {"gaya": {"eye": {"platform": "girder", "path": "673da5d4b0e95728eb010c1d"}}}, "feelpp_kub_cem": {"gaya": {"poznan": {"platform": "girder", "path": "673da5a6b0e95728eb010c1a"}}}}, "machines": {"gaya": {"display_name": "Gaya", "description": "Strasbourg, France"}, "discoverer": {"display_name": "Discoverer", "description": "Sofia, Bulgaria"}, "meluxina": {"display_name": "MeluXina", "description": "Bissen, Luxembourg"}, "lumi": {"display_name": "Lumi", "description": "Kajaani, Finland"}, "karolina": {"display_name": "Karolina", "description": "Ostrava, Czech Republic"}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}, "busbar2d": {"display_name": "Busbar 2D", "description": "Busbar 2D simulation"}, "busbar3d": {"display_name": "Busbar 3D", "description": "Busbar 3D simulation"}, "thermal_bridges_case_1": {"display_name": "Thermal Bridges Case 1", "description": "Thermal bridges case 1 simulation"}, "thermal_bridges_case_2": {"display_name": "Thermal Bridges Case 2", "description": "Thermal bridges case 2 simulation"}, "thermal_bridges_case_3": {"display_name": "Thermal Bridges Case 3", "description": "Thermal bridges case 3 simulation"}, "thermal_bridges_case_4": {"display_name": "Thermal Bridges Case 4", "description": "Thermal bridges case 4 simulation"}, "laminar2d": {"display_name": "Laminar 2D", "description": "Laminar 2D simulation"}, "laminar3d": {"display_name": "Laminar 3D", "description": "Laminar 3D simulation"}, "quarter_turn_3d": {"display_name": "Quarter Turn 3D", "description": "Quarter Turn 3D simulation"}, "HL_31": {"display_name": "HL-31", "description": "HL-31"}, "poznan": {"display_name": "Poznan", "description": "Poznan, Poland"}, "eye": {"display_name": "Eye", "description": "steady aqueous humor (AH) flow in the posterior and anterior chambers of the human eyeball, coupled with the overall heat transfer"}, "default": {"display_name": "Default", "description": "Unkown test case"}}, "applications": {"feelpp_toolbox_heat": {"display_name": "Feel++ heat toolbox", "description": "Feel++ heat toolbox", "main_variables": ["Solve_solve", "Constructor_init", "PostProcessing_exportResults"]}, "feelpp_toolbox_electric": {"display_name": "Feel++ electric toolbox", "description": "Feel++ electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_heatfluid": {"display_name": "Feel++ Heat Fluid toolbox", "description": "Feel++ Heat Fluid toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_thermoelectric": {"display_name": "Feel++ Thermo-electric toolbox", "description": "Feel++ Thermo-electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_kub_cem": {"display_name": "KUB", "description": "Ktirio Urban Buildings", "main_variables": ["constructor", "updateForUse", "execute", "simulate", "postprocess"]}}} \ No newline at end of file diff --git a/girder_deploy_config.sh b/girder_deploy_config.sh new file mode 100644 index 00000000..7ba9f78e --- /dev/null +++ b/girder_deploy_config.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +production_website_config_id=6752f0734c9ccbdde21a48ca +staging_folder_id=6752b5194c9ccbdde21a48b8 +production_folder_id=6752efed4c9ccbdde21a48c7 +accepted_folder_id=67580d534c9ccbdde21a4a75 +denied_folder_id=6752b66d4c9ccbdde21a48b9 \ No newline at end of file diff --git a/netlify-build.sh b/netlify-build.sh index 4b6bff38..ffd2b5a6 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -8,15 +8,14 @@ pip3 install -r requirements.txt pip3 wheel --no-deps --wheel-dir dist . pip3 install dist/*.whl npm i +source ./girder_deploy_config.sh if [[ $HEAD == *"new-benchmark"* ]]; then echo "Downloading Staging benchmarks" - config_id=6752b5194c9ccbdde21a48b8 - girder-download -gid $config_id -o ./tmp/ -d + girder-download -gid $staging_folder_id -o ./tmp/ -d merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u else echo "Downloading Production benchmarks" - config_id=6752f0734c9ccbdde21a48ca - girder-download -gid $config_id -o ./tmp/ -fn website_config.json + girder-download -gid $production_website_config_id -o ./tmp/ -fn website_config.json fi render-benchmarks --config_file=./tmp/website_config.json npx antora --stacktrace generate --cache-dir cache --clean --html-url-extension-style=indexify site.yml From 306908e059777c98ad6283ddbe0520827fb87101 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 12:33:31 +0100 Subject: [PATCH 50/56] fix sourced var [ci skip] --- .github/workflows/benchmark_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml index ac0bba79..624d74e7 100644 --- a/.github/workflows/benchmark_deploy.yml +++ b/.github/workflows/benchmark_deploy.yml @@ -73,6 +73,7 @@ jobs: - name: Upload New Website config run: | source .venv/bin/activate + source ./girder_deploy_config.sh girder-upload --item ./configs_tmp/website_config.json --girder_id $production_folder_id env: GIRDER_API_KEY: ${{ secrets.GIRDER }} From 3b62c8a55fbe4b5f2dbfc9b6044074dd70a726a2 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Tue, 10 Dec 2024 12:59:52 +0100 Subject: [PATCH 51/56] rm tmp files from git tree... #181 [ci skip] (Oopsie) --- configs_tmp/production_config.json | 1 - configs_tmp/stage_config.json | 1 - configs_tmp/website_config.json | 1 - 3 files changed, 3 deletions(-) delete mode 100644 configs_tmp/production_config.json delete mode 100644 configs_tmp/stage_config.json delete mode 100644 configs_tmp/website_config.json diff --git a/configs_tmp/production_config.json b/configs_tmp/production_config.json deleted file mode 100644 index 845e7fde..00000000 --- a/configs_tmp/production_config.json +++ /dev/null @@ -1 +0,0 @@ -{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}, "thermal_bridges_case_2": {"platform": "girder", "path": "673da5c0b0e95728eb010c1b"}, "thermal_bridges_case_3": {"platform": "girder", "path": "673da5c7b0e95728eb010c1c"}}, "discoverer": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49ab"}}}, "feelpp_toolbox_thermoelectric": {"gaya": {"HL_31": {"platform": "girder", "path": "673da5e2b0e95728eb010c1e"}}}, "feelpp_toolbox_heatfluid": {"gaya": {"eye": {"platform": "girder", "path": "673da5d4b0e95728eb010c1d"}}}, "feelpp_kub_cem": {"gaya": {"poznan": {"platform": "girder", "path": "673da5a6b0e95728eb010c1a"}}}}, "machines": {"gaya": {"display_name": "Gaya", "description": "Strasbourg, France"}, "discoverer": {"display_name": "Discoverer", "description": "Sofia, Bulgaria"}, "meluxina": {"display_name": "MeluXina", "description": "Bissen, Luxembourg"}, "lumi": {"display_name": "Lumi", "description": "Kajaani, Finland"}, "karolina": {"display_name": "Karolina", "description": "Ostrava, Czech Republic"}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}, "busbar2d": {"display_name": "Busbar 2D", "description": "Busbar 2D simulation"}, "busbar3d": {"display_name": "Busbar 3D", "description": "Busbar 3D simulation"}, "thermal_bridges_case_1": {"display_name": "Thermal Bridges Case 1", "description": "Thermal bridges case 1 simulation"}, "thermal_bridges_case_2": {"display_name": "Thermal Bridges Case 2", "description": "Thermal bridges case 2 simulation"}, "thermal_bridges_case_3": {"display_name": "Thermal Bridges Case 3", "description": "Thermal bridges case 3 simulation"}, "thermal_bridges_case_4": {"display_name": "Thermal Bridges Case 4", "description": "Thermal bridges case 4 simulation"}, "laminar2d": {"display_name": "Laminar 2D", "description": "Laminar 2D simulation"}, "laminar3d": {"display_name": "Laminar 3D", "description": "Laminar 3D simulation"}, "quarter_turn_3d": {"display_name": "Quarter Turn 3D", "description": "Quarter Turn 3D simulation"}, "HL_31": {"display_name": "HL-31", "description": "HL-31"}, "poznan": {"display_name": "Poznan", "description": "Poznan, Poland"}, "eye": {"display_name": "Eye", "description": "steady aqueous humor (AH) flow in the posterior and anterior chambers of the human eyeball, coupled with the overall heat transfer"}, "default": {"display_name": "Default", "description": "Unkown test case"}}, "applications": {"feelpp_toolbox_heat": {"display_name": "Feel++ heat toolbox", "description": "Feel++ heat toolbox", "main_variables": ["Solve_solve", "PostProcessing_exportResults", "Constructor_init"]}, "feelpp_toolbox_electric": {"display_name": "Feel++ electric toolbox", "description": "Feel++ electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_heatfluid": {"display_name": "Feel++ Heat Fluid toolbox", "description": "Feel++ Heat Fluid toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_thermoelectric": {"display_name": "Feel++ Thermo-electric toolbox", "description": "Feel++ Thermo-electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_kub_cem": {"display_name": "KUB", "description": "Ktirio Urban Buildings", "main_variables": ["constructor", "updateForUse", "execute", "simulate", "postprocess"]}}} \ No newline at end of file diff --git a/configs_tmp/stage_config.json b/configs_tmp/stage_config.json deleted file mode 100644 index be1d113a..00000000 --- a/configs_tmp/stage_config.json +++ /dev/null @@ -1 +0,0 @@ -{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}}}}, "machines": {"gaya": {"display_name": "gaya", "description": ""}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}}, "applications": {"feelpp_toolbox_heat": {"display_name": "feelpp_toolbox_heat", "description": "", "main_variables": []}}} \ No newline at end of file diff --git a/configs_tmp/website_config.json b/configs_tmp/website_config.json deleted file mode 100644 index 24bee281..00000000 --- a/configs_tmp/website_config.json +++ /dev/null @@ -1 +0,0 @@ -{"execution_mapping": {"feelpp_toolbox_heat": {"gaya": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49a0"}, "thermal_bridges_case_2": {"platform": "girder", "path": "673da5c0b0e95728eb010c1b"}, "thermal_bridges_case_3": {"platform": "girder", "path": "673da5c7b0e95728eb010c1c"}}, "discoverer": {"ThermalBridgesENISO10211": {"platform": "girder", "path": "675801b64c9ccbdde21a49ab"}}}, "feelpp_toolbox_thermoelectric": {"gaya": {"HL_31": {"platform": "girder", "path": "673da5e2b0e95728eb010c1e"}}}, "feelpp_toolbox_heatfluid": {"gaya": {"eye": {"platform": "girder", "path": "673da5d4b0e95728eb010c1d"}}}, "feelpp_kub_cem": {"gaya": {"poznan": {"platform": "girder", "path": "673da5a6b0e95728eb010c1a"}}}}, "machines": {"gaya": {"display_name": "Gaya", "description": "Strasbourg, France"}, "discoverer": {"display_name": "Discoverer", "description": "Sofia, Bulgaria"}, "meluxina": {"display_name": "MeluXina", "description": "Bissen, Luxembourg"}, "lumi": {"display_name": "Lumi", "description": "Kajaani, Finland"}, "karolina": {"display_name": "Karolina", "description": "Ostrava, Czech Republic"}}, "use_cases": {"ThermalBridgesENISO10211": {"display_name": "ThermalBridgesENISO10211", "description": ""}, "busbar2d": {"display_name": "Busbar 2D", "description": "Busbar 2D simulation"}, "busbar3d": {"display_name": "Busbar 3D", "description": "Busbar 3D simulation"}, "thermal_bridges_case_1": {"display_name": "Thermal Bridges Case 1", "description": "Thermal bridges case 1 simulation"}, "thermal_bridges_case_2": {"display_name": "Thermal Bridges Case 2", "description": "Thermal bridges case 2 simulation"}, "thermal_bridges_case_3": {"display_name": "Thermal Bridges Case 3", "description": "Thermal bridges case 3 simulation"}, "thermal_bridges_case_4": {"display_name": "Thermal Bridges Case 4", "description": "Thermal bridges case 4 simulation"}, "laminar2d": {"display_name": "Laminar 2D", "description": "Laminar 2D simulation"}, "laminar3d": {"display_name": "Laminar 3D", "description": "Laminar 3D simulation"}, "quarter_turn_3d": {"display_name": "Quarter Turn 3D", "description": "Quarter Turn 3D simulation"}, "HL_31": {"display_name": "HL-31", "description": "HL-31"}, "poznan": {"display_name": "Poznan", "description": "Poznan, Poland"}, "eye": {"display_name": "Eye", "description": "steady aqueous humor (AH) flow in the posterior and anterior chambers of the human eyeball, coupled with the overall heat transfer"}, "default": {"display_name": "Default", "description": "Unkown test case"}}, "applications": {"feelpp_toolbox_heat": {"display_name": "Feel++ heat toolbox", "description": "Feel++ heat toolbox", "main_variables": ["Solve_solve", "Constructor_init", "PostProcessing_exportResults"]}, "feelpp_toolbox_electric": {"display_name": "Feel++ electric toolbox", "description": "Feel++ electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_heatfluid": {"display_name": "Feel++ Heat Fluid toolbox", "description": "Feel++ Heat Fluid toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_toolbox_thermoelectric": {"display_name": "Feel++ Thermo-electric toolbox", "description": "Feel++ Thermo-electric toolbox", "main_variables": ["Constructor_init", "PostProcessing_exportResults", "Solve_solve"]}, "feelpp_kub_cem": {"display_name": "KUB", "description": "Ktirio Urban Buildings", "main_variables": ["constructor", "updateForUse", "execute", "simulate", "postprocess"]}}} \ No newline at end of file From 80bc4a62ec18cb26d4f3d76f9922d083f97ed968 Mon Sep 17 00:00:00 2001 From: JavierCladellas <82624449+JavierCladellas@users.noreply.github.com> Date: Thu, 12 Dec 2024 09:10:55 +0100 Subject: [PATCH 52/56] [create-pull-request] automated change --- docs/modules/ROOT/pages/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 736ae7fe..dbef1161 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -4,4 +4,4 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/] = Benchmarking :page-layout: toolboxes :page-tags: catalog, catalog-index -:docdatetime: 2024-12-10T11:52:09 \ No newline at end of file +:docdatetime: 2024-12-12T09:10:54 \ No newline at end of file From 7b71257d11c0bee468af9a706d81706f03600f1f Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Thu, 12 Dec 2024 10:39:07 +0100 Subject: [PATCH 53/56] trigger netlify From d61ca70423368bb5a7c49ab850ddc6769bb66efd Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Thu, 12 Dec 2024 10:46:02 +0100 Subject: [PATCH 54/56] fix merge-json-config args #181 --- .github/workflows/benchmark_deploy.yml | 4 ++-- netlify-build.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml index 624d74e7..67f0f193 100644 --- a/.github/workflows/benchmark_deploy.yml +++ b/.github/workflows/benchmark_deploy.yml @@ -58,7 +58,7 @@ jobs: - name: Merge stage configs run: | source .venv/bin/activate - merge-json-configs -fp ./tmp/**/website_config.json -o ./configs_tmp/stage_config.json -u + merge-json-configs -fp "./tmp/**/website_config.json" -o ./configs_tmp/stage_config.json -u - name: Update Stage Config run: | source .venv/bin/activate @@ -69,7 +69,7 @@ jobs: - name: Merge stage and prod configs run: | source .venv/bin/activate - merge-json-configs -fp ./configs_tmp/**/*.json -o ./configs_tmp/website_config.json + merge-json-configs -fp "./configs_tmp/**/*.json" -o ./configs_tmp/website_config.json - name: Upload New Website config run: | source .venv/bin/activate diff --git a/netlify-build.sh b/netlify-build.sh index ffd2b5a6..245f45fa 100755 --- a/netlify-build.sh +++ b/netlify-build.sh @@ -12,7 +12,7 @@ source ./girder_deploy_config.sh if [[ $HEAD == *"new-benchmark"* ]]; then echo "Downloading Staging benchmarks" girder-download -gid $staging_folder_id -o ./tmp/ -d - merge-json-configs -fp ./tmp/**/website_config.json -o ./tmp/website_config.json -u + merge-json-configs -fp "./tmp/**/website_config.json" -o ./tmp/website_config.json -u else echo "Downloading Production benchmarks" girder-download -gid $production_website_config_id -o ./tmp/ -fn website_config.json From 0e91697b3876c553363beb0901b28e6bdf8caa66 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Thu, 12 Dec 2024 10:59:52 +0100 Subject: [PATCH 55/56] try fix label recognition [ci skip] --- .github/workflows/benchmark_deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml index 67f0f193..1579f68c 100644 --- a/.github/workflows/benchmark_deploy.yml +++ b/.github/workflows/benchmark_deploy.yml @@ -8,7 +8,7 @@ jobs: build_wheel: runs-on: self-ubuntu-22.04 name: Build wheel package - if: contains(github.event.pull_request.labels, 'new-benchmark') + if: contains(github.event.pull_request.labels.*.name, 'new-benchmark') steps: - uses: actions/checkout@v4 - name: Build wheel @@ -25,7 +25,7 @@ jobs: path: dist/*.whl accepted: - if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels, 'new-benchmark') + if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'new-benchmark') runs-on: self-ubuntu-22.04 name: Move to production needs: build_wheel @@ -84,7 +84,7 @@ jobs: source ./girder_deploy_config.sh girder-move -oid $staging_folder_id -nid $accepted_folder_id denied: - if: github.event.pull_request.merged == false && contains(github.event.pull_request.labels, 'new-benchmark') + if: github.event.pull_request.merged == false && contains(github.event.pull_request.labels.*.name, 'new-benchmark') runs-on: self-ubuntu-22.04 name: Move to denied needs: build_wheel From 9b82dc2c4efdcddefaf2eb296c7b3e70e18b1f35 Mon Sep 17 00:00:00 2001 From: Javier Cladellas Date: Thu, 12 Dec 2024 11:22:27 +0100 Subject: [PATCH 56/56] forgot girder api key #181 :p [ci skip] --- .github/workflows/benchmark_deploy.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark_deploy.yml b/.github/workflows/benchmark_deploy.yml index 1579f68c..7e7cd6f4 100644 --- a/.github/workflows/benchmark_deploy.yml +++ b/.github/workflows/benchmark_deploy.yml @@ -83,6 +83,8 @@ jobs: source .venv/bin/activate source ./girder_deploy_config.sh girder-move -oid $staging_folder_id -nid $accepted_folder_id + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} denied: if: github.event.pull_request.merged == false && contains(github.event.pull_request.labels.*.name, 'new-benchmark') runs-on: self-ubuntu-22.04 @@ -104,4 +106,6 @@ jobs: echo "Moving all folders from staging/ to denied/" source .venv/bin/activate source ./girder_deploy_config.sh - girder-move -oid $staging_folder_id -nid $denied_folder_id \ No newline at end of file + girder-move -oid $staging_folder_id -nid $denied_folder_id + env: + GIRDER_API_KEY: ${{ secrets.GIRDER }} \ No newline at end of file