Add benchmark for feelpp_toolbox_heat - ThermalBridgesENISO10211

config/kub/kub_plots.json

@@ -2,11 +2,25 @@
 
     "plots": [
         {
-            "title": "Absolute performance",
-            "plot_types": [ "grouped_bar", "table" ],
+            "title": "General Relative performance",
+            "plot_types": [ "stacked_bar", "grouped_bar" ],
+            "transformation": "relative_performance",
+            "variables": [ "constructor", "simulation", "postprocess" ],
+            "names": ["Constructor", "Simulation", "PostProcess"],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            }
+        },
+        {
+            "title": "General Absolute performance",
+            "plot_types": [ "grouped_bar","stacked_bar"],
             "transformation": "performance",
-            "variables": [ "execute", "postprocess", "constructor", "updateForUse", "simulation" ],
-            "names": [],
+            "variables": [ "constructor", "simulation", "postprocess" ],
+            "names": ["Constructor", "Simulation", "PostProcess"],
             "xaxis": {
                 "parameter": "resources.tasks",
                 "label": "Number of tasks"
@@ -16,11 +30,69 @@
             }
         },
         {
-            "title": "Relative performance",
-            "plot_types": [ "stacked_bar", "grouped_bar" ],
-            "transformation": "relative_performance",
-            "variables": [ "execute", "postprocess", "constructor", "updateForUse", "simulation" ],
-            "names": [],
+            "title": "General Speedup",
+            "plot_types": [ "scatter" ],
+            "transformation": "speedup",
+            "variables": [ "constructor", "simulation","postprocess" ],
+            "names": ["Constructor", "Simulation", "PostProcess"],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            }
+        },
+        {
+            "title": "Constructor UpdateForUse Absolute performance",
+            "plot_types": [ "grouped_bar" ],
+            "transformation": "performance",
+            "variables": [
+                "constructor.updateForUse.SunModel", "constructor.updateForUse.updateMeshBuilding", "constructor.updateForUse.weather",
+                "constructor.updateForUse.buildingMap", "constructor.updateForUse.gisMeshInit", "constructor.updateForUse.gisPartition",
+                "constructor.updateForUse.initBuildingModels", "constructor.updateForUse.scenarios"
+            ],
+            "names": [
+                "SunModel", "updateMeshBuilding", "weather",
+                "buildingMap", "gisMeshInit", "gisPartition",
+                "initBuildingModels", "scenarios"
+            ],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            }
+        },
+        {
+            "title": "Constructor Speedup",
+            "plot_types": [ "scatter" ],
+            "transformation": "speedup",
+            "variables": [
+                "constructor.updateForUse.SunModel", "constructor.updateForUse.updateMeshBuilding", "constructor.updateForUse.weather",
+                "constructor.updateForUse.buildingMap", "constructor.updateForUse.gisMeshInit", "constructor.updateForUse.gisPartition",
+                "constructor.updateForUse.initBuildingModels", "constructor.updateForUse.scenarios"
+            ],
+            "names": [
+                "SunModel", "updateMeshBuilding", "weather",
+                "buildingMap", "gisMeshInit", "gisPartition",
+                "initBuildingModels", "scenarios"
+            ],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            }
+        },
+        {
+            "title": "Simulation Absolute performance",
+            "plot_types": [ "grouped_bar" ],
+            "transformation": "performance",
+            "variables": [ "simulation.solarMasks", "simulation.buildingExecuteStep" ],
+            "names": ["Solar Masks","Building Execute Step"],
             "xaxis": {
                 "parameter": "resources.tasks",
                 "label": "Number of tasks"
@@ -30,11 +102,11 @@
             }
         },
         {
-            "title": "Speedup",
+            "title": "Simulation Speedup",
             "plot_types": [ "scatter" ],
             "transformation": "speedup",
-            "variables": [ "execute", "postprocess", "constructor", "updateForUse", "simulation" ],
-            "names": [],
+            "variables": [ "simulation.solarMasks", "simulation.buildingExecuteStep" ],
+            "names": ["Solar Masks","Building Execute Step"],
             "xaxis": {
                 "parameter": "resources.tasks",
                 "label": "Number of tasks"

config/kub/poznan.json

@@ -1,6 +1,6 @@
 {
     "executable": "feelpp_kub_cem",
-    "output_directory": "{{machine.output_app_dir}}/kub",
+    "output_directory": "{{machine.output_app_dir}}/javier_test/kub",
     "use_case_name": "poznan",
     "timeout":"0-0:15:0",
     "resources":{
@@ -10,47 +10,44 @@
     "platforms": {
         "apptainer":{
             "image": {
+                "remote":"oras://ghcr.io/feelpp/ktirio-urban-building:master-sif",
                 "name":"{{machine.containers.apptainer.image_base_dir}}/kub.sif"
             },
-            "input_dir":"/input_data/",
+            "input_dir":"/input_data",
             "options": [
                 "--home {{machine.output_app_dir}}",
-                "--bind {{machine.input_dataset_base_dir}}/kub/cfg/:{{platforms.apptainer.input_dir}}"
+                "--bind {{machine.input_dataset_base_dir}}:{{platforms.apptainer.input_dir}}",
+                "--env OMP_NUM_THREADS=1"
             ],
             "append_app_option":[]
         },
         "builtin":{
-            "input_dir":"{{machine.input_dataset_base_dir}}/kub/cfg/",
+            "input_dir":"{{machine.input_dataset_base_dir}}",
             "append_app_option":[]
         }
     },
+    "input_file_dependencies":{
+        "case_cfg":"kub/cfg/Poznan.cfg"
+    },
     "options": [
-        "--config-file {{platforms.{{machine.platform}}.input_dir}}/Poznan.cfg",
-        "--cem.database.directory {{output_directory}}/{{use_case_name}}/{{instance}}"
+        "--config-file {{platforms.{{machine.platform}}.input_dir}}/{{input_file_dependencies.case_cfg}}",
+        "--cem.database.directory {{output_directory}}/{{instance}}/{{use_case_name}}",
+        "--directory {{output_directory}}/{{instance}}/{{use_case_name}}"
     ],
-    "outputs": [],
     "scalability": {
-        "directory": "{{output_directory}}/{{use_case_name}}/{{instance}}/cem/",
+        "directory": "{{output_directory}}/{{instance}}/{{use_case_name}}/cem/",
         "stages": [
             {
                 "name":"",
                 "filepath": "instances/np_{{parameters.resources.tasks.value}}/logs/timers.json",
                 "format": "json",
-                "variables_path":["*.timeElapsed.max","execute.subtimers.*.subtimers.*.timeElapsed.max"],
-                "units":{
-                    "*":"custom instance unit",
-                    "postprocess.exportOutputs":"exportOutputs unit special",
-                    "simulation.buildingExecuteStep":"buildingExecuteStep unit special"
-                }
+                "variables_path":["*.timeElapsed.max","execute.subtimers.*.subtimers.*.timeElapsed.max","execute.subtimers.*.timeElapsed.max"]
             },
             {
                 "name":"",
                 "filepath": "logs/timers.json",
                 "format": "json",
-                "variables_path":["*.timeElapsed.max","*.subtimers.updateForUse.subtimers.*.timeElapsed.max"],
-                "units":{
-                    "*":"custom unit"
-                }
+                "variables_path":["*.timeElapsed.max","*.subtimers.*.subtimers.*.timeElapsed.max"]
             }
         ]
     },
@@ -64,7 +61,10 @@
             "sequence": [
                 {"tasks":32, "exclusive_access":true},
                 {"tasks":64, "exclusive_access":true},
-                {"tasks":128, "exclusive_access":true}
+                {"tasks":128, "exclusive_access":true},
+                {"tasks":256, "exclusive_access":true},
+                {"tasks":384, "exclusive_access":true},
+                {"tasks":512, "exclusive_access":true}
             ]
         }
     ]

config/machines/gaya.json

@@ -1,18 +1,18 @@
 {
     "machine": "gaya",
-    "targets":["production:apptainer:"],
-    "execution_policy": "serial",
+    "targets":["public:builtin:hpcx"],
+    "execution_policy": "async",
     "reframe_base_dir":"$PWD/build/reframe",
     "reports_base_dir":"$PWD/reports/",
     //If input_user_dir exists, input_dataset_base_dir is supposed to be temporary. Will be deleted after test.
     // If input_user_dir DOES NOT exist, input_datates_base_dir will not be deleted
     "input_dataset_base_dir":"/data/scratch/cladellash/feelppdb/input_data",
-    "input_user_dir":"/home/u2/cladellash/largedataexample/input_data/",
+    // "input_user_dir":"/home/u2/cladellash/largedataexample/input_data/",
     "output_app_dir":"/data/scratch/cladellash/feelppdb",
     "containers":{
         "apptainer":{
-            // "cachedir": "/home/u2/cladellash/.apptainer/cache",
-            // "tmpdir": "/data/scratch/cladellash/images/tmp",
+            "cachedir": "/home/u2/cladellash/.apptainer/cache",
+            "tmpdir": "/data/scratch/cladellash/images/tmp",
             "image_base_dir":"/data/scratch/cladellash/images",
             "options":[
                 "--sharens",

config/toolbox_heat/heat_plots.json

@@ -28,7 +28,7 @@
                 "label": "Measures"
             },
             "aggregations":[
-                {"column":"resources.tasks","agg":"filter:640"}
+                {"column":"resources.tasks","agg":"filter:768"}
             ]
         },
         {
@@ -57,6 +57,7 @@
                 "label":"Performance variable"
             },
             "aggregations":[
+                {"column":"repeat", "agg":"min"},
                 {"column":"discretization","agg":"filter:P1"}
             ]
         },
@@ -204,6 +205,58 @@
             "aggregations":[
                 {"column":"discretization","agg":"filter:P3"}
             ]
+        },
+        {
+            "title": "Algebraic-solve execution time",
+            "plot_types": [ "grouped_bar" ],
+            "transformation": "performance",
+            "variables": [
+                "Solve_algebraic-solve"
+            ],
+            "names": [ ],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            },
+            "color_axis":{
+                "parameter":"mesh",
+                "label":"Mesh"
+            },
+            "secondary_axis":{
+                "parameter":"discretization",
+                "label":"Discretization"
+            },
+            "aggregations":[
+                {"column":"performance_variable","agg":"filter:Solve_algebraic-solve"}
+            ]
+        },
+        {
+            "title":"Number of iterations of GMRES",
+            "plot_types":["scatter"],
+            "transformation":"performance",
+            "variables":["Solve_ksp-niter"],
+            "names":[],
+            "xaxis":{
+                "parameter":"resources.tasks",
+                "label":"Number of tasks"
+            },
+            "yaxis":{
+                "label":"Number of iterations"
+            },
+            "color_axis":{
+                "parameter":"mesh",
+                "label":"Mesh"
+            },
+            "secondary_axis":{
+                "parameter":"discretization",
+                "label":"Discretization"
+            },
+            "aggregations":[
+                {"column":"performance_variable","agg":"filter:Solve_ksp-niter"}
+            ]
         }
     ]
 }

docs/modules/ROOT/pages/index.adoc

@@ -4,4 +4,4 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/]
 = Benchmarking
 :page-layout: toolboxes
 :page-tags: catalog, catalog-index
-:docdatetime: 2025-01-22T11:42:45
+:docdatetime: 2025-02-10T15:33:05

docs/modules/tutorial/pages/configurationfiles/plots.adoc

@@ -120,7 +120,7 @@ plot_config = Plot(**{
     "plot_types": [ "stacked_bar", "grouped_bar" ],
     "transformation": "performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "xaxis":{
         "parameter":"nb_tasks.tasks",
         "label":"Number of tasks"
@@ -148,7 +148,7 @@ plot_config = Plot(**{
     "plot_types": [ "stacked_bar", "grouped_bar" ],
     "transformation": "relative_performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "xaxis":{
         "parameter":"nb_tasks.tasks",
         "label":"Number of tasks"
@@ -179,7 +179,7 @@ plot_config = Plot(**{
     "plot_types": [ "stacked_bar", "grouped_bar" ],
     "transformation": "speedup",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "xaxis":{
         "parameter":"nb_tasks.tasks",
         "label":"Number of tasks"
@@ -208,7 +208,7 @@ figures = FigureFactory.create(Plot(**{
     "plot_types": [ "scatter" ],
     "transformation": "performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "color_axis":{
     "parameter":"performance_variable",
     "label":"Performance variable"
@@ -236,7 +236,7 @@ figures = FigureFactory.create(Plot(**{
     "plot_types": [ "stacked_bar" ],
     "transformation": "performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "color_axis":{
     "parameter":"performance_variable",
     "label":"Performance variable"
@@ -265,7 +265,7 @@ figures = FigureFactory.create(Plot(**{
     "plot_types": [ "grouped_bar" ],
     "transformation": "performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "color_axis":{
     "parameter":"performance_variable",
     "label":"Performance variable"
@@ -293,7 +293,7 @@ figures = FigureFactory.create(Plot(**{
     "plot_types": [ "table" ],
     "transformation": "performance",
     "variables": [ "computation_time","communication_time" ],
-    "names": ["Time"],
+    "names": ["Computation","Comunication"],
     "color_axis":{
     "parameter":"performance_variable",
     "label":"Performance variable"

src/feelpp/benchmarking/reframe/__main__.py

@@ -43,6 +43,18 @@ def main_cli():
         app_reader.updateConfig(machine_reader.processor.flattenDict(machine_reader.config,"machine"))
         app_reader.updateConfig() #Update with own field
 
+        #PULL IMAGES
+        if not parser.args.dry_run:
+            for platform_name, platform_field in app_reader.config.platforms.items():
+                if not platform_field.image or not platform_field.image.remote:
+                    continue
+
+                if platform_name == "apptainer":
+                    subprocess.run(f"apptainer pull -F {platform_field.image.name} {platform_field.image.remote}", shell=True)
+                elif platform_name == "docker":
+                    raise NotImplementedError("Pulling docker image is not yet implemented")
+
+
         reframe_cmd = cmd_builder.buildCommand( app_reader.config.timeout)
 
         exit_code = subprocess.run(reframe_cmd, shell=True)