From 63b304e14a0fd7fee6e43b726cc9ad6d3e68ba7e Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 11:48:44 +0100
Subject: [PATCH 01/47] Add initial ASV CI

---
 asv.conf.json                    | 203 +++++++++++++++++++++++++++++++
 asv_util/asv-machine.santis.json |  11 ++
 asv_util/run_tests.py            |  31 +++++
 benchmarks/__init__.py           |  20 +++
 benchmarks/setup_tests.py        |  70 +++++++++++
 ci/benchmark_asv.yml             |  26 ++++
 6 files changed, 361 insertions(+)
 create mode 100644 asv.conf.json
 create mode 100644 asv_util/asv-machine.santis.json
 create mode 100644 asv_util/run_tests.py
 create mode 100644 benchmarks/__init__.py
 create mode 100644 benchmarks/setup_tests.py
 create mode 100644 ci/benchmark_asv.yml

diff --git a/asv.conf.json b/asv.conf.json
new file mode 100644
index 000000000..9571d6e54
--- /dev/null
+++ b/asv.conf.json
@@ -0,0 +1,203 @@
+{
+    // The version of the config file format.  Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+
+    // The name of the project being benchmarked
+    "project": "icon4py",
+
+    // The project's homepage
+    "project_url": "https://github.com/C2SM/icon4py",
+
+    // The URL or local path of the source code repository for the
+    // project being benchmarked
+    "repo": ".",
+
+    // The Python project's subdirectory in your repo.  If missing or
+    // the empty string, the project is assumed to be located at the root
+    // of the repository.
+    // "repo_subdir": "",
+
+    // Customizable commands for building the project.
+    // See asv.conf.json documentation.
+    // To build the package using pyproject.toml (PEP518), uncomment the following lines
+    "build_command": ["uv pip install ."
+        // "python3.11 -c \"import os; print(os.environ.get('GT4PY_BACKEND'))\"",
+        // "python3.11 {conf_dir}/asv_util/run_tests.py"
+    ],
+    // To build the package using setuptools and a setup.py file, uncomment the following lines
+    // "build_command": [
+    //     "python setup.py build",
+    //     "python -mpip wheel -w {build_cache_dir} {build_dir}"
+    // ],
+
+    // Customizable commands for installing and uninstalling the project.
+    // See asv.conf.json documentation.
+    "install_command": [
+        // "uv pip install .",
+        // "python3.11 -c \"import os; print(os.environ.get('GT4PY_BACKEND'))\"",
+        // "pytest model/atmosphere/dycore/tests --benchmark-json {build_dir}/benchmark_{commit}_{GT4PY_BACKEND}_{ICON4PY_GRID}.json --benchmark-only --memray --backend {GT4PY_BACKEND} --grid {ICON4PY_GRID} -k TestFusedSolveNonhydroStencil39To40 --benchmark-min-rounds=1"
+        "python {conf_dir}/asv_util/run_tests.py"
+    ],
+    // "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"],
+
+    // List of branches to benchmark. If not provided, defaults to "main"
+    // (for git) or "default" (for mercurial).
+    "branches": ["asv_test"], // for git
+    // "branches": ["default"],    // for mercurial
+
+    // The DVCS being used.  If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    "dvcs": "git",
+
+    // The tool to use to create environments.  May be "conda",
+    // "virtualenv", "mamba" (above 3.8)
+    // or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "virtualenv",
+
+    // timeout in seconds for installing any dependencies in environment
+    // defaults to 10 min
+    "install_timeout": 2400,
+
+    // the base URL to show a commit for the project.
+    // "show_commit_url": "http://github.com/owner/project/commit/",
+
+    // The Pythons you'd like to test against.  If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    // "pythons": ["3.8", "3.12"],
+
+    // The list of conda channel names to be searched for benchmark
+    // dependency packages in the specified order
+    // "conda_channels": ["conda-forge", "defaults"],
+
+    // A conda environment file that is used for environment creation.
+    // "conda_environment_file": "environment.yml",
+
+    // The matrix of dependencies to test.  Each key of the "req"
+    // requirements dictionary is the name of a package (in PyPI) and
+    // the values are version numbers.  An empty list or empty string
+    // indicates to just test against the default (latest)
+    // version. null indicates that the package is to not be
+    // installed. If the package to be tested is only available from
+    // PyPi, and the 'environment_type' is conda, then you can preface
+    // the package name by 'pip+', and the package will be installed
+    // via pip (with all the conda available packages installed first,
+    // followed by the pip installed packages).
+    //
+    // The ``@env`` and ``@env_nobuild`` keys contain the matrix of
+    // environment variables to pass to build and benchmark commands.
+    // An environment will be created for every combination of the
+    // cartesian product of the "@env" variables in this matrix.
+    // Variables in "@env_nobuild" will be passed to every environment
+    // during the benchmark phase, but will not trigger creation of
+    // new environments.  A value of ``null`` means that the variable
+    // will not be set for the current combination.
+    //
+    "matrix": {
+        "req": {
+            "uv": [],
+            "pytest": [],
+            "pytest-benchmark": [],
+            "git+https://github.com/iomaganaris/pytest-memray.git@pytest_json": [],
+            "nvidia-cuda-runtime-cu12==12.6.77": [],
+            "cupy-cuda12x": [],
+        },
+        "env": {"GT4PY_BACKEND": ["gtfn_cpu", "gtfn_gpu"], "ICON4PY_GRID": ["icon_grid", "icon_grid_global"], "MEMRAY": ["ON", null]},
+        // "env_nobuild": {"ENV_VAR_2": ["val3", null]},
+    },
+
+    
+    // Combinations of libraries/python versions can be excluded/included
+    // from the set to test. Each entry is a dictionary containing additional
+    // key-value pairs to include/exclude.
+    //
+    // An exclude entry excludes entries where all values match. The
+    // values are regexps that should match the whole string.
+    //
+    // An include entry adds an environment. Only the packages listed
+    // are installed. The 'python' key is required. The exclude rules
+    // do not apply to includes.
+    //
+    // In addition to package names, the following keys are available:
+    //
+    // - python
+    //     Python version, as in the *pythons* variable above.
+    // - environment_type
+    //     Environment type, as above.
+    // - sys_platform
+    //     Platform, as in sys.platform. Possible values for the common
+    //     cases: 'linux2', 'win32', 'cygwin', 'darwin'.
+    // - req
+    //     Required packages
+    // - env
+    //     Environment variables
+    // - env_nobuild
+    //     Non-build environment variables
+    //
+    "exclude": [
+    //     {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows
+    //     {"environment_type": "conda", "req": {"six": null}}, // don't run without six on conda
+    //    {"env": {"ENV_VAR_1": "val2"}}, // skip val2 for ENV_VAR_1
+        {"env": {"GT4PY_BACKEND": "gtfn_gpu", "MEMRAY": "ON"}},
+    ],
+    //
+    // "include": [
+    //     // additional env for python3.12
+    //     {"python": "3.12", "req": {"numpy": "1.26"}, "env_nobuild": {"FOO": "123"}},
+    //     // additional env if run on windows+conda
+    //     {"platform": "win32", "environment_type": "conda", "python": "3.12", "req": {"libpython": ""}},
+    // ],
+
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in.  If not provided, defaults to "benchmarks"
+    // "benchmark_dir": "benchmarks",
+
+    // The directory (relative to the current directory) to cache the Python
+    // environments in.  If not provided, defaults to "env"
+    "env_dir": ".asv/env",
+
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in.  If not provided, defaults to "results".
+    "results_dir": ".asv/results",
+
+    // The directory (relative to the current directory) that the html tree
+    // should be written to.  If not provided, defaults to "html".
+    "html_dir": ".asv/html",
+
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+
+    // `asv` will cache results of the recent builds in each
+    // environment, making them faster to install next time.  This is
+    // the number of builds to keep, per environment.
+    // "build_cache_size": 2,
+
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions.  The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // },
+
+    // The thresholds for relative change in results, after which `asv
+    // publish` starts reporting regressions. Dictionary of the same
+    // form as in ``regressions_first_commits``, with values
+    // indicating the thresholds.  If multiple entries match, the
+    // maximum is taken. If no entry matches, the default is 5%.
+    //
+    // "regressions_thresholds": {
+    //    "some_benchmark": 0.01,     // Threshold of 1%
+    //    "another_benchmark": 0.5,   // Threshold of 50%
+    // },
+}
diff --git a/asv_util/asv-machine.santis.json b/asv_util/asv-machine.santis.json
new file mode 100644
index 000000000..dacf8d723
--- /dev/null
+++ b/asv_util/asv-machine.santis.json
@@ -0,0 +1,11 @@
+{
+    "gh200": {
+        "arch": "aarch64",
+        "cpu": "GH200 (72 cores)",
+        "machine": "gh200",
+        "num_cpu": "288",
+        "os": "Linux 5.14.21",
+        "ram": "854 GB"
+    },
+    "version": 1
+}
\ No newline at end of file
diff --git a/asv_util/run_tests.py b/asv_util/run_tests.py
new file mode 100644
index 000000000..fd166f67e
--- /dev/null
+++ b/asv_util/run_tests.py
@@ -0,0 +1,31 @@
+import pytest
+import os
+
+GT4PY_BACKEND = os.environ.get("GT4PY_BACKEND", None)
+if GT4PY_BACKEND is None:
+    print("Error: GT4PY_BACKEND environment variable is not set.")
+    exit(1)
+ICON4PY_GRID = os.environ.get("ICON4PY_GRID", None)
+if ICON4PY_GRID is None:
+    print("Error: ICON4PY_GRID environment variable is not set.")
+    exit(1)
+BENCHMARK_DIR = os.environ.get("ASV_BUILD_DIR", None)
+if BENCHMARK_DIR is None:
+    print("Error: ASV_BUILD_DIR environment variable is not set.")
+    exit(1)
+COMMIT_HASH = os.environ.get("ASV_COMMIT", None)
+if COMMIT_HASH is None:
+    print("Error: COMMIT_HASH environment variable is not set.")
+    exit(1)
+MEMRAY = os.environ.get("MEMRAY", None)
+BENCHMARK_RUNTIME_FILENAME = "benchmark_runtime_{}_{}_{}.json".format(COMMIT_HASH, GT4PY_BACKEND, ICON4PY_GRID)
+BENCHMARK_MEMRAY_FILENAME = "benchmark_memray_{}_{}_{}.json".format(COMMIT_HASH, GT4PY_BACKEND, ICON4PY_GRID)
+
+benchmark_runtime_file_path = os.path.join(BENCHMARK_DIR, BENCHMARK_RUNTIME_FILENAME) if BENCHMARK_DIR else BENCHMARK_RUNTIME_FILENAME
+benchmark_memray_file_path = os.path.join(BENCHMARK_DIR, BENCHMARK_MEMRAY_FILENAME) if BENCHMARK_DIR else BENCHMARK_MEMRAY_FILENAME
+
+if os.environ.get("MEMRAY", None) is None:
+    pytest.main([os.path.join(os.path.dirname(__file__), "../model/atmosphere/dycore/tests"), "--benchmark-json", benchmark_runtime_file_path, "--benchmark-only", "--backend", GT4PY_BACKEND, "--grid", ICON4PY_GRID, "-k", "test_fused_velocity_advection_stencil_15_to_18", "--benchmark-min-rounds=1"])
+else:
+    pytest.main([os.path.join(os.path.dirname(__file__), "../model/atmosphere/dycore/tests"), "--benchmark-json", benchmark_memray_file_path, "--benchmark-only", "--backend", GT4PY_BACKEND, "--grid", ICON4PY_GRID, "-k", "test_fused_velocity_advection_stencil_15_to_18", "--benchmark-min-rounds=1", "--memray"])
+
diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py
new file mode 100644
index 000000000..78ab72088
--- /dev/null
+++ b/benchmarks/__init__.py
@@ -0,0 +1,20 @@
+# Write the benchmarking functions here.
+# See "Writing benchmarks" in the asv docs for more information.
+
+import benchmarks.setup_tests as setup_tests
+
+class BenchmarkMetaclass(type):
+    def __dir__(cls):
+        return list(setup_tests.BENCHMARKS.keys())
+    def __getattr__(cls, name):
+        if not name.startswith(setup_tests.PREFIX):
+            raise AttributeError
+        setattr(cls, name, setup_tests.BENCHMARKS[name])
+        return getattr(cls, name)
+
+class Benchmarks(metaclass=BenchmarkMetaclass):
+    def __getattr__(self, name):
+        if not name.startswith(setup_tests.PREFIX):
+            raise AttributeError
+        setattr(type(self), name, setup_tests.BENCHMARKS[name])
+        return getattr(self, name)
diff --git a/benchmarks/setup_tests.py b/benchmarks/setup_tests.py
new file mode 100644
index 000000000..f0a9a9451
--- /dev/null
+++ b/benchmarks/setup_tests.py
@@ -0,0 +1,70 @@
+import pytest
+import json
+import os
+import traceback
+
+BENCHMARKS = {}
+PREFIX = "track_"
+BENCHMARK_DIR = os.environ.get("ASV_BUILD_DIR", os.path.join(os.environ.get("ASV_ENV_DIR"), "project"))
+COMMIT_HASH = os.environ.get("ASV_COMMIT", None)
+GT4PY_BACKEND = os.environ.get("GT4PY_BACKEND", None)
+ICON4PY_GRID = os.environ.get("ICON4PY_GRID", None)
+BENCHMARK_RUNTIME_FILENAME = "benchmark_runtime_{}_{}_{}.json".format(COMMIT_HASH, GT4PY_BACKEND, ICON4PY_GRID)
+BENCHMARK_MEMRAY_FILENAME = "benchmark_memray_{}_{}_{}.json".format(COMMIT_HASH, GT4PY_BACKEND, ICON4PY_GRID)
+MEMRAY = os.environ.get("MEMRAY", None)
+
+benchmark_runtime_file_path = os.path.join(BENCHMARK_DIR, BENCHMARK_RUNTIME_FILENAME) if BENCHMARK_DIR else BENCHMARK_RUNTIME_FILENAME
+benchmark_memray_file_path = os.path.join(BENCHMARK_DIR, BENCHMARK_MEMRAY_FILENAME) if BENCHMARK_DIR else BENCHMARK_MEMRAY_FILENAME
+
+if MEMRAY is not None:
+    if not os.path.exists(benchmark_memray_file_path):
+        print(f"Error: Benchmark memray file {benchmark_memray_file_path} does not exist.")
+        exit(1)
+    else:
+        with open(benchmark_memray_file_path, "r") as f:
+            benchmark_data = json.load(f)
+
+            for benchmark in benchmark_data["benchmarks"]:
+                benchmark_name = benchmark["name"]
+                if GT4PY_BACKEND in benchmark_name:
+                    if ICON4PY_GRID in benchmark_name:
+                        filtered_name = benchmark_name.replace("-", "_").replace("=", "_").replace("[", "_").replace("]", "_").replace(f"_benchmark_backend_{GT4PY_BACKEND}_", "_").replace(f"_grid_{ICON4PY_GRID}_", "_").replace("test_", "_")
+                        print("Filtered name: {}".format(filtered_name))
+                        mem_high_watermark = benchmark["extra_info"]["memory_high_watermark"]
+                        asv_mem_name = "{}memory{}".format(PREFIX, filtered_name)
+                        print("Asv mem name: {}".format(asv_mem_name))
+                        def asv_memray_method(self, mem_high_watermark=mem_high_watermark): return mem_high_watermark
+                        asv_memray_method.unit = "MB"
+                        asv_memray_method.number = 1
+                        asv_memray_method.repeat = 1
+                        BENCHMARKS[asv_mem_name] = asv_memray_method
+        benchmark_runtime_file_path = benchmark_memray_file_path
+
+if not os.path.exists(benchmark_runtime_file_path):
+    print(f"Error: Benchmark runtime file {benchmark_runtime_file_path} does not exist.")
+    exit(1)
+
+with open(benchmark_runtime_file_path, "r") as f:
+    benchmark_data = json.load(f)
+
+    for benchmark in benchmark_data["benchmarks"]:
+        benchmark_name = benchmark["name"]
+        if GT4PY_BACKEND in benchmark_name:
+            if ICON4PY_GRID in benchmark_name:
+                filtered_name = benchmark_name.replace("-", "_").replace("=", "_").replace("[", "_").replace("]", "_").replace(f"_benchmark_backend_{GT4PY_BACKEND}_", "_").replace(f"_grid_{ICON4PY_GRID}_", "_").replace("test_", "_")
+                print("Filtered name: {}".format(filtered_name))
+                time = benchmark["stats"]["median"]
+                asv_runtime_name = "{}runtime{}".format(PREFIX, filtered_name)
+                print("Asv runtime name: {}".format(asv_runtime_name))
+                def asv_runtime_method(self, t=time): return t if MEMRAY is None else None
+                asv_runtime_method.unit = "s"
+                asv_runtime_method.number = 1
+                asv_runtime_method.repeat = 1
+                BENCHMARKS[asv_runtime_name] = asv_runtime_method
+                if MEMRAY is None:
+                    asv_mem_name = "{}memory{}".format(PREFIX, filtered_name)
+                    def asv_memray_method(self): return None
+                    asv_memray_method.unit = "MB"
+                    asv_memray_method.number = 1
+                    asv_memray_method.repeat = 1
+                    BENCHMARKS[asv_mem_name] = asv_memray_method
diff --git a/ci/benchmark_asv.yml b/ci/benchmark_asv.yml
new file mode 100644
index 000000000..59bd20860
--- /dev/null
+++ b/ci/benchmark_asv.yml
@@ -0,0 +1,26 @@
+include:
+  - local: 'ci/base.yml'
+
+.benchmark_model_stencils:
+  stage: benchmark
+  script:
+    - echo "Running benchmarks"
+    - pip install asv
+    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
+    - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
+    - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
+    - cp asv_util/asv-machine.santis.json .asv-machine.json
+    - asv run asv_test^!
+    - pushd .asv/results
+    - git add .
+    - git commit -m "Benchmark results machine: $CI_COMMIT_SHA commit: Santis (GH200)"
+    - git fetch origin
+    - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
+    - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
+    - popd
+    - asv gh-pages --no-push
+    - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
+# benchmark_model_stencils_x86_64:
+#   extends: [.benchmark_model_stencils, .test_template_x86_64]
+benchmark_model_stencils_aarch64:
+  extends: [.benchmark_model_stencils, .test_template_aarch64]

From 6d8a6e8c66e39bad9891facffff48469e5b52ad7 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 11:51:31 +0100
Subject: [PATCH 02/47] Use benchmark.yml for asv for testing

---
 ci/benchmark.yml     | 21 ++++++++++++++++-----
 ci/benchmark_asv.yml | 26 --------------------------
 2 files changed, 16 insertions(+), 31 deletions(-)
 delete mode 100644 ci/benchmark_asv.yml

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 885027667..59bd20860 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -4,11 +4,22 @@ include:
 .benchmark_model_stencils:
   stage: benchmark
   script:
-    - nox -s benchmark_model-3.10  -- --backend=$BACKEND --grid=$GRID
-  parallel:
-    matrix:
-      - BACKEND: [gtfn_cpu, gtfn_gpu]
-        GRID: [icon_grid, icon_grid_global]
+    - echo "Running benchmarks"
+    - pip install asv
+    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
+    - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
+    - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
+    - cp asv_util/asv-machine.santis.json .asv-machine.json
+    - asv run asv_test^!
+    - pushd .asv/results
+    - git add .
+    - git commit -m "Benchmark results machine: $CI_COMMIT_SHA commit: Santis (GH200)"
+    - git fetch origin
+    - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
+    - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
+    - popd
+    - asv gh-pages --no-push
+    - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
 # benchmark_model_stencils_x86_64:
 #   extends: [.benchmark_model_stencils, .test_template_x86_64]
 benchmark_model_stencils_aarch64:
diff --git a/ci/benchmark_asv.yml b/ci/benchmark_asv.yml
deleted file mode 100644
index 59bd20860..000000000
--- a/ci/benchmark_asv.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-include:
-  - local: 'ci/base.yml'
-
-.benchmark_model_stencils:
-  stage: benchmark
-  script:
-    - echo "Running benchmarks"
-    - pip install asv
-    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
-    - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
-    - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
-    - cp asv_util/asv-machine.santis.json .asv-machine.json
-    - asv run asv_test^!
-    - pushd .asv/results
-    - git add .
-    - git commit -m "Benchmark results machine: $CI_COMMIT_SHA commit: Santis (GH200)"
-    - git fetch origin
-    - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
-    - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
-    - popd
-    - asv gh-pages --no-push
-    - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-# benchmark_model_stencils_x86_64:
-#   extends: [.benchmark_model_stencils, .test_template_x86_64]
-benchmark_model_stencils_aarch64:
-  extends: [.benchmark_model_stencils, .test_template_aarch64]

From 4d4f1242468ce68ba7fe02da1e85462a42b39528 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 11:55:24 +0100
Subject: [PATCH 03/47] Fixing $ in yml

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 59bd20860..2bd75d961 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -13,7 +13,7 @@ include:
     - asv run asv_test^!
     - pushd .asv/results
     - git add .
-    - git commit -m "Benchmark results machine: $CI_COMMIT_SHA commit: Santis (GH200)"
+    - git commit -m "Benchmark results machine: '$CI_COMMIT_SHA' commit: Santis (GH200)"
     - git fetch origin
     - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
     - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH

From ee35a833e81639a55eda9a56656df5b014827edd Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 11:56:53 +0100
Subject: [PATCH 04/47] Avoid using env variable in git commit for now

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 2bd75d961..c8d015ec3 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -13,7 +13,7 @@ include:
     - asv run asv_test^!
     - pushd .asv/results
     - git add .
-    - git commit -m "Benchmark results machine: '$CI_COMMIT_SHA' commit: Santis (GH200)"
+    - git commit -m "Benchmark results"
     - git fetch origin
     - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
     - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH

From e437665f12e7e4cf140aba884e2c351d001abf9e Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:04:02 +0100
Subject: [PATCH 05/47] Place .asv-machine.json in right path

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index c8d015ec3..1ee0edab7 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -9,7 +9,7 @@ include:
     - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
-    - cp asv_util/asv-machine.santis.json .asv-machine.json
+    - cp asv_util/asv-machine.santis.json ~/.asv-machine.json
     - asv run asv_test^!
     - pushd .asv/results
     - git add .

From 9f0d2c2700890c74cc85a3fe0d72bb0c428cc8f7 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:06:11 +0100
Subject: [PATCH 06/47] Remove unnecessary CI

---
 .github/workflows/deploy-docs.yml | 38 -------------------------------
 .github/workflows/icon4py-qa.yml  | 36 -----------------------------
 2 files changed, 74 deletions(-)
 delete mode 100644 .github/workflows/deploy-docs.yml
 delete mode 100644 .github/workflows/icon4py-qa.yml

diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
deleted file mode 100644
index 5a80356b2..000000000
--- a/.github/workflows/deploy-docs.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-name: Deploy Documentation
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  deploy-docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libboost-all-dev
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: 'pip'
-          cache-dependency-path: |
-            **/pyproject.toml
-            uv.lock
-      - name: Install all icon4py namespace packages
-        run: |
-          uv sync --extra all
-      - name: Build documentation
-        run: |
-          source .venv/bin/activate
-          cd model/atmosphere/dycore/docs
-          make html
-      - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v4
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: model/atmosphere/dycore/docs/_build/html
diff --git a/.github/workflows/icon4py-qa.yml b/.github/workflows/icon4py-qa.yml
deleted file mode 100644
index 29b555946..000000000
--- a/.github/workflows/icon4py-qa.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-name: ICON4Py Quality Assurance
-
-on:
-    workflow_dispatch:
-    push:
-        branches:
-            - main
-    pull_request:
-        branches:
-            - main
-        types: [opened, reopened, synchronize]
-jobs:
-    pre-commit-icon4py-model:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-            -   name: Install system dependencies
-                run: |
-                    sudo apt-get update
-                    sudo apt-get install libboost-all-dev
-                    curl -LsSf https://astral.sh/uv/install.sh | sh
-            - name: Set up Python
-              uses: actions/setup-python@v4
-              with:
-                  python-version: "3.10"
-                  cache: 'pip'
-                  cache-dependency-path: |
-                    **/pyproject.toml
-                    uv.lock
-            - name: Install venv and icon4py namespace packages
-              run: |
-                  uv sync --extra all
-            - name: Run checks
-              run: |
-                  source .venv/bin/activate
-                  pre-commit run

From 7599cd8f7f6daf7046571e3e3824d5048b634bcb Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:08:10 +0100
Subject: [PATCH 07/47] Use /Users/ioannmag instead of ~

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 1ee0edab7..8fddf1909 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -9,7 +9,7 @@ include:
     - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
-    - cp asv_util/asv-machine.santis.json ~/.asv-machine.json
+    - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json'
     - asv run asv_test^!
     - pushd .asv/results
     - git add .

From f34bb1a46b8f8f4e6b7d2a08bae5c13951acbeaa Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:12:26 +0100
Subject: [PATCH 08/47] Use https for results repo

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 8fddf1909..9838ba35c 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,7 +6,7 @@ include:
   script:
     - echo "Running benchmarks"
     - pip install asv
-    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
+    - ASV_RESULTS_REPOSITORY="https://github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
     - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json'

From e17e76ad53e7d35855e971947f2dcb64553cca00 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:12:46 +0100
Subject: [PATCH 09/47] Delete commit posting ci too

---
 .../mandatory_and_optional_test_reminder.yml  | 36 -------------------
 1 file changed, 36 deletions(-)
 delete mode 100644 .github/workflows/mandatory_and_optional_test_reminder.yml

diff --git a/.github/workflows/mandatory_and_optional_test_reminder.yml b/.github/workflows/mandatory_and_optional_test_reminder.yml
deleted file mode 100644
index e76e6fbb1..000000000
--- a/.github/workflows/mandatory_and_optional_test_reminder.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-on:
-    pull_request:
-
-jobs:
-    test:
-        runs-on: ubuntu-latest
-        permissions:
-            pull-requests: write
-        steps:
-            - uses: mshick/add-pr-comment@v2
-              with:
-                  refresh-message-position: true
-                  message: |
-
-                      **Mandatory Tests**
-
-                      Please make sure you run these tests via comment before you merge!
-
-                      * `cscs-ci run default`
-                      * `launch jenkins spack`
-
-                      **Optional Tests**
-
-                      To run benchmarks you can use:
-
-                      * `cscs-ci run benchmark`
-
-                      To run tests and benchmarks with the DaCe backend you can use:
-
-                      * `cscs-ci run dace`
-
-                      In case your change might affect downstream [icon-exclaim](https://github.com/C2SM/icon-exclaim), please consider running
-
-                      * `launch jenkins icon`
-
-                      For more detailed information please look at [CI in the EXCLAIM universe](https://github.com/C2SM/icon-exclaim/wiki).

From 33edc225a3631b1121c6bab6e6a9e9192a84beda Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:46:14 +0100
Subject: [PATCH 10/47] Fix typo

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 9838ba35c..f53f4b56e 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -9,7 +9,7 @@ include:
     - ASV_RESULTS_REPOSITORY="https://github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
-    - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json'
+    - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json
     - asv run asv_test^!
     - pushd .asv/results
     - git add .

From 71f95073850a412a51f4140a5506dd1f1a8131e8 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 12:59:43 +0100
Subject: [PATCH 11/47] Set correct cli args for asv run

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index f53f4b56e..daffefb99 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -10,7 +10,7 @@ include:
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
     - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json
-    - asv run asv_test^!
+    - asv run --environment virtualenv:3.10 -v asv_test^!
     - pushd .asv/results
     - git add .
     - git commit -m "Benchmark results"

From 04eb10c01cfab46baf5588691ffbceeaece0e4e4 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 13:15:11 +0100
Subject: [PATCH 12/47] Trying to fix git issues

---
 ci/benchmark.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index daffefb99..35f6718f9 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -8,6 +8,10 @@ include:
     - pip install asv
     - ASV_RESULTS_REPOSITORY="https://github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
+    - git config --global user.name 'CI Benchmark'
+    - git config --global user.email 'benchmark@ci.invalid'
+    - git config --global push.default nothing
+    - git checkout asv_test
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
     - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json
     - asv run --environment virtualenv:3.10 -v asv_test^!

From c8e9cc0b83df0956f397b00026cfeebcbfa9945a Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 13:45:00 +0100
Subject: [PATCH 13/47] Clone repo

---
 ci/benchmark.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 35f6718f9..69e53ae49 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -3,6 +3,8 @@ include:
 
 .benchmark_model_stencils:
   stage: benchmark
+  variables:
+    GIT_STRATEGY: clone
   script:
     - echo "Running benchmarks"
     - pip install asv

From 01b63c056cab4503779497e9357e1519d25501ac Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 16:02:31 +0100
Subject: [PATCH 14/47] Clone again icon4py to fix issue with gitlabci not
 cloning again the project

---
 ci/benchmark.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 69e53ae49..71702fc51 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,6 +6,8 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
+    - git checkout -b asv_test https://github.com/C2SM/icon4py.git
+    - cd icon4py
     - echo "Running benchmarks"
     - pip install asv
     - ASV_RESULTS_REPOSITORY="https://github.com/iomaganaris/icon4py_asv.git"

From 33cd06f8fd5039c0b71c45af8a2d9efbe4040fb5 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 16:35:33 +0100
Subject: [PATCH 15/47] Fix typo

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 71702fc51..d67c1d963 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,7 +6,7 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
-    - git checkout -b asv_test https://github.com/C2SM/icon4py.git
+    - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
     - pip install asv

From 1aee1864f1b1cbe8009f60ccb7837f50e2560756 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 18:06:47 +0100
Subject: [PATCH 16/47] Use ssh url

---
 ci/benchmark.yml          | 2 +-
 ci/docker/base.Dockerfile | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index d67c1d963..f8a4ecb50 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -10,7 +10,7 @@ include:
     - cd icon4py
     - echo "Running benchmarks"
     - pip install asv
-    - ASV_RESULTS_REPOSITORY="https://github.com/iomaganaris/icon4py_asv.git"
+    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git config --global user.name 'CI Benchmark'
     - git config --global user.email 'benchmark@ci.invalid'
diff --git a/ci/docker/base.Dockerfile b/ci/docker/base.Dockerfile
index 5542f1a21..71869cd67 100644
--- a/ci/docker/base.Dockerfile
+++ b/ci/docker/base.Dockerfile
@@ -27,6 +27,7 @@ RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
     python-openssl \
     libreadline-dev \
     git \
+    openssh-client \
     rustc \
     htop && \
     rm -rf /var/lib/apt/lists/*

From 61f5f7aa4421c2d8452c007747811f5b9d962f5c Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 22:44:36 +0100
Subject: [PATCH 17/47] Try adding proxies for cloning git repo with ssh

---
 ci/benchmark.yml | 1 +
 ci/proxies.txt   | 2 ++
 2 files changed, 3 insertions(+)
 create mode 100644 ci/proxies.txt

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index f8a4ecb50..41ea6fc8b 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,6 +6,7 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
+    - cat ci/proxies.txt >> $HOME/.ssh/config
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
diff --git a/ci/proxies.txt b/ci/proxies.txt
new file mode 100644
index 000000000..42aea7256
--- /dev/null
+++ b/ci/proxies.txt
@@ -0,0 +1,2 @@
+Match Host *,!148.187.0.0/16,!172.28.0.0/16,!10.0.0.0/8
+    ProxyCommand nc -X connect -x proxy.cscs.ch:8080 %h %p
\ No newline at end of file

From 29522907cd41928f0d47bc9748486b9c3d681c3c Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Fri, 7 Feb 2025 23:45:23 +0100
Subject: [PATCH 18/47] Cp instead of cat

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 41ea6fc8b..779584a5d 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,7 +6,7 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
-    - cat ci/proxies.txt >> $HOME/.ssh/config
+    - cp ci/proxies.txt $HOME/.ssh/config
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"

From 5e2634ea6dc2e151014253d79ca71915bc4ad1af Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Sat, 8 Feb 2025 11:14:15 +0100
Subject: [PATCH 19/47] Create .ssh folder

---
 ci/benchmark.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 779584a5d..8cec40c69 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,6 +6,7 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
+    - mkdir -p $HOME/.ssh
     - cp ci/proxies.txt $HOME/.ssh/config
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py

From 885a56f54d46d5059e065f50d3527643c9f5cf8e Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Sat, 8 Feb 2025 14:33:16 +0100
Subject: [PATCH 20/47] Install netcat in docker file as well

---
 ci/docker/base.Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/docker/base.Dockerfile b/ci/docker/base.Dockerfile
index 71869cd67..db4f69abe 100644
--- a/ci/docker/base.Dockerfile
+++ b/ci/docker/base.Dockerfile
@@ -28,6 +28,7 @@ RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
     libreadline-dev \
     git \
     openssh-client \
+    netcat \
     rustc \
     htop && \
     rm -rf /var/lib/apt/lists/*

From a9d0c7c59989f849fa0c01f563404f56d0c5b078 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Mon, 10 Feb 2025 13:47:55 +0100
Subject: [PATCH 21/47] Trying to use github token for pushing gh-pages to repo

---
 ci/benchmark.yml          | 7 +++----
 ci/docker/base.Dockerfile | 2 --
 ci/proxies.txt            | 2 --
 3 files changed, 3 insertions(+), 8 deletions(-)
 delete mode 100644 ci/proxies.txt

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 8cec40c69..32160e9b7 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -7,15 +7,14 @@ include:
     GIT_STRATEGY: clone
   script:
     - mkdir -p $HOME/.ssh
-    - cp ci/proxies.txt $HOME/.ssh/config
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
     - pip install asv
-    - ASV_RESULTS_REPOSITORY="git@github.com:iomaganaris/icon4py_asv.git"
+    - ASV_RESULTS_REPOSITORY="https://iomaganaris:$ICON4PY_ASV_TOKEN@github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
-    - git config --global user.name 'CI Benchmark'
-    - git config --global user.email 'benchmark@ci.invalid'
+    - git config --global user.name 'icon4py CI'
+    - git config --global user.email 'icon4py@C2SM'
     - git config --global push.default nothing
     - git checkout asv_test
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
diff --git a/ci/docker/base.Dockerfile b/ci/docker/base.Dockerfile
index db4f69abe..5542f1a21 100644
--- a/ci/docker/base.Dockerfile
+++ b/ci/docker/base.Dockerfile
@@ -27,8 +27,6 @@ RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
     python-openssl \
     libreadline-dev \
     git \
-    openssh-client \
-    netcat \
     rustc \
     htop && \
     rm -rf /var/lib/apt/lists/*
diff --git a/ci/proxies.txt b/ci/proxies.txt
deleted file mode 100644
index 42aea7256..000000000
--- a/ci/proxies.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Match Host *,!148.187.0.0/16,!172.28.0.0/16,!10.0.0.0/8
-    ProxyCommand nc -X connect -x proxy.cscs.ch:8080 %h %p
\ No newline at end of file

From 6fb96e94281c9ddbaa74eab7fa7bdfe047231e80 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Mon, 10 Feb 2025 15:00:27 +0100
Subject: [PATCH 22/47] Uninstall uv to avoid issues with asv

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 32160e9b7..c115b5803 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,10 +6,10 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
-    - mkdir -p $HOME/.ssh
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
+    - pip uninstall uv || true
     - pip install asv
     - ASV_RESULTS_REPOSITORY="https://iomaganaris:$ICON4PY_ASV_TOKEN@github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"

From c714693db8fc9c21d4802b6245e6514eacc91ff3 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Mon, 10 Feb 2025 15:15:13 +0100
Subject: [PATCH 23/47] Add -y in pip uninstall

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index c115b5803..159ecc0c0 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -9,7 +9,7 @@ include:
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
-    - pip uninstall uv || true
+    - pip uninstall -y uv || true
     - pip install asv
     - ASV_RESULTS_REPOSITORY="https://iomaganaris:$ICON4PY_ASV_TOKEN@github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"

From 7833091e3f154751a22cbd5fcf5658d875eb1e7a Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 11:08:20 +0100
Subject: [PATCH 24/47] Trying to post comment with changes in performance and
 make CI fail if there is big difference

---
 ci/benchmark.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 159ecc0c0..0035cfeb1 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,6 +6,7 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
+    - export
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
@@ -29,6 +30,27 @@ include:
     - popd
     - asv gh-pages --no-push
     - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
+    - COMMENT=$(asv compare $(git log -2 --reverse --format=%H) -m gh200)
+    - echo $COMMENT
+    - |
+      if [[ "$CI_EXTERNAL_PULL_REQUEST_IID" != "null" ]]; then
+        curl -X POST -H "Authorization: token $GITHUB_TOKEN" \
+          -H "Accept: application/vnd.github.v3+json" \
+          -d "{\"body\": \"$COMMENT\"}" \
+          "https://api.github.com/repos/C2SM/icon4py/issues/$CI_EXTERNAL_PULL_REQUEST_IID/comments"
+      else
+        echo "No open pull requests found."
+      fi
+    - |
+      if [[ "$(echo $COMMENT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+        exit 1
+      fi
+    - |
+      if [[ "$(echo $COMMENT | grep -e "memory" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+        exit 1
+      fi
+
+
 # benchmark_model_stencils_x86_64:
 #   extends: [.benchmark_model_stencils, .test_template_x86_64]
 benchmark_model_stencils_aarch64:

From 7736433efa8a4fc1fbb1b72a2cfb89d45673c3f0 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 11:10:16 +0100
Subject: [PATCH 25/47] Fix used token

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 0035cfeb1..dd66b4177 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -34,7 +34,7 @@ include:
     - echo $COMMENT
     - |
       if [[ "$CI_EXTERNAL_PULL_REQUEST_IID" != "null" ]]; then
-        curl -X POST -H "Authorization: token $GITHUB_TOKEN" \
+        curl -X POST -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "Accept: application/vnd.github.v3+json" \
           -d "{\"body\": \"$COMMENT\"}" \
           "https://api.github.com/repos/C2SM/icon4py/issues/$CI_EXTERNAL_PULL_REQUEST_IID/comments"

From f64c79305c33bd6bc9ca5a5e3598e27d614e5cd2 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 13:08:19 +0100
Subject: [PATCH 26/47] Set PR ID properly

---
 ci/benchmark.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index dd66b4177..95e233ba8 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -32,12 +32,13 @@ include:
     - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare $(git log -2 --reverse --format=%H) -m gh200)
     - echo $COMMENT
+    - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - |
-      if [[ "$CI_EXTERNAL_PULL_REQUEST_IID" != "null" ]]; then
+      if [[ "$PR_ID" != "" ]]; then
         curl -X POST -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "Accept: application/vnd.github.v3+json" \
           -d "{\"body\": \"$COMMENT\"}" \
-          "https://api.github.com/repos/C2SM/icon4py/issues/$CI_EXTERNAL_PULL_REQUEST_IID/comments"
+          "https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments"
       else
         echo "No open pull requests found."
       fi

From be90316ee5b2e2617fad393d44979f4e78267145 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 13:10:45 +0100
Subject: [PATCH 27/47] Compare latest commit with certain commit I have
 generated results for (c714693d)

---
 ci/benchmark.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 95e233ba8..d4a816732 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -23,14 +23,14 @@ include:
     - asv run --environment virtualenv:3.10 -v asv_test^!
     - pushd .asv/results
     - git add .
-    - git commit -m "Benchmark results"
+    - git commit -m "Benchmark results for commit $CI_COMMIT_SHA"
     - git fetch origin
     - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
     - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
     - popd
     - asv gh-pages --no-push
     - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - COMMENT=$(asv compare $(git log -2 --reverse --format=%H) -m gh200)
+    - COMMENT=$(asv compare c714693d $(git log -1 --format=%H) -m gh200)
     - echo $COMMENT
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - |

From e238b628d1cb245fd188ea256f42aaeab3961cf3 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 13:11:22 +0100
Subject: [PATCH 28/47] Remove export command

---
 ci/benchmark.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index d4a816732..87360e2f8 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,7 +6,6 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
-    - export
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"

From ea4e210654abe9aa6858ffa4ccb0716632c34adc Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 15:39:16 +0100
Subject: [PATCH 29/47] Just try to comment on the PR

---
 ci/benchmark.yml | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 87360e2f8..82d0451eb 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -19,25 +19,29 @@ include:
     - git checkout asv_test
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
     - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json
-    - asv run --environment virtualenv:3.10 -v asv_test^!
-    - pushd .asv/results
-    - git add .
-    - git commit -m "Benchmark results for commit $CI_COMMIT_SHA"
-    - git fetch origin
-    - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
-    - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
-    - popd
-    - asv gh-pages --no-push
-    - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
+    # - asv run --environment virtualenv:3.10 -v asv_test^!
+    # - pushd .asv/results
+    # - git add .
+    # - git commit -m "Benchmark results for commit $CI_COMMIT_SHA"
+    # - git fetch origin
+    # - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
+    # - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
+    # - popd
+    # - asv gh-pages --no-push
+    # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare c714693d $(git log -1 --format=%H) -m gh200)
     - echo $COMMENT
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
+    - echo $PR_ID
     - |
       if [[ "$PR_ID" != "" ]]; then
-        curl -X POST -H "Authorization: token $ICON4PY_ASV_TOKEN" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -d "{\"body\": \"$COMMENT\"}" \
-          "https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments"
+        curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer $ICON4PY_ASV_TOKEN" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/C2SM/icon4py/pulls/$PR_ID/comments \
+          -d '{"body":"$COMMENT"}'
       else
         echo "No open pull requests found."
       fi

From 8730061b4218dccf46f2a56fa245451d20a637d4 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 15:40:55 +0100
Subject: [PATCH 30/47] Use right git commit ids for comparisson

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 82d0451eb..ded682de2 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -29,7 +29,7 @@ include:
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - COMMENT=$(asv compare c714693d $(git log -1 --format=%H) -m gh200)
+    - COMMENT=$(asv compare c714693d e238b628 -m gh200)
     - echo $COMMENT
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID

From 337bd8da5e655115b275fea5bdf402e366324c15 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 15:56:46 +0100
Subject: [PATCH 31/47] Trying a bit different comment command

---
 ci/benchmark.yml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index ded682de2..b08946a0c 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -35,13 +35,10 @@ include:
     - echo $PR_ID
     - |
       if [[ "$PR_ID" != "" ]]; then
-        curl -L \
-          -X POST \
-          -H "Accept: application/vnd.github+json" \
-          -H "Authorization: Bearer $ICON4PY_ASV_TOKEN" \
-          -H "X-GitHub-Api-Version: 2022-11-28" \
-          https://api.github.com/repos/C2SM/icon4py/pulls/$PR_ID/comments \
-          -d '{"body":"$COMMENT"}'
+        curl -H "Authorization: token $ICON4PY_ASV_TOKEN" \
+             -X POST \
+             -d "{\"body\": \"$COMMENT\"}" \
+             "https://api.github.com/repos/C2SM/icon4py/pulls/$PR_ID/comments"
       else
         echo "No open pull requests found."
       fi

From 41d1f475970bb9bbc4634b91e91104a87e47b701 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Tue, 11 Feb 2025 17:41:34 +0100
Subject: [PATCH 32/47] Trying a different way to write a comment

---
 ci/benchmark.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index b08946a0c..0599fc43b 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -35,10 +35,13 @@ include:
     - echo $PR_ID
     - |
       if [[ "$PR_ID" != "" ]]; then
-        curl -H "Authorization: token $ICON4PY_ASV_TOKEN" \
-             -X POST \
-             -d "{\"body\": \"$COMMENT\"}" \
-             "https://api.github.com/repos/C2SM/icon4py/pulls/$PR_ID/comments"
+        curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: token $ICON4PY_ASV_TOKEN" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
+          -d '{"body":"$COMMENT"}'
       else
         echo "No open pull requests found."
       fi

From 52a4cd0d9fd86a69c76465454755db417db3feac Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 09:11:16 +0100
Subject: [PATCH 33/47] Trying to fix comment style

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 0599fc43b..2a450c58d 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -41,7 +41,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d '{"body":"$COMMENT"}'
+          -d "{\"body\":\"$COMMENT\"}"
       else
         echo "No open pull requests found."
       fi

From 2240115c54978e3bf529cee873d0f303f19ef3cd Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 09:40:00 +0100
Subject: [PATCH 34/47] Trying with split in asv compare and -x for debug

---
 ci/benchmark.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 2a450c58d..9c41770ee 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -29,12 +29,13 @@ include:
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - COMMENT=$(asv compare c714693d e238b628 -m gh200)
+    - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
     - echo $COMMENT
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |
       if [[ "$PR_ID" != "" ]]; then
+        set -x
         curl -L \
           -X POST \
           -H "Accept: application/vnd.github+json" \
@@ -42,6 +43,7 @@ include:
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
           -d "{\"body\":\"$COMMENT\"}"
+        set +x
       else
         echo "No open pull requests found."
       fi

From 786fbace6710827f0769cbf8005225120cee271e Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 11:01:45 +0100
Subject: [PATCH 35/47] Trying to format properly the comment

---
 ci/benchmark.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 9c41770ee..208d67777 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -31,6 +31,7 @@ include:
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
     - echo $COMMENT
+    - FILTERED_COMMENT_FOR_JSON=$(echo $new_comment | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |
@@ -42,7 +43,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "{\"body\":\"$COMMENT\"}"
+          -d "{\"body\":\"$FILTERED_COMMENT_FOR_JSON\"}"
         set +x
       else
         echo "No open pull requests found."

From c59038b06a12b363497fcec97581fe4d68dd759d Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 11:19:58 +0100
Subject: [PATCH 36/47] Fix typo

---
 ci/benchmark.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 208d67777..b1fe0d659 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -31,7 +31,8 @@ include:
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
     - echo $COMMENT
-    - FILTERED_COMMENT_FOR_JSON=$(echo $new_comment | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)
+    - FILTERED_COMMENT_FOR_JSON=$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)
+    - echo $FILTERED_COMMENT_FOR_JSON
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |

From c5f7fc83b28336b4312812fe74246d4a57c0db88 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 11:54:20 +0100
Subject: [PATCH 37/47] Trying different way to format the comment

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index b1fe0d659..23af4db7d 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -44,7 +44,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "{\"body\":\"$FILTERED_COMMENT_FOR_JSON\"}"
+          -d "{\"body\":\"$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)\"}"
         set +x
       else
         echo "No open pull requests found."

From 4862025037491f0b6f19f854f046263395ebfdd0 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 13:03:26 +0100
Subject: [PATCH 38/47] Try formatting with "

---
 ci/benchmark.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 23af4db7d..6f00bec0b 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -31,8 +31,8 @@ include:
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
     - echo $COMMENT
-    - FILTERED_COMMENT_FOR_JSON=$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)
-    - echo $FILTERED_COMMENT_FOR_JSON
+    - FILTERED_COMMENT_FOR_JSON="$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)""
+    - echo "$FILTERED_COMMENT_FOR_JSON"
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |
@@ -44,7 +44,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "{\"body\":\"$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)\"}"
+          -d "{\"body\":\"$FILTERED_COMMENT_FOR_JSON\"}"
         set +x
       else
         echo "No open pull requests found."

From 9450648ac814dff9ebe8d9aaa1ce8ca0f150dc05 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 13:23:33 +0100
Subject: [PATCH 39/47] Remove typo

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 6f00bec0b..c8f79fc9f 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -31,7 +31,7 @@ include:
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
     - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
     - echo $COMMENT
-    - FILTERED_COMMENT_FOR_JSON="$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)""
+    - FILTERED_COMMENT_FOR_JSON="$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)"
     - echo "$FILTERED_COMMENT_FOR_JSON"
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID

From 8083f97b651e451ff0beb224bbb24a4b273e25e1 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 14:49:48 +0100
Subject: [PATCH 40/47] Trying json format fix and make factor in comparison
 lower

---
 ci/benchmark.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index c8f79fc9f..8c453da2c 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -29,10 +29,8 @@ include:
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s)
+    - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s -f 1.01)
     - echo $COMMENT
-    - FILTERED_COMMENT_FOR_JSON="$(echo $COMMENT | sed 's/$/\\n/g' | tr -d '\n' | rev | cut -c 3- | rev)"
-    - echo "$FILTERED_COMMENT_FOR_JSON"
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |
@@ -44,7 +42,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "{\"body\":\"$FILTERED_COMMENT_FOR_JSON\"}"
+          -d "$(jq -nc --arg body "$COMMENT" '{body: $body}')"
         set +x
       else
         echo "No open pull requests found."

From ce73ffa0eec89077b6ef2970d4ef3e2da3e2c25a Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 14:52:12 +0100
Subject: [PATCH 41/47] Fix comment without jq

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 8c453da2c..7f23f9665 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -42,7 +42,7 @@ include:
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "$(jq -nc --arg body "$COMMENT" '{body: $body}')"
+          -d "{ \"body\": \"${COMMENT//$'\n'/\\n}\" }"
         set +x
       else
         echo "No open pull requests found."

From e4555ccc257c1ac1982984b4931c9fd1b7b8fcc7 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 15:24:10 +0100
Subject: [PATCH 42/47] Rename COMMIT variable name and improve a bit printing
 on the CI logs

---
 ci/benchmark.yml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 7f23f9665..18058648e 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -29,8 +29,7 @@ include:
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - COMMENT=$(asv compare c714693d e238b628 -m gh200 -s -f 1.01)
-    - echo $COMMENT
+    - ASV_COMPARE_OUTPUT=$(asv compare c714693d e238b628 -m gh200 -s -f 1.01 --only-changed)
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |
@@ -41,18 +40,22 @@ include:
           -H "Accept: application/vnd.github+json" \
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
-          https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
-          -d "{ \"body\": \"${COMMENT//$'\n'/\\n}\" }"
+          https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/ASV_COMPARE_OUTPUTs \
+          -d "{ \"body\": \"${ASV_COMPARE_OUTPUT//$'\n'/\\n}\" }"
         set +x
       else
         echo "No open pull requests found."
       fi
     - |
-      if [[ "$(echo $COMMENT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+        echo "The runtime of the following benchmarks has increased"
+        echo $ASV_COMPARE_OUTPUT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|" -B 1
         exit 1
       fi
     - |
-      if [[ "$(echo $COMMENT | grep -e "memory" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "memory" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+        echo "The memory high watermark of the following benchmarks has increased"
+        echo $ASV_COMPARE_OUTPUT | grep -e "memory" | grep "|[\ ]*+[\ ]*|"
         exit 1
       fi
 

From 72d25a54901d9a5944dae76138c31760587befdd Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 15:24:32 +0100
Subject: [PATCH 43/47] Also show only changed results

---
 ci/benchmark.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 18058648e..6e4926e7c 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -29,7 +29,7 @@ include:
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
-    - ASV_COMPARE_OUTPUT=$(asv compare c714693d e238b628 -m gh200 -s -f 1.01 --only-changed)
+    - ASV_COMPARE_OUTPUT=$(asv compare c714693d e238b628 -m gh200 --only-changed -s -f 1.01)
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID
     - |

From 24c7ae4749e209fb856a01d2375061fd251def25 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 15:55:01 +0100
Subject: [PATCH 44/47] Added comments with TODOs and trying to fix PR comment
 style with custom asv

---
 ci/benchmark.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 6e4926e7c..e14467409 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -6,29 +6,39 @@ include:
   variables:
     GIT_STRATEGY: clone
   script:
+    # instead of this branch we should checkout $CI_COMMIT_SHA
     - git clone -b asv_test https://github.com/C2SM/icon4py.git
     - cd icon4py
     - echo "Running benchmarks"
     - pip uninstall -y uv || true
-    - pip install asv
+    - pip install git+https://github.com/iomaganaris/asv.git
     - ASV_RESULTS_REPOSITORY="https://iomaganaris:$ICON4PY_ASV_TOKEN@github.com/iomaganaris/icon4py_asv.git"
     - ASV_RESULTS_REPOSITORY_BRANCH="icon4py_upstream_santis"
     - git config --global user.name 'icon4py CI'
     - git config --global user.email 'icon4py@C2SM'
     - git config --global push.default nothing
     - git checkout asv_test
+    # .asv/results has the results of the benchmarks from 'main'
+    # from there get the last commit for which we have results by looking at the comming message of HEAD
     - git clone -b $ASV_RESULTS_REPOSITORY_BRANCH $ASV_RESULTS_REPOSITORY .asv/results
     - cp asv_util/asv-machine.santis.json $HOME/.asv-machine.json
+    # Run below only on pushes to main branch
+    ############################################################################################################
+    # Instead of "asv_test^!" use $CI_COMMIT_SHA
     # - asv run --environment virtualenv:3.10 -v asv_test^!
     # - pushd .asv/results
     # - git add .
-    # - git commit -m "Benchmark results for commit $CI_COMMIT_SHA"
+    # Commit name should be $CI_COMMIT_SHA so that we can have a reference sha to test new commits against
+    # - git commit -m "$CI_COMMIT_SHA"
     # - git fetch origin
     # - git rebase origin/$ASV_RESULTS_REPOSITORY_BRANCH
+    # Need to figure out how to allow only the CI to push to the results repository automatically. Otherwise a PR is necessary to avoid losing the data
     # - git push origin HEAD:$ASV_RESULTS_REPOSITORY_BRANCH
     # - popd
     # - asv gh-pages --no-push
     # - git push $ASV_RESULTS_REPOSITORY gh-pages:gh-pages -f
+    ############################################################################################################
+    # First commit id should be the commit id of the results we read above. Second commit id should be $CI_COMMIT_SHA
     - ASV_COMPARE_OUTPUT=$(asv compare c714693d e238b628 -m gh200 --only-changed -s -f 1.01)
     - PR_ID=$(echo "${CI_COMMIT_BRANCH}" | grep -o 'pr[0-9]*' | grep -o '[0-9]*')
     - echo $PR_ID

From f50fa80352ccaae82784cd30ea1898d814d1bd12 Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 16:13:48 +0100
Subject: [PATCH 45/47] Fix commit and CI log output

---
 ci/benchmark.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index e14467409..6f441a3ca 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -44,28 +44,26 @@ include:
     - echo $PR_ID
     - |
       if [[ "$PR_ID" != "" ]]; then
-        set -x
         curl -L \
           -X POST \
           -H "Accept: application/vnd.github+json" \
           -H "Authorization: token $ICON4PY_ASV_TOKEN" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
-          https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/ASV_COMPARE_OUTPUTs \
+          https://api.github.com/repos/C2SM/icon4py/issues/$PR_ID/comments \
           -d "{ \"body\": \"${ASV_COMPARE_OUTPUT//$'\n'/\\n}\" }"
-        set +x
       else
         echo "No open pull requests found."
       fi
     - |
-      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | "|[\ ]*+[\ ]*|.*runtime")" != "" ]]; then
         echo "The runtime of the following benchmarks has increased"
-        echo $ASV_COMPARE_OUTPUT | grep -e "runtime" | grep "|[\ ]*+[\ ]*|" -B 1
+        echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*runtime" -B 2 | tr '\n' '\r\n'
         exit 1
       fi
     - |
-      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "memory" | grep "|[\ ]*+[\ ]*|")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep "|[\ ]*+[\ ]*|.*memory")" != "" ]]; then
         echo "The memory high watermark of the following benchmarks has increased"
-        echo $ASV_COMPARE_OUTPUT | grep -e "memory" | grep "|[\ ]*+[\ ]*|"
+        echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*memory" -B 2 | tr '\n' '\r\n'
         exit 1
       fi
 

From 68866460dd1baeb42a05c2a98cf86a7d6b2aefaf Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 16:26:06 +0100
Subject: [PATCH 46/47] Fix grep command

---
 ci/benchmark.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index 6f441a3ca..c2435ca6a 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -55,13 +55,13 @@ include:
         echo "No open pull requests found."
       fi
     - |
-      if [[ "$(echo $ASV_COMPARE_OUTPUT | "|[\ ]*+[\ ]*|.*runtime")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*runtime")" != "" ]]; then
         echo "The runtime of the following benchmarks has increased"
         echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*runtime" -B 2 | tr '\n' '\r\n'
         exit 1
       fi
     - |
-      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep "|[\ ]*+[\ ]*|.*memory")" != "" ]]; then
+      if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*memory")" != "" ]]; then
         echo "The memory high watermark of the following benchmarks has increased"
         echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*memory" -B 2 | tr '\n' '\r\n'
         exit 1

From b0643921339ce014314d33bc7c503b6f547ba13f Mon Sep 17 00:00:00 2001
From: Ioannis Magkanaris <ioannis.magkanaris@cscs.ch>
Date: Wed, 12 Feb 2025 16:45:11 +0100
Subject: [PATCH 47/47] Trying to fix ci log

---
 ci/benchmark.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/benchmark.yml b/ci/benchmark.yml
index c2435ca6a..3c4809771 100644
--- a/ci/benchmark.yml
+++ b/ci/benchmark.yml
@@ -57,13 +57,13 @@ include:
     - |
       if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*runtime")" != "" ]]; then
         echo "The runtime of the following benchmarks has increased"
-        echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*runtime" -B 2 | tr '\n' '\r\n'
+        echo "$ASV_COMPARE_OUTPUT" | grep -e "|[\ ]*+[\ ]*|.*runtime.*" -B 2
         exit 1
       fi
     - |
       if [[ "$(echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*memory")" != "" ]]; then
         echo "The memory high watermark of the following benchmarks has increased"
-        echo $ASV_COMPARE_OUTPUT | grep -e "|[\ ]*+[\ ]*|.*memory" -B 2 | tr '\n' '\r\n'
+        echo "$ASV_COMPARE_OUTPUT" | grep -e "|[\ ]*+[\ ]*|.*memory.*" -B 2
         exit 1
       fi