refactor[next]: Refactor workflow based executors and move the backend class. (#1470)

## Changed

- `OTFCompileExecutor` and `CachedOTFCompileExecutor` merged into
`ModularExecutor`

## Moved

- `OTFBackend` moved and renamed to `next.backend.Backend`

## Reasoning

The two `*CompileExecutor` classes were identical, except for type
hints. Since they are now almost exclusively used inside `OTFBackend`,
which does not retain typing information about which of them it
contains, this distinction is no longer helpful for static type
checking.

`OTFBackend` has always been more general than it's naming and location
suggested. It is the de-facto definition of a backend within
`gt4py.next`: a wrapper around an executor and an allocator. This change
makes the status quo visible.

Author: DropD

docs/development/ADRs/0015-Test_Exclusion_Matrices.md

@@ -49,9 +49,9 @@ The following backend processors are defined:
 ```python
 DACE_CPU = "dace_iterator.run_dace_cpu"
 DACE_GPU = "dace_iterator.run_dace_gpu"
-GTFN_CPU = "otf_compile_executor.run_gtfn"
-GTFN_CPU_IMPERATIVE = "otf_compile_executor.run_gtfn_imperative"
-GTFN_CPU_WITH_TEMPORARIES = "otf_compile_executor.run_gtfn_with_temporaries"
+GTFN_CPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn"
+GTFN_CPU_IMPERATIVE = "gt4py.next.program_processors.runners.gtfn.run_gtfn_imperative"
+GTFN_CPU_WITH_TEMPORARIES = "gt4py.next.program_processors.runners.gtfn.run_gtfn_with_temporaries"
 GTFN_GPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn_gpu"
 ```
 

src/gt4py/next/backend.py

@@ -0,0 +1,46 @@
+# GT4Py - GridTools Framework
+#
+# Copyright (c) 2014-2023, ETH Zurich
+# All rights reserved.
+#
+# This file is part of the GT4Py project and the GridTools framework.
+# GT4Py is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Generic
+
+from gt4py._core import definitions as core_defs
+from gt4py.next import allocators as next_allocators
+from gt4py.next.iterator import ir as itir
+from gt4py.next.program_processors import processor_interface as ppi
+
+
+@dataclasses.dataclass(frozen=True)
+class Backend(Generic[core_defs.DeviceTypeT]):
+    executor: ppi.ProgramExecutor
+    allocator: next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]
+
+    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
+        self.executor.__call__(program, *args, **kwargs)
+
+    @property
+    def __name__(self) -> str:
+        return getattr(self.executor, "__name__", None) or repr(self)
+
+    @property
+    def kind(self) -> type[ppi.ProgramExecutor]:
+        return self.executor.kind
+
+    @property
+    def __gt_allocator__(
+        self,
+    ) -> next_allocators.FieldBufferAllocatorProtocol[core_defs.DeviceTypeT]:
+        return self.allocator

src/gt4py/next/program_processors/modular_executor.py

@@ -0,0 +1,43 @@
+# GT4Py - GridTools Framework
+#
+# Copyright (c) 2014-2023, ETH Zurich
+# All rights reserved.
+#
+# This file is part of the GT4Py project and the GridTools framework.
+# GT4Py is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or any later
+# version. See the LICENSE.txt file at the top-level directory of this
+# distribution for a copy of the license or check <https://www.gnu.org/licenses/>.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Optional, TypeVar
+
+import gt4py.next.iterator.ir as itir
+import gt4py.next.program_processors.processor_interface as ppi
+from gt4py.next.otf import languages, stages, workflow
+
+
+SrcL = TypeVar("SrcL", bound=languages.NanobindSrcL)
+TgtL = TypeVar("TgtL", bound=languages.LanguageTag)
+LS = TypeVar("LS", bound=languages.LanguageSettings)
+HashT = TypeVar("HashT")
+
+
+@dataclasses.dataclass(frozen=True)
+class ModularExecutor(ppi.ProgramExecutor):
+    otf_workflow: workflow.Workflow[stages.ProgramCall, stages.CompiledProgram]
+    name: Optional[str] = None
+
+    def __call__(self, program: itir.FencilDefinition, *args, **kwargs: Any) -> None:
+        self.otf_workflow(stages.ProgramCall(program, args, kwargs))(
+            *args, offset_provider=kwargs["offset_provider"]
+        )
+
+    @property
+    def __name__(self) -> str:
+        return self.name or repr(self)

src/gt4py/next/program_processors/otf_compile_executor.py

@@ -25,9 +25,8 @@
 
 import gt4py.next.allocators as next_allocators
 import gt4py.next.iterator.ir as itir
-import gt4py.next.program_processors.otf_compile_executor as otf_exec
 import gt4py.next.program_processors.processor_interface as ppi
-from gt4py.next import common
+from gt4py.next import backend, common
 from gt4py.next.iterator import transforms as itir_transforms
 from gt4py.next.otf.compilation import cache as compilation_cache
 from gt4py.next.type_system import type_specifications as ts, type_translation
@@ -437,7 +436,7 @@ def _run_dace_cpu(program: itir.FencilDefinition, *args, **kwargs) -> None:
     )
 
 
-run_dace_cpu = otf_exec.OTFBackend(
+run_dace_cpu = backend.Backend(
     executor=ppi.program_executor(_run_dace_cpu, name="run_dace_cpu"),
     allocator=next_allocators.StandardCPUFieldBufferAllocator(),
 )
@@ -460,7 +459,7 @@ def _run_dace_gpu(program: itir.FencilDefinition, *args, **kwargs) -> None:
         raise RuntimeError("Missing 'cupy' dependency for GPU execution.")
 
 
-run_dace_gpu = otf_exec.OTFBackend(
+run_dace_gpu = backend.Backend(
     executor=ppi.program_executor(_run_dace_gpu, name="run_dace_gpu"),
     allocator=next_allocators.StandardGPUFieldBufferAllocator(),
 )

src/gt4py/next/program_processors/runners/dace_iterator/__init__.py

@@ -16,9 +16,9 @@
 
 from typing import TYPE_CHECKING, Any
 
-import gt4py.next.program_processors.otf_compile_executor as otf_compile_executor
 import gt4py.next.program_processors.processor_interface as ppi
-import gt4py.next.program_processors.runners.roundtrip as roundtrip
+from gt4py.next import backend as next_backend
+from gt4py.next.program_processors.runners import roundtrip
 
 
 if TYPE_CHECKING:
@@ -30,7 +30,7 @@ def executor(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> None:
     roundtrip.execute_roundtrip(program, *args, dispatch_backend=roundtrip.executor, **kwargs)
 
 
-backend = otf_compile_executor.OTFBackend(
+backend = next_backend.Backend(
     executor=executor,
     allocator=roundtrip.backend.allocator,
 )

src/gt4py/next/program_processors/runners/double_roundtrip.py

@@ -22,14 +22,14 @@
 import gt4py._core.definitions as core_defs
 import gt4py.next.allocators as next_allocators
 from gt4py.eve.utils import content_hash
-from gt4py.next import common, config
+from gt4py.next import backend, common, config
 from gt4py.next.iterator import transforms
 from gt4py.next.iterator.transforms import global_tmps
 from gt4py.next.otf import recipes, stages, workflow
 from gt4py.next.otf.binding import nanobind
 from gt4py.next.otf.compilation import compiler
 from gt4py.next.otf.compilation.build_systems import compiledb
-from gt4py.next.program_processors import otf_compile_executor
+from gt4py.next.program_processors import modular_executor
 from gt4py.next.program_processors.codegens.gtfn import gtfn_module
 from gt4py.next.type_system.type_translation import from_value
 
@@ -145,7 +145,7 @@ class Params:
 
 class GTFNBackendFactory(factory.Factory):
     class Meta:
-        model = otf_compile_executor.OTFBackend
+        model = backend.Backend
 
     class Params:
         name_device = "cpu"
@@ -159,7 +159,7 @@ class Params:
         )
         cached = factory.Trait(
             executor=factory.LazyAttribute(
-                lambda o: otf_compile_executor.CachedOTFCompileExecutor(
+                lambda o: modular_executor.ModularExecutor(
                     otf_workflow=workflow.CachedStep(o.otf_workflow, hash_function=o.hash_function),
                     name=o.name,
                 )
@@ -181,7 +181,7 @@ class Params:
         )
 
     executor = factory.LazyAttribute(
-        lambda o: otf_compile_executor.OTFCompileExecutor(otf_workflow=o.otf_workflow, name=o.name)
+        lambda o: modular_executor.ModularExecutor(otf_workflow=o.otf_workflow, name=o.name)
     )
     allocator = next_allocators.StandardCPUFieldBufferAllocator()
 

src/gt4py/next/program_processors/runners/gtfn.py

@@ -21,16 +21,16 @@
 from collections.abc import Callable, Iterable
 from typing import Any, Optional
 
-import gt4py.eve.codegen as codegen
 import gt4py.next.allocators as next_allocators
 import gt4py.next.common as common
 import gt4py.next.iterator.embedded as embedded
 import gt4py.next.iterator.ir as itir
 import gt4py.next.iterator.transforms as itir_transforms
 import gt4py.next.iterator.transforms.global_tmps as gtmps_transform
-import gt4py.next.program_processors.otf_compile_executor as otf_compile_executor
 import gt4py.next.program_processors.processor_interface as ppi
+from gt4py.eve import codegen
 from gt4py.eve.codegen import FormatTemplate as as_fmt, MakoTemplate as as_mako
+from gt4py.next import backend as next_backend
 
 
 def _create_tmp(axes, origin, shape, dtype):
@@ -229,6 +229,6 @@ def execute_roundtrip(
 
 executor = ppi.program_executor(execute_roundtrip)  # type: ignore[arg-type]
 
-backend = otf_compile_executor.OTFBackend(
+backend = next_backend.Backend(
     executor=executor, allocator=next_allocators.StandardCPUFieldBufferAllocator()
 )

src/gt4py/next/program_processors/runners/roundtrip.py

@@ -16,9 +16,9 @@
 from numpy import int32, int64
 
 from gt4py import next as gtx
-from gt4py.next import common
+from gt4py.next import backend, common
 from gt4py.next.iterator.transforms import LiftMode, apply_common_transforms
-from gt4py.next.program_processors import otf_compile_executor
+from gt4py.next.program_processors import modular_executor
 from gt4py.next.program_processors.runners.gtfn import run_gtfn_with_temporaries
 
 from next_tests.integration_tests import cases
@@ -38,8 +38,8 @@
 
 @pytest.fixture
 def run_gtfn_with_temporaries_and_symbolic_sizes():
-    return otf_compile_executor.OTFBackend(
-        executor=otf_compile_executor.OTFCompileExecutor(
+    return backend.Backend(
+        executor=modular_executor.ModularExecutor(
             name="run_gtfn_with_temporaries_and_sizes",
             otf_workflow=run_gtfn_with_temporaries.executor.otf_workflow.replace(
                 translation=run_gtfn_with_temporaries.executor.otf_workflow.translation.replace(