From 1216a88ad4dcd41cee1f8061c3e6232c49ba2837 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Tue, 28 Nov 2023 17:01:15 +0100 Subject: [PATCH 01/19] make common.field private --- src/gt4py/next/common.py | 6 +-- src/gt4py/next/constructors.py | 4 +- src/gt4py/next/embedded/nd_array_field.py | 12 ++--- src/gt4py/next/ffront/fbuiltins.py | 2 +- src/gt4py/next/iterator/embedded.py | 2 +- .../embedded_tests/test_nd_array_field.py | 54 +++++++++---------- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index 7f1ad8c0bb..9b517df388 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -758,7 +758,7 @@ def is_connectivity_field( @functools.singledispatch -def field( +def _field( definition: Any, /, *, @@ -769,7 +769,7 @@ def field( @functools.singledispatch -def connectivity( +def _connectivity( definition: Any, /, codomain: Dimension, @@ -898,7 +898,7 @@ def restrict(self, index: AnyIndexSpec) -> core_defs.IntegralScalar: __getitem__ = restrict -connectivity.register(numbers.Integral, CartesianConnectivity.from_offset) +_connectivity.register(numbers.Integral, CartesianConnectivity.from_offset) @enum.unique diff --git a/src/gt4py/next/constructors.py b/src/gt4py/next/constructors.py index 63fde1cfde..8e8604a6bc 100644 --- a/src/gt4py/next/constructors.py +++ b/src/gt4py/next/constructors.py @@ -87,7 +87,7 @@ def empty( buffer = next_allocators.allocate( domain, dtype, aligned_index=aligned_index, allocator=allocator, device=device ) - res = common.field(buffer.ndarray, domain=domain) + res = common._field(buffer.ndarray, domain=domain) assert common.is_mutable_field(res) assert isinstance(res, nd_array_field.NdArrayField) return res @@ -357,7 +357,7 @@ def as_connectivity( device = core_defs.Device(*data.__dlpack_device__()) buffer = next_allocators.allocate(actual_domain, dtype, allocator=allocator, device=device) buffer.ndarray[...] = storage_utils.asarray(data) # type: ignore[index] # TODO(havogt): consider addin MutableNDArrayObject - connectivity_field = common.connectivity( + connectivity_field = common._connectivity( buffer.ndarray, codomain=codomain, domain=actual_domain ) assert isinstance(connectivity_field, nd_array_field.NdArrayConnectivityField) diff --git a/src/gt4py/next/embedded/nd_array_field.py b/src/gt4py/next/embedded/nd_array_field.py index ff6a2ceac7..334db5e96f 100644 --- a/src/gt4py/next/embedded/nd_array_field.py +++ b/src/gt4py/next/embedded/nd_array_field.py @@ -509,7 +509,7 @@ class NumPyArrayField(NdArrayField): array_ns: ClassVar[ModuleType] = np -common.field.register(np.ndarray, NumPyArrayField.from_array) +common._field.register(np.ndarray, NumPyArrayField.from_array) @dataclasses.dataclass(frozen=True, eq=False) @@ -517,7 +517,7 @@ class NumPyArrayConnectivityField(NdArrayConnectivityField): array_ns: ClassVar[ModuleType] = np -common.connectivity.register(np.ndarray, NumPyArrayConnectivityField.from_array) +common._connectivity.register(np.ndarray, NumPyArrayConnectivityField.from_array) # CuPy if cp: @@ -527,13 +527,13 @@ class NumPyArrayConnectivityField(NdArrayConnectivityField): class CuPyArrayField(NdArrayField): array_ns: ClassVar[ModuleType] = cp - common.field.register(cp.ndarray, CuPyArrayField.from_array) + common._field.register(cp.ndarray, CuPyArrayField.from_array) @dataclasses.dataclass(frozen=True, eq=False) class CuPyArrayConnectivityField(NdArrayConnectivityField): array_ns: ClassVar[ModuleType] = cp - common.connectivity.register(cp.ndarray, CuPyArrayConnectivityField.from_array) + common._connectivity.register(cp.ndarray, CuPyArrayConnectivityField.from_array) # JAX if jnp: @@ -551,7 +551,7 @@ def __setitem__( # TODO(havogt): use something like `self.ndarray = self.ndarray.at(index).set(value)` raise NotImplementedError("`__setitem__` for JaxArrayField not yet implemented.") - common.field.register(jnp.ndarray, JaxArrayField.from_array) + common._field.register(jnp.ndarray, JaxArrayField.from_array) def _broadcast(field: common.Field, new_dimensions: tuple[common.Dimension, ...]) -> common.Field: @@ -566,7 +566,7 @@ def _broadcast(field: common.Field, new_dimensions: tuple[common.Dimension, ...] named_ranges.append( (dim, common.UnitRange(common.Infinity.negative(), common.Infinity.positive())) ) - return common.field(field.ndarray[tuple(domain_slice)], domain=common.Domain(*named_ranges)) + return common._field(field.ndarray[tuple(domain_slice)], domain=common.Domain(*named_ranges)) def _builtins_broadcast( diff --git a/src/gt4py/next/ffront/fbuiltins.py b/src/gt4py/next/ffront/fbuiltins.py index 8230e35a35..cc0ab21f02 100644 --- a/src/gt4py/next/ffront/fbuiltins.py +++ b/src/gt4py/next/ffront/fbuiltins.py @@ -185,7 +185,7 @@ def broadcast( assert core_defs.is_scalar_type( field ) # default implementation for scalars, Fields are handled via dispatch - return common.field( + return common._field( np.asarray(field)[ tuple([np.newaxis] * len(dims)) ], # TODO(havogt) use FunctionField once available diff --git a/src/gt4py/next/iterator/embedded.py b/src/gt4py/next/iterator/embedded.py index b02d6c8d72..b5ef88dd95 100644 --- a/src/gt4py/next/iterator/embedded.py +++ b/src/gt4py/next/iterator/embedded.py @@ -1029,7 +1029,7 @@ def _maker(a) -> common.Field: offset = origin.get(d, 0) ranges.append(common.UnitRange(-offset, s - offset)) - res = common.field(a, domain=common.Domain(dims=tuple(axes), ranges=tuple(ranges))) + res = common._field(a, domain=common.Domain(dims=tuple(axes), ranges=tuple(ranges))) return res return _maker diff --git a/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py b/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py index 2b78eb9114..cb9691976d 100644 --- a/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py +++ b/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py @@ -78,7 +78,7 @@ def _make_field(lst: Iterable, nd_array_implementation, *, domain=None, dtype=No domain = tuple( (common.Dimension(f"D{i}"), common.UnitRange(0, s)) for i, s in enumerate(buffer.shape) ) - return common.field( + return common._field( buffer, domain=domain, ) @@ -120,14 +120,14 @@ def test_where_builtin_different_domain(nd_array_implementation): true_ = np.asarray([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32) false_ = np.asarray([7.0, 8.0, 9.0, 10.0], dtype=np.float32) - cond_field = common.field( + cond_field = common._field( nd_array_implementation.asarray(cond), domain=common.domain({JDim: 2}) ) - true_field = common.field( + true_field = common._field( nd_array_implementation.asarray(true_), domain=common.domain({IDim: common.UnitRange(0, 2), JDim: common.UnitRange(-1, 2)}), ) - false_field = common.field( + false_field = common._field( nd_array_implementation.asarray(false_), domain=common.domain({JDim: common.UnitRange(-1, 3)}), ) @@ -226,8 +226,8 @@ def test_binary_operations_with_intersection(binary_arithmetic_op, dims, expecte arr2 = np.ones((5, 5)) arr2_domain = common.Domain(dims=(IDim, JDim), ranges=(UnitRange(5, 10), UnitRange(5, 10))) - field1 = common.field(arr1, domain=arr1_domain) - field2 = common.field(arr2, domain=arr2_domain) + field1 = common._field(arr1, domain=arr1_domain) + field2 = common._field(arr2, domain=arr2_domain) op_result = binary_arithmetic_op(field1, field2) expected_result = binary_arithmetic_op(arr1[expected_indices[0], expected_indices[1]], arr2) @@ -288,11 +288,11 @@ def test_remap_implementation(): V_START, V_STOP = 2, 7 E_START, E_STOP = 0, 10 - v_field = common.field( + v_field = common._field( -0.1 * np.arange(V_START, V_STOP), domain=common.Domain(dims=(V,), ranges=(UnitRange(V_START, V_STOP),)), ) - e2v_conn = common.connectivity( + e2v_conn = common._connectivity( np.arange(E_START, E_STOP), domain=common.Domain( dims=(E,), @@ -304,7 +304,7 @@ def test_remap_implementation(): ) result = v_field.remap(e2v_conn) - expected = common.field( + expected = common._field( -0.1 * np.arange(V_START, V_STOP), domain=common.Domain(dims=(E,), ranges=(UnitRange(V_START, V_STOP),)), ) @@ -319,14 +319,14 @@ def test_cartesian_remap_implementation(): V_START, V_STOP = 2, 7 OFFSET = 2 - v_field = common.field( + v_field = common._field( -0.1 * np.arange(V_START, V_STOP), domain=common.Domain(dims=(V,), ranges=(UnitRange(V_START, V_STOP),)), ) - v2_conn = common.connectivity(OFFSET, V) + v2_conn = common._connectivity(OFFSET, V) result = v_field.remap(v2_conn) - expected = common.field( + expected = common._field( v_field.ndarray, domain=common.Domain(dims=(V,), ranges=(UnitRange(V_START - OFFSET, V_STOP - OFFSET),)), ) @@ -341,7 +341,7 @@ def test_cartesian_remap_implementation(): ( ( (IDim,), - common.field( + common._field( np.arange(10), domain=common.Domain(dims=(IDim,), ranges=(UnitRange(0, 10),)) ), Domain(dims=(IDim,), ranges=(UnitRange(0, 10),)), @@ -350,7 +350,7 @@ def test_cartesian_remap_implementation(): ( ( (IDim, JDim), - common.field( + common._field( np.arange(10), domain=common.Domain(dims=(IDim,), ranges=(UnitRange(0, 10),)) ), Domain(dims=(IDim, JDim), ranges=(UnitRange(0, 10), UnitRange.infinity())), @@ -359,7 +359,7 @@ def test_cartesian_remap_implementation(): ( ( (IDim, JDim), - common.field( + common._field( np.arange(10), domain=common.Domain(dims=(JDim,), ranges=(UnitRange(0, 10),)) ), Domain(dims=(IDim, JDim), ranges=(UnitRange.infinity(), UnitRange(0, 10))), @@ -368,7 +368,7 @@ def test_cartesian_remap_implementation(): ( ( (IDim, JDim, KDim), - common.field( + common._field( np.arange(10), domain=common.Domain(dims=(JDim,), ranges=(UnitRange(0, 10),)) ), Domain( @@ -456,7 +456,7 @@ def test_absolute_indexing(domain_slice, expected_dimensions, expected_shape): domain = common.Domain( dims=(IDim, JDim, KDim), ranges=(UnitRange(5, 10), UnitRange(5, 15), UnitRange(10, 25)) ) - field = common.field(np.ones((5, 10, 15)), domain=domain) + field = common._field(np.ones((5, 10, 15)), domain=domain) indexed_field = field[domain_slice] assert common.is_field(indexed_field) @@ -466,7 +466,7 @@ def test_absolute_indexing(domain_slice, expected_dimensions, expected_shape): def test_absolute_indexing_value_return(): domain = common.Domain(dims=(IDim, JDim), ranges=(UnitRange(10, 20), UnitRange(5, 15))) - field = common.field(np.reshape(np.arange(100, dtype=np.int32), (10, 10)), domain=domain) + field = common._field(np.reshape(np.arange(100, dtype=np.int32), (10, 10)), domain=domain) named_index = ((IDim, 12), (JDim, 6)) value = field[named_index] @@ -503,7 +503,7 @@ def test_absolute_indexing_value_return(): ) def test_relative_indexing_slice_2D(index, expected_shape, expected_domain): domain = common.Domain(dims=(IDim, JDim), ranges=(UnitRange(5, 15), UnitRange(2, 12))) - field = common.field(np.ones((10, 10)), domain=domain) + field = common._field(np.ones((10, 10)), domain=domain) indexed_field = field[index] assert common.is_field(indexed_field) @@ -559,7 +559,7 @@ def test_relative_indexing_slice_3D(index, expected_shape, expected_domain): domain = common.Domain( dims=(IDim, JDim, KDim), ranges=(UnitRange(5, 15), UnitRange(10, 25), UnitRange(10, 20)) ) - field = common.field(np.ones((10, 15, 10)), domain=domain) + field = common._field(np.ones((10, 15, 10)), domain=domain) indexed_field = field[index] assert common.is_field(indexed_field) @@ -573,7 +573,7 @@ def test_relative_indexing_slice_3D(index, expected_shape, expected_domain): ) def test_relative_indexing_value_return(index, expected_value): domain = common.Domain(dims=(IDim, JDim), ranges=(UnitRange(5, 15), UnitRange(2, 12))) - field = common.field(np.reshape(np.arange(100, dtype=int), (10, 10)), domain=domain) + field = common._field(np.reshape(np.arange(100, dtype=int), (10, 10)), domain=domain) indexed_field = field[index] assert indexed_field == expected_value @@ -582,7 +582,7 @@ def test_relative_indexing_value_return(index, expected_value): @pytest.mark.parametrize("lazy_slice", [lambda f: f[13], lambda f: f[:5, :3, :2]]) def test_relative_indexing_out_of_bounds(lazy_slice): domain = common.Domain(dims=(IDim, JDim), ranges=(UnitRange(3, 13), UnitRange(-5, 5))) - field = common.field(np.ones((10, 10)), domain=domain) + field = common._field(np.ones((10, 10)), domain=domain) with pytest.raises((embedded_exceptions.IndexOutOfBounds, IndexError)): lazy_slice(field) @@ -591,7 +591,7 @@ def test_relative_indexing_out_of_bounds(lazy_slice): @pytest.mark.parametrize("index", [IDim, "1", (IDim, JDim)]) def test_field_unsupported_index(index): domain = common.Domain(dims=(IDim,), ranges=(UnitRange(0, 10),)) - field = common.field(np.ones((10,)), domain=domain) + field = common._field(np.ones((10,)), domain=domain) with pytest.raises(IndexError, match="Unsupported index type"): field[index] @@ -603,12 +603,12 @@ def test_field_unsupported_index(index): ((1, slice(None)), np.ones((10,)) * 42.0), ( (1, slice(None)), - common.field(np.ones((10,)) * 42.0, domain=common.Domain((JDim, UnitRange(0, 10)))), + common._field(np.ones((10,)) * 42.0, domain=common.Domain((JDim, UnitRange(0, 10)))), ), ], ) def test_setitem(index, value): - field = common.field( + field = common._field( np.arange(100).reshape(10, 10), domain=common.Domain(dims=(IDim, JDim), ranges=(UnitRange(0, 10), UnitRange(0, 10))), ) @@ -622,12 +622,12 @@ def test_setitem(index, value): def test_setitem_wrong_domain(): - field = common.field( + field = common._field( np.arange(100).reshape(10, 10), domain=common.Domain(dims=(IDim, JDim), ranges=(UnitRange(0, 10), UnitRange(0, 10))), ) - value_incompatible = common.field( + value_incompatible = common._field( np.ones((10,)) * 42.0, domain=common.Domain((JDim, UnitRange(-5, 5))) ) From fa191a98b12604566271678176f4b0edc3f7cafa Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Tue, 28 Nov 2023 17:09:16 +0100 Subject: [PATCH 02/19] add allocators to test matrix --- tests/next_tests/exclusion_matrices.py | 19 +++++++++++++- tests/next_tests/integration_tests/cases.py | 26 ++++++++++--------- .../ffront_tests/ffront_test_utils.py | 14 +++++----- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/tests/next_tests/exclusion_matrices.py b/tests/next_tests/exclusion_matrices.py index a6a302e143..396883a0c8 100644 --- a/tests/next_tests/exclusion_matrices.py +++ b/tests/next_tests/exclusion_matrices.py @@ -19,6 +19,8 @@ import pytest +from gt4py.next import allocators as next_allocators + # Skip definitions XFAIL = pytest.xfail @@ -44,6 +46,11 @@ def short_id(self, num_components: int = 2) -> str: return ".".join(self.value.split(".")[-num_components:]) +class _PythonObjectIdMixinForAllocator(_PythonObjectIdMixin): + def short_id(self, num_components: int = 1) -> str: + return "None-" + super().short_id(num_components) + + class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): GTFN_CPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn" GTFN_CPU_IMPERATIVE = "gt4py.next.program_processors.runners.gtfn.run_gtfn_imperative" @@ -55,6 +62,15 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): DOUBLE_ROUNDTRIP = "gt4py.next.program_processors.runners.double_roundtrip.backend" +cpu_allocator = next_allocators.StandardCPUFieldBufferAllocator() +gpu_allocator = next_allocators.StandardGPUFieldBufferAllocator() + + +class AllocatorId(_PythonObjectIdMixinForAllocator, str, enum.Enum): + CPU_ALLOCATOR = "next_tests.exclusion_matrices.cpu_allocator" + GPU_ALLOCATOR = "next_tests.exclusion_matrices.gpu_allocator" + + class OptionalProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): DACE_CPU = "gt4py.next.program_processors.runners.dace_iterator.run_dace_cpu" DACE_GPU = "gt4py.next.program_processors.runners.dace_iterator.run_dace_gpu" @@ -146,7 +162,8 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): #: Skip matrix, contains for each backend processor a list of tuples with following fields: #: (, ) BACKEND_SKIP_TEST_MATRIX = { - None: EMBEDDED_SKIP_LIST, + AllocatorId.CPU_ALLOCATOR: EMBEDDED_SKIP_LIST, + AllocatorId.GPU_ALLOCATOR: EMBEDDED_SKIP_LIST, OptionalProgramBackendId.DACE_CPU: DACE_SKIP_TEST_LIST, OptionalProgramBackendId.DACE_GPU: DACE_SKIP_TEST_LIST + [ diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index 81f216397b..b1b70fe37d 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -28,7 +28,7 @@ from gt4py._core import definitions as core_defs from gt4py.eve import extended_typing as xtyping from gt4py.eve.extended_typing import Self -from gt4py.next import common, constructors, utils +from gt4py.next import allocators as next_allocators, common, constructors, utils from gt4py.next.ffront import decorator from gt4py.next.program_processors import processor_interface as ppi from gt4py.next.type_system import type_specifications as ts, type_translation @@ -103,7 +103,7 @@ def scalar(self, dtype: np.typing.DTypeLike) -> ScalarValue: def field( self, - backend: ppi.ProgramProcessor, + allocator: next_allocators.FieldBufferAllocatorProtocol, sizes: dict[gtx.Dimension, int], dtype: np.typing.DTypeLike, ) -> FieldValue: @@ -137,7 +137,7 @@ def scalar_value(self) -> ScalarValue: def field( self, - backend: ppi.ProgramExecutor, + allocator: next_allocators.FieldBufferAllocatorProtocol, sizes: dict[gtx.Dimension, int], dtype: np.typing.DTypeLike, ) -> FieldValue: @@ -145,7 +145,7 @@ def field( domain=common.domain(sizes), fill_value=self.value, dtype=dtype, - allocator=backend, + allocator=allocator, ) @@ -166,7 +166,7 @@ def scalar_value(self) -> ScalarValue: def field( self, - backend: ppi.ProgramExecutor, + allocator: next_allocators.FieldBufferAllocatorProtocol, sizes: dict[gtx.Dimension, int], dtype: np.typing.DTypeLike, ) -> FieldValue: @@ -176,7 +176,7 @@ def field( ) n_data = list(sizes.values())[0] return constructors.as_field( - domain=common.domain(sizes), data=np.arange(0, n_data, dtype=dtype), allocator=backend + domain=common.domain(sizes), data=np.arange(0, n_data, dtype=dtype), allocator=allocator ) def from_case( @@ -207,7 +207,7 @@ def scalar_value(self) -> ScalarValue: def field( self, - backend: ppi.ProgramProcessor, + allocator: next_allocators.FieldBufferAllocatorProtocol, sizes: dict[gtx.Dimension, int], dtype: np.typing.DTypeLike, ) -> FieldValue: @@ -218,7 +218,7 @@ def field( return constructors.as_field( common.domain(sizes), np.arange(start, start + n_data, dtype=dtype).reshape(svals), - allocator=backend, + allocator=allocator, ) def from_case( @@ -482,10 +482,11 @@ def verify_with_default_data( @pytest.fixture def cartesian_case(fieldview_backend): # noqa: F811 # fixtures yield Case( - fieldview_backend, + fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, offset_provider={"Ioff": IDim, "Joff": JDim, "Koff": KDim}, default_sizes={IDim: 10, JDim: 10, KDim: 10}, grid_type=common.GridType.CARTESIAN, + allocator=fieldview_backend, ) @@ -516,7 +517,7 @@ def _allocate_from_type( match arg_type: case ts.FieldType(dims=dims, dtype=arg_dtype): return strategy.field( - backend=case.backend, + allocator=case.allocator, sizes={dim: sizes[dim] for dim in dims}, dtype=dtype or arg_dtype.kind.name.lower(), ) @@ -601,11 +602,12 @@ def get_default_data( class Case: """Parametrizable components for single feature integration tests.""" - backend: ppi.ProgramProcessor + backend: Optional[ppi.ProgramProcessor] offset_provider: dict[str, common.Connectivity | gtx.Dimension] default_sizes: dict[gtx.Dimension, int] grid_type: common.GridType + allocator: next_allocators.FieldBufferAllocatorFactoryProtocol @property def as_field(self): - return constructors.as_field.partial(allocator=self.backend) + return constructors.as_field.partial(allocator=self.allocator) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index f8a3f6a975..aa40b93c9f 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -57,7 +57,9 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, pytest.param(definitions.ProgramBackendId.GTFN_GPU, marks=pytest.mark.requires_gpu), - None, + # will use the default (embedded) execution, but input/output allocated with the provided allocator + definitions.AllocatorId.CPU_ALLOCATOR, + definitions.AllocatorId.GPU_ALLOCATOR, ] + OPTIONAL_PROCESSORS, ids=lambda p: p.short_id() if p is not None else "None", @@ -69,18 +71,18 @@ def fieldview_backend(request): Notes: Check ADR 15 for details on the test-exclusion matrices. """ - backend_id = request.param - backend = None if backend_id is None else backend_id.load() + backend_or_allocator_id = request.param + backend_or_allocator = backend_or_allocator_id.load() for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( - backend_id, [] + backend_or_allocator_id, [] ): if request.node.get_closest_marker(marker): - skip_mark(msg.format(marker=marker, backend=backend_id)) + skip_mark(msg.format(marker=marker, backend=backend_or_allocator_id)) backup_backend = decorator.DEFAULT_BACKEND decorator.DEFAULT_BACKEND = no_backend - yield backend + yield backend_or_allocator decorator.DEFAULT_BACKEND = backup_backend From f0b6bb6487f72b35fc79974e481b684ba08f7f90 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Wed, 29 Nov 2023 08:29:09 +0100 Subject: [PATCH 03/19] rename exclusion_matrices to definitions --- tests/next_tests/__init__.py | 4 +-- .../{exclusion_matrices.py => definitions.py} | 4 +-- tests/next_tests/integration_tests/cases.py | 3 ++- .../ffront_tests/ffront_test_utils.py | 25 +++++++++++-------- .../feature_tests/test_util_cases.py | 2 +- tests/next_tests/unit_tests/conftest.py | 24 +++++++++--------- 6 files changed, 33 insertions(+), 29 deletions(-) rename tests/next_tests/{exclusion_matrices.py => definitions.py} (98%) diff --git a/tests/next_tests/__init__.py b/tests/next_tests/__init__.py index e2905ab49a..1745dac6ef 100644 --- a/tests/next_tests/__init__.py +++ b/tests/next_tests/__init__.py @@ -12,10 +12,10 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -from . import exclusion_matrices +from . import definitions -__all__ = ["exclusion_matrices", "get_processor_id"] +__all__ = ["definitions", "get_processor_id"] def get_processor_id(processor): diff --git a/tests/next_tests/exclusion_matrices.py b/tests/next_tests/definitions.py similarity index 98% rename from tests/next_tests/exclusion_matrices.py rename to tests/next_tests/definitions.py index 396883a0c8..94b7ad9cab 100644 --- a/tests/next_tests/exclusion_matrices.py +++ b/tests/next_tests/definitions.py @@ -67,8 +67,8 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): class AllocatorId(_PythonObjectIdMixinForAllocator, str, enum.Enum): - CPU_ALLOCATOR = "next_tests.exclusion_matrices.cpu_allocator" - GPU_ALLOCATOR = "next_tests.exclusion_matrices.gpu_allocator" + CPU_ALLOCATOR = "next_tests.definitions.cpu_allocator" + GPU_ALLOCATOR = "next_tests.definitions.gpu_allocator" class OptionalProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index b1b70fe37d..c77225b955 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -493,7 +493,7 @@ def cartesian_case(fieldview_backend): # noqa: F811 # fixtures @pytest.fixture def unstructured_case(reduction_setup, fieldview_backend): # noqa: F811 # fixtures yield Case( - fieldview_backend, + fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, offset_provider=reduction_setup.offset_provider, default_sizes={ Vertex: reduction_setup.num_vertices, @@ -502,6 +502,7 @@ def unstructured_case(reduction_setup, fieldview_backend): # noqa: F811 # fixtu KDim: reduction_setup.k_levels, }, grid_type=common.GridType.UNSTRUCTURED, + allocator=fieldview_backend, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index aa40b93c9f..a0f58ebe21 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -34,7 +34,6 @@ raise e import next_tests -import next_tests.exclusion_matrices as definitions def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> None: @@ -44,22 +43,26 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non OPTIONAL_PROCESSORS = [] if dace_iterator: - OPTIONAL_PROCESSORS.append(definitions.OptionalProgramBackendId.DACE_CPU) + OPTIONAL_PROCESSORS.append(next_tests.definitions.OptionalProgramBackendId.DACE_CPU) OPTIONAL_PROCESSORS.append( - pytest.param(definitions.OptionalProgramBackendId.DACE_GPU, marks=pytest.mark.requires_gpu) + pytest.param( + next_tests.definitions.OptionalProgramBackendId.DACE_GPU, marks=pytest.mark.requires_gpu + ) ), @pytest.fixture( params=[ - definitions.ProgramBackendId.ROUNDTRIP, - definitions.ProgramBackendId.GTFN_CPU, - definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, - definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, - pytest.param(definitions.ProgramBackendId.GTFN_GPU, marks=pytest.mark.requires_gpu), + next_tests.definitions.ProgramBackendId.ROUNDTRIP, + next_tests.definitions.ProgramBackendId.GTFN_CPU, + next_tests.definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, + next_tests.definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, + pytest.param( + next_tests.definitions.ProgramBackendId.GTFN_GPU, marks=pytest.mark.requires_gpu + ), # will use the default (embedded) execution, but input/output allocated with the provided allocator - definitions.AllocatorId.CPU_ALLOCATOR, - definitions.AllocatorId.GPU_ALLOCATOR, + next_tests.definitions.AllocatorId.CPU_ALLOCATOR, + next_tests.definitions.AllocatorId.GPU_ALLOCATOR, ] + OPTIONAL_PROCESSORS, ids=lambda p: p.short_id() if p is not None else "None", @@ -74,7 +77,7 @@ def fieldview_backend(request): backend_or_allocator_id = request.param backend_or_allocator = backend_or_allocator_id.load() - for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( + for marker, skip_mark, msg in next_tests.definitions.BACKEND_SKIP_TEST_MATRIX.get( backend_or_allocator_id, [] ): if request.node.get_closest_marker(marker): diff --git a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py index 579dec11f8..530233dc0e 100644 --- a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py +++ b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py @@ -18,7 +18,7 @@ import gt4py.next as gtx from gt4py.next import errors -import next_tests.exclusion_matrices as definitions +from next_tests import definitions from next_tests.integration_tests import cases from next_tests.integration_tests.cases import ( # noqa: F401 # fixtures cartesian_case, diff --git a/tests/next_tests/unit_tests/conftest.py b/tests/next_tests/unit_tests/conftest.py index 6f91557e46..de5dbb4196 100644 --- a/tests/next_tests/unit_tests/conftest.py +++ b/tests/next_tests/unit_tests/conftest.py @@ -31,8 +31,8 @@ else: raise e + import next_tests -import next_tests.exclusion_matrices as definitions @pytest.fixture( @@ -49,7 +49,7 @@ def lift_mode(request): OPTIONAL_PROCESSORS = [] if dace_iterator: - OPTIONAL_PROCESSORS.append((definitions.OptionalProgramBackendId.DACE_CPU, True)) + OPTIONAL_PROCESSORS.append((next_tests.definitions.OptionalProgramBackendId.DACE_CPU, True)) # TODO(havogt): update tests to use proper allocation # OPTIONAL_PROCESSORS.append( # pytest.param( @@ -61,16 +61,16 @@ def lift_mode(request): @pytest.fixture( params=[ (None, True), - (definitions.ProgramBackendId.ROUNDTRIP, True), - (definitions.ProgramBackendId.DOUBLE_ROUNDTRIP, True), - (definitions.ProgramBackendId.GTFN_CPU, True), - (definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, True), - (definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, True), + (next_tests.definitions.ProgramBackendId.ROUNDTRIP, True), + (next_tests.definitions.ProgramBackendId.DOUBLE_ROUNDTRIP, True), + (next_tests.definitions.ProgramBackendId.GTFN_CPU, True), + (next_tests.definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, True), + (next_tests.definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, True), # pytest.param((definitions.ProgramBackendId.GTFN_GPU, True), marks=pytest.mark.requires_gpu), # TODO(havogt): update tests to use proper allocation - (definitions.ProgramFormatterId.LISP_FORMATTER, False), - (definitions.ProgramFormatterId.ITIR_PRETTY_PRINTER, False), - (definitions.ProgramFormatterId.ITIR_TYPE_CHECKER, False), - (definitions.ProgramFormatterId.GTFN_CPP_FORMATTER, False), + (next_tests.definitions.ProgramFormatterId.LISP_FORMATTER, False), + (next_tests.definitions.ProgramFormatterId.ITIR_PRETTY_PRINTER, False), + (next_tests.definitions.ProgramFormatterId.ITIR_TYPE_CHECKER, False), + (next_tests.definitions.ProgramFormatterId.GTFN_CPP_FORMATTER, False), ] + OPTIONAL_PROCESSORS, ids=lambda p: p[0].short_id() if p[0] is not None else "None", @@ -89,7 +89,7 @@ def program_processor(request) -> tuple[ppi.ProgramProcessor, bool]: processor = processor_id.load() assert is_backend == ppi.is_program_backend(processor) - for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( + for marker, skip_mark, msg in next_tests.definitions.BACKEND_SKIP_TEST_MATRIX.get( processor_id, [] ): if request.node.get_closest_marker(marker): From d92803932202e96f2a448ce5f0811b7880ea365a Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Wed, 24 Jan 2024 16:35:53 +0000 Subject: [PATCH 04/19] store allocator --- src/gt4py/next/common.py | 2 ++ src/gt4py/next/constructors.py | 11 +++++--- src/gt4py/next/embedded/nd_array_field.py | 28 +++++++++++++------ .../ffront_tests/test_icon_like_scan.py | 4 ++- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index 6e0ca14cb0..56983c09c7 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -848,6 +848,7 @@ def _field( *, domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, + allocator=None, #: Optional[next_allocators.FieldBufferAllocationUtil] = None, ) -> Field: raise NotImplementedError @@ -860,6 +861,7 @@ def _connectivity( *, domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, + allocator=None, #: Optional[next_allocators.FieldBufferAllocationUtil] = None, ) -> ConnectivityField: raise NotImplementedError diff --git a/src/gt4py/next/constructors.py b/src/gt4py/next/constructors.py index 8b41bf7cba..d00537f5cf 100644 --- a/src/gt4py/next/constructors.py +++ b/src/gt4py/next/constructors.py @@ -15,17 +15,20 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import Optional, cast +from typing import TYPE_CHECKING, Optional, cast import gt4py._core.definitions as core_defs import gt4py.eve as eve import gt4py.eve.extended_typing as xtyping import gt4py.next.allocators as next_allocators import gt4py.next.common as common -import gt4py.next.embedded.nd_array_field as nd_array_field import gt4py.storage.cartesian.utils as storage_utils +if TYPE_CHECKING: + import gt4py.next.embedded.nd_array_field as nd_array_field + + @eve.utils.with_fluid_partial def empty( domain: common.DomainLike, @@ -89,7 +92,7 @@ def empty( ) res = common._field(buffer.ndarray, domain=domain) assert common.is_mutable_field(res) - assert isinstance(res, nd_array_field.NdArrayField) + # assert isinstance(res, nd_array_field.NdArrayField) return res @@ -361,6 +364,6 @@ def as_connectivity( connectivity_field = common._connectivity( buffer.ndarray, codomain=codomain, domain=actual_domain ) - assert isinstance(connectivity_field, nd_array_field.NdArrayConnectivityField) + # assert isinstance(connectivity_field, nd_array_field.NdArrayConnectivityField) return connectivity_field diff --git a/src/gt4py/next/embedded/nd_array_field.py b/src/gt4py/next/embedded/nd_array_field.py index c25e1b69c8..b9b4e2c44c 100644 --- a/src/gt4py/next/embedded/nd_array_field.py +++ b/src/gt4py/next/embedded/nd_array_field.py @@ -25,7 +25,7 @@ from gt4py._core import definitions as core_defs from gt4py.eve.extended_typing import Any, Never, Optional, ParamSpec, TypeAlias, TypeVar -from gt4py.next import common +from gt4py.next import allocators, common, constructors from gt4py.next.embedded import common as embedded_common from gt4py.next.ffront import fbuiltins @@ -94,6 +94,7 @@ class NdArrayField( _domain: common.Domain _ndarray: core_defs.NDArrayObject + _allocator: Optional[allocators.FieldBufferAllocationUtil] array_ns: ClassVar[ ModuleType @@ -143,6 +144,7 @@ def from_array( *, domain: common.DomainLike, dtype: Optional[core_defs.DTypeLike] = None, + allocator: Optional[allocators.FieldBufferAllocationUtil] = None, ) -> NdArrayField: domain = common.domain(domain) xp = cls.array_ns @@ -159,7 +161,7 @@ def from_array( assert len(domain) == array.ndim assert all(s == 1 or len(r) == s for r, s in zip(domain.ranges, array.shape)) - return cls(domain, array) + return cls(domain, array, allocator) def remap( self: NdArrayField, connectivity: common.ConnectivityField | fbuiltins.FieldOffset @@ -201,7 +203,9 @@ def remap( # finally, take the new array new_buffer = xp.take(self._ndarray, new_idx_array, axis=dim_idx) - return self.__class__.from_array(new_buffer, domain=new_domain, dtype=self.dtype) + return self.__class__.from_array( + new_buffer, domain=new_domain, dtype=self.dtype, allocator=self._allocator + ) # TODO __call__ = remap # type: ignore[assignment] @@ -213,7 +217,9 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Scala # TODO: assert core_defs.is_scalar_type(new_buffer), new_buffer return new_buffer # type: ignore[return-value] # I don't think we can express that we return `ScalarT` here else: - return self.__class__.from_array(new_buffer, domain=new_domain) + return self.__class__.from_array( + new_buffer, domain=new_domain, allocator=self._allocator + ) # TODO __getitem__ = restrict @@ -347,6 +353,7 @@ def from_array( # type: ignore[override] *, domain: common.DomainLike, dtype: Optional[core_defs.DTypeLike] = None, + allocator: Optional[allocators.FieldBufferAllocationUtil] = None, ) -> NdArrayConnectivityField: domain = common.domain(domain) xp = cls.array_ns @@ -365,7 +372,7 @@ def from_array( # type: ignore[override] assert isinstance(codomain, common.Dimension) - return cls(domain, array, codomain) + return cls(domain, array, allocator, codomain) def inverse_image( self, image_range: common.UnitRange | common.NamedRange @@ -435,11 +442,14 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Integ cache_key = (id(self.ndarray), self.domain, index) if (restricted_connectivity := self._cache.get(cache_key, None)) is None: - cls = self.__class__ - xp = cls.array_ns new_domain, buffer_slice = self._slice(index) - new_buffer = xp.asarray(self.ndarray[buffer_slice]) - restricted_connectivity = cls(new_domain, new_buffer, self.codomain) + restricted_connectivity = constructors.as_connectivity( + new_domain, + self.codomain, + self.ndarray[buffer_slice], + self.dtype, + allocator=self._allocator, + ) # TODO does it make sense to go through constructors here which does a lot of checks? self._cache[cache_key] = restricted_connectivity return restricted_connectivity diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index 130f6bd29c..6eb65f4a68 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -19,6 +19,7 @@ import gt4py.next as gtx from gt4py.next import common +from gt4py.next.program_processors import processor_interface as ppi from gt4py.next.program_processors.runners import gtfn, roundtrip from next_tests.integration_tests import cases @@ -195,10 +196,11 @@ def reference( @pytest.fixture def test_setup(fieldview_backend): test_case = cases.Case( - fieldview_backend, + fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, offset_provider={"Koff": KDim}, default_sizes={Cell: 14, KDim: 10}, grid_type=common.GridType.UNSTRUCTURED, + allocator=fieldview_backend, ) @dataclass(frozen=True) From 6fa9f8f9ad2926c4cb2ed6c9f1c7ba1a3fe8eae3 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Thu, 25 Jan 2024 13:51:17 +0100 Subject: [PATCH 05/19] based on array_ns --- pyproject.toml | 3 +- src/gt4py/next/common.py | 2 -- src/gt4py/next/constructors.py | 11 +++---- src/gt4py/next/embedded/nd_array_field.py | 24 +++++--------- src/gt4py/next/embedded/operators.py | 33 +++++++++++++++---- src/gt4py/next/ffront/fbuiltins.py | 8 ++--- tests/next_tests/definitions.py | 9 +++-- .../ffront_tests/test_execution.py | 2 ++ 8 files changed, 53 insertions(+), 39 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5d7a2f2cb6..7d499b1f3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -340,7 +340,9 @@ markers = [ 'uses_origin: tests that require backend support for domain origin', 'uses_reduction_over_lift_expressions: tests that require backend support for reduction over lift expressions', 'uses_reduction_with_only_sparse_fields: tests that require backend support for with sparse fields', + 'uses_scan: tests that uses scan', 'uses_scan_in_field_operator: tests that require backend support for scan in field operator', + 'uses_scan_without_field_args: tests that require calls to scan that do not have any fields as arguments', 'uses_sparse_fields: tests that require backend support for sparse fields', 'uses_sparse_fields_as_output: tests that require backend support for writing sparse fields', 'uses_strided_neighbor_offset: tests that require backend support for strided neighbor offset', @@ -349,7 +351,6 @@ markers = [ 'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields', 'uses_cartesian_shift: tests that use a Cartesian connectivity', 'uses_unstructured_shift: tests that use a unstructured connectivity', - 'uses_scan: tests that uses scan', 'checks_specific_error: tests that rely on the backend to produce a specific error message' ] norecursedirs = ['dist', 'build', 'cpp_backend_tests/build*', '_local/*', '.*'] diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index 56983c09c7..6e0ca14cb0 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -848,7 +848,6 @@ def _field( *, domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, - allocator=None, #: Optional[next_allocators.FieldBufferAllocationUtil] = None, ) -> Field: raise NotImplementedError @@ -861,7 +860,6 @@ def _connectivity( *, domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, - allocator=None, #: Optional[next_allocators.FieldBufferAllocationUtil] = None, ) -> ConnectivityField: raise NotImplementedError diff --git a/src/gt4py/next/constructors.py b/src/gt4py/next/constructors.py index d00537f5cf..8b41bf7cba 100644 --- a/src/gt4py/next/constructors.py +++ b/src/gt4py/next/constructors.py @@ -15,20 +15,17 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Optional, cast +from typing import Optional, cast import gt4py._core.definitions as core_defs import gt4py.eve as eve import gt4py.eve.extended_typing as xtyping import gt4py.next.allocators as next_allocators import gt4py.next.common as common +import gt4py.next.embedded.nd_array_field as nd_array_field import gt4py.storage.cartesian.utils as storage_utils -if TYPE_CHECKING: - import gt4py.next.embedded.nd_array_field as nd_array_field - - @eve.utils.with_fluid_partial def empty( domain: common.DomainLike, @@ -92,7 +89,7 @@ def empty( ) res = common._field(buffer.ndarray, domain=domain) assert common.is_mutable_field(res) - # assert isinstance(res, nd_array_field.NdArrayField) + assert isinstance(res, nd_array_field.NdArrayField) return res @@ -364,6 +361,6 @@ def as_connectivity( connectivity_field = common._connectivity( buffer.ndarray, codomain=codomain, domain=actual_domain ) - # assert isinstance(connectivity_field, nd_array_field.NdArrayConnectivityField) + assert isinstance(connectivity_field, nd_array_field.NdArrayConnectivityField) return connectivity_field diff --git a/src/gt4py/next/embedded/nd_array_field.py b/src/gt4py/next/embedded/nd_array_field.py index b9b4e2c44c..8140f0a140 100644 --- a/src/gt4py/next/embedded/nd_array_field.py +++ b/src/gt4py/next/embedded/nd_array_field.py @@ -25,7 +25,7 @@ from gt4py._core import definitions as core_defs from gt4py.eve.extended_typing import Any, Never, Optional, ParamSpec, TypeAlias, TypeVar -from gt4py.next import allocators, common, constructors +from gt4py.next import common, constructors from gt4py.next.embedded import common as embedded_common from gt4py.next.ffront import fbuiltins @@ -94,11 +94,8 @@ class NdArrayField( _domain: common.Domain _ndarray: core_defs.NDArrayObject - _allocator: Optional[allocators.FieldBufferAllocationUtil] - array_ns: ClassVar[ - ModuleType - ] # TODO(havogt) after storage PR is merged, update to the NDArrayNamespace protocol + array_ns: ClassVar[ModuleType] # TODO(havogt) introduce a NDArrayNamespace protocol @property def domain(self) -> common.Domain: @@ -144,7 +141,6 @@ def from_array( *, domain: common.DomainLike, dtype: Optional[core_defs.DTypeLike] = None, - allocator: Optional[allocators.FieldBufferAllocationUtil] = None, ) -> NdArrayField: domain = common.domain(domain) xp = cls.array_ns @@ -161,7 +157,7 @@ def from_array( assert len(domain) == array.ndim assert all(s == 1 or len(r) == s for r, s in zip(domain.ranges, array.shape)) - return cls(domain, array, allocator) + return cls(domain, array) def remap( self: NdArrayField, connectivity: common.ConnectivityField | fbuiltins.FieldOffset @@ -204,8 +200,10 @@ def remap( new_buffer = xp.take(self._ndarray, new_idx_array, axis=dim_idx) return self.__class__.from_array( - new_buffer, domain=new_domain, dtype=self.dtype, allocator=self._allocator - ) # TODO + new_buffer, + domain=new_domain, + dtype=self.dtype, + ) __call__ = remap # type: ignore[assignment] @@ -217,9 +215,7 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Scala # TODO: assert core_defs.is_scalar_type(new_buffer), new_buffer return new_buffer # type: ignore[return-value] # I don't think we can express that we return `ScalarT` here else: - return self.__class__.from_array( - new_buffer, domain=new_domain, allocator=self._allocator - ) # TODO + return self.__class__.from_array(new_buffer, domain=new_domain) __getitem__ = restrict @@ -353,7 +349,6 @@ def from_array( # type: ignore[override] *, domain: common.DomainLike, dtype: Optional[core_defs.DTypeLike] = None, - allocator: Optional[allocators.FieldBufferAllocationUtil] = None, ) -> NdArrayConnectivityField: domain = common.domain(domain) xp = cls.array_ns @@ -372,7 +367,7 @@ def from_array( # type: ignore[override] assert isinstance(codomain, common.Dimension) - return cls(domain, array, allocator, codomain) + return cls(domain, array, codomain) def inverse_image( self, image_range: common.UnitRange | common.NamedRange @@ -448,7 +443,6 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Integ self.codomain, self.ndarray[buffer_slice], self.dtype, - allocator=self._allocator, ) # TODO does it make sense to go through constructors here which does a lot of checks? self._cache[cache_key] = restricted_connectivity diff --git a/src/gt4py/next/embedded/operators.py b/src/gt4py/next/embedded/operators.py index 0992401ebb..6320e74cc6 100644 --- a/src/gt4py/next/embedded/operators.py +++ b/src/gt4py/next/embedded/operators.py @@ -13,11 +13,14 @@ # SPDX-License-Identifier: GPL-3.0-or-later import dataclasses +from types import ModuleType from typing import Any, Callable, Generic, ParamSpec, Sequence, TypeVar +import numpy as np + from gt4py import eve from gt4py._core import definitions as core_defs -from gt4py.next import common, constructors, errors, utils +from gt4py.next import common, errors, utils from gt4py.next.embedded import common as embedded_common, context as embedded_context @@ -43,7 +46,8 @@ def __call__(self, *args: common.Field | core_defs.Scalar, **kwargs: common.Fiel scan_range = embedded_context.closure_column_range.get() assert self.axis == scan_range[0] scan_axis = scan_range[0] - domain_intersection = _intersect_scan_args(*args, *kwargs.values()) + all_args = [*args, *kwargs.values()] + domain_intersection = _intersect_scan_args(*all_args) non_scan_domain = common.Domain(*[nr for nr in domain_intersection if nr[0] != scan_axis]) out_domain = common.Domain( @@ -53,7 +57,8 @@ def __call__(self, *args: common.Field | core_defs.Scalar, **kwargs: common.Fiel # even if the scan dimension is not in the input, we can scan over it out_domain = common.Domain(*out_domain, (scan_range)) - res = _construct_scan_array(out_domain)(self.init) + xp = _get_array_ns(*all_args) + res = _construct_scan_array(out_domain, xp)(self.init) def scan_loop(hpos): acc = self.init @@ -128,7 +133,11 @@ def _tuple_assign_field( ): @utils.tree_map def impl(target: common.MutableField, source: common.Field): - target[domain] = source[domain] + if common.is_field(source): + target[domain] = source[domain] + else: + assert core_defs.is_scalar_type(source) + target[domain] = source impl(target, source) @@ -141,10 +150,21 @@ def _intersect_scan_args( ) -def _construct_scan_array(domain: common.Domain): +def _get_array_ns( + *args: core_defs.Scalar | common.Field | tuple[core_defs.Scalar | common.Field | tuple, ...] +): + for arg in utils.flatten_nested_tuple(args): + if hasattr(arg, "array_ns"): + return arg.array_ns + return np + + +def _construct_scan_array( + domain: common.Domain, xp: ModuleType +): # TODO(havogt) introduce a NDArrayNamespace protocol @utils.tree_map def impl(init: core_defs.Scalar) -> common.Field: - return constructors.empty(domain, dtype=type(init)) + return common._field(xp.empty(domain.shape, dtype=type(init)), domain=domain) return impl @@ -168,6 +188,7 @@ def _tuple_at( @utils.tree_map def impl(field: common.Field | core_defs.Scalar) -> core_defs.Scalar: res = field[pos] if common.is_field(field) else field + res = res.item() if hasattr(res, "item") else res # extract scalar value from array assert core_defs.is_scalar_type(res) return res diff --git a/src/gt4py/next/ffront/fbuiltins.py b/src/gt4py/next/ffront/fbuiltins.py index 66ad6fd84a..9a6e321f9a 100644 --- a/src/gt4py/next/ffront/fbuiltins.py +++ b/src/gt4py/next/ffront/fbuiltins.py @@ -188,12 +188,8 @@ def broadcast( assert core_defs.is_scalar_type( field ) # default implementation for scalars, Fields are handled via dispatch - return common._field( - np.asarray(field)[ - tuple([np.newaxis] * len(dims)) - ], # TODO(havogt) use FunctionField once available - domain=common.Domain(dims=dims, ranges=tuple([common.UnitRange.infinite()] * len(dims))), - ) + # TODO(havogt) implement with FunctionField, the workaround is to ignore broadcasting on scalars as they broadcast automatically, but we lose the check for compatible dimensions + return field # type: ignore[return-value] # see comment above @WhereBuiltinFunction diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index 69f8c94ba1..cf5f16c4ed 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -109,7 +109,9 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): USES_NEGATIVE_MODULO = "uses_negative_modulo" USES_ORIGIN = "uses_origin" USES_REDUCTION_OVER_LIFT_EXPRESSIONS = "uses_reduction_over_lift_expressions" +USES_SCAN = "uses_scan" USES_SCAN_IN_FIELD_OPERATOR = "uses_scan_in_field_operator" +USES_SCAN_WITHOUT_FIELD_ARGS = "uses_scan_without_field_args" USES_SPARSE_FIELDS = "uses_sparse_fields" USES_SPARSE_FIELDS_AS_OUTPUT = "uses_sparse_fields_as_output" USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS = "uses_reduction_with_only_sparse_fields" @@ -119,7 +121,6 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields" USES_CARTESIAN_SHIFT = "uses_cartesian_shift" USES_UNSTRUCTURED_SHIFT = "uses_unstructured_shift" -USES_SCAN = "uses_scan" CHECKS_SPECIFIC_ERROR = "checks_specific_error" # Skip messages (available format keys: 'marker', 'backend') @@ -161,7 +162,11 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): #: (, ) BACKEND_SKIP_TEST_MATRIX = { AllocatorId.CPU_ALLOCATOR: EMBEDDED_SKIP_LIST, - AllocatorId.GPU_ALLOCATOR: EMBEDDED_SKIP_LIST, + AllocatorId.GPU_ALLOCATOR: EMBEDDED_SKIP_LIST + + [ + # we can't extract the type of the output field + (USES_SCAN_WITHOUT_FIELD_ARGS, XFAIL, UNSUPPORTED_MESSAGE) + ], OptionalProgramBackendId.DACE_CPU: DACE_SKIP_TEST_LIST, OptionalProgramBackendId.DACE_GPU: DACE_SKIP_TEST_LIST + [ diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py index a08931628b..992f13ea67 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py @@ -589,6 +589,7 @@ def testee(a: tuple[tuple[cases.IField, cases.IField], cases.IField]) -> cases.I @pytest.mark.uses_scan +@pytest.mark.uses_scan_without_field_args @pytest.mark.parametrize("forward", [True, False]) def test_fieldop_from_scan(cartesian_case, forward): init = 1.0 @@ -743,6 +744,7 @@ def simple_scan_operator(carry: float, a: float) -> float: @pytest.mark.parametrize("forward", [True, False]) @pytest.mark.uses_scan +@pytest.mark.uses_scan_without_field_args @pytest.mark.uses_tuple_returns def test_scan_nested_tuple_output(forward, cartesian_case): if cartesian_case.backend in [gtfn.run_gtfn_with_temporaries]: From 816dafb29fcf70eacb0651923ab9fe8717353a22 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Thu, 25 Jan 2024 13:58:05 +0100 Subject: [PATCH 06/19] cleanup --- src/gt4py/next/embedded/nd_array_field.py | 12 +++++------- src/gt4py/next/embedded/operators.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/gt4py/next/embedded/nd_array_field.py b/src/gt4py/next/embedded/nd_array_field.py index 8140f0a140..e8bdfc9d7f 100644 --- a/src/gt4py/next/embedded/nd_array_field.py +++ b/src/gt4py/next/embedded/nd_array_field.py @@ -25,7 +25,7 @@ from gt4py._core import definitions as core_defs from gt4py.eve.extended_typing import Any, Never, Optional, ParamSpec, TypeAlias, TypeVar -from gt4py.next import common, constructors +from gt4py.next import common from gt4py.next.embedded import common as embedded_common from gt4py.next.ffront import fbuiltins @@ -437,13 +437,11 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Integ cache_key = (id(self.ndarray), self.domain, index) if (restricted_connectivity := self._cache.get(cache_key, None)) is None: + cls = self.__class__ + xp = cls.array_ns new_domain, buffer_slice = self._slice(index) - restricted_connectivity = constructors.as_connectivity( - new_domain, - self.codomain, - self.ndarray[buffer_slice], - self.dtype, - ) # TODO does it make sense to go through constructors here which does a lot of checks? + new_buffer = xp.asarray(self.ndarray[buffer_slice]) + restricted_connectivity = cls(new_domain, new_buffer, self.codomain) self._cache[cache_key] = restricted_connectivity return restricted_connectivity diff --git a/src/gt4py/next/embedded/operators.py b/src/gt4py/next/embedded/operators.py index 6320e74cc6..cb03373b41 100644 --- a/src/gt4py/next/embedded/operators.py +++ b/src/gt4py/next/embedded/operators.py @@ -152,7 +152,7 @@ def _intersect_scan_args( def _get_array_ns( *args: core_defs.Scalar | common.Field | tuple[core_defs.Scalar | common.Field | tuple, ...] -): +) -> ModuleType: for arg in utils.flatten_nested_tuple(args): if hasattr(arg, "array_ns"): return arg.array_ns From ca7aa6c15597779a4ea1027460813620aa88ff3c Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Thu, 25 Jan 2024 16:08:58 +0100 Subject: [PATCH 07/19] requires_gpu --- .../feature_tests/ffront_tests/ffront_test_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index ed8c91e41c..00be56ab91 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -64,7 +64,9 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non ), # will use the default (embedded) execution, but input/output allocated with the provided allocator next_tests.definitions.AllocatorId.CPU_ALLOCATOR, - next_tests.definitions.AllocatorId.GPU_ALLOCATOR, + pytest.param( + next_tests.definitions.AllocatorId.GPU_ALLOCATOR, marks=pytest.mark.requires_gpu + ), ] + OPTIONAL_PROCESSORS, ids=lambda p: p.short_id() if p is not None else "None", From 0a0dc509fbaa8708b5e34bfaa622b791ac5f643a Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Thu, 25 Jan 2024 22:31:19 +0100 Subject: [PATCH 08/19] refactor backend selection --- .../ADRs/0015-Test_Exclusion_Matrices.md | 6 +-- src/gt4py/next/allocators.py | 13 ++++++ tests/next_tests/definitions.py | 17 ++------ tests/next_tests/integration_tests/cases.py | 8 ++-- .../ffront_tests/ffront_test_utils.py | 43 +++++++++++++++---- .../feature_tests/test_util_cases.py | 13 ++++-- .../ffront_tests/test_icon_like_scan.py | 4 +- 7 files changed, 70 insertions(+), 34 deletions(-) diff --git a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md index b338169d61..7f55604774 100644 --- a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md +++ b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md @@ -1,5 +1,5 @@ --- -tags: [] +tags: [testing] --- # Test-Exclusion Matrices @@ -7,7 +7,7 @@ tags: [] - **Status**: valid - **Authors**: Edoardo Paone (@edopao), Enrique G. Paredes (@egparedes) - **Created**: 2023-09-21 -- **Updated**: 2023-09-21 +- **Updated**: 2024-01-25 In the context of Field View testing, lacking support for specific ITIR features while a certain backend is being developed, we decided to use `pytest` fixtures to exclude unsupported tests. @@ -33,7 +33,7 @@ In the example below, `test_offset_field` requires the backend to support dynami def test_offset_field(cartesian_case): ``` -In order to selectively enable the backends, the dictionary `next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX` +In order to selectively enable the backends, the dictionary `next_tests.definitions.BACKEND_SKIP_TEST_MATRIX` lists for each backend the features that are not supported. The fixture will check if the annotated feature is present in the exclusion-matrix for the selected backend. If so, the exclusion matrix will also specify the action `pytest` should take (e.g. `SKIP` or `XFAIL`). diff --git a/src/gt4py/next/allocators.py b/src/gt4py/next/allocators.py index 97e83276fe..3388a29cd6 100644 --- a/src/gt4py/next/allocators.py +++ b/src/gt4py/next/allocators.py @@ -231,6 +231,7 @@ def __init__(self) -> None: device_allocators[core_defs.DeviceType.CPU] = StandardCPUFieldBufferAllocator() + assert is_field_allocator(device_allocators[core_defs.DeviceType.CPU]) @@ -349,3 +350,15 @@ def allocate( device_id=device.device_id, aligned_index=aligned_index, ) + + +def __getattr__(name: str) -> Any: + if name == "default_cpu_allocator": + return device_allocators[core_defs.DeviceType.CPU] + if name == "default_gpu_allocator": + if CUPY_DEVICE is core_defs.DeviceType.CUDA: + return device_allocators[core_defs.DeviceType.CUDA] + elif CUPY_DEVICE is core_defs.DeviceType.ROCM: + return device_allocators[core_defs.DeviceType.ROCM] + # else: fall through + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index cf5f16c4ed..c193a05057 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -19,8 +19,6 @@ import pytest -from gt4py.next import allocators as next_allocators - # Skip definitions XFAIL = pytest.xfail @@ -46,11 +44,6 @@ def short_id(self, num_components: int = 2) -> str: return ".".join(self.value.split(".")[-num_components:]) -class _PythonObjectIdMixinForAllocator(_PythonObjectIdMixin): - def short_id(self, num_components: int = 1) -> str: - return "None-" + super().short_id(num_components) - - class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): GTFN_CPU = "gt4py.next.program_processors.runners.gtfn.run_gtfn" GTFN_CPU_IMPERATIVE = "gt4py.next.program_processors.runners.gtfn.run_gtfn_imperative" @@ -62,13 +55,9 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): DOUBLE_ROUNDTRIP = "gt4py.next.program_processors.runners.double_roundtrip.backend" -cpu_allocator = next_allocators.StandardCPUFieldBufferAllocator() -gpu_allocator = next_allocators.StandardGPUFieldBufferAllocator() - - -class AllocatorId(_PythonObjectIdMixinForAllocator, str, enum.Enum): - CPU_ALLOCATOR = "next_tests.definitions.cpu_allocator" - GPU_ALLOCATOR = "next_tests.definitions.gpu_allocator" +class AllocatorId(_PythonObjectIdMixin, str, enum.Enum): + CPU_ALLOCATOR = "gt4py.next.allocators.default_cpu_allocator" + GPU_ALLOCATOR = "gt4py.next.allocators.default_gpu_allocator" class OptionalProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index 1975d96b09..d5a28fcca3 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -482,18 +482,18 @@ def verify_with_default_data( @pytest.fixture def cartesian_case(fieldview_backend): # noqa: F811 # fixtures yield Case( - fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, + fieldview_backend.executor, offset_provider={"Ioff": IDim, "Joff": JDim, "Koff": KDim}, default_sizes={IDim: 10, JDim: 10, KDim: 10}, grid_type=common.GridType.CARTESIAN, - allocator=fieldview_backend, + allocator=fieldview_backend.allocator, ) @pytest.fixture def unstructured_case(reduction_setup, fieldview_backend): # noqa: F811 # fixtures yield Case( - fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, + fieldview_backend.executor, offset_provider=reduction_setup.offset_provider, default_sizes={ Vertex: reduction_setup.num_vertices, @@ -502,7 +502,7 @@ def unstructured_case(reduction_setup, fieldview_backend): # noqa: F811 # fixtu KDim: reduction_setup.k_levels, }, grid_type=common.GridType.UNSTRUCTURED, - allocator=fieldview_backend, + allocator=fieldview_backend.allocator, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index 00be56ab91..1612cd8671 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -14,7 +14,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from collections import namedtuple -from typing import Any, TypeVar +from typing import Any, Optional, TypeVar import numpy as np import pytest @@ -52,6 +52,23 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non ) ), +_ExecutorAndAllocator = namedtuple("_ExecutorAndAllocator", ["executor", "allocator"]) + + +def _backend_name( + p: next_tests.definitions._PythonObjectIdMixin + | tuple[ + Optional[next_tests.definitions._PythonObjectIdMixin], + next_tests.definitions._PythonObjectIdMixin, + ] +): + if isinstance(p, tuple): + executor = p[0].short_id() if p[0] is not None else "None" + allocator = p[1].short_id() + return f"{executor}-{allocator}" + else: + return p.short_id() + @pytest.fixture( params=[ @@ -63,13 +80,13 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non next_tests.definitions.ProgramBackendId.GTFN_GPU, marks=pytest.mark.requires_gpu ), # will use the default (embedded) execution, but input/output allocated with the provided allocator - next_tests.definitions.AllocatorId.CPU_ALLOCATOR, + (None, next_tests.definitions.AllocatorId.CPU_ALLOCATOR), pytest.param( - next_tests.definitions.AllocatorId.GPU_ALLOCATOR, marks=pytest.mark.requires_gpu + (None, next_tests.definitions.AllocatorId.GPU_ALLOCATOR), marks=pytest.mark.requires_gpu ), ] + OPTIONAL_PROCESSORS, - ids=lambda p: p.short_id() if p is not None else "None", + ids=_backend_name, ) def fieldview_backend(request): """ @@ -79,17 +96,27 @@ def fieldview_backend(request): Check ADR 15 for details on the test-exclusion matrices. """ backend_or_allocator_id = request.param - backend_or_allocator = backend_or_allocator_id.load() + if isinstance(backend_or_allocator_id, tuple): + backend_id, allocator_id = backend_or_allocator_id + assert backend_id is None # revisit if we have need for backend_id != None + executor = None + allocator = allocator_id.load() + exclusion_matrix_id = backend_or_allocator_id[1] + else: + backend = backend_or_allocator_id.load() + executor = backend.executor + allocator = backend + exclusion_matrix_id = backend_or_allocator_id for marker, skip_mark, msg in next_tests.definitions.BACKEND_SKIP_TEST_MATRIX.get( - backend_or_allocator_id, [] + exclusion_matrix_id, [] ): if request.node.get_closest_marker(marker): - skip_mark(msg.format(marker=marker, backend=backend_or_allocator_id)) + skip_mark(msg.format(marker=marker, backend=_backend_name(backend_or_allocator_id))) backup_backend = decorator.DEFAULT_BACKEND decorator.DEFAULT_BACKEND = no_backend - yield backend_or_allocator + yield _ExecutorAndAllocator(executor, allocator) decorator.DEFAULT_BACKEND = backup_backend diff --git a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py index 530233dc0e..89854fe62e 100644 --- a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py +++ b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py @@ -24,6 +24,7 @@ cartesian_case, fieldview_backend, ) +from next_tests.integration_tests.feature_tests.ffront_tests import ffront_test_utils @gtx.field_operator @@ -70,7 +71,13 @@ def test_allocate_const(cartesian_case): # noqa: F811 # fixtures assert b == 42.0 -@pytest.mark.parametrize("fieldview_backend", [~definitions.ProgramBackendId.ROUNDTRIP]) +_roundtrip_executor_and_allocator = ffront_test_utils._ExecutorAndAllocator( + definitions.ProgramBackendId.ROUNDTRIP.load().executor, + definitions.ProgramBackendId.ROUNDTRIP.load(), +) + + +@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a")() b = cases.allocate(cartesian_case, addition, "b")() @@ -81,7 +88,7 @@ def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixt cases.verify(cartesian_case, addition, a, b, out=out, ref=wrong_ref) -@pytest.mark.parametrize("fieldview_backend", [~definitions.ProgramBackendId.ROUNDTRIP]) +@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a").dtype(np.float32)() b = cases.allocate(cartesian_case, addition, "b")() @@ -91,7 +98,7 @@ def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures cases.verify(cartesian_case, addition, a, b, out=out, ref=a + b) -@pytest.mark.parametrize("fieldview_backend", [~definitions.ProgramBackendId.ROUNDTRIP]) +@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) def test_verify_with_default_data_fails_with_wrong_reference( cartesian_case, # noqa: F811 # fixtures ): diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index 39e6609879..dbb4ad7782 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -196,11 +196,11 @@ def reference( @pytest.fixture def test_setup(fieldview_backend): test_case = cases.Case( - fieldview_backend if isinstance(fieldview_backend, ppi.ProgramExecutor) else None, + fieldview_backend.executor, offset_provider={"Koff": KDim}, default_sizes={Cell: 14, KDim: 10}, grid_type=common.GridType.UNSTRUCTURED, - allocator=fieldview_backend, + allocator=fieldview_backend.allocator, ) @dataclasses.dataclass(frozen=True) From a29c9cace2968bd0191afd44c6e219d53087e6e6 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Thu, 25 Jan 2024 23:05:04 +0100 Subject: [PATCH 09/19] refactor weirdness in skip list for embedded --- src/gt4py/next/common.py | 2 + tests/next_tests/definitions.py | 25 +++++++---- .../ffront_tests/ffront_test_utils.py | 42 ++++++------------- .../ffront_tests/test_embedded_regression.py | 2 +- 4 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index 840b0e8bbc..62ba4a7967 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -851,6 +851,8 @@ def _field( domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, ) -> Field: + # Utility function to construct a `Field` from different buffer representations. + # Consider removing this function and using `Field` constructor directly. raise NotImplementedError diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index c193a05057..e8c3460ccf 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -19,6 +19,8 @@ import pytest +from gt4py.next import allocators as next_allocators + # Skip definitions XFAIL = pytest.xfail @@ -55,9 +57,13 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): DOUBLE_ROUNDTRIP = "gt4py.next.program_processors.runners.double_roundtrip.backend" -class AllocatorId(_PythonObjectIdMixin, str, enum.Enum): - CPU_ALLOCATOR = "gt4py.next.allocators.default_cpu_allocator" - GPU_ALLOCATOR = "gt4py.next.allocators.default_gpu_allocator" +numpy_execution = (None, next_allocators.default_cpu_allocator) +cupy_execution = (None, next_allocators.default_gpu_allocator) + + +class EmbeddedIds(_PythonObjectIdMixin, str, enum.Enum): + NUMPY_EXECUTION = "next_tests.definitions.numpy_execution" + CUPY_EXECUTION = "next_tests.definitions.cupy_execution" class OptionalProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): @@ -140,6 +146,11 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): EMBEDDED_SKIP_LIST = [ (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE), (CHECKS_SPECIFIC_ERROR, XFAIL, UNSUPPORTED_MESSAGE), + ( + USES_SCAN_WITHOUT_FIELD_ARGS, + XFAIL, + UNSUPPORTED_MESSAGE, + ), # we can't extract the field type from scan args ] GTFN_SKIP_TEST_LIST = COMMON_SKIP_TEST_LIST + [ # floordiv not yet supported, see https://github.com/GridTools/gt4py/issues/1136 @@ -150,12 +161,8 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): #: Skip matrix, contains for each backend processor a list of tuples with following fields: #: (, ) BACKEND_SKIP_TEST_MATRIX = { - AllocatorId.CPU_ALLOCATOR: EMBEDDED_SKIP_LIST, - AllocatorId.GPU_ALLOCATOR: EMBEDDED_SKIP_LIST - + [ - # we can't extract the type of the output field - (USES_SCAN_WITHOUT_FIELD_ARGS, XFAIL, UNSUPPORTED_MESSAGE) - ], + EmbeddedIds.NUMPY_EXECUTION: EMBEDDED_SKIP_LIST, + EmbeddedIds.CUPY_EXECUTION: EMBEDDED_SKIP_LIST, OptionalProgramBackendId.DACE_CPU: DACE_SKIP_TEST_LIST, OptionalProgramBackendId.DACE_GPU: DACE_SKIP_TEST_LIST + [ diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index 1612cd8671..9649f38c3e 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -55,21 +55,6 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non _ExecutorAndAllocator = namedtuple("_ExecutorAndAllocator", ["executor", "allocator"]) -def _backend_name( - p: next_tests.definitions._PythonObjectIdMixin - | tuple[ - Optional[next_tests.definitions._PythonObjectIdMixin], - next_tests.definitions._PythonObjectIdMixin, - ] -): - if isinstance(p, tuple): - executor = p[0].short_id() if p[0] is not None else "None" - allocator = p[1].short_id() - return f"{executor}-{allocator}" - else: - return p.short_id() - - @pytest.fixture( params=[ next_tests.definitions.ProgramBackendId.ROUNDTRIP, @@ -80,13 +65,13 @@ def _backend_name( next_tests.definitions.ProgramBackendId.GTFN_GPU, marks=pytest.mark.requires_gpu ), # will use the default (embedded) execution, but input/output allocated with the provided allocator - (None, next_tests.definitions.AllocatorId.CPU_ALLOCATOR), + next_tests.definitions.EmbeddedIds.NUMPY_EXECUTION, pytest.param( - (None, next_tests.definitions.AllocatorId.GPU_ALLOCATOR), marks=pytest.mark.requires_gpu + next_tests.definitions.EmbeddedIds.CUPY_EXECUTION, marks=pytest.mark.requires_gpu ), ] + OPTIONAL_PROCESSORS, - ids=_backend_name, + ids=lambda p: p.short_id(), ) def fieldview_backend(request): """ @@ -95,24 +80,21 @@ def fieldview_backend(request): Notes: Check ADR 15 for details on the test-exclusion matrices. """ - backend_or_allocator_id = request.param - if isinstance(backend_or_allocator_id, tuple): - backend_id, allocator_id = backend_or_allocator_id - assert backend_id is None # revisit if we have need for backend_id != None - executor = None - allocator = allocator_id.load() - exclusion_matrix_id = backend_or_allocator_id[1] + backend_id = request.param + backend = backend_id.load() + # we support ppi.ProgramBackends and a tuple of executor and allocator + # the use case for the latter is embedded with executor == None + if isinstance(backend, tuple): + executor, allocator = backend else: - backend = backend_or_allocator_id.load() - executor = backend.executor + executor = backend allocator = backend - exclusion_matrix_id = backend_or_allocator_id for marker, skip_mark, msg in next_tests.definitions.BACKEND_SKIP_TEST_MATRIX.get( - exclusion_matrix_id, [] + backend_id, [] ): if request.node.get_closest_marker(marker): - skip_mark(msg.format(marker=marker, backend=_backend_name(backend_or_allocator_id))) + skip_mark(msg.format(marker=marker, backend=backend_id)) backup_backend = decorator.DEFAULT_BACKEND decorator.DEFAULT_BACKEND = no_backend diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py index ba4b1b0cdb..6a8f30bdb4 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py @@ -51,7 +51,7 @@ def test_default_backend_is_respected_scan_operator(cartesian_case): # noqa: F8 def sum(state: float, a: float) -> float: return state + a - a = gtx.ones({KDim: 10}, allocator=cartesian_case.backend) + a = gtx.ones({KDim: 10}, allocator=cartesian_case.allocator) with pytest.raises(ValueError, match="No backend selected!"): # see comment in field_operator test From 034016b222ec9a9b3f09b8d1a4654ceec73adf3e Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 09:36:32 +0100 Subject: [PATCH 10/19] fix import --- tests/next_tests/definitions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index e8c3460ccf..639af94550 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -58,7 +58,8 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): numpy_execution = (None, next_allocators.default_cpu_allocator) -cupy_execution = (None, next_allocators.default_gpu_allocator) +if hasattr(next_allocators, "default_gpu_allocator"): + cupy_execution = (None, next_allocators.default_gpu_allocator) class EmbeddedIds(_PythonObjectIdMixin, str, enum.Enum): From 2f63bace9a7b262990e08686bd93515d06f96611 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 09:42:06 +0100 Subject: [PATCH 11/19] fix testcase --- .../feature_tests/ffront_tests/test_temporaries_with_sizes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py index 788081b81e..e57da01175 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py @@ -86,6 +86,7 @@ def test_verification(testee, run_gtfn_with_temporaries_and_symbolic_sizes, redu KDim: reduction_setup.k_levels, }, grid_type=common.GridType.UNSTRUCTURED, + allocator=run_gtfn_with_temporaries_and_symbolic_sizes, ) a = cases.allocate(unstructured_case, testee, "a")() From 2d7b0cf4497c834f440e0b99c50b425e1a5cc89a Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 11:52:45 +0100 Subject: [PATCH 12/19] implement execution and allocator descriptor for tests --- tests/next_tests/definitions.py | 28 ++++++++++++++++--- tests/next_tests/integration_tests/cases.py | 10 +++++-- .../ffront_tests/ffront_test_utils.py | 11 +------- .../test_temporaries_with_sizes.py | 4 +-- .../feature_tests/test_util_cases.py | 13 ++------- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index 639af94550..e18159ab5d 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -14,12 +14,16 @@ """Contains definition of test-exclusion matrices, see ADR 15.""" +import dataclasses import enum import importlib +from types import NoneType +from typing import Optional, Protocol import pytest from gt4py.next import allocators as next_allocators +from gt4py.next.program_processors import processor_interface as ppi # Skip definitions @@ -40,8 +44,6 @@ def load(self) -> object: obj = eval(f"_m.{obj}", globs) return obj - __invert__ = load - def short_id(self, num_components: int = 2) -> str: return ".".join(self.value.split(".")[-num_components:]) @@ -57,9 +59,27 @@ class ProgramBackendId(_PythonObjectIdMixin, str, enum.Enum): DOUBLE_ROUNDTRIP = "gt4py.next.program_processors.runners.double_roundtrip.backend" -numpy_execution = (None, next_allocators.default_cpu_allocator) +class ExecutionAndAllocatorDescriptor(Protocol): + # Used for test infrastructure, consider implementing this in gt4py when refactoring otf + @property + def executor(self) -> Optional[ppi.ProgramExecutor]: + ... + + @property + def allocator(self) -> next_allocators.FieldBufferAllocatorProtocol: + ... + + +@dataclasses.dataclass # (frozen=True) +class EmbeddedExecutionDescriptor: + allocator: next_allocators.FieldBufferAllocatorProtocol + executor: NoneType = dataclasses.field(default=None, init=False) + + +numpy_execution = EmbeddedExecutionDescriptor(allocator=next_allocators.default_cpu_allocator) + if hasattr(next_allocators, "default_gpu_allocator"): - cupy_execution = (None, next_allocators.default_gpu_allocator) + cupy_execution = EmbeddedExecutionDescriptor(next_allocators.default_gpu_allocator) class EmbeddedIds(_PythonObjectIdMixin, str, enum.Enum): diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index d5a28fcca3..a529298259 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -33,6 +33,7 @@ from gt4py.next.program_processors import processor_interface as ppi from gt4py.next.type_system import type_specifications as ts, type_translation +from next_tests import definitions as test_definitions from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( # noqa: F401 # fixture and aliases Cell, Edge, @@ -480,7 +481,9 @@ def verify_with_default_data( @pytest.fixture -def cartesian_case(fieldview_backend): # noqa: F811 # fixtures +def cartesian_case( + fieldview_backend: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures +): yield Case( fieldview_backend.executor, offset_provider={"Ioff": IDim, "Joff": JDim, "Koff": KDim}, @@ -491,7 +494,10 @@ def cartesian_case(fieldview_backend): # noqa: F811 # fixtures @pytest.fixture -def unstructured_case(reduction_setup, fieldview_backend): # noqa: F811 # fixtures +def unstructured_case( + reduction_setup, # noqa: F811 # fixtures + fieldview_backend: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures +): yield Case( fieldview_backend.executor, offset_provider=reduction_setup.offset_provider, diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index 9649f38c3e..4a02af1ede 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -52,8 +52,6 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non ) ), -_ExecutorAndAllocator = namedtuple("_ExecutorAndAllocator", ["executor", "allocator"]) - @pytest.fixture( params=[ @@ -82,13 +80,6 @@ def fieldview_backend(request): """ backend_id = request.param backend = backend_id.load() - # we support ppi.ProgramBackends and a tuple of executor and allocator - # the use case for the latter is embedded with executor == None - if isinstance(backend, tuple): - executor, allocator = backend - else: - executor = backend - allocator = backend for marker, skip_mark, msg in next_tests.definitions.BACKEND_SKIP_TEST_MATRIX.get( backend_id, [] @@ -98,7 +89,7 @@ def fieldview_backend(request): backup_backend = decorator.DEFAULT_BACKEND decorator.DEFAULT_BACKEND = no_backend - yield _ExecutorAndAllocator(executor, allocator) + yield backend decorator.DEFAULT_BACKEND = backup_backend diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py index e57da01175..c4cdd8a4be 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_temporaries_with_sizes.py @@ -77,7 +77,7 @@ def prog( def test_verification(testee, run_gtfn_with_temporaries_and_symbolic_sizes, reduction_setup): unstructured_case = Case( - run_gtfn_with_temporaries_and_symbolic_sizes, + run_gtfn_with_temporaries_and_symbolic_sizes.executor, offset_provider=reduction_setup.offset_provider, default_sizes={ Vertex: reduction_setup.num_vertices, @@ -86,7 +86,7 @@ def test_verification(testee, run_gtfn_with_temporaries_and_symbolic_sizes, redu KDim: reduction_setup.k_levels, }, grid_type=common.GridType.UNSTRUCTURED, - allocator=run_gtfn_with_temporaries_and_symbolic_sizes, + allocator=run_gtfn_with_temporaries_and_symbolic_sizes.allocator, ) a = cases.allocate(unstructured_case, testee, "a")() diff --git a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py index 89854fe62e..9ea1cdeb80 100644 --- a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py +++ b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py @@ -24,7 +24,6 @@ cartesian_case, fieldview_backend, ) -from next_tests.integration_tests.feature_tests.ffront_tests import ffront_test_utils @gtx.field_operator @@ -71,13 +70,7 @@ def test_allocate_const(cartesian_case): # noqa: F811 # fixtures assert b == 42.0 -_roundtrip_executor_and_allocator = ffront_test_utils._ExecutorAndAllocator( - definitions.ProgramBackendId.ROUNDTRIP.load().executor, - definitions.ProgramBackendId.ROUNDTRIP.load(), -) - - -@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) +@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a")() b = cases.allocate(cartesian_case, addition, "b")() @@ -88,7 +81,7 @@ def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixt cases.verify(cartesian_case, addition, a, b, out=out, ref=wrong_ref) -@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) +@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a").dtype(np.float32)() b = cases.allocate(cartesian_case, addition, "b")() @@ -98,7 +91,7 @@ def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures cases.verify(cartesian_case, addition, a, b, out=out, ref=a + b) -@pytest.mark.parametrize("fieldview_backend", [_roundtrip_executor_and_allocator]) +@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_with_default_data_fails_with_wrong_reference( cartesian_case, # noqa: F811 # fixtures ): From 845810f7ecd6ed8855796825bf76c5fa8ff8f2be Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 13:57:01 +0100 Subject: [PATCH 13/19] cleanups --- pyproject.toml | 3 ++ src/gt4py/next/allocators.py | 12 ------- tests/next_tests/definitions.py | 23 +++++++++----- .../ffront_tests/test_execution.py | 17 ++-------- .../ffront_tests/test_gt4py_builtins.py | 9 +----- .../ffront_tests/test_icon_like_scan.py | 31 ++++++++++--------- 6 files changed, 37 insertions(+), 58 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87934b11fb..51cfc267d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -343,6 +343,8 @@ markers = [ 'uses_scan: tests that uses scan', 'uses_scan_in_field_operator: tests that require backend support for scan in field operator', 'uses_scan_without_field_args: tests that require calls to scan that do not have any fields as arguments', + 'uses_scan_nested: tests that use nested scans', + 'uses_scan_requiring_projector: tests need a projector implementation in gtfn', 'uses_sparse_fields: tests that require backend support for sparse fields', 'uses_sparse_fields_as_output: tests that require backend support for writing sparse fields', 'uses_strided_neighbor_offset: tests that require backend support for strided neighbor offset', @@ -351,6 +353,7 @@ markers = [ 'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields', 'uses_cartesian_shift: tests that use a Cartesian connectivity', 'uses_unstructured_shift: tests that use a unstructured connectivity', + 'uses_max_over: tests that use the max_over builtin', 'checks_specific_error: tests that rely on the backend to produce a specific error message' ] norecursedirs = ['dist', 'build', 'cpp_backend_tests/build*', '_local/*', '.*'] diff --git a/src/gt4py/next/allocators.py b/src/gt4py/next/allocators.py index 3388a29cd6..44203bf6d8 100644 --- a/src/gt4py/next/allocators.py +++ b/src/gt4py/next/allocators.py @@ -350,15 +350,3 @@ def allocate( device_id=device.device_id, aligned_index=aligned_index, ) - - -def __getattr__(name: str) -> Any: - if name == "default_cpu_allocator": - return device_allocators[core_defs.DeviceType.CPU] - if name == "default_gpu_allocator": - if CUPY_DEVICE is core_defs.DeviceType.CUDA: - return device_allocators[core_defs.DeviceType.CUDA] - elif CUPY_DEVICE is core_defs.DeviceType.ROCM: - return device_allocators[core_defs.DeviceType.ROCM] - # else: fall through - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index e18159ab5d..fd6d439850 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -70,16 +70,14 @@ def allocator(self) -> next_allocators.FieldBufferAllocatorProtocol: ... -@dataclasses.dataclass # (frozen=True) +@dataclasses.dataclass(frozen=True) class EmbeddedExecutionDescriptor: allocator: next_allocators.FieldBufferAllocatorProtocol executor: NoneType = dataclasses.field(default=None, init=False) -numpy_execution = EmbeddedExecutionDescriptor(allocator=next_allocators.default_cpu_allocator) - -if hasattr(next_allocators, "default_gpu_allocator"): - cupy_execution = EmbeddedExecutionDescriptor(next_allocators.default_gpu_allocator) +numpy_execution = EmbeddedExecutionDescriptor(next_allocators.StandardCPUFieldBufferAllocator()) +cupy_execution = EmbeddedExecutionDescriptor(next_allocators.StandardGPUFieldBufferAllocator()) class EmbeddedIds(_PythonObjectIdMixin, str, enum.Enum): @@ -128,6 +126,8 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): USES_SCAN = "uses_scan" USES_SCAN_IN_FIELD_OPERATOR = "uses_scan_in_field_operator" USES_SCAN_WITHOUT_FIELD_ARGS = "uses_scan_without_field_args" +USES_SCAN_NESTED = "uses_scan_nested" +USES_SCAN_REQUIRING_PROJECTOR = "uses_scan_requiring_projector" USES_SPARSE_FIELDS = "uses_sparse_fields" USES_SPARSE_FIELDS_AS_OUTPUT = "uses_sparse_fields_as_output" USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS = "uses_reduction_with_only_sparse_fields" @@ -137,6 +137,7 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields" USES_CARTESIAN_SHIFT = "uses_cartesian_shift" USES_UNSTRUCTURED_SHIFT = "uses_unstructured_shift" +USES_MAX_OVER = "uses_max_over" CHECKS_SPECIFIC_ERROR = "checks_specific_error" # Skip messages (available format keys: 'marker', 'backend') @@ -177,6 +178,9 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): # floordiv not yet supported, see https://github.com/GridTools/gt4py/issues/1136 (USES_FLOORDIV, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE), (USES_STRIDED_NEIGHBOR_OFFSET, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE), + # max_over broken, see https://github.com/GridTools/gt4py/issues/1289 + (USES_MAX_OVER, XFAIL, UNSUPPORTED_MESSAGE), + (USES_SCAN_REQUIRING_PROJECTOR, XFAIL, UNSUPPORTED_MESSAGE), ] #: Skip matrix, contains for each backend processor a list of tuples with following fields: @@ -190,9 +194,12 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): # awaiting dace fix, see https://github.com/spcl/dace/pull/1442 (USES_FLOORDIV, XFAIL, BINDINGS_UNSUPPORTED_MESSAGE), ], - ProgramBackendId.GTFN_CPU: GTFN_SKIP_TEST_LIST, - ProgramBackendId.GTFN_CPU_IMPERATIVE: GTFN_SKIP_TEST_LIST, - ProgramBackendId.GTFN_GPU: GTFN_SKIP_TEST_LIST, + ProgramBackendId.GTFN_CPU: GTFN_SKIP_TEST_LIST + + [(USES_SCAN_NESTED, XFAIL, UNSUPPORTED_MESSAGE)], + ProgramBackendId.GTFN_CPU_IMPERATIVE: GTFN_SKIP_TEST_LIST + + [(USES_SCAN_NESTED, XFAIL, UNSUPPORTED_MESSAGE)], + ProgramBackendId.GTFN_GPU: GTFN_SKIP_TEST_LIST + + [(USES_SCAN_NESTED, XFAIL, UNSUPPORTED_MESSAGE)], ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES: GTFN_SKIP_TEST_LIST + [ (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE), diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py index ca2e7c2932..bbac04ac20 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py @@ -612,14 +612,8 @@ def simple_scan_operator(carry: float) -> float: @pytest.mark.uses_scan @pytest.mark.uses_lift_expressions +@pytest.mark.uses_scan_nested def test_solve_triag(cartesian_case): - if cartesian_case.backend in [ - gtfn.run_gtfn, - gtfn.run_gtfn_gpu, - gtfn.run_gtfn_imperative, - gtfn.run_gtfn_with_temporaries, - ]: - pytest.xfail("Nested `scan`s requires creating temporaries.") if cartesian_case.backend == gtfn.run_gtfn_with_temporaries: pytest.xfail("Temporary extraction does not work correctly in combination with scans.") @@ -918,15 +912,8 @@ def program_domain(a: cases.IField, out: cases.IField): cases.verify(cartesian_case, program_domain, a, out, inout=out, ref=ref) +@pytest.mark.uses_floordiv def test_domain_input_bounds(cartesian_case): - if cartesian_case.backend in [ - gtfn.run_gtfn, - gtfn.run_gtfn_gpu, - gtfn.run_gtfn_imperative, - gtfn.run_gtfn_with_temporaries, - ]: - pytest.xfail("FloorDiv not fully supported in gtfn.") - lower_i = 1 upper_i = 10 diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py index e2434d860a..ccd0c872f8 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py @@ -40,20 +40,13 @@ @pytest.mark.uses_unstructured_shift +@pytest.mark.uses_max_over @pytest.mark.parametrize( "strategy", [cases.UniqueInitializer(1), cases.UniqueInitializer(-100)], ids=["positive_values", "negative_values"], ) def test_maxover_execution_(unstructured_case, strategy): - if unstructured_case.backend in [ - gtfn.run_gtfn, - gtfn.run_gtfn_gpu, - gtfn.run_gtfn_imperative, - gtfn.run_gtfn_with_temporaries, - ]: - pytest.xfail("`maxover` broken in gtfn, see #1289.") - @gtx.field_operator def testee(edge_f: cases.EField) -> cases.VField: out = max_over(edge_f(V2E), axis=V2EDim) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index dbb4ad7782..8b68eff189 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -19,9 +19,8 @@ import gt4py.next as gtx from gt4py.next import common -from gt4py.next.program_processors import processor_interface as ppi -from gt4py.next.program_processors.runners import gtfn, roundtrip +from next_tests import definitions as test_definitions from next_tests.integration_tests import cases from next_tests.integration_tests.cases import Cell, KDim, Koff from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( @@ -228,15 +227,8 @@ class setup: @pytest.mark.uses_tuple_returns +@pytest.mark.uses_scan_requiring_projector def test_solve_nonhydro_stencil_52_like_z_q(test_setup): - if test_setup.case.backend in [ - gtfn.run_gtfn, - gtfn.run_gtfn_gpu, - gtfn.run_gtfn_imperative, - gtfn.run_gtfn_with_temporaries, - ]: - pytest.xfail("Needs implementation of scan projector.") - cases.verify( test_setup.case, solve_nonhydro_stencil_52_like_z_q, @@ -255,12 +247,15 @@ def test_solve_nonhydro_stencil_52_like_z_q(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like_z_q_tup(test_setup): - if test_setup.case.backend in [gtfn.run_gtfn_with_temporaries]: + if ( + test_setup.case.backend + == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor + ): pytest.xfail( "Needs implementation of scan projector. Breaks in type inference as executed" "again after CollapseTuple." ) - if test_setup.case.backend == roundtrip.backend: + if test_setup.case.backend == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: pytest.xfail("Needs proper handling of tuple[Column] <-> Column[tuple].") cases.verify( @@ -279,7 +274,10 @@ def test_solve_nonhydro_stencil_52_like_z_q_tup(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like(test_setup): - if test_setup.case.backend in [gtfn.run_gtfn_with_temporaries]: + if ( + test_setup.case.backend + == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor + ): pytest.xfail("Temporary extraction does not work correctly in combination with scans.") cases.run( @@ -298,9 +296,12 @@ def test_solve_nonhydro_stencil_52_like(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like_with_gtfn_tuple_merge(test_setup): - if test_setup.case.backend in [gtfn.run_gtfn_with_temporaries]: + if ( + test_setup.case.backend + == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor + ): pytest.xfail("Temporary extraction does not work correctly in combination with scans.") - if test_setup.case.backend == roundtrip.backend: + if test_setup.case.backend == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: pytest.xfail("Needs proper handling of tuple[Column] <-> Column[tuple].") cases.run( From 8aba24da99251ef1bd2b598b4c95d38365aa88cf Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 14:00:44 +0100 Subject: [PATCH 14/19] renaming --- .../ADRs/0015-Test_Exclusion_Matrices.md | 2 +- tests/next_tests/integration_tests/cases.py | 14 +++++++------- .../ffront_tests/ffront_test_utils.py | 4 ++-- .../ffront_tests/test_arg_call_interface.py | 2 +- .../feature_tests/ffront_tests/test_bound_args.py | 2 +- .../feature_tests/ffront_tests/test_execution.py | 2 +- .../ffront_tests/test_external_local_field.py | 2 +- .../ffront_tests/test_gt4py_builtins.py | 2 +- .../ffront_tests/test_math_builtin_execution.py | 2 +- .../ffront_tests/test_math_unary_builtins.py | 2 +- .../feature_tests/ffront_tests/test_program.py | 8 +++++++- .../feature_tests/ffront_tests/test_scalar_if.py | 2 +- .../feature_tests/test_util_cases.py | 8 ++++---- .../ffront_tests/test_embedded_regression.py | 4 ++-- .../ffront_tests/test_icon_like_scan.py | 8 ++++---- .../ffront_tests/test_laplacian.py | 2 +- 16 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md index 7f55604774..c868757905 100644 --- a/docs/development/ADRs/0015-Test_Exclusion_Matrices.md +++ b/docs/development/ADRs/0015-Test_Exclusion_Matrices.md @@ -22,7 +22,7 @@ the supported backends, while keeping the test code clean. ## Decision It was decided to apply fixtures and markers from `pytest` module. The fixture is the same used to execute the test -on different backends (`fieldview_backend` and `program_processor`), but it is extended with a check on the available feature markers. +on different backends (`exec_alloc_descriptor` and `program_processor`), but it is extended with a check on the available feature markers. If a test is annotated with a feature marker, the fixture will check if this feature is supported on the selected backend. If no marker is specified, the test is supposed to run on all backends. diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index a529298259..4abfcc0727 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -44,7 +44,7 @@ KDim, Koff, Vertex, - fieldview_backend, + exec_alloc_descriptor, reduction_setup, ) @@ -482,24 +482,24 @@ def verify_with_default_data( @pytest.fixture def cartesian_case( - fieldview_backend: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures + exec_alloc_descriptor: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures ): yield Case( - fieldview_backend.executor, + exec_alloc_descriptor.executor, offset_provider={"Ioff": IDim, "Joff": JDim, "Koff": KDim}, default_sizes={IDim: 10, JDim: 10, KDim: 10}, grid_type=common.GridType.CARTESIAN, - allocator=fieldview_backend.allocator, + allocator=exec_alloc_descriptor.allocator, ) @pytest.fixture def unstructured_case( reduction_setup, # noqa: F811 # fixtures - fieldview_backend: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures + exec_alloc_descriptor: test_definitions.ExecutionAndAllocatorDescriptor, # noqa: F811 # fixtures ): yield Case( - fieldview_backend.executor, + exec_alloc_descriptor.executor, offset_provider=reduction_setup.offset_provider, default_sizes={ Vertex: reduction_setup.num_vertices, @@ -508,7 +508,7 @@ def unstructured_case( KDim: reduction_setup.k_levels, }, grid_type=common.GridType.UNSTRUCTURED, - allocator=fieldview_backend.allocator, + allocator=exec_alloc_descriptor.allocator, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index 4a02af1ede..e421763699 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -71,7 +71,7 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non + OPTIONAL_PROCESSORS, ids=lambda p: p.short_id(), ) -def fieldview_backend(request): +def exec_alloc_descriptor(request): """ Fixture creating field-view operator backend on-demand for tests. @@ -232,7 +232,7 @@ def reduction_setup(): __all__ = [ - "fieldview_backend", + "exec_alloc_descriptor", "reduction_setup", "debug_itir", "DimsType", diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py index b41696a36b..ddeb8f8ee6 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py @@ -26,7 +26,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import IDim, IField, IJKFloatField, KDim, cartesian_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py index 0de953d85f..355cff9774 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py @@ -21,7 +21,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import cartesian_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, reduction_setup, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py index bbac04ac20..7822b4385a 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py @@ -51,7 +51,7 @@ unstructured_case, ) from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, reduction_setup, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py index d100cd380c..bb1d878a6a 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py @@ -21,7 +21,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import V2E, Edge, V2EDim, Vertex, unstructured_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, reduction_setup, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py index ccd0c872f8..90d07f360d 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py @@ -34,7 +34,7 @@ unstructured_case, ) from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, reduction_setup, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py index 4444742c66..96200e4b0b 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py @@ -28,7 +28,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import IDim, cartesian_case, unstructured_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, ) from next_tests.integration_tests.feature_tests.math_builtin_test_data import math_builtin_test_data diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py index f5bf453a09..b21f29c9bc 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py @@ -41,7 +41,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import IDim, cartesian_case, unstructured_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py index 938c69fb52..073bbb9550 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py @@ -23,7 +23,13 @@ from gt4py.next import errors from next_tests.integration_tests import cases -from next_tests.integration_tests.cases import IDim, Ioff, JDim, cartesian_case, fieldview_backend +from next_tests.integration_tests.cases import ( + IDim, + Ioff, + JDim, + cartesian_case, + exec_alloc_descriptor, +) from next_tests.past_common_fixtures import ( copy_program_def, copy_restrict_program_def, diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_scalar_if.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_scalar_if.py index af06da3e29..834966c125 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_scalar_if.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_scalar_if.py @@ -40,7 +40,7 @@ ) from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( Cell, - fieldview_backend, + exec_alloc_descriptor, size, ) diff --git a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py index 9ea1cdeb80..59c72bbf3f 100644 --- a/tests/next_tests/integration_tests/feature_tests/test_util_cases.py +++ b/tests/next_tests/integration_tests/feature_tests/test_util_cases.py @@ -22,7 +22,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import ( # noqa: F401 # fixtures cartesian_case, - fieldview_backend, + exec_alloc_descriptor, ) @@ -70,7 +70,7 @@ def test_allocate_const(cartesian_case): # noqa: F811 # fixtures assert b == 42.0 -@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) +@pytest.mark.parametrize("exec_alloc_descriptor", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a")() b = cases.allocate(cartesian_case, addition, "b")() @@ -81,7 +81,7 @@ def test_verify_fails_with_wrong_reference(cartesian_case): # noqa: F811 # fixt cases.verify(cartesian_case, addition, a, b, out=out, ref=wrong_ref) -@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) +@pytest.mark.parametrize("exec_alloc_descriptor", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures a = cases.allocate(cartesian_case, addition, "a").dtype(np.float32)() b = cases.allocate(cartesian_case, addition, "b")() @@ -91,7 +91,7 @@ def test_verify_fails_with_wrong_type(cartesian_case): # noqa: F811 # fixtures cases.verify(cartesian_case, addition, a, b, out=out, ref=a + b) -@pytest.mark.parametrize("fieldview_backend", [definitions.ProgramBackendId.ROUNDTRIP.load()]) +@pytest.mark.parametrize("exec_alloc_descriptor", [definitions.ProgramBackendId.ROUNDTRIP.load()]) def test_verify_with_default_data_fails_with_wrong_reference( cartesian_case, # noqa: F811 # fixtures ): diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py index 6a8f30bdb4..f4b1cf7bd3 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py @@ -22,7 +22,7 @@ from next_tests.integration_tests.cases import IField, cartesian_case # noqa: F401 # fixtures from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( # noqa: F401 # fixtures KDim, - fieldview_backend, + exec_alloc_descriptor, ) @@ -38,7 +38,7 @@ def copy(a: IField) -> IField: with pytest.raises(ValueError, match="No backend selected!"): # Calling this should fail if the default backend is respected - # due to `fieldview_backend` fixture (dependency of `cartesian_case`) + # due to `exec_alloc_descriptor` fixture (dependency of `cartesian_case`) # setting the default backend to something invalid. _ = copy(a, out=a, offset_provider={}) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index 8b68eff189..8cdb037c08 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -24,7 +24,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import Cell, KDim, Koff from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, ) @@ -193,13 +193,13 @@ def reference( @pytest.fixture -def test_setup(fieldview_backend): +def test_setup(exec_alloc_descriptor): test_case = cases.Case( - fieldview_backend.executor, + exec_alloc_descriptor.executor, offset_provider={"Koff": KDim}, default_sizes={Cell: 14, KDim: 10}, grid_type=common.GridType.UNSTRUCTURED, - allocator=fieldview_backend.allocator, + allocator=exec_alloc_descriptor.allocator, ) @dataclasses.dataclass(frozen=True) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py index 4f4d4969a9..6784857211 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py @@ -20,7 +20,7 @@ from next_tests.integration_tests import cases from next_tests.integration_tests.cases import IDim, Ioff, JDim, Joff, cartesian_case from next_tests.integration_tests.feature_tests.ffront_tests.ffront_test_utils import ( - fieldview_backend, + exec_alloc_descriptor, ) From c907349cfd87a6286284eb243ccfbcc04eb968a6 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 14:09:41 +0100 Subject: [PATCH 15/19] missing file --- tests/next_tests/integration_tests/cases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/next_tests/integration_tests/cases.py b/tests/next_tests/integration_tests/cases.py index 4abfcc0727..03a0a9f5a7 100644 --- a/tests/next_tests/integration_tests/cases.py +++ b/tests/next_tests/integration_tests/cases.py @@ -383,7 +383,7 @@ def run( """Run fieldview code in the context of a given test case.""" if kwargs.get("offset_provider", None) is None: kwargs["offset_provider"] = case.offset_provider - fieldview_prog.with_grid_type(case.grid_type).with_backend(case.backend)(*args, **kwargs) + fieldview_prog.with_grid_type(case.grid_type).with_backend(case.executor)(*args, **kwargs) def verify( @@ -609,7 +609,7 @@ def get_default_data( class Case: """Parametrizable components for single feature integration tests.""" - backend: Optional[ppi.ProgramProcessor] + executor: Optional[ppi.ProgramProcessor] offset_provider: dict[str, common.Connectivity | gtx.Dimension] default_sizes: dict[gtx.Dimension, int] grid_type: common.GridType From 8238e8dd7d6146d488afc41c2699bc1315935f60 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Fri, 26 Jan 2024 14:31:44 +0100 Subject: [PATCH 16/19] more renames --- .../ffront_tests/test_arg_call_interface.py | 4 ++-- .../feature_tests/ffront_tests/test_bound_args.py | 2 +- .../feature_tests/ffront_tests/test_execution.py | 14 +++++++------- .../ffront_tests/test_math_builtin_execution.py | 4 ++-- .../feature_tests/ffront_tests/test_program.py | 10 +++++----- .../ffront_tests/test_embedded_regression.py | 8 ++++---- .../ffront_tests/test_icon_like_scan.py | 10 +++++----- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py index ddeb8f8ee6..354323afeb 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py @@ -59,7 +59,7 @@ def testee(a: IField, b: IField, c: IField) -> IField: pos_args = [args[name] for name in arg_names] kw_args = {name: args[name] for name in kwarg_names} - testee.with_backend(cartesian_case.backend)( + testee.with_backend(cartesian_case.executor)( *pos_args, **kw_args, out=out, offset_provider=cartesian_case.offset_provider ) @@ -85,7 +85,7 @@ def testee(a: IField, b: IField, out: IField): pos_args = [args[name] for name in arg_names] kw_args = {name: args[name] for name in kwarg_names} - testee.with_backend(cartesian_case.backend)( + testee.with_backend(cartesian_case.executor)( *pos_args, **kw_args, offset_provider=cartesian_case.offset_provider ) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py index 355cff9774..e4baedc6ee 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_bound_args.py @@ -52,7 +52,7 @@ def fieldop_args(a: cases.IField, condition: bool, scalar: int32) -> cases.IFiel scalar = 0 if not condition else scalar return a + scalar - @gtx.program(backend=cartesian_case.backend) + @gtx.program(backend=cartesian_case.executor) def program_args(a: cases.IField, condition: bool, scalar: int32, out: cases.IField): fieldop_args(a, condition, scalar, out=out) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py index 7822b4385a..9482860d13 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py @@ -614,7 +614,7 @@ def simple_scan_operator(carry: float) -> float: @pytest.mark.uses_lift_expressions @pytest.mark.uses_scan_nested def test_solve_triag(cartesian_case): - if cartesian_case.backend == gtfn.run_gtfn_with_temporaries: + if cartesian_case.executor == gtfn.run_gtfn_with_temporaries: pytest.xfail("Temporary extraction does not work correctly in combination with scans.") @gtx.scan_operator(axis=KDim, forward=True, init=(0.0, 0.0)) @@ -716,7 +716,7 @@ def testee(a: cases.EField, b: cases.EField) -> cases.VField: @pytest.mark.uses_scan def test_ternary_scan(cartesian_case): - if cartesian_case.backend in [gtfn.run_gtfn_with_temporaries]: + if cartesian_case.executor in [gtfn.run_gtfn_with_temporaries]: pytest.xfail("Temporary extraction does not work correctly in combination with scans.") @gtx.scan_operator(axis=KDim, forward=True, init=0.0) @@ -741,7 +741,7 @@ def simple_scan_operator(carry: float, a: float) -> float: @pytest.mark.uses_scan_without_field_args @pytest.mark.uses_tuple_returns def test_scan_nested_tuple_output(forward, cartesian_case): - if cartesian_case.backend in [gtfn.run_gtfn_with_temporaries]: + if cartesian_case.executor in [gtfn.run_gtfn_with_temporaries]: pytest.xfail("Temporary extraction does not work correctly in combination with scans.") init = (1, (2, 3)) @@ -959,7 +959,7 @@ def test_domain_input_bounds_1(cartesian_case): def fieldop_domain(a: cases.IJField) -> cases.IJField: return a + a - @gtx.program(backend=cartesian_case.backend) + @gtx.program(backend=cartesian_case.executor) def program_domain( a: cases.IJField, out: cases.IJField, @@ -1060,7 +1060,7 @@ def prog(inp: cases.IKField, k_index: gtx.Field[[KDim], gtx.IndexType], out: cas def test_undefined_symbols(cartesian_case): with pytest.raises(errors.DSLError, match="Undeclared symbol"): - @gtx.field_operator(backend=cartesian_case.backend) + @gtx.field_operator(backend=cartesian_case.executor) def return_undefined(): return undefined_symbol @@ -1160,7 +1160,7 @@ def test_tuple_unpacking_too_many_values(cartesian_case): match=(r"Too many values to unpack \(expected 3\)."), ): - @gtx.field_operator(backend=cartesian_case.backend) + @gtx.field_operator(backend=cartesian_case.executor) def _star_unpack() -> tuple[int32, float64, int32]: a, b, c = (1, 2.0, 3, 4, 5, 6, 7.0) return a, b, c @@ -1171,7 +1171,7 @@ def test_tuple_unpacking_too_few_values(cartesian_case): errors.DSLError, match=(r"Assignment value must be of type tuple, got 'int32'.") ): - @gtx.field_operator(backend=cartesian_case.backend) + @gtx.field_operator(backend=cartesian_case.executor) def _invalid_unpack() -> tuple[int32, float64, int32]: a, b, c = 1 return a diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py index 96200e4b0b..e076ec4227 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py @@ -118,7 +118,7 @@ def make_builtin_field_operator(builtin_name: str, backend: Optional[ppi.Program @pytest.mark.parametrize("builtin_name, inputs", math_builtin_test_data()) def test_math_function_builtins_execution(cartesian_case, builtin_name: str, inputs): - if cartesian_case.backend is None: + if cartesian_case.executor is None: # TODO(havogt) find a way that works for embedded pytest.xfail("Test does not have a field view program.") if builtin_name == "gamma": @@ -131,7 +131,7 @@ def test_math_function_builtins_execution(cartesian_case, builtin_name: str, inp expected = ref_impl(*inputs) out = cartesian_case.as_field([IDim], np.zeros_like(expected)) - builtin_field_op = make_builtin_field_operator(builtin_name, cartesian_case.backend) + builtin_field_op = make_builtin_field_operator(builtin_name, cartesian_case.executor) builtin_field_op(*inps, out=out, offset_provider={}) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py index 073bbb9550..df0009d0d4 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py @@ -40,7 +40,7 @@ def test_identity_fo_execution(cartesian_case, identity_def): - identity = gtx.field_operator(identity_def, backend=cartesian_case.backend) + identity = gtx.field_operator(identity_def, backend=cartesian_case.executor) in_field = cases.allocate(cartesian_case, identity, "in_field").strategy( cases.ConstInitializer(1) @@ -88,13 +88,13 @@ def shift_by_one_program(in_field: cases.IFloatField, out_field: cases.IFloatFie def test_copy_execution(cartesian_case, copy_program_def): - copy_program = gtx.program(copy_program_def, backend=cartesian_case.backend) + copy_program = gtx.program(copy_program_def, backend=cartesian_case.executor) cases.verify_with_default_data(cartesian_case, copy_program, ref=lambda in_field: in_field) def test_double_copy_execution(cartesian_case, double_copy_program_def): - double_copy_program = gtx.program(double_copy_program_def, backend=cartesian_case.backend) + double_copy_program = gtx.program(double_copy_program_def, backend=cartesian_case.executor) cases.verify_with_default_data( cartesian_case, double_copy_program, ref=lambda in_field, intermediate_field: in_field @@ -102,7 +102,7 @@ def test_double_copy_execution(cartesian_case, double_copy_program_def): def test_copy_restricted_execution(cartesian_case, copy_restrict_program_def): - copy_restrict_program = gtx.program(copy_restrict_program_def, backend=cartesian_case.backend) + copy_restrict_program = gtx.program(copy_restrict_program_def, backend=cartesian_case.executor) cases.verify_with_default_data( cartesian_case, @@ -224,7 +224,7 @@ def prog( def test_wrong_argument_type(cartesian_case, copy_program_def): - copy_program = gtx.program(copy_program_def, backend=cartesian_case.backend) + copy_program = gtx.program(copy_program_def, backend=cartesian_case.executor) inp = cartesian_case.as_field([JDim], np.ones((cartesian_case.default_sizes[JDim],))) out = cases.allocate(cartesian_case, copy_program, "out").strategy(cases.ConstInitializer(1))() diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py index f4b1cf7bd3..65f017a518 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_embedded_regression.py @@ -81,7 +81,7 @@ def copy_program(a: IField, b: IField) -> IField: def test_missing_arg_field_operator(cartesian_case): # noqa: F811 # fixtures """Test that calling a field_operator without required args raises an error.""" - @gtx.field_operator(backend=cartesian_case.backend) + @gtx.field_operator(backend=cartesian_case.executor) def copy(a: IField) -> IField: return a @@ -97,7 +97,7 @@ def copy(a: IField) -> IField: def test_missing_arg_scan_operator(cartesian_case): # noqa: F811 # fixtures """Test that calling a scan_operator without required args raises an error.""" - @gtx.scan_operator(backend=cartesian_case.backend, axis=KDim, init=0.0, forward=True) + @gtx.scan_operator(backend=cartesian_case.executor, axis=KDim, init=0.0, forward=True) def sum(state: float, a: float) -> float: return state + a @@ -122,7 +122,7 @@ def copy(a: IField) -> IField: with pytest.raises(errors.DSLError, match="Invalid call"): - @gtx.program(backend=cartesian_case.backend) + @gtx.program(backend=cartesian_case.executor) def copy_program(a: IField, b: IField) -> IField: copy(a) @@ -130,7 +130,7 @@ def copy_program(a: IField, b: IField) -> IField: with pytest.raises(TypeError, match="'offset_provider'"): - @gtx.program(backend=cartesian_case.backend) + @gtx.program(backend=cartesian_case.executor) def copy_program(a: IField, b: IField) -> IField: copy(a, out=b) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index 8cdb037c08..f1a5b41f81 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -248,14 +248,14 @@ def test_solve_nonhydro_stencil_52_like_z_q(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like_z_q_tup(test_setup): if ( - test_setup.case.backend + test_setup.case.executor == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor ): pytest.xfail( "Needs implementation of scan projector. Breaks in type inference as executed" "again after CollapseTuple." ) - if test_setup.case.backend == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: + if test_setup.case.executor == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: pytest.xfail("Needs proper handling of tuple[Column] <-> Column[tuple].") cases.verify( @@ -275,7 +275,7 @@ def test_solve_nonhydro_stencil_52_like_z_q_tup(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like(test_setup): if ( - test_setup.case.backend + test_setup.case.executor == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor ): pytest.xfail("Temporary extraction does not work correctly in combination with scans.") @@ -297,11 +297,11 @@ def test_solve_nonhydro_stencil_52_like(test_setup): @pytest.mark.uses_tuple_returns def test_solve_nonhydro_stencil_52_like_with_gtfn_tuple_merge(test_setup): if ( - test_setup.case.backend + test_setup.case.executor == test_definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES.load().executor ): pytest.xfail("Temporary extraction does not work correctly in combination with scans.") - if test_setup.case.backend == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: + if test_setup.case.executor == test_definitions.ProgramBackendId.ROUNDTRIP.load().executor: pytest.xfail("Needs proper handling of tuple[Column] <-> Column[tuple].") cases.run( From 470a36ec0a2410665e2d5f190d6232addf9c534a Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Mon, 29 Jan 2024 12:59:33 +0100 Subject: [PATCH 17/19] Update tests/next_tests/definitions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Enrique González Paredes --- tests/next_tests/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index fd6d439850..3702decc2b 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -73,7 +73,7 @@ def allocator(self) -> next_allocators.FieldBufferAllocatorProtocol: @dataclasses.dataclass(frozen=True) class EmbeddedExecutionDescriptor: allocator: next_allocators.FieldBufferAllocatorProtocol - executor: NoneType = dataclasses.field(default=None, init=False) + executor: Final = None numpy_execution = EmbeddedExecutionDescriptor(next_allocators.StandardCPUFieldBufferAllocator()) From b0bf88f95764307a4ad6ae3b67ffdde8ef2ec935 Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Mon, 29 Jan 2024 13:03:49 +0100 Subject: [PATCH 18/19] address review comment --- src/gt4py/next/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index 62ba4a7967..33a0591813 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -843,6 +843,8 @@ def is_connectivity_field( return isinstance(v, ConnectivityField) # type: ignore[misc] # we use extended_runtime_checkable +# Utility function to construct a `Field` from different buffer representations. +# Consider removing this function and using `Field` constructor directly. See also `_connectivity`. @functools.singledispatch def _field( definition: Any, @@ -851,11 +853,10 @@ def _field( domain: Optional[DomainLike] = None, dtype: Optional[core_defs.DType] = None, ) -> Field: - # Utility function to construct a `Field` from different buffer representations. - # Consider removing this function and using `Field` constructor directly. raise NotImplementedError +# See comment for `_field`. @functools.singledispatch def _connectivity( definition: Any, From cdf0a32a311360f9616cd592d1eea9b585d22cfa Mon Sep 17 00:00:00 2001 From: Hannes Vogt Date: Mon, 29 Jan 2024 13:10:12 +0100 Subject: [PATCH 19/19] fix import --- tests/next_tests/definitions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/next_tests/definitions.py b/tests/next_tests/definitions.py index 3702decc2b..dbb2366f47 100644 --- a/tests/next_tests/definitions.py +++ b/tests/next_tests/definitions.py @@ -17,8 +17,7 @@ import dataclasses import enum import importlib -from types import NoneType -from typing import Optional, Protocol +from typing import Final, Optional, Protocol import pytest