diff --git a/pyproject.toml b/pyproject.toml index 7690ae583e..041448e17d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -343,7 +343,11 @@ markers = [ 'uses_strided_neighbor_offset: tests that require backend support for strided neighbor offset', 'uses_tuple_args: tests that require backend support for tuple arguments', 'uses_tuple_returns: tests that require backend support for tuple results', - 'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields' + 'uses_zero_dimensional_fields: tests that require backend support for zero-dimensional fields', + 'uses_cartesian_shift: tests that use a Cartesian connectivity', + 'uses_unstructured_shift: tests that use a unstructured connectivity', + 'uses_scan: tests that uses scan', + 'checks_specific_error: tests that rely on the backend to produce a specific error message' ] norecursedirs = ['dist', 'build', 'cpp_backend_tests/build*', '_local/*', '.*'] testpaths = 'tests' diff --git a/src/gt4py/_core/definitions.py b/src/gt4py/_core/definitions.py index 7b318bc2de..79543a1849 100644 --- a/src/gt4py/_core/definitions.py +++ b/src/gt4py/_core/definitions.py @@ -446,6 +446,9 @@ def shape(self) -> tuple[int, ...]: def dtype(self) -> Any: ... + def astype(self, dtype: npt.DTypeLike) -> NDArrayObject: + ... + def __getitem__(self, item: Any) -> NDArrayObject: ... diff --git a/src/gt4py/next/common.py b/src/gt4py/next/common.py index ffaa410563..66766be76b 100644 --- a/src/gt4py/next/common.py +++ b/src/gt4py/next/common.py @@ -133,7 +133,7 @@ def __getitem__(self, index: int | slice) -> int | UnitRange: else: raise IndexError("UnitRange index out of range") - def __and__(self, other: Set[Any]) -> UnitRange: + def __and__(self, other: Set[int]) -> UnitRange: if isinstance(other, UnitRange): start = max(self.start, other.start) stop = min(self.stop, other.stop) @@ -141,6 +141,16 @@ def __and__(self, other: Set[Any]) -> UnitRange: else: raise NotImplementedError("Can only find the intersection between UnitRange instances.") + def __le__(self, other: Set[int]): + if isinstance(other, UnitRange): + return self.start >= other.start and self.stop <= other.stop + elif len(self) == Infinity.positive(): + return False + else: + return Set.__le__(self, other) + + __ge__ = __lt__ = __gt__ = lambda self, other: NotImplemented + def __str__(self) -> str: return f"({self.start}:{self.stop})" @@ -486,6 +496,14 @@ def __neg__(self) -> Field: def __invert__(self) -> Field: """Only defined for `Field` of value type `bool`.""" + @abc.abstractmethod + def __eq__(self, other: Any) -> Field: # type: ignore[override] # mypy wants return `bool` + ... + + @abc.abstractmethod + def __ne__(self, other: Any) -> Field: # type: ignore[override] # mypy wants return `bool` + ... + @abc.abstractmethod def __add__(self, other: Field | core_defs.ScalarT) -> Field: ... diff --git a/src/gt4py/next/constructors.py b/src/gt4py/next/constructors.py index 30ef8452aa..42b0bcda90 100644 --- a/src/gt4py/next/constructors.py +++ b/src/gt4py/next/constructors.py @@ -82,6 +82,8 @@ def empty( (3, 3) """ dtype = core_defs.dtype(dtype) + if allocator is None and device is None: + device = core_defs.Device(core_defs.DeviceType.CPU, device_id=0) buffer = next_allocators.allocate( domain, dtype, aligned_index=aligned_index, allocator=allocator, device=device ) diff --git a/src/gt4py/next/embedded/nd_array_field.py b/src/gt4py/next/embedded/nd_array_field.py index ea88948841..51e613ef81 100644 --- a/src/gt4py/next/embedded/nd_array_field.py +++ b/src/gt4py/next/embedded/nd_array_field.py @@ -135,25 +135,22 @@ def from_array( /, *, domain: common.DomainLike, - dtype_like: Optional[core_defs.DType] = None, # TODO define DTypeLike + dtype: Optional[core_defs.DTypeLike] = None, ) -> NdArrayField: domain = common.domain(domain) xp = cls.array_ns - xp_dtype = None if dtype_like is None else xp.dtype(core_defs.dtype(dtype_like).scalar_type) + xp_dtype = None if dtype is None else xp.dtype(core_defs.dtype(dtype).scalar_type) array = xp.asarray(data, dtype=xp_dtype) - if dtype_like is not None: - assert array.dtype.type == core_defs.dtype(dtype_like).scalar_type + if dtype is not None: + assert array.dtype.type == core_defs.dtype(dtype).scalar_type assert issubclass(array.dtype.type, core_defs.SCALAR_TYPES) assert all(isinstance(d, common.Dimension) for d in domain.dims), domain assert len(domain) == array.ndim - assert all( - len(r) == s or (s == 1 and r == common.UnitRange.infinity()) - for r, s in zip(domain.ranges, array.shape) - ) + assert all(len(r) == s or s == 1 for r, s in zip(domain.ranges, array.shape)) return cls(domain, array) @@ -194,6 +191,10 @@ def restrict(self, index: common.AnyIndexSpec) -> common.Field | core_defs.Scala __mod__ = __rmod__ = _make_builtin("mod", "mod") + __ne__ = _make_builtin("not_equal", "not_equal") # type: ignore[assignment] # mypy wants return `bool` + + __eq__ = _make_builtin("equal", "equal") # type: ignore[assignment] # mypy wants return `bool` + def __and__(self, other: common.Field | core_defs.ScalarT) -> NdArrayField: if self.dtype == core_defs.BoolDType(): return _make_builtin("logical_and", "logical_and")(self, other) @@ -285,7 +286,7 @@ def _np_cp_setitem( _nd_array_implementations = [np] -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, eq=False) class NumPyArrayField(NdArrayField): array_ns: ClassVar[ModuleType] = np @@ -298,7 +299,7 @@ class NumPyArrayField(NdArrayField): if cp: _nd_array_implementations.append(cp) - @dataclasses.dataclass(frozen=True) + @dataclasses.dataclass(frozen=True, eq=False) class CuPyArrayField(NdArrayField): array_ns: ClassVar[ModuleType] = cp @@ -310,7 +311,7 @@ class CuPyArrayField(NdArrayField): if jnp: _nd_array_implementations.append(jnp) - @dataclasses.dataclass(frozen=True) + @dataclasses.dataclass(frozen=True, eq=False) class JaxArrayField(NdArrayField): array_ns: ClassVar[ModuleType] = jnp @@ -351,6 +352,13 @@ def _builtins_broadcast( NdArrayField.register_builtin_func(fbuiltins.broadcast, _builtins_broadcast) +def _astype(field: NdArrayField, type_: type) -> NdArrayField: + return field.__class__.from_array(field.ndarray.astype(type_), domain=field.domain) + + +NdArrayField.register_builtin_func(fbuiltins.astype, _astype) # type: ignore[arg-type] # TODO(havogt) the registry should not be for any Field + + def _get_slices_from_domain_slice( domain: common.Domain, domain_slice: common.Domain | Sequence[common.NamedRange | common.NamedIndex | Any], diff --git a/src/gt4py/next/ffront/decorator.py b/src/gt4py/next/ffront/decorator.py index 2d12331513..107415eb06 100644 --- a/src/gt4py/next/ffront/decorator.py +++ b/src/gt4py/next/ffront/decorator.py @@ -32,7 +32,7 @@ from gt4py._core import definitions as core_defs from gt4py.eve import utils as eve_utils from gt4py.eve.extended_typing import Any, Optional -from gt4py.next import allocators as next_allocators +from gt4py.next import allocators as next_allocators, common from gt4py.next.common import Dimension, DimensionKind, GridType from gt4py.next.ffront import ( dialect_ast_enums, @@ -171,14 +171,14 @@ class Program: past_node: past.Program closure_vars: dict[str, Any] definition: Optional[types.FunctionType] = None - backend: Optional[ppi.ProgramExecutor] = None + backend: Optional[ppi.ProgramExecutor] = DEFAULT_BACKEND grid_type: Optional[GridType] = None @classmethod def from_function( cls, definition: types.FunctionType, - backend: Optional[ppi.ProgramExecutor] = None, + backend: Optional[ppi.ProgramExecutor] = DEFAULT_BACKEND, grid_type: Optional[GridType] = None, ) -> Program: source_def = SourceDefinition.from_function(definition) @@ -282,27 +282,23 @@ def itir(self) -> itir.FencilDefinition: ) def __call__(self, *args, offset_provider: dict[str, Dimension], **kwargs) -> None: - if ( - self.backend is None and DEFAULT_BACKEND is None - ): # TODO(havogt): for now enable embedded execution by setting DEFAULT_BACKEND to None - self.definition(*args, **kwargs) - return - rewritten_args, size_args, kwargs = self._process_args(args, kwargs) - if not self.backend: + if self.backend is None: warnings.warn( UserWarning( - f"Field View Program '{self.itir.id}': Using default ({DEFAULT_BACKEND}) backend." + f"Field View Program '{self.itir.id}': Using Python execution, consider selecting a perfomance backend." ) ) - backend = self.backend or DEFAULT_BACKEND - ppi.ensure_processor_kind(backend, ppi.ProgramExecutor) + self.definition(*rewritten_args, **kwargs) + return + + ppi.ensure_processor_kind(self.backend, ppi.ProgramExecutor) if "debug" in kwargs: debug(self.itir) - backend( + self.backend( self.itir, *rewritten_args, *size_args, @@ -547,14 +543,14 @@ class FieldOperator(GTCallable, Generic[OperatorNodeT]): foast_node: OperatorNodeT closure_vars: dict[str, Any] definition: Optional[types.FunctionType] = None - backend: Optional[ppi.ProgramExecutor] = None + backend: Optional[ppi.ProgramExecutor] = DEFAULT_BACKEND grid_type: Optional[GridType] = None @classmethod def from_function( cls, definition: types.FunctionType, - backend: Optional[ppi.ProgramExecutor] = None, + backend: Optional[ppi.ProgramExecutor] = DEFAULT_BACKEND, grid_type: Optional[GridType] = None, *, operator_node_cls: type[OperatorNodeT] = foast.FieldOperator, @@ -687,9 +683,9 @@ def __call__( # if we are reaching this from a program call. if "out" in kwargs: out = kwargs.pop("out") - if "offset_provider" in kwargs: + offset_provider = kwargs.pop("offset_provider", None) + if self.backend is not None: # "out" and "offset_provider" -> field_operator as program - offset_provider = kwargs.pop("offset_provider") args, kwargs = type_info.canonicalize_arguments(self.foast_node.type, args, kwargs) # TODO(tehrengruber): check all offset providers are given # deduce argument types @@ -705,13 +701,34 @@ def __call__( ) else: # "out" -> field_operator called from program in embedded execution - out.ndarray[:] = self.definition(*args, **kwargs).ndarray[:] + # TODO(egparedes): put offset_provider in ctxt var here when implementing remap + domain = kwargs.pop("domain", None) + res = self.definition(*args, **kwargs) + _tuple_assign_field( + out, res, domain=None if domain is None else common.domain(domain) + ) return else: # field_operator called from other field_operator in embedded execution + assert self.backend is None return self.definition(*args, **kwargs) +def _tuple_assign_field( + target: tuple[common.Field | tuple, ...] | common.Field, + source: tuple[common.Field | tuple, ...] | common.Field, + domain: Optional[common.Domain], +): + if isinstance(target, tuple): + if not isinstance(source, tuple): + raise RuntimeError(f"Cannot assign {source} to {target}.") + for t, s in zip(target, source): + _tuple_assign_field(t, s, domain) + else: + domain = domain or target.domain + target[domain] = source[domain] + + @typing.overload def field_operator( definition: types.FunctionType, *, backend: Optional[ppi.ProgramExecutor] diff --git a/src/gt4py/next/ffront/fbuiltins.py b/src/gt4py/next/ffront/fbuiltins.py index 7b96de8e89..706b6a4606 100644 --- a/src/gt4py/next/ffront/fbuiltins.py +++ b/src/gt4py/next/ffront/fbuiltins.py @@ -28,10 +28,12 @@ cast, ) +import numpy as np from numpy import float32, float64, int32, int64 -from gt4py._core import definitions as gt4py_defs -from gt4py.next.common import Dimension, DimensionKind, Field +from gt4py._core import definitions as core_defs +from gt4py.next import common +from gt4py.next.common import Dimension, Field # direct import for TYPE_BUILTINS from gt4py.next.ffront.experimental import as_offset # noqa F401 from gt4py.next.iterator import runtime from gt4py.next.type_system import type_specifications as ts @@ -40,7 +42,14 @@ PYTHON_TYPE_BUILTINS = [bool, int, float, tuple] PYTHON_TYPE_BUILTIN_NAMES = [t.__name__ for t in PYTHON_TYPE_BUILTINS] -TYPE_BUILTINS = [Field, Dimension, int32, int64, float32, float64] + PYTHON_TYPE_BUILTINS +TYPE_BUILTINS = [ + Field, + Dimension, + int32, + int64, + float32, + float64, +] + PYTHON_TYPE_BUILTINS TYPE_BUILTIN_NAMES = [t.__name__ for t in TYPE_BUILTINS] # Be aware: Type aliases are not fully supported in the frontend yet, e.g. `IndexType(1)` will not @@ -54,11 +63,11 @@ def _type_conversion_helper(t: type) -> type[ts.TypeSpec] | tuple[type[ts.TypeSpec], ...]: - if t is Field: + if t is common.Field: return ts.FieldType - elif t is Dimension: + elif t is common.Dimension: return ts.DimensionType - elif t is gt4py_defs.ScalarT: + elif t is core_defs.ScalarT: return ts.ScalarType elif t is type: return ( @@ -128,12 +137,8 @@ def __gt_type__(self) -> ts.FunctionType: ) -def builtin_function(fun: Callable[_P, _R]) -> BuiltInFunction[_R, _P]: - return BuiltInFunction(fun) - - -MaskT = TypeVar("MaskT", bound=Field) -FieldT = TypeVar("FieldT", bound=Union[Field, gt4py_defs.Scalar, Tuple]) +MaskT = TypeVar("MaskT", bound=common.Field) +FieldT = TypeVar("FieldT", bound=Union[common.Field, core_defs.Scalar, Tuple]) class WhereBuiltinFunction( @@ -153,55 +158,71 @@ def __call__(self, mask: MaskT, true_field: FieldT, false_field: FieldT) -> _R: return super().__call__(mask, true_field, false_field) -@builtin_function +@BuiltInFunction def neighbor_sum( - field: Field, + field: common.Field, /, - axis: Dimension, -) -> Field: + axis: common.Dimension, +) -> common.Field: raise NotImplementedError() -@builtin_function +@BuiltInFunction def max_over( - field: Field, + field: common.Field, /, - axis: Dimension, -) -> Field: + axis: common.Dimension, +) -> common.Field: raise NotImplementedError() -@builtin_function +@BuiltInFunction def min_over( - field: Field, + field: common.Field, /, - axis: Dimension, -) -> Field: + axis: common.Dimension, +) -> common.Field: raise NotImplementedError() -@builtin_function -def broadcast(field: Field | gt4py_defs.ScalarT, dims: Tuple[Dimension, ...], /) -> Field: - raise NotImplementedError() +@BuiltInFunction +def broadcast( + field: common.Field | core_defs.ScalarT, + dims: tuple[common.Dimension, ...], + /, +) -> common.Field: + assert core_defs.is_scalar_type( + field + ) # default implementation for scalars, Fields are handled via dispatch + return common.field( + np.asarray(field)[ + tuple([np.newaxis] * len(dims)) + ], # TODO(havogt) use FunctionField once available + domain=common.Domain(dims=dims, ranges=tuple([common.UnitRange.infinity()] * len(dims))), + ) @WhereBuiltinFunction def where( - mask: Field, - true_field: Field | gt4py_defs.ScalarT | Tuple, - false_field: Field | gt4py_defs.ScalarT | Tuple, + mask: common.Field, + true_field: common.Field | core_defs.ScalarT | Tuple, + false_field: common.Field | core_defs.ScalarT | Tuple, /, -) -> Field | Tuple: +) -> common.Field | Tuple: raise NotImplementedError() -@builtin_function +@BuiltInFunction def astype( - field: Field | gt4py_defs.ScalarT | Tuple[Field, ...], + value: Field | core_defs.ScalarT | Tuple, type_: type, /, -) -> Field | Tuple[Field, ...]: - raise NotImplementedError() +) -> Field | core_defs.ScalarT | Tuple: + if isinstance(value, tuple): + return tuple(astype(v, type_) for v in value) + # default implementation for scalars, Fields are handled via dispatch + assert core_defs.is_scalar_type(value) + return core_defs.dtype(type_).scalar_type(value) UNARY_MATH_NUMBER_BUILTIN_NAMES = ["abs"] @@ -233,11 +254,14 @@ def astype( def _make_unary_math_builtin(name): - def impl(value: Field | gt4py_defs.ScalarT, /) -> Field | gt4py_defs.ScalarT: + def impl(value: common.Field | core_defs.ScalarT, /) -> common.Field | core_defs.ScalarT: + # TODO(havogt): enable once we have a failing test (see `test_math_builtin_execution.py`) + # assert core_defs.is_scalar_type(value) # default implementation for scalars, Fields are handled via dispatch # noqa: E800 # commented code + # return getattr(math, name)(value)# noqa: E800 # commented code raise NotImplementedError() impl.__name__ = name - globals()[name] = builtin_function(impl) + globals()[name] = BuiltInFunction(impl) for f in ( @@ -252,14 +276,17 @@ def impl(value: Field | gt4py_defs.ScalarT, /) -> Field | gt4py_defs.ScalarT: def _make_binary_math_builtin(name): def impl( - lhs: Field | gt4py_defs.ScalarT, - rhs: Field | gt4py_defs.ScalarT, + lhs: common.Field | core_defs.ScalarT, + rhs: common.Field | core_defs.ScalarT, /, - ) -> Field | gt4py_defs.ScalarT: - raise NotImplementedError() + ) -> common.Field | core_defs.ScalarT: + # default implementation for scalars, Fields are handled via dispatch + assert core_defs.is_scalar_type(lhs) + assert core_defs.is_scalar_type(rhs) + return getattr(np, name)(lhs, rhs) impl.__name__ = name - globals()[name] = builtin_function(impl) + globals()[name] = BuiltInFunction(impl) for f in BINARY_MATH_NUMBER_BUILTIN_NAMES: @@ -295,12 +322,12 @@ def impl( # guidelines for decision. @dataclasses.dataclass(frozen=True) class FieldOffset(runtime.Offset): - source: Dimension - target: tuple[Dimension] | tuple[Dimension, Dimension] + source: common.Dimension + target: tuple[common.Dimension] | tuple[common.Dimension, common.Dimension] connectivity: Optional[Any] = None # TODO def __post_init__(self): - if len(self.target) == 2 and self.target[1].kind != DimensionKind.LOCAL: + if len(self.target) == 2 and self.target[1].kind != common.DimensionKind.LOCAL: raise ValueError("Second dimension in offset must be a local dimension.") def __gt_type__(self): diff --git a/src/gt4py/next/iterator/embedded.py b/src/gt4py/next/iterator/embedded.py index 674f99f61c..44294a3a71 100644 --- a/src/gt4py/next/iterator/embedded.py +++ b/src/gt4py/next/iterator/embedded.py @@ -1093,6 +1093,12 @@ def __neg__(self) -> common.Field: def __invert__(self) -> common.Field: raise NotImplementedError() + def __eq__(self, other: Any) -> common.Field: # type: ignore[override] # mypy wants return `bool` + raise NotImplementedError() + + def __ne__(self, other: Any) -> common.Field: # type: ignore[override] # mypy wants return `bool` + raise NotImplementedError() + def __add__(self, other: common.Field | core_defs.ScalarT) -> common.Field: raise NotImplementedError() @@ -1194,6 +1200,12 @@ def __neg__(self) -> common.Field: def __invert__(self) -> common.Field: raise NotImplementedError() + def __eq__(self, other: Any) -> common.Field: # type: ignore[override] # mypy wants return `bool` + raise NotImplementedError() + + def __ne__(self, other: Any) -> common.Field: # type: ignore[override] # mypy wants return `bool` + raise NotImplementedError() + def __add__(self, other: common.Field | core_defs.ScalarT) -> common.Field: raise NotImplementedError() diff --git a/tests/next_tests/exclusion_matrices.py b/tests/next_tests/exclusion_matrices.py index ddea04649f..249e17d358 100644 --- a/tests/next_tests/exclusion_matrices.py +++ b/tests/next_tests/exclusion_matrices.py @@ -98,6 +98,10 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): USES_TUPLE_ARGS = "uses_tuple_args" USES_TUPLE_RETURNS = "uses_tuple_returns" USES_ZERO_DIMENSIONAL_FIELDS = "uses_zero_dimensional_fields" +USES_CARTESIAN_SHIFT = "uses_cartesian_shift" +USES_UNSTRUCTURED_SHIFT = "uses_unstructured_shift" +USES_SCAN = "uses_scan" +CHECKS_SPECIFIC_ERROR = "checks_specific_error" # Skip messages (available format keys: 'marker', 'backend') UNSUPPORTED_MESSAGE = "'{marker}' tests not supported by '{backend}' backend" @@ -114,10 +118,18 @@ class ProgramFormatterId(_PythonObjectIdMixin, str, enum.Enum): (USES_REDUCTION_WITH_ONLY_SPARSE_FIELDS, XFAIL, REDUCTION_WITH_ONLY_SPARSE_FIELDS_MESSAGE), (USES_SCAN_IN_FIELD_OPERATOR, XFAIL, UNSUPPORTED_MESSAGE), ] +EMBEDDED_SKIP_LIST = [ + (USES_CARTESIAN_SHIFT, XFAIL, UNSUPPORTED_MESSAGE), + (USES_UNSTRUCTURED_SHIFT, XFAIL, UNSUPPORTED_MESSAGE), + (USES_SCAN, XFAIL, UNSUPPORTED_MESSAGE), + (USES_DYNAMIC_OFFSETS, XFAIL, UNSUPPORTED_MESSAGE), + (CHECKS_SPECIFIC_ERROR, XFAIL, UNSUPPORTED_MESSAGE), +] #: Skip matrix, contains for each backend processor a list of tuples with following fields: #: (, ) BACKEND_SKIP_TEST_MATRIX = { + None: EMBEDDED_SKIP_LIST, OptionalProgramBackendId.DACE_CPU: GTFN_SKIP_TEST_LIST + [ (USES_CAN_DEREF, XFAIL, UNSUPPORTED_MESSAGE), diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py index 386e64451d..fb753bf169 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/ffront_test_utils.py @@ -53,6 +53,7 @@ def no_backend(program: itir.FencilDefinition, *args: Any, **kwargs: Any) -> Non definitions.ProgramBackendId.GTFN_CPU, definitions.ProgramBackendId.GTFN_CPU_IMPERATIVE, definitions.ProgramBackendId.GTFN_CPU_WITH_TEMPORARIES, + None, ] + OPTIONAL_PROCESSORS, ids=lambda p: p.short_id() if p is not None else "None", @@ -65,19 +66,15 @@ def fieldview_backend(request): Check ADR 15 for details on the test-exclusion matrices. """ backend_id = request.param - if backend_id is None: - backend = None - else: - backend = backend_id.load() - - for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( - backend_id, [] - ): - if request.node.get_closest_marker(marker): - skip_mark(msg.format(marker=marker, backend=backend_id)) + backend = None if backend_id is None else backend_id.load() - backup_backend = decorator.DEFAULT_BACKEND + for marker, skip_mark, msg in next_tests.exclusion_matrices.BACKEND_SKIP_TEST_MATRIX.get( + backend_id, [] + ): + if request.node.get_closest_marker(marker): + skip_mark(msg.format(marker=marker, backend=backend_id)) + backup_backend = decorator.DEFAULT_BACKEND decorator.DEFAULT_BACKEND = no_backend yield backend decorator.DEFAULT_BACKEND = backup_backend diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py index deb1382dfb..6957e628bb 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_arg_call_interface.py @@ -158,6 +158,7 @@ def testee( ) +@pytest.mark.uses_scan @pytest.mark.uses_scan_in_field_operator def test_call_scan_operator_from_field_operator(cartesian_case): @scan_operator(axis=KDim, forward=True, init=0.0) @@ -183,6 +184,7 @@ def testee(a: IJKFloatField, b: IJKFloatField) -> IJKFloatField: cases.verify(cartesian_case, testee, a, b, out=out, ref=expected) +@pytest.mark.uses_scan def test_call_scan_operator_from_program(cartesian_case): @scan_operator(axis=KDim, forward=True, init=0.0) def testee_scan(state: float, x: float, y: float) -> float: @@ -222,6 +224,7 @@ def testee( ) +@pytest.mark.uses_scan def test_scan_wrong_return_type(cartesian_case): with pytest.raises( errors.DSLError, @@ -239,6 +242,7 @@ def testee(qc: cases.IKFloatField, param_1: int32, param_2: float, scalar: float testee_scan(qc, param_1, param_2, scalar, out=(qc, param_1, param_2)) +@pytest.mark.uses_scan def test_scan_wrong_state_type(cartesian_case): with pytest.raises( errors.DSLError, diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py index 8787b7d7bc..8036c22670 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_execution.py @@ -76,6 +76,7 @@ def testee(a: cases.IJKField, b: cases.IJKField) -> tuple[cases.IJKField, cases. cases.verify_with_default_data(cartesian_case, testee, ref=lambda a, b: (a, b)) +@pytest.mark.uses_cartesian_shift def test_cartesian_shift(cartesian_case): # noqa: F811 # fixtures @gtx.field_operator def testee(a: cases.IJKField) -> cases.IJKField: @@ -87,6 +88,7 @@ def testee(a: cases.IJKField) -> cases.IJKField: cases.verify(cartesian_case, testee, a, out=out, ref=a[1:]) +@pytest.mark.uses_unstructured_shift def test_unstructured_shift(unstructured_case): # noqa: F811 # fixtures @gtx.field_operator def testee(a: cases.VField) -> cases.EField: @@ -99,6 +101,7 @@ def testee(a: cases.VField) -> cases.EField: ) +@pytest.mark.uses_unstructured_shift def test_composed_unstructured_shift(unstructured_case): @gtx.field_operator def composed_shift_unstructured_flat(inp: cases.VField) -> cases.CField: @@ -143,6 +146,7 @@ def composed_shift_unstructured(inp: cases.VField) -> cases.CField: ) +@pytest.mark.uses_cartesian_shift def test_fold_shifts(cartesian_case): # noqa: F811 # fixtures """Shifting the result of an addition should work.""" @@ -206,6 +210,7 @@ def testee(a: int32) -> cases.VField: @pytest.mark.uses_index_fields +@pytest.mark.uses_cartesian_shift def test_scalar_arg_with_field(cartesian_case): # noqa: F811 # fixtures @gtx.field_operator def testee(a: cases.IJKField, b: int32) -> cases.IJKField: @@ -246,6 +251,7 @@ def testee(size: gtx.IndexType, out: gtx.Field[[IDim], gtx.IndexType]): ) +@pytest.mark.uses_scan def test_scalar_scan(cartesian_case): # noqa: F811 # fixtures @gtx.scan_operator(axis=KDim, forward=True, init=(0.0)) def testee_scan(state: float, qc_in: float, scalar: float) -> float: @@ -264,6 +270,7 @@ def testee(qc: cases.IKFloatField, scalar: float): cases.verify(cartesian_case, testee, qc, scalar, inout=qc, ref=expected) +@pytest.mark.uses_scan @pytest.mark.uses_scan_in_field_operator def test_tuple_scalar_scan(cartesian_case): # noqa: F811 # fixtures @gtx.scan_operator(axis=KDim, forward=True, init=0.0) @@ -285,6 +292,7 @@ def testee_op( cases.verify(cartesian_case, testee_op, qc, tuple_scalar, out=qc, ref=expected) +@pytest.mark.uses_scan @pytest.mark.uses_index_fields def test_scalar_scan_vertical_offset(cartesian_case): # noqa: F811 # fixtures @gtx.scan_operator(axis=KDim, forward=True, init=(0.0)) @@ -363,8 +371,8 @@ def cast_nested_tuple( a = cases.allocate(cartesian_case, cast_tuple, "a")() b = cases.allocate(cartesian_case, cast_tuple, "b")() - a_asint = gtx.np_as_located_field(IDim)(np.asarray(a).astype(int32)) - b_asint = gtx.np_as_located_field(IDim)(np.asarray(b).astype(int32)) + a_asint = gtx.as_field([IDim], np.asarray(a).astype(int32)) + b_asint = gtx.as_field([IDim], np.asarray(b).astype(int32)) out_tuple = cases.allocate(cartesian_case, cast_tuple, cases.RETURN)() out_nested_tuple = cases.allocate(cartesian_case, cast_nested_tuple, cases.RETURN)() @@ -483,6 +491,7 @@ def combine(a: cases.IField, b: cases.IField) -> cases.IField: cases.verify_with_default_data(cartesian_case, combine, ref=lambda a, b: a + a + b) +@pytest.mark.uses_unstructured_shift @pytest.mark.uses_reduction_over_lift_expressions def test_nested_reduction(unstructured_case): @gtx.field_operator @@ -504,6 +513,7 @@ def testee(a: cases.EField) -> cases.EField: ) +@pytest.mark.uses_unstructured_shift @pytest.mark.xfail(reason="Not yet supported in lowering, requires `map_`ing of inner reduce op.") def test_nested_reduction_shift_first(unstructured_case): @gtx.field_operator @@ -524,6 +534,7 @@ def testee(inp: cases.EField) -> cases.EField: ) +@pytest.mark.uses_unstructured_shift @pytest.mark.uses_tuple_returns def test_tuple_return_2(unstructured_case): @gtx.field_operator @@ -543,6 +554,7 @@ def testee(a: cases.EField, b: cases.EField) -> tuple[cases.VField, cases.VField ) +@pytest.mark.uses_unstructured_shift @pytest.mark.uses_constant_fields def test_tuple_with_local_field_in_reduction_shifted(unstructured_case): @gtx.field_operator @@ -572,6 +584,7 @@ def testee(a: tuple[tuple[cases.IField, cases.IField], cases.IField]) -> cases.I ) +@pytest.mark.uses_scan @pytest.mark.parametrize("forward", [True, False]) def test_fieldop_from_scan(cartesian_case, forward): init = 1.0 @@ -592,6 +605,7 @@ def simple_scan_operator(carry: float) -> float: cases.verify(cartesian_case, simple_scan_operator, out=out, ref=expected) +@pytest.mark.uses_scan @pytest.mark.uses_lift_expressions def test_solve_triag(cartesian_case): if cartesian_case.backend in [ @@ -680,6 +694,7 @@ def testee( ) +@pytest.mark.uses_unstructured_shift @pytest.mark.uses_reduction_over_lift_expressions def test_ternary_builtin_neighbor_sum(unstructured_case): @gtx.field_operator @@ -698,6 +713,7 @@ def testee(a: cases.EField, b: cases.EField) -> cases.VField: ) +@pytest.mark.uses_scan def test_ternary_scan(cartesian_case): if cartesian_case.backend in [gtfn.run_gtfn_with_temporaries]: pytest.xfail("Temporary extraction does not work correctly in combination with scans.") @@ -720,6 +736,7 @@ def simple_scan_operator(carry: float, a: float) -> float: @pytest.mark.parametrize("forward", [True, False]) +@pytest.mark.uses_scan @pytest.mark.uses_tuple_returns def test_scan_nested_tuple_output(forward, cartesian_case): if cartesian_case.backend in [gtfn.run_gtfn_with_temporaries]: @@ -745,13 +762,14 @@ def testee(out: tuple[cases.KField, tuple[cases.KField, cases.KField]]): cartesian_case, testee, ref=lambda: (expected + 1.0, (expected + 2.0, expected + 3.0)), - comparison=lambda ref, out: np.all(out[0] == ref[0]) - and np.all(out[1][0] == ref[1][0]) - and np.all(out[1][1] == ref[1][1]), + comparison=lambda ref, out: np.all(np.asarray(out[0]) == ref[0]) + and np.all(np.asarray(out[1][0]) == ref[1][0]) + and np.all(np.asarray(out[1][1]) == ref[1][1]), ) @pytest.mark.uses_tuple_args +@pytest.mark.uses_scan def test_scan_nested_tuple_input(cartesian_case): init = 1.0 k_size = cartesian_case.default_sizes[KDim] @@ -824,7 +842,10 @@ def program_domain(a: cases.IField, out: cases.IField): a = cases.allocate(cartesian_case, program_domain, "a")() out = cases.allocate(cartesian_case, program_domain, "out")() - cases.verify(cartesian_case, program_domain, a, out, inout=out[1:9], ref=a[1:9] * 2) + ref = out.ndarray.copy() # ensure we are not overwriting out outside of the domain + ref[1:9] = a[1:9] * 2 + + cases.verify(cartesian_case, program_domain, a, out, inout=out, ref=ref) def test_domain_input_bounds(cartesian_case): @@ -855,6 +876,9 @@ def program_domain( inp = cases.allocate(cartesian_case, program_domain, "inp")() out = cases.allocate(cartesian_case, fieldop_domain, cases.RETURN)() + ref = out.ndarray.copy() + ref[lower_i : int(upper_i / 2)] = inp[lower_i : int(upper_i / 2)] * 2 + cases.verify( cartesian_case, program_domain, @@ -862,8 +886,8 @@ def program_domain( out, lower_i, upper_i, - inout=out[lower_i : int(upper_i / 2)], - ref=inp[lower_i : int(upper_i / 2)] * 2, + inout=out, + ref=ref, ) @@ -895,6 +919,11 @@ def program_domain( a = cases.allocate(cartesian_case, program_domain, "a")() out = cases.allocate(cartesian_case, program_domain, "out")() + ref = out.ndarray.copy() + ref[1 * lower_i : upper_i + 0, lower_j - 0 : upper_j] = ( + a[1 * lower_i : upper_i + 0, lower_j - 0 : upper_j] * 2 + ) + cases.verify( cartesian_case, program_domain, @@ -904,8 +933,8 @@ def program_domain( upper_i, lower_j, upper_j, - inout=out[1 * lower_i : upper_i + 0, lower_j - 0 : upper_j], - ref=a[1 * lower_i : upper_i + 0, lower_j - 0 : upper_j] * 2, + inout=out, + ref=ref, ) @@ -930,6 +959,11 @@ def program_domain_tuple( out0 = cases.allocate(cartesian_case, program_domain_tuple, "out0")() out1 = cases.allocate(cartesian_case, program_domain_tuple, "out1")() + ref0 = out0.ndarray.copy() + ref0[1:9, 4:6] = inp0[1:9, 4:6] + inp1[1:9, 4:6] + ref1 = out1.ndarray.copy() + ref1[1:9, 4:6] = inp1[1:9, 4:6] + cases.verify( cartesian_case, program_domain_tuple, @@ -937,11 +971,12 @@ def program_domain_tuple( inp1, out0, out1, - inout=(out0[1:9, 4:6], out1[1:9, 4:6]), - ref=(inp0[1:9, 4:6] + inp1[1:9, 4:6], inp1[1:9, 4:6]), + inout=(out0, out1), + ref=(ref0, ref1), ) +@pytest.mark.uses_cartesian_shift def test_where_k_offset(cartesian_case): @gtx.field_operator def fieldop_where_k_offset( @@ -1079,6 +1114,13 @@ def _invalid_unpack() -> tuple[int32, float64, int32]: def test_constant_closure_vars(cartesian_case): + if cartesian_case.backend is None: + # >>> field = gtx.zeros(domain) + # >>> np.int32(1)*field # steals the buffer from the field + # array([0.]) + + # TODO(havogt): remove `__array__`` from `NdArrayField` + pytest.xfail("Bug: Binary operation between np datatype and Field returns ndarray.") from gt4py.eve.utils import FrozenNamespace constants = FrozenNamespace( diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py index 04b27c6c17..5135b3d47a 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_external_local_field.py @@ -26,6 +26,9 @@ ) +pytestmark = pytest.mark.uses_unstructured_shift + + def test_external_local_field(unstructured_case): @gtx.field_operator def testee( diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py index 8213f54a45..1eba95e880 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_gt4py_builtins.py @@ -39,6 +39,7 @@ ) +@pytest.mark.uses_unstructured_shift @pytest.mark.parametrize( "strategy", [cases.UniqueInitializer(1), cases.UniqueInitializer(-100)], @@ -65,6 +66,7 @@ def testee(edge_f: cases.EField) -> cases.VField: cases.verify(unstructured_case, testee, inp, ref=ref, out=out) +@pytest.mark.uses_unstructured_shift def test_minover_execution(unstructured_case): @gtx.field_operator def minover(edge_f: cases.EField) -> cases.VField: @@ -77,6 +79,7 @@ def minover(edge_f: cases.EField) -> cases.VField: ) +@pytest.mark.uses_unstructured_shift def test_reduction_execution(unstructured_case): @gtx.field_operator def reduction(edge_f: cases.EField) -> cases.VField: @@ -93,6 +96,7 @@ def fencil(edge_f: cases.EField, out: cases.VField): ) +@pytest.mark.uses_unstructured_shift @pytest.mark.uses_constant_fields def test_reduction_expression_in_call(unstructured_case): @gtx.field_operator @@ -113,6 +117,7 @@ def fencil(edge_f: cases.EField, out: cases.VField): ) +@pytest.mark.uses_unstructured_shift def test_reduction_with_common_expression(unstructured_case): @gtx.field_operator def testee(flux: cases.EField) -> cases.VField: @@ -191,6 +196,7 @@ def broadcast_two_fields(inp1: cases.IField, inp2: gtx.Field[[JDim], int32]) -> ) +@pytest.mark.uses_cartesian_shift def test_broadcast_shifted(cartesian_case): @gtx.field_operator def simple_broadcast(inp: cases.IField) -> cases.IJField: @@ -249,6 +255,7 @@ def conditional_promotion(a: cases.IFloatField) -> cases.IFloatField: ) +@pytest.mark.uses_cartesian_shift def test_conditional_shifted(cartesian_case): @gtx.field_operator def conditional_shifted( diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py index a5d2b92719..a1839b8e17 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_builtin_execution.py @@ -116,6 +116,9 @@ def make_builtin_field_operator(builtin_name: str): @pytest.mark.parametrize("builtin_name, inputs", math_builtin_test_data()) def test_math_function_builtins_execution(cartesian_case, builtin_name: str, inputs): + if cartesian_case.backend is None: + # TODO(havogt) find a way that works for embedded + pytest.xfail("Test does not have a field view program.") if builtin_name == "gamma": # numpy has no gamma function ref_impl: Callable = np.vectorize(math.gamma) diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py index 5a277f9440..59e11a7de8 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_math_unary_builtins.py @@ -173,17 +173,14 @@ def tilde_fieldop(inp1: cases.IBoolField) -> cases.IBoolField: def test_unary_not(cartesian_case): - @gtx.field_operator - def not_fieldop(inp1: cases.IBoolField) -> cases.IBoolField: - return not inp1 + pytest.xfail( + "We accidentally supported `not` on fields. This is wrong, we should raise an error." + ) + with pytest.raises: # TODO `not` on a field should be illegal - size = cartesian_case.default_sizes[IDim] - bool_field = np.random.choice(a=[False, True], size=(size)) - inp1 = cases.allocate(cartesian_case, not_fieldop, "inp1").strategy( - cases.ConstInitializer(bool_field) - )() - out = cases.allocate(cartesian_case, not_fieldop, cases.RETURN)() - cases.verify(cartesian_case, not_fieldop, inp1, out=out, ref=~inp1) + @gtx.field_operator + def not_fieldop(inp1: cases.IBoolField) -> cases.IBoolField: + return not inp1 # Trig builtins diff --git a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py index 7a1c827a0d..545abd2825 100644 --- a/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py +++ b/tests/next_tests/integration_tests/feature_tests/ffront_tests/test_program.py @@ -51,6 +51,7 @@ def test_identity_fo_execution(cartesian_case, identity_def): ) +@pytest.mark.uses_cartesian_shift def test_shift_by_one_execution(cartesian_case): @gtx.field_operator def shift_by_one(in_field: cases.IFloatField) -> cases.IFloatField: @@ -230,6 +231,7 @@ def test_wrong_argument_type(cartesian_case, copy_program_def): assert re.search(msg, exc_info.value.__cause__.args[0]) is not None +@pytest.mark.checks_specific_error def test_dimensions_domain(cartesian_case): @gtx.field_operator def empty_domain_fieldop(a: cases.IJField): @@ -246,4 +248,4 @@ def empty_domain_program(a: cases.IJField, out_field: cases.IJField): ValueError, match=(r"Dimensions in out field and field domain are not equivalent"), ): - empty_domain_program(a, out_field, offset_provider={}) + cases.run(cartesian_case, empty_domain_program, a, out_field, offset_provider={}) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py index 108ee25862..eaae9a2a3e 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_icon_like_scan.py @@ -25,6 +25,9 @@ ) +pytestmark = pytest.mark.uses_unstructured_shift + + Cell = gtx.Dimension("Cell") KDim = gtx.Dimension("KDim", kind=gtx.DimensionKind.VERTICAL) Koff = gtx.FieldOffset("Koff", KDim, (KDim,)) diff --git a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py index d275a977dd..9a1e968de0 100644 --- a/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py +++ b/tests/next_tests/integration_tests/multi_feature_tests/ffront_tests/test_laplacian.py @@ -13,6 +13,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import numpy as np +import pytest import gt4py.next as gtx @@ -23,6 +24,9 @@ ) +pytestmark = pytest.mark.uses_cartesian_shift + + @gtx.field_operator def lap(in_field: gtx.Field[[IDim, JDim], "float"]) -> gtx.Field[[IDim, JDim], "float"]: return ( diff --git a/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py b/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py index 49aeece87e..00dbf68274 100644 --- a/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py +++ b/tests/next_tests/unit_tests/embedded_tests/test_nd_array_field.py @@ -259,7 +259,7 @@ def test_mixed_fields(product_nd_array_implementation): def test_non_dispatched_function(): - @fbuiltins.builtin_function + @fbuiltins.BuiltInFunction def fma(a: common.Field, b: common.Field, c: common.Field, /) -> common.Field: return a * b + c diff --git a/tests/next_tests/unit_tests/test_common.py b/tests/next_tests/unit_tests/test_common.py index 31e35221ab..84008eb99c 100644 --- a/tests/next_tests/unit_tests/test_common.py +++ b/tests/next_tests/unit_tests/test_common.py @@ -11,6 +11,7 @@ # distribution for a copy of the license or check . # # SPDX-License-Identifier: GPL-3.0-or-later +import operator from typing import Optional, Pattern import pytest @@ -150,6 +151,21 @@ def test_mixed_infinity_range(): assert len(mixed_inf_range) == Infinity.positive() +@pytest.mark.parametrize( + "op, rng1, rng2, expected", + [ + (operator.le, UnitRange(-1, 2), UnitRange(-2, 3), True), + (operator.le, UnitRange(-1, 2), {-1, 0, 1}, True), + (operator.le, UnitRange(-1, 2), {-1, 0}, False), + (operator.le, UnitRange(-1, 2), {-2, -1, 0, 1, 2}, True), + (operator.le, UnitRange(Infinity.negative(), 2), UnitRange(Infinity.negative(), 3), True), + (operator.le, UnitRange(Infinity.negative(), 2), {1, 2, 3}, False), + ], +) +def test_range_comparison(op, rng1, rng2, expected): + assert op(rng1, rng2) == expected + + @pytest.mark.parametrize( "named_rng_like", [