From 7ad244a795a72a8b5ba14f5829e8fb5a3d2beda1 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Fri, 5 Jul 2024 09:43:26 +0200 Subject: [PATCH 01/40] Ignore virtual env --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 60608eb..15f0073 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /Manifest.toml docs/build/ /.DS_Store +.venv +env_setup.sh From 2c76baea418c556319ec9507ba1c89cf1632601b Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Mon, 8 Jul 2024 10:07:44 +0200 Subject: [PATCH 02/40] Add notes for future improvements --- src/gt2py/gt2py.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gt2py/gt2py.jl b/src/gt2py/gt2py.jl index e091a45..9e44765 100644 --- a/src/gt2py/gt2py.jl +++ b/src/gt2py/gt2py.jl @@ -241,6 +241,7 @@ function postprocess_definition(foast_node, closure_vars, annotations) return foast_node end +# TODO(tehrengruber): unify with convert_type py_args(args::Union{Base.Pairs, Dict}) = Dict(i.first => convert_type(i.second) for i in args) py_args(args::Tuple) = [convert_type(arg) for arg in args] @@ -282,6 +283,7 @@ function convert_type(a::Connectivity) # account for different indexing in python return gtx.NeighborTableOffsetProvider( + #TODO(lorenzovarese): fix performance (conversion from 0-index to 1-index) (caching or directly store the 0-index version of the connectivity) ifelse.(a.data .!= -1, a.data .- 1, a.data), target_dim, source_dim, From f9d9118c86c74a2ac718ae2206bddf8245ffd34c Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Mon, 8 Jul 2024 10:08:12 +0200 Subject: [PATCH 03/40] Ignore all .DS_Store --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 15f0073..627be74 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ /Manifest.toml docs/build/ -/.DS_Store +.DS_Store .venv env_setup.sh From 109e2854a1844a3bc6f61c3bee412242922e1070 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Mon, 8 Jul 2024 10:08:57 +0200 Subject: [PATCH 04/40] Ignore configuration files in the .vscode folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 627be74..ab83538 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ docs/build/ .DS_Store .venv env_setup.sh +.vscode \ No newline at end of file From 43626a3a757417a0feca040898af31a616c6797e Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 13:07:27 +0200 Subject: [PATCH 05/40] Ignore .python-version specified by pyenv --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ab83538..00cd533 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ docs/build/ .DS_Store .venv env_setup.sh -.vscode \ No newline at end of file +.vscode +.python-version \ No newline at end of file From 95681e1f59f196c64a584e2f9c9f01d4346df840 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 13:49:11 +0200 Subject: [PATCH 06/40] Refine .gitignore --- .gitignore | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 00cd533..0e02d77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,33 @@ -/Manifest.toml +# Files generated by invoking Julia with --code-coverage +*.jl.cov +*.jl.*.cov + +# Files generated by invoking Julia with --track-allocation +*.jl.mem + +# System-specific files and directories generated by the BinaryProvider and BinDeps packages +# They contain absolute paths specific to the host computer, and so should not be committed +deps/deps.jl +deps/build.log +deps/downloads/ +deps/usr/ +deps/src/ + +# Build artifacts for creating documentation generated by the Documenter package docs/build/ -.DS_Store +docs/site/ + +# File generated by Pkg, the package manager, based on a corresponding Project.toml +# It records a fixed state of all packages used by the project. As such, it should not be +# committed for packages, but should be committed for applications that require a static +# environment. +Manifest.toml + +# Python Env .venv env_setup.sh +.python-version + +# Misc +.DS_Store .vscode -.python-version \ No newline at end of file From 5dcf2db0e1b975eb690ce112432168c875c7aee1 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 13:53:06 +0200 Subject: [PATCH 07/40] Add first tests with ground truth --- test/gt2py_fo_exec.jl | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index c084765..2f99a0e 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -19,7 +19,9 @@ macro to_py(expr::Expr) return res end -# Setup ------------------------------------------------------------------------------------------ +# ======================================== +# ============== Setup =================== +# ======================================== edge_to_cell_table = [ [1 -1]; @@ -45,7 +47,6 @@ cell_to_edge_table = [ [6 7 12] ] - E2C_offset_provider = Connectivity(edge_to_cell_table, Cell, Edge, 2) C2E_offset_provider = Connectivity(cell_to_edge_table, Edge, Cell, 3) @@ -54,8 +55,10 @@ offset_provider = Dict{String, Connectivity}( "C2E" => C2E_offset_provider ) +# ======================================== +# ============== Tests =================== +# ======================================== -# Tests ------------------------------------------------ function test_gt4py_fo_exec() a = Field(Cell, collect(1.:15.)) b = Field(Cell, collect(-1.:-1:-15.)) @@ -75,17 +78,17 @@ function test_gt4py_fo_exec() @field_operator function fo_nested_if_else(f::Field{Tuple{Cell_}, Int32})::Field{Tuple{Cell_}, Int32} tmp = f - if 1. .< 10.0 - tmp = f .+ 1 + if 1. < 10.0 + tmp = f .+ Int32(1) if 30 > 5 - tmp = tmp .+ 20 - tmp = tmp .- 10 + tmp = tmp .+ Int32(20) + tmp = tmp .- Int32(10) elseif 40 < 4 tmp = 4 == 5 ? tmp : tmp .- 100 else tmp = tmp .* 5 end - tmp = tmp .+ 10 + tmp = tmp .+ Int32(10) elseif 10 < 20 tmp = f .- 1 else @@ -96,17 +99,20 @@ function test_gt4py_fo_exec() return tmp end - @test @to_py fo_nested_if_else(a, backend = "py", out = out) + @test @to_py fo_nested_if_else(a, backend = "embedded", out = out) + @test all(out.data .== collect(22:36)) # ------------------------------------------------ a = Field(Cell, collect(1.:15.)) out = Field(Edge, zeros(Float64, 12)) + expected_output = a[edge_to_cell_table[:, 1]] # First column of the edge to cell connectivity table @field_operator function fo_remapping(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} return a(E2C[1]) end - @test @to_py fo_remapping(a, offset_provider=offset_provider, backend = "py", out = out) + fo_remapping(a, offset_provider=offset_provider, backend = "embedded", out = out) + @test all(out.data .== expected_output) # ------------------------------------------------ a = Field(Cell, collect(1.:15.)) From 9f881694e130c97e2c0ee74a5c9fbf6df2d14262 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 15:19:24 +0200 Subject: [PATCH 08/40] Separate the setup and the backend selection in the gt2py_fo_exec tests --- test/gt2py_fo_exec.jl | 189 +++++++++++++++++++++++++++++++++--------- 1 file changed, 148 insertions(+), 41 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 2f99a0e..7becec2 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -19,47 +19,72 @@ macro to_py(expr::Expr) return res end +# ======================================== +# ============== Utility ================= +# ======================================== + +struct ConnectivityData + edge_to_cell_table::Matrix{Integer} + cell_to_edge_table::Matrix{Integer} + E2C_offset_provider::Connectivity + C2E_offset_provider::Connectivity + offset_provider::Dict{String, Connectivity} +end + +function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) + if setupfunc === nothing + testfunc(args...) + else + data = setupfunc() + testfunc(data, args...) + end +end + # ======================================== # ============== Setup =================== # ======================================== -edge_to_cell_table = [ - [1 -1]; - [3 -1]; - [3 -1]; - [4 -1]; - [5 -1]; - [6 -1]; - [1 6]; - [1 2]; - [2 3]; - [2 4]; - [4 5]; - [5 6] -] - -cell_to_edge_table = [ - [1 7 8]; - [8 9 10]; - [2 3 9]; - [4 10 11]; - [5 11 12]; - [6 7 12] -] - -E2C_offset_provider = Connectivity(edge_to_cell_table, Cell, Edge, 2) -C2E_offset_provider = Connectivity(cell_to_edge_table, Edge, Cell, 3) - -offset_provider = Dict{String, Connectivity}( - "E2C" => E2C_offset_provider, - "C2E" => C2E_offset_provider - ) +function setup_simple_connectivity() + edge_to_cell_table = [ + [1 -1]; + [3 -1]; + [3 -1]; + [4 -1]; + [5 -1]; + [6 -1]; + [1 6]; + [1 2]; + [2 3]; + [2 4]; + [4 5]; + [5 6] + ] + + cell_to_edge_table = [ + [1 7 8]; + [8 9 10]; + [2 3 9]; + [4 10 11]; + [5 11 12]; + [6 7 12] + ] + + E2C_offset_provider = Connectivity(edge_to_cell_table, Cell, Edge, 2) + C2E_offset_provider = Connectivity(cell_to_edge_table, Edge, Cell, 3) + + offset_provider = Dict{String, Connectivity}( + "E2C" => E2C_offset_provider, + "C2E" => C2E_offset_provider + ) + + return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) +end # ======================================== -# ============== Tests =================== +# ========= Tests Definition ============= # ======================================== -function test_gt4py_fo_exec() +function test_fo_addition(backend::String) a = Field(Cell, collect(1.:15.)) b = Field(Cell, collect(-1.:-1:-15.)) out = Field(Cell, zeros(Float64, 15)) @@ -68,11 +93,11 @@ function test_gt4py_fo_exec() return a .+ b end - fo_addition(a, b, backend = "py", out = out) + fo_addition(a, b, backend = backend, out = out) @test all(out.data .== 0) +end - # ------------------------------------------------ - +function test_fo_nested_if_else(backend::String) a = Field(Cell, collect(Int32, 1:15)) # TODO(tehrengruber): if we don't use the right dtype here we get a horrible error in python out = Field(Cell, zeros(Int32, 15)) @@ -99,20 +124,93 @@ function test_gt4py_fo_exec() return tmp end - @test @to_py fo_nested_if_else(a, backend = "embedded", out = out) + fo_nested_if_else(a, backend = backend, out = out) @test all(out.data .== collect(22:36)) +end - # ------------------------------------------------ +function test_fo_remapping(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.:15.)) out = Field(Edge, zeros(Float64, 12)) - expected_output = a[edge_to_cell_table[:, 1]] # First column of the edge to cell connectivity table + expected_output = a[data.edge_to_cell_table[:, 1]] # First column of the edge to cell connectivity table @field_operator function fo_remapping(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} return a(E2C[1]) end - fo_remapping(a, offset_provider=offset_provider, backend = "embedded", out = out) + fo_remapping(a, offset_provider = data.offset_provider, backend = backend, out = out) @test all(out.data .== expected_output) +end + +function test_fo_neighbor_sum(backend::String) + a = Field(Cell, collect(1.:15.)) + out = Field(Edge, zeros(Float64, 12)) + + @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + return neighbor_sum(a(E2C), axis=E2CDim) + end + + @test @to_py fo_neighbor_sum(a, offset_provider=offset_provider, backend = "py", out = out) +end + +function test_fo_max_over(backend::String) + +end + +function test_fo_min_over(backend::String) + +end + +function test_fo_simple_broadcast(backend::String) + +end + +function test_fo_scalar_broadcast(backend::String) + +end + +function test_fo_where(backend::String) + +end + +function test_fo_astype(backend::String) + +end + +function test_fo_sin(backend::String) + +end + +function test_fo_asinh(backend::String) + +end + +function test_fo_offset_array(backend::String) + +end + +function test_nested_fo(backend::String) + +end + +# ======================================== +# ========== Test Executions ============= +# ======================================== + +function test_gt4py_fo_exec() + testwrapper(nothing, (args...) -> test_fo_addition(args...), "embedded") + testwrapper(nothing, (args...) -> test_fo_addition(args...), "py") + + testwrapper(nothing, (args...) -> test_fo_nested_if_else(args...), "embedded") + testwrapper(nothing, (args...) -> test_fo_nested_if_else(args...), "py") + + testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "embedded") + testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "py") +end + +function test_gt4py_fo_exec_legacy() + # Set up locally to mimic the previous global behavior + data = setup_simple_connectivity() + offset_provider = data.offset_provider # ------------------------------------------------ a = Field(Cell, collect(1.:15.)) @@ -235,10 +333,19 @@ function test_gt4py_fo_exec() b = Field(Cell, ones(15)) out = Field(Cell, zeros(15)) + @field_operator function fo_addition(a::Field{Tuple{Cell_}, Float64}, b::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_}, Float64} + return a .+ b + end + @field_operator function nested_fo(a::Field{Tuple{Cell_}, Float64}, b::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_}, Float64} res = fo_addition(a, b) return res .+ a end end -@testset "Testset GT2Py fo exec" test_gt4py_fo_exec() \ No newline at end of file +function run_all() + test_gt4py_fo_exec() + test_gt4py_fo_exec_legacy() +end + +@testset "Testset GT2Py fo exec" run_all() From 11e637b0c1c6d6b3eeb70f228a77682afdae61b7 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 16:44:45 +0200 Subject: [PATCH 09/40] Reformat the code --- test/gt2py_fo_exec.jl | 195 ++++++++++++++++++++++-------------------- 1 file changed, 101 insertions(+), 94 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 7becec2..f710950 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -9,7 +9,8 @@ struct TestFailedException <: Exception end macro to_py(expr::Expr) - res = quote try + res = quote + try $(esc(expr)) true catch e @@ -28,7 +29,7 @@ struct ConnectivityData cell_to_edge_table::Matrix{Integer} E2C_offset_provider::Connectivity C2E_offset_provider::Connectivity - offset_provider::Dict{String, Connectivity} + offset_provider::Dict{String,Connectivity} end function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) @@ -46,36 +47,36 @@ end function setup_simple_connectivity() edge_to_cell_table = [ - [1 -1]; - [3 -1]; - [3 -1]; - [4 -1]; - [5 -1]; - [6 -1]; - [1 6]; - [1 2]; - [2 3]; - [2 4]; - [4 5]; - [5 6] + [1 -1]; + [3 -1]; + [3 -1]; + [4 -1]; + [5 -1]; + [6 -1]; + [1 6]; + [1 2]; + [2 3]; + [2 4]; + [4 5]; + [5 6] ] cell_to_edge_table = [ - [1 7 8]; - [8 9 10]; - [2 3 9]; - [4 10 11]; - [5 11 12]; - [6 7 12] + [1 7 8]; + [8 9 10]; + [2 3 9]; + [4 10 11]; + [5 11 12]; + [6 7 12] ] E2C_offset_provider = Connectivity(edge_to_cell_table, Cell, Edge, 2) C2E_offset_provider = Connectivity(cell_to_edge_table, Edge, Cell, 3) - offset_provider = Dict{String, Connectivity}( - "E2C" => E2C_offset_provider, - "C2E" => C2E_offset_provider - ) + offset_provider = Dict{String,Connectivity}( + "E2C" => E2C_offset_provider, + "C2E" => C2E_offset_provider + ) return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) end @@ -85,15 +86,15 @@ end # ======================================== function test_fo_addition(backend::String) - a = Field(Cell, collect(1.:15.)) - b = Field(Cell, collect(-1.:-1:-15.)) + a = Field(Cell, collect(1.0:15.0)) + b = Field(Cell, collect(-1.0:-1:-15.0)) out = Field(Cell, zeros(Float64, 15)) - @field_operator function fo_addition(a::Field{Tuple{Cell_}, Float64}, b::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_}, Float64} + @field_operator function fo_addition(a::Field{Tuple{Cell_},Float64}, b::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_},Float64} return a .+ b end - fo_addition(a, b, backend = backend, out = out) + fo_addition(a, b, backend=backend, out=out) @test all(out.data .== 0) end @@ -101,16 +102,16 @@ function test_fo_nested_if_else(backend::String) a = Field(Cell, collect(Int32, 1:15)) # TODO(tehrengruber): if we don't use the right dtype here we get a horrible error in python out = Field(Cell, zeros(Int32, 15)) - @field_operator function fo_nested_if_else(f::Field{Tuple{Cell_}, Int32})::Field{Tuple{Cell_}, Int32} + @field_operator function fo_nested_if_else(f::Field{Tuple{Cell_},Int32})::Field{Tuple{Cell_},Int32} tmp = f - if 1. < 10.0 + if 1.0 < 10.0 tmp = f .+ Int32(1) if 30 > 5 tmp = tmp .+ Int32(20) tmp = tmp .- Int32(10) elseif 40 < 4 tmp = 4 == 5 ? tmp : tmp .- 100 - else + else tmp = tmp .* 5 end tmp = tmp .+ Int32(10) @@ -124,72 +125,73 @@ function test_fo_nested_if_else(backend::String) return tmp end - fo_nested_if_else(a, backend = backend, out = out) + fo_nested_if_else(a, backend=backend, out=out) @test all(out.data .== collect(22:36)) end function test_fo_remapping(data::ConnectivityData, backend::String) - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) expected_output = a[data.edge_to_cell_table[:, 1]] # First column of the edge to cell connectivity table - @field_operator function fo_remapping(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + @field_operator function fo_remapping(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return a(E2C[1]) end - fo_remapping(a, offset_provider = data.offset_provider, backend = backend, out = out) + fo_remapping(a, offset_provider=data.offset_provider, backend=backend, out=out) @test all(out.data .== expected_output) end -function test_fo_neighbor_sum(backend::String) - a = Field(Cell, collect(1.:15.)) +function test_fo_neighbor_sum(data::ConnectivityData, backend::String) + a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return neighbor_sum(a(E2C), axis=E2CDim) end - @test @to_py fo_neighbor_sum(a, offset_provider=offset_provider, backend = "py", out = out) + fo_neighbor_sum(a, offset_provider=data.offset_provider, backend=backend, out=out) + @test 1 == 1 # TODO(lorenzovarese) check the correct output end function test_fo_max_over(backend::String) - + end function test_fo_min_over(backend::String) - + end function test_fo_simple_broadcast(backend::String) - + end function test_fo_scalar_broadcast(backend::String) - + end function test_fo_where(backend::String) - + end function test_fo_astype(backend::String) - + end function test_fo_sin(backend::String) - + end function test_fo_asinh(backend::String) - + end function test_fo_offset_array(backend::String) - + end function test_nested_fo(backend::String) - + end # ======================================== @@ -197,6 +199,8 @@ end # ======================================== function test_gt4py_fo_exec() + include("mesh_definitions.jl") + testwrapper(nothing, (args...) -> test_fo_addition(args...), "embedded") testwrapper(nothing, (args...) -> test_fo_addition(args...), "py") @@ -205,6 +209,9 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "embedded") testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "py") + + # testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "embedded") #TODO(lorenzovarese) fixing access to global mesh_definitions + # testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "py") end function test_gt4py_fo_exec_legacy() @@ -213,131 +220,131 @@ function test_gt4py_fo_exec_legacy() offset_provider = data.offset_provider # ------------------------------------------------ - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return neighbor_sum(a(E2C), axis=E2CDim) end - @test @to_py fo_neighbor_sum(a, offset_provider=offset_provider, backend = "py", out = out) + @test @to_py fo_neighbor_sum(a, offset_provider=offset_provider, backend="py", out=out) # ------------------------------------------------ - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - @field_operator function fo_max_over(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + @field_operator function fo_max_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return max_over(a(E2C), axis=E2CDim) end - @test @to_py fo_max_over(a, offset_provider=offset_provider, backend = "py", out = out) + @test @to_py fo_max_over(a, offset_provider=offset_provider, backend="py", out=out) # ------------------------------------------------ - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - @field_operator function fo_min_over(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Edge_}, Float64} + @field_operator function fo_min_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return min_over(a(E2C), axis=E2CDim) end - @test @to_py fo_min_over(a, offset_provider=offset_provider, backend = "py", out = out) + @test @to_py fo_min_over(a, offset_provider=offset_provider, backend="py", out=out) # ------------------------------------------------ - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) out = Field((Cell, K), zeros(15, 5)) - @field_operator function fo_simple_broadcast(a::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_, K_}, Float64} - return broadcast(a, (Cell, K)) + @field_operator function fo_simple_broadcast(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_,K_},Float64} + return broadcast(a, (Cell, K)) end - - @test @to_py fo_simple_broadcast(a, backend = "py", out = out) + + @test @to_py fo_simple_broadcast(a, backend="py", out=out) # ------------------------------------------------ - j_out = Field((), fill(0.), (Cell, K)) - py_out = Field((Cell, K), fill(0., (10, 10))) + j_out = Field((), fill(0.0), (Cell, K)) + py_out = Field((Cell, K), fill(0.0, (10, 10))) - @field_operator function fo_scalar_broadcast()::Field{Tuple{Cell_, K_}, Float64} - return broadcast(5., (Cell, K)) + @field_operator function fo_scalar_broadcast()::Field{Tuple{Cell_,K_},Float64} + return broadcast(5.0, (Cell, K)) end - @test @to_py fo_scalar_broadcast(backend = "py", out = py_out) + @test @to_py fo_scalar_broadcast(backend="py", out=py_out) # ------------------------------------------------ - a = Field((Cell, K), reshape(collect(1.:12.), (6, 2))) - b = Field((Cell, K), fill(-1., (6, 2))) + a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) + b = Field((Cell, K), fill(-1.0, (6, 2))) mask = Field((Cell, K), rand(Bool, (6, 2))) out = Field((Cell, K), zeros(6, 2)) - @field_operator function fo_where(mask::Field{Tuple{Cell_, K_}, Bool}, a::Field{Tuple{Cell_, K_}, Float64}, b::Field{Tuple{Cell_, K_}, Float64})::Field{Tuple{Cell_, K_}, Float64} - return where(mask, a, b) + @field_operator function fo_where(mask::Field{Tuple{Cell_,K_},Bool}, a::Field{Tuple{Cell_,K_},Float64}, b::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} + return where(mask, a, b) end - @test @to_py fo_where(mask, a, b, backend = "py", out = out) + @test @to_py fo_where(mask, a, b, backend="py", out=out) # ------------------------------------------------- - a = Field((Cell, K), reshape(collect(1.:12.), (6, 2))) + a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros(Int64, (6, 2))) - @field_operator function fo_astype(a::Field{Tuple{Cell_, K_}, Float64})::Field{Tuple{Cell_, K_}, Int64} - return convert(Int64, a) + @field_operator function fo_astype(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Int64} + return convert(Int64, a) end - @test @to_py fo_astype(a, backend = "py", out = out) + @test @to_py fo_astype(a, backend="py", out=out) # ------------------------------------------------- - a = Field((Cell, K), reshape(collect(1.:12.), (6, 2))) + a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) - @field_operator function fo_sin(a::Field{Tuple{Cell_, K_}, Float64})::Field{Tuple{Cell_, K_}, Float64} - return sin.(a) + @field_operator function fo_sin(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} + return sin.(a) end - @test @to_py fo_sin(a, backend = "py", out = out) + @test @to_py fo_sin(a, backend="py", out=out) # ------------------------------------------------- - a = Field((Cell, K), reshape(collect(1.:12.), (6, 2))) + a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) - @field_operator function fo_asinh(a::Field{Tuple{Cell_, K_}, Float64})::Field{Tuple{Cell_, K_}, Float64} - return asinh.(a) + @field_operator function fo_asinh(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} + return asinh.(a) end - @test @to_py fo_asinh(a, backend = "py", out = out) + @test @to_py fo_asinh(a, backend="py", out=out) # ------------------------------------------------- # TODO OffsetArray is ignored for the moment - A = Field((Vertex, K), reshape(collect(1.:15.), 3, 5), origin = Dict(Vertex => -2, K => -1)) + A = Field((Vertex, K), reshape(collect(1.0:15.0), 3, 5), origin=Dict(Vertex => -2, K => -1)) B = Field((K, Edge), reshape(ones(6), 3, 2)) - out = Field((Vertex, K, Edge), zeros(3,3,2)) + out = Field((Vertex, K, Edge), zeros(3, 3, 2)) - @field_operator function fo_offset_array(A::Field{Tuple{Vertex_, K_}, Float64}, B::Field{Tuple{K_, Edge_}, Float64})::Field{Tuple{Vertex_, K_, Edge_}, Float64} - return A .+ B - end + @field_operator function fo_offset_array(A::Field{Tuple{Vertex_,K_},Float64}, B::Field{Tuple{K_,Edge_},Float64})::Field{Tuple{Vertex_,K_,Edge_},Float64} + return A .+ B + end @test @to_py fo_offset_array(A, B, backend="py", out=out) # ------------------------------------------------- - a = Field(Cell, collect(1.:15.)) + a = Field(Cell, collect(1.0:15.0)) b = Field(Cell, ones(15)) out = Field(Cell, zeros(15)) - @field_operator function fo_addition(a::Field{Tuple{Cell_}, Float64}, b::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_}, Float64} + @field_operator function fo_addition(a::Field{Tuple{Cell_},Float64}, b::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_},Float64} return a .+ b end - @field_operator function nested_fo(a::Field{Tuple{Cell_}, Float64}, b::Field{Tuple{Cell_}, Float64})::Field{Tuple{Cell_}, Float64} + @field_operator function nested_fo(a::Field{Tuple{Cell_},Float64}, b::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_},Float64} res = fo_addition(a, b) return res .+ a end @@ -345,7 +352,7 @@ end function run_all() test_gt4py_fo_exec() - test_gt4py_fo_exec_legacy() + # test_gt4py_fo_exec_legacy() end @testset "Testset GT2Py fo exec" run_all() From 8c3cbd09f7a6d9d16ce9757baccffc5d86a9132a Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Tue, 9 Jul 2024 17:34:04 +0200 Subject: [PATCH 10/40] Fix the offset provider of the embedded execution that requires E2CDim in gt2py_fo_exec --- test/gt2py_fo_exec.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index f710950..3e49016 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -75,7 +75,8 @@ function setup_simple_connectivity() offset_provider = Dict{String,Connectivity}( "E2C" => E2C_offset_provider, - "C2E" => C2E_offset_provider + "C2E" => C2E_offset_provider, + "E2CDim" => E2C_offset_provider #TODO(lorenzovarese) this is required for the embedded backend and the python already uses E2C ) return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) @@ -199,8 +200,6 @@ end # ======================================== function test_gt4py_fo_exec() - include("mesh_definitions.jl") - testwrapper(nothing, (args...) -> test_fo_addition(args...), "embedded") testwrapper(nothing, (args...) -> test_fo_addition(args...), "py") @@ -210,7 +209,7 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "embedded") testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "py") - # testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "embedded") #TODO(lorenzovarese) fixing access to global mesh_definitions + testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "embedded") #TODO(lorenzovarese) fixing access to global mesh_definitions # testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "py") end From 154e571cafc98b5024727a3b983219c5a9e74416 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 11:09:07 +0200 Subject: [PATCH 11/40] Clean lambda functions in testwrapper --- test/gt2py_fo_exec.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 3e49016..824fdf0 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -200,17 +200,17 @@ end # ======================================== function test_gt4py_fo_exec() - testwrapper(nothing, (args...) -> test_fo_addition(args...), "embedded") - testwrapper(nothing, (args...) -> test_fo_addition(args...), "py") + testwrapper(nothing, test_fo_addition, "embedded") + testwrapper(nothing, test_fo_addition, "py") - testwrapper(nothing, (args...) -> test_fo_nested_if_else(args...), "embedded") - testwrapper(nothing, (args...) -> test_fo_nested_if_else(args...), "py") + testwrapper(nothing, test_fo_nested_if_else, "embedded") + testwrapper(nothing, test_fo_nested_if_else, "py") - testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "embedded") - testwrapper(setup_simple_connectivity, (args...) -> test_fo_remapping(args...), "py") + testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") + testwrapper(setup_simple_connectivity, test_fo_remapping, "py") - testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "embedded") #TODO(lorenzovarese) fixing access to global mesh_definitions - # testwrapper(setup_simple_connectivity, (args...) -> test_fo_neighbor_sum(args...), "py") + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") end function test_gt4py_fo_exec_legacy() From e6f7128b2e7c45d74b33051996cd76c8f9e91df4 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 11:33:55 +0200 Subject: [PATCH 12/40] Add ground truth computation for test_fo_neighbor_sum test --- test/gt2py_fo_exec.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 824fdf0..197dcae 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -147,12 +147,18 @@ function test_fo_neighbor_sum(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) + # Function to sum only the positive elements of each inner vector (to exclude the -1 in the connectivity) + sum_positive_elements(v) = sum(x -> x > 0 ? x : 0, v) + + # Compute the ground truth manually computing the sum on that dimension + expected_output = a[Integer.(map(sum_positive_elements, eachrow(data.edge_to_cell_table)))] + @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return neighbor_sum(a(E2C), axis=E2CDim) end fo_neighbor_sum(a, offset_provider=data.offset_provider, backend=backend, out=out) - @test 1 == 1 # TODO(lorenzovarese) check the correct output + @test out == expected_output end function test_fo_max_over(backend::String) From 9cdf9b95b6211052008ee26f2bdec8d5384dd02b Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 11:48:21 +0200 Subject: [PATCH 13/40] Setup the implementation of the following tests --- test/gt2py_fo_exec.jl | 164 +++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 105 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 197dcae..8b4fedc 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -161,80 +161,7 @@ function test_fo_neighbor_sum(data::ConnectivityData, backend::String) @test out == expected_output end -function test_fo_max_over(backend::String) - -end - -function test_fo_min_over(backend::String) - -end - -function test_fo_simple_broadcast(backend::String) - -end - -function test_fo_scalar_broadcast(backend::String) - -end - -function test_fo_where(backend::String) - -end - -function test_fo_astype(backend::String) - -end - -function test_fo_sin(backend::String) - -end - -function test_fo_asinh(backend::String) - -end - -function test_fo_offset_array(backend::String) - -end - -function test_nested_fo(backend::String) - -end - -# ======================================== -# ========== Test Executions ============= -# ======================================== - -function test_gt4py_fo_exec() - testwrapper(nothing, test_fo_addition, "embedded") - testwrapper(nothing, test_fo_addition, "py") - - testwrapper(nothing, test_fo_nested_if_else, "embedded") - testwrapper(nothing, test_fo_nested_if_else, "py") - - testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") - testwrapper(setup_simple_connectivity, test_fo_remapping, "py") - - testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") - testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") -end - -function test_gt4py_fo_exec_legacy() - # Set up locally to mimic the previous global behavior - data = setup_simple_connectivity() - offset_provider = data.offset_provider - - # ------------------------------------------------ - a = Field(Cell, collect(1.0:15.0)) - out = Field(Edge, zeros(Float64, 12)) - - @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} - return neighbor_sum(a(E2C), axis=E2CDim) - end - - @test @to_py fo_neighbor_sum(a, offset_provider=offset_provider, backend="py", out=out) - - # ------------------------------------------------ +function test_fo_max_over(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) @@ -242,10 +169,11 @@ function test_gt4py_fo_exec_legacy() return max_over(a(E2C), axis=E2CDim) end - @test @to_py fo_max_over(a, offset_provider=offset_provider, backend="py", out=out) - - # ------------------------------------------------ + fo_max_over(a, offset_provider=data.offset_provider, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_min_over(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) @@ -253,10 +181,11 @@ function test_gt4py_fo_exec_legacy() return min_over(a(E2C), axis=E2CDim) end - @test @to_py fo_min_over(a, offset_provider=offset_provider, backend="py", out=out) - - # ------------------------------------------------ + fo_min_over(a, offset_provider=data.offset_provider, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_simple_broadcast(backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field((Cell, K), zeros(15, 5)) @@ -264,10 +193,11 @@ function test_gt4py_fo_exec_legacy() return broadcast(a, (Cell, K)) end - @test @to_py fo_simple_broadcast(a, backend="py", out=out) - - # ------------------------------------------------ + fo_simple_broadcast(a, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_scalar_broadcast(backend::String) j_out = Field((), fill(0.0), (Cell, K)) py_out = Field((Cell, K), fill(0.0, (10, 10))) @@ -276,10 +206,11 @@ function test_gt4py_fo_exec_legacy() return broadcast(5.0, (Cell, K)) end - @test @to_py fo_scalar_broadcast(backend="py", out=py_out) - - # ------------------------------------------------ + fo_scalar_broadcast(backend=backend, out=py_out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_where(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) b = Field((Cell, K), fill(-1.0, (6, 2))) mask = Field((Cell, K), rand(Bool, (6, 2))) @@ -289,10 +220,11 @@ function test_gt4py_fo_exec_legacy() return where(mask, a, b) end - @test @to_py fo_where(mask, a, b, backend="py", out=out) - - # ------------------------------------------------- + fo_where(mask, a, b, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_astype(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros(Int64, (6, 2))) @@ -300,10 +232,11 @@ function test_gt4py_fo_exec_legacy() return convert(Int64, a) end - @test @to_py fo_astype(a, backend="py", out=out) - - # ------------------------------------------------- + fo_astype(a, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_sin(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) @@ -311,10 +244,11 @@ function test_gt4py_fo_exec_legacy() return sin.(a) end - @test @to_py fo_sin(a, backend="py", out=out) - - # ------------------------------------------------- + fo_sin(a, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_asinh(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) @@ -322,10 +256,11 @@ function test_gt4py_fo_exec_legacy() return asinh.(a) end - @test @to_py fo_asinh(a, backend="py", out=out) - - # ------------------------------------------------- + fo_asinh(a, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_fo_offset_array(backend::String) # TODO OffsetArray is ignored for the moment A = Field((Vertex, K), reshape(collect(1.0:15.0), 3, 5), origin=Dict(Vertex => -2, K => -1)) @@ -337,10 +272,11 @@ function test_gt4py_fo_exec_legacy() return A .+ B end - @test @to_py fo_offset_array(A, B, backend="py", out=out) - - # ------------------------------------------------- + fo_offset_array(A, B, backend=backend, out=out) + @test out == out # TODO(lorenzovarese): identify ground truth +end +function test_nested_fo(backend::String) a = Field(Cell, collect(1.0:15.0)) b = Field(Cell, ones(15)) out = Field(Cell, zeros(15)) @@ -353,11 +289,29 @@ function test_gt4py_fo_exec_legacy() res = fo_addition(a, b) return res .+ a end + + # Add invocation + @test out == out # TODO(lorenzovarese): identify ground truth end -function run_all() - test_gt4py_fo_exec() - # test_gt4py_fo_exec_legacy() +# ======================================== +# ========== Test Executions ============= +# ======================================== + +function test_gt4py_fo_exec() + testwrapper(nothing, test_fo_addition, "embedded") + testwrapper(nothing, test_fo_addition, "py") + + testwrapper(nothing, test_fo_nested_if_else, "embedded") + testwrapper(nothing, test_fo_nested_if_else, "py") + + testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") + testwrapper(setup_simple_connectivity, test_fo_remapping, "py") + + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") + + # TODO(lorenzovarese): add the missing ones end -@testset "Testset GT2Py fo exec" run_all() +@testset "Testset GT2Py fo exec" test_gt4py_fo_exec() From 6f7fd528a30b4cd73f59b567861c4b447742bb42 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 11:58:54 +0200 Subject: [PATCH 14/40] Improve fo_max_over test --- test/gt2py_fo_exec.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 8b4fedc..4203ae5 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -165,12 +165,15 @@ function test_fo_max_over(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) + # Compute the ground truth manually computing max on that dimension + expected_output = a[Integer.(map(maximum, eachrow(data.edge_to_cell_table)))] + @field_operator function fo_max_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return max_over(a(E2C), axis=E2CDim) end fo_max_over(a, offset_provider=data.offset_provider, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test out == expected_output end function test_fo_min_over(data::ConnectivityData, backend::String) @@ -311,6 +314,9 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") + testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") + testwrapper(setup_simple_connectivity, test_fo_max_over, "py") + # TODO(lorenzovarese): add the missing ones end From c6d89c7b36a7b6e2bb2dec504311bf80b4e171ae Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 12:07:22 +0200 Subject: [PATCH 15/40] Improve test for min --- test/gt2py_fo_exec.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 4203ae5..cdaa7fa 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -180,12 +180,18 @@ function test_fo_min_over(data::ConnectivityData, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) + # Function to return the minimum positive element of each inner vector + mim_positive_element(v) = minimum(filter(x -> x > 0, v)) + + # Compute the ground truth manually computing min on that dimension + expected_output = a[Integer.(map(mim_positive_element, eachrow(data.edge_to_cell_table)))] # We exclude the -1 + @field_operator function fo_min_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return min_over(a(E2C), axis=E2CDim) end fo_min_over(a, offset_provider=data.offset_provider, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test out == expected_output end function test_fo_simple_broadcast(backend::String) @@ -317,6 +323,9 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") testwrapper(setup_simple_connectivity, test_fo_max_over, "py") + testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") + testwrapper(setup_simple_connectivity, test_fo_min_over, "py") + # TODO(lorenzovarese): add the missing ones end From 7ca03d5720b25536542b9a715bc1d64ab3f49767 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 12:56:42 +0200 Subject: [PATCH 16/40] Improve test for simple broadcast --- test/gt2py_fo_exec.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index cdaa7fa..34ce862 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -195,15 +195,20 @@ function test_fo_min_over(data::ConnectivityData, backend::String) end function test_fo_simple_broadcast(backend::String) - a = Field(Cell, collect(1.0:15.0)) - out = Field((Cell, K), zeros(15, 5)) + data = collect(1.0:15.0) + broadcast_num_dims = 5 + a = Field(Cell, data) + out = Field((Cell, K), zeros(15, broadcast_num_dims)) + + # Compute the expected output by broadcasting a + expected_output = [a[i] for i in 1:15, j in 1:broadcast_num_dims] @field_operator function fo_simple_broadcast(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_,K_},Float64} return broadcast(a, (Cell, K)) end fo_simple_broadcast(a, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test out == expected_output end function test_fo_scalar_broadcast(backend::String) @@ -326,6 +331,8 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") testwrapper(setup_simple_connectivity, test_fo_min_over, "py") + testwrapper(nothing, test_fo_simple_broadcast, "embedded") + testwrapper(nothing, test_fo_simple_broadcast, "py") # TODO(lorenzovarese): add the missing ones end From 500d57ca937f4994e9d8d857624d83a6cf869f02 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 12:59:58 +0200 Subject: [PATCH 17/40] Fix Typos --- test/gt2py_fo_exec.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 34ce862..0dc288a 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -76,14 +76,14 @@ function setup_simple_connectivity() offset_provider = Dict{String,Connectivity}( "E2C" => E2C_offset_provider, "C2E" => C2E_offset_provider, - "E2CDim" => E2C_offset_provider #TODO(lorenzovarese) this is required for the embedded backend and the python already uses E2C + "E2CDim" => E2C_offset_provider #TODO(lorenzovarese) this is required for the embedded backend (note: python already uses E2C) ) return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) end # ======================================== -# ========= Tests Definition ============= +# ========= Test Definitions ============= # ======================================== function test_fo_addition(backend::String) From 48605623deeb9fb7f0b3520e9358e9d7a56d1a51 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 13:16:24 +0200 Subject: [PATCH 18/40] Improve scalar broadcast test --- test/gt2py_fo_exec.jl | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 0dc288a..ec04363 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -212,16 +212,17 @@ function test_fo_simple_broadcast(backend::String) end function test_fo_scalar_broadcast(backend::String) - j_out = Field((), fill(0.0), (Cell, K)) - py_out = Field((Cell, K), fill(0.0, (10, 10))) + out = Field((Cell, K), fill(0.0, (10, 10))) + # Compute the expected output by broadcasting the value + expected_output = fill(5.0, (10, 10)) @field_operator function fo_scalar_broadcast()::Field{Tuple{Cell_,K_},Float64} return broadcast(5.0, (Cell, K)) end - fo_scalar_broadcast(backend=backend, out=py_out) - @test out == out # TODO(lorenzovarese): identify ground truth + fo_scalar_broadcast(backend=backend, out=out) + @test out == expected_output end function test_fo_where(backend::String) @@ -313,26 +314,29 @@ end # ======================================== function test_gt4py_fo_exec() - testwrapper(nothing, test_fo_addition, "embedded") - testwrapper(nothing, test_fo_addition, "py") + # testwrapper(nothing, test_fo_addition, "embedded") + # testwrapper(nothing, test_fo_addition, "py") + + # testwrapper(nothing, test_fo_nested_if_else, "embedded") + # testwrapper(nothing, test_fo_nested_if_else, "py") - testwrapper(nothing, test_fo_nested_if_else, "embedded") - testwrapper(nothing, test_fo_nested_if_else, "py") + # testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") + # testwrapper(setup_simple_connectivity, test_fo_remapping, "py") - testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") - testwrapper(setup_simple_connectivity, test_fo_remapping, "py") + # testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") + # testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") - testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") - testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") + # testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") + # testwrapper(setup_simple_connectivity, test_fo_max_over, "py") - testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") - testwrapper(setup_simple_connectivity, test_fo_max_over, "py") + # testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") + # testwrapper(setup_simple_connectivity, test_fo_min_over, "py") - testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") - testwrapper(setup_simple_connectivity, test_fo_min_over, "py") + # testwrapper(nothing, test_fo_simple_broadcast, "embedded") + # testwrapper(nothing, test_fo_simple_broadcast, "py") - testwrapper(nothing, test_fo_simple_broadcast, "embedded") - testwrapper(nothing, test_fo_simple_broadcast, "py") + testwrapper(nothing, test_fo_scalar_broadcast, "embedded") + testwrapper(nothing, test_fo_scalar_broadcast, "py") # TODO(lorenzovarese): add the missing ones end From d3f9b92e9366ba3f1cd47cdffe2ce85b2e0af3b5 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 13:37:51 +0200 Subject: [PATCH 19/40] Fix `where` example in builtins.jl --- src/embedded/builtins.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/embedded/builtins.jl b/src/embedded/builtins.jl index 493dd60..6ddf639 100644 --- a/src/embedded/builtins.jl +++ b/src/embedded/builtins.jl @@ -78,9 +78,9 @@ julia> a = Field((Cell, K), fill(1.0, (3,3))); julia> b = Field((Cell, K), fill(2.0, (3,3))); julia> where(mask, a, b) 3x3 Field with dimensions ("Cell", "K") with indices 1:3×1:3: - 1.0 2.0 2.0 - 2.0 1.0 2.0 + 2.0 2.0 1.0 1.0 1.0 1.0 + 1.0 2.0 2.0 ``` The `where` function builtin also allows for nesting of tuples. In this scenario, it will first perform an unrolling: From 3fcc315fe029ab8b904a69be24ee92c6e5fd7418 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 13:47:04 +0200 Subject: [PATCH 20/40] Improve `where` test --- test/gt2py_fo_exec.jl | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index ec04363..5ee7ca4 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -226,17 +226,27 @@ function test_fo_scalar_broadcast(backend::String) end function test_fo_where(backend::String) - a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) - b = Field((Cell, K), fill(-1.0, (6, 2))) - mask = Field((Cell, K), rand(Bool, (6, 2))) - out = Field((Cell, K), zeros(6, 2)) + a = Field((Cell, K), reshape(collect(1.0:10.0), (5, 2))) # The matrix is filled column major + b = Field((Cell, K), fill(-1.0, (5, 2))) + mask = Field((Cell, K), [true false; + false true; + true false; + false false; + true true ]) + out = Field((Cell, K), zeros(5, 2)) + + expected_output = [ 1 -1 + -1 7 + 3 -1 + -1 -1 + 5 10 ] @field_operator function fo_where(mask::Field{Tuple{Cell_,K_},Bool}, a::Field{Tuple{Cell_,K_},Float64}, b::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} return where(mask, a, b) end fo_where(mask, a, b, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test out == expected_output end function test_fo_astype(backend::String) @@ -335,8 +345,11 @@ function test_gt4py_fo_exec() # testwrapper(nothing, test_fo_simple_broadcast, "embedded") # testwrapper(nothing, test_fo_simple_broadcast, "py") - testwrapper(nothing, test_fo_scalar_broadcast, "embedded") - testwrapper(nothing, test_fo_scalar_broadcast, "py") + # testwrapper(nothing, test_fo_scalar_broadcast, "embedded") + # testwrapper(nothing, test_fo_scalar_broadcast, "py") + + testwrapper(nothing, test_fo_where, "embedded") + testwrapper(nothing, test_fo_where, "py") # TODO(lorenzovarese): add the missing ones end From 16f3f89e46396fd0b04a8617831993cab88b968f Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 14:19:57 +0200 Subject: [PATCH 21/40] Improve test on field `convert` (casting) --- test/gt2py_fo_exec.jl | 45 +++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 5ee7ca4..f4bc646 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -250,15 +250,19 @@ function test_fo_where(backend::String) end function test_fo_astype(backend::String) - a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) + a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) # Floating Point out = Field((Cell, K), zeros(Int64, (6, 2))) + expected_values = reshape(collect(1.0:12.0), (6, 2)) + @field_operator function fo_astype(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Int64} - return convert(Int64, a) + return convert(Int64, a) # Integer end fo_astype(a, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test out == expected_values + @test eltype(out.data) == Int64 + @test eltype(a.data) == Float64 end function test_fo_sin(backend::String) @@ -324,32 +328,35 @@ end # ======================================== function test_gt4py_fo_exec() - # testwrapper(nothing, test_fo_addition, "embedded") - # testwrapper(nothing, test_fo_addition, "py") + testwrapper(nothing, test_fo_addition, "embedded") + testwrapper(nothing, test_fo_addition, "py") - # testwrapper(nothing, test_fo_nested_if_else, "embedded") - # testwrapper(nothing, test_fo_nested_if_else, "py") + testwrapper(nothing, test_fo_nested_if_else, "embedded") + testwrapper(nothing, test_fo_nested_if_else, "py") - # testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") - # testwrapper(setup_simple_connectivity, test_fo_remapping, "py") + testwrapper(setup_simple_connectivity, test_fo_remapping, "embedded") + testwrapper(setup_simple_connectivity, test_fo_remapping, "py") - # testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") - # testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") + testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") - # testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") - # testwrapper(setup_simple_connectivity, test_fo_max_over, "py") + testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") + testwrapper(setup_simple_connectivity, test_fo_max_over, "py") - # testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") - # testwrapper(setup_simple_connectivity, test_fo_min_over, "py") + testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") + testwrapper(setup_simple_connectivity, test_fo_min_over, "py") - # testwrapper(nothing, test_fo_simple_broadcast, "embedded") - # testwrapper(nothing, test_fo_simple_broadcast, "py") + testwrapper(nothing, test_fo_simple_broadcast, "embedded") + testwrapper(nothing, test_fo_simple_broadcast, "py") - # testwrapper(nothing, test_fo_scalar_broadcast, "embedded") - # testwrapper(nothing, test_fo_scalar_broadcast, "py") + testwrapper(nothing, test_fo_scalar_broadcast, "embedded") + testwrapper(nothing, test_fo_scalar_broadcast, "py") testwrapper(nothing, test_fo_where, "embedded") testwrapper(nothing, test_fo_where, "py") + + testwrapper(nothing, test_fo_astype, "embedded") + testwrapper(nothing, test_fo_astype, "py") # TODO(lorenzovarese): add the missing ones end From 4a2aeabe1978b1405fec89f2faf1cd8baa1bd303 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 14:27:16 +0200 Subject: [PATCH 22/40] Improve tests for sin and asinh --- test/gt2py_fo_exec.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index f4bc646..9d08093 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -269,24 +269,30 @@ function test_fo_sin(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) + # Compute the expected output using the sin function + expected_output = sin.(reshape(collect(1.0:12.0), (6, 2))) + @field_operator function fo_sin(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} return sin.(a) end fo_sin(a, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test isapprox(out.data, expected_output, atol=1e-6) end function test_fo_asinh(backend::String) a = Field((Cell, K), reshape(collect(1.0:12.0), (6, 2))) out = Field((Cell, K), zeros((6, 2))) + # Compute the expected output using the asinh function + expected_output = asinh.(reshape(collect(1.0:12.0), (6, 2))) + @field_operator function fo_asinh(a::Field{Tuple{Cell_,K_},Float64})::Field{Tuple{Cell_,K_},Float64} return asinh.(a) end fo_asinh(a, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test isapprox(out.data, expected_output, atol=1e-6) end function test_fo_offset_array(backend::String) @@ -357,6 +363,12 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_astype, "embedded") testwrapper(nothing, test_fo_astype, "py") + + testwrapper(nothing, test_fo_sin, "embedded") + testwrapper(nothing, test_fo_sin, "py") + + testwrapper(nothing, test_fo_asinh, "embedded") + testwrapper(nothing, test_fo_asinh, "py") # TODO(lorenzovarese): add the missing ones end From 8e258daf48cd464814b7ae50a1268a79f55cce5d Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 14:31:52 +0200 Subject: [PATCH 23/40] Add documentation to the testwrapper utility function --- test/gt2py_fo_exec.jl | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 9d08093..cd1f041 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -32,6 +32,51 @@ struct ConnectivityData offset_provider::Dict{String,Connectivity} end +""" + testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) + +Wrapper function to facilitate testing with optional setup. + +# Arguments +- `setupfunc::Union{Function,Nothing}`: An optional setup function. If provided, it will be called before the test function. + If `nothing`, the test function is called directly. +- `testfunc::Function`: The test function to be executed. +- `args...`: Additional arguments to be passed to the test function. + +# Usage +- If `setupfunc` is provided, it should return the necessary data that `testfunc` will use. + The returned data will be passed as the first argument to `testfunc`, followed by `args...`. +- If `setupfunc` is `nothing`, `testfunc` will be called directly with `args...`. + +# Examples + +## Example 1: Without Setup Function +```julia +function mytest(args...) + println("Running test with arguments: ", args) +end + +testwrapper(nothing, mytest, 1, 2, 3) +# Output: Running test with arguments: (1, 2, 3) +``` + +## Example 2: With Setup Function +```julia +function setup() + return "setup data" +end + +function mytest(data, args...) + println("Setup data: ", data) + println("Running test with arguments: ", args) +end + +testwrapper(setup, mytest, 1, 2, 3) +# Output: +# Setup data: setup data +# Running test with arguments: (1, 2, 3) +``` +""" function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) if setupfunc === nothing testfunc(args...) From c57646fe808c53d4164fd7cac9cb293244d8292b Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 14:51:49 +0200 Subject: [PATCH 24/40] Improve nested field_operator test and clean the missing offset array test --- test/gt2py_fo_exec.jl | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index cd1f041..d73fa1a 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -352,8 +352,9 @@ function test_fo_offset_array(backend::String) return A .+ B end - fo_offset_array(A, B, backend=backend, out=out) - @test out == out # TODO(lorenzovarese): identify ground truth + @test @to_py fo_offset_array(A, B, backend=backend, out=out) + println("test_fo_offset_array - backend->[", backend, "] - output: ", out.data) + # @test out == expected_output # TODO: identify ground truth end function test_nested_fo(backend::String) @@ -361,6 +362,10 @@ function test_nested_fo(backend::String) b = Field(Cell, ones(15)) out = Field(Cell, zeros(15)) + # Compute the Ground Truth + intermediate_result = a.data .+ b.data + expected_output = intermediate_result .+ a.data + @field_operator function fo_addition(a::Field{Tuple{Cell_},Float64}, b::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_},Float64} return a .+ b end @@ -370,8 +375,10 @@ function test_nested_fo(backend::String) return res .+ a end - # Add invocation - @test out == out # TODO(lorenzovarese): identify ground truth + nested_fo(a, b, backend=backend, out=out) + + # Test against the Ground Truth + @test out.data == expected_output end # ======================================== @@ -414,7 +421,12 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_asinh, "embedded") testwrapper(nothing, test_fo_asinh, "py") - # TODO(lorenzovarese): add the missing ones + + testwrapper(nothing, test_fo_offset_array, "embedded") # TODO: implementation is missing + testwrapper(nothing, test_fo_offset_array, "py") + + testwrapper(nothing, test_nested_fo, "embedded") + testwrapper(nothing, test_nested_fo, "py") end @testset "Testset GT2Py fo exec" test_gt4py_fo_exec() From 461814f453c7263c8f1a53af7d9dad0954f70aeb Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 15:09:09 +0200 Subject: [PATCH 25/40] Add notes on offset array --- test/gt2py_fo_exec.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index d73fa1a..43e2c4f 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -341,7 +341,7 @@ function test_fo_asinh(backend::String) end function test_fo_offset_array(backend::String) - # TODO OffsetArray is ignored for the moment + # TODO OffsetArray is ignored and will be removed in the future A = Field((Vertex, K), reshape(collect(1.0:15.0), 3, 5), origin=Dict(Vertex => -2, K => -1)) B = Field((K, Edge), reshape(ones(6), 3, 2)) @@ -352,7 +352,7 @@ function test_fo_offset_array(backend::String) return A .+ B end - @test @to_py fo_offset_array(A, B, backend=backend, out=out) + @test @to_py fo_offset_array(A, B, backend=backend, out=out) # Simply check if the execution is performed println("test_fo_offset_array - backend->[", backend, "] - output: ", out.data) # @test out == expected_output # TODO: identify ground truth end From 5a1384212f16c16156138bd65879bd5e5b1944a8 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 15:10:43 +0200 Subject: [PATCH 26/40] Cleanup before submission --- test/gt2py_fo_exec.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 43e2c4f..9288ecd 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -352,9 +352,9 @@ function test_fo_offset_array(backend::String) return A .+ B end - @test @to_py fo_offset_array(A, B, backend=backend, out=out) # Simply check if the execution is performed + @test fo_offset_array(A, B, backend=backend, out=out) # Simply check if the execution is performed println("test_fo_offset_array - backend->[", backend, "] - output: ", out.data) - # @test out == expected_output # TODO: identify ground truth + # @test out == expected_output end function test_nested_fo(backend::String) @@ -422,7 +422,7 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_asinh, "embedded") testwrapper(nothing, test_fo_asinh, "py") - testwrapper(nothing, test_fo_offset_array, "embedded") # TODO: implementation is missing + testwrapper(nothing, test_fo_offset_array, "embedded") testwrapper(nothing, test_fo_offset_array, "py") testwrapper(nothing, test_nested_fo, "embedded") From 8420fe9623d77e9003fc40298c5dbe93d15e5214 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 10 Jul 2024 15:28:35 +0200 Subject: [PATCH 27/40] Fix offset array test --- test/gt2py_fo_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 9288ecd..3627994 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -352,7 +352,7 @@ function test_fo_offset_array(backend::String) return A .+ B end - @test fo_offset_array(A, B, backend=backend, out=out) # Simply check if the execution is performed + @test @to_py fo_offset_array(A, B, backend=backend, out=out) # Simply check if the execution is performed println("test_fo_offset_array - backend->[", backend, "] - output: ", out.data) # @test out == expected_output end From b1cda4d1925d2aae515b7c1ba6d572cb63b664f8 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Thu, 11 Jul 2024 09:06:52 +0200 Subject: [PATCH 28/40] Minor fixes --- test/gt2py_fo_exec.jl | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 3627994..2e61c30 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -20,9 +20,7 @@ macro to_py(expr::Expr) return res end -# ======================================== -# ============== Utility ================= -# ======================================== +# Utility ---------------------------------------------------------------------------------------------------- struct ConnectivityData edge_to_cell_table::Matrix{Integer} @@ -86,11 +84,9 @@ function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, arg end end -# ======================================== -# ============== Setup =================== -# ======================================== +# Setup ------------------------------------------------------------------------------------------------------ -function setup_simple_connectivity() +function setup_simple_connectivity()::ConnectivityData edge_to_cell_table = [ [1 -1]; [3 -1]; @@ -127,9 +123,7 @@ function setup_simple_connectivity() return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) end -# ======================================== -# ========= Test Definitions ============= -# ======================================== +# Test Definitions ------------------------------------------------------------------------------------------- function test_fo_addition(backend::String) a = Field(Cell, collect(1.0:15.0)) @@ -381,9 +375,7 @@ function test_nested_fo(backend::String) @test out.data == expected_output end -# ======================================== -# ========== Test Executions ============= -# ======================================== +# Test Executions -------------------------------------------------------------------------------------------- function test_gt4py_fo_exec() testwrapper(nothing, test_fo_addition, "embedded") From 26f7686140d50dd33e6902a785cdc66f59a18187 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Thu, 11 Jul 2024 09:45:11 +0200 Subject: [PATCH 29/40] Remove struct for connectivity data (ensure usage of offset_provider in all cases) --- src/GridTools.jl | 10 ++++++++++ test/gt2py_fo_exec.jl | 36 ++++++++++++++---------------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/GridTools.jl b/src/GridTools.jl index 31b80ba..71e911a 100644 --- a/src/GridTools.jl +++ b/src/GridTools.jl @@ -455,6 +455,16 @@ struct Connectivity dims::Integer end +@generated function Base.getindex(conn::Connectivity, row::Union{Integer, Colon}, col::Integer) + if row <: Integer + return :(conn.data[row, col]) + elseif row <: Colon + return :(conn.data[:, col]) + else + throw(ArgumentError("Unsupported index type")) + end +end + # Field operator ---------------------------------------------------------------------- struct FieldOp diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 2e61c30..3df2814 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -22,14 +22,6 @@ end # Utility ---------------------------------------------------------------------------------------------------- -struct ConnectivityData - edge_to_cell_table::Matrix{Integer} - cell_to_edge_table::Matrix{Integer} - E2C_offset_provider::Connectivity - C2E_offset_provider::Connectivity - offset_provider::Dict{String,Connectivity} -end - """ testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) @@ -86,7 +78,7 @@ end # Setup ------------------------------------------------------------------------------------------------------ -function setup_simple_connectivity()::ConnectivityData +function setup_simple_connectivity()::Dict{String,Connectivity} edge_to_cell_table = [ [1 -1]; [3 -1]; @@ -120,7 +112,7 @@ function setup_simple_connectivity()::ConnectivityData "E2CDim" => E2C_offset_provider #TODO(lorenzovarese) this is required for the embedded backend (note: python already uses E2C) ) - return ConnectivityData(edge_to_cell_table, cell_to_edge_table, E2C_offset_provider, C2E_offset_provider, offset_provider) + return offset_provider end # Test Definitions ------------------------------------------------------------------------------------------- @@ -169,20 +161,20 @@ function test_fo_nested_if_else(backend::String) @test all(out.data .== collect(22:36)) end -function test_fo_remapping(data::ConnectivityData, backend::String) +function test_fo_remapping(offset_provider::Dict{String,Connectivity}, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - expected_output = a[data.edge_to_cell_table[:, 1]] # First column of the edge to cell connectivity table + expected_output = a[offset_provider["E2C"][:, 1]] # First column of the edge to cell connectivity table @field_operator function fo_remapping(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return a(E2C[1]) end - fo_remapping(a, offset_provider=data.offset_provider, backend=backend, out=out) + fo_remapping(a, offset_provider=offset_provider, backend=backend, out=out) @test all(out.data .== expected_output) end -function test_fo_neighbor_sum(data::ConnectivityData, backend::String) +function test_fo_neighbor_sum(offset_provider::Dict{String,Connectivity}, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) @@ -190,32 +182,32 @@ function test_fo_neighbor_sum(data::ConnectivityData, backend::String) sum_positive_elements(v) = sum(x -> x > 0 ? x : 0, v) # Compute the ground truth manually computing the sum on that dimension - expected_output = a[Integer.(map(sum_positive_elements, eachrow(data.edge_to_cell_table)))] + expected_output = a[Integer.(map(sum_positive_elements, eachrow(offset_provider["E2C"].data)))] @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return neighbor_sum(a(E2C), axis=E2CDim) end - fo_neighbor_sum(a, offset_provider=data.offset_provider, backend=backend, out=out) + fo_neighbor_sum(a, offset_provider=offset_provider, backend=backend, out=out) @test out == expected_output end -function test_fo_max_over(data::ConnectivityData, backend::String) +function test_fo_max_over(offset_provider::Dict{String,Connectivity}, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) # Compute the ground truth manually computing max on that dimension - expected_output = a[Integer.(map(maximum, eachrow(data.edge_to_cell_table)))] + expected_output = a[Integer.(map(maximum, eachrow(offset_provider["E2C"].data)))] @field_operator function fo_max_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return max_over(a(E2C), axis=E2CDim) end - fo_max_over(a, offset_provider=data.offset_provider, backend=backend, out=out) + fo_max_over(a, offset_provider=offset_provider, backend=backend, out=out) @test out == expected_output end -function test_fo_min_over(data::ConnectivityData, backend::String) +function test_fo_min_over(offset_provider::Dict{String,Connectivity}, backend::String) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) @@ -223,13 +215,13 @@ function test_fo_min_over(data::ConnectivityData, backend::String) mim_positive_element(v) = minimum(filter(x -> x > 0, v)) # Compute the ground truth manually computing min on that dimension - expected_output = a[Integer.(map(mim_positive_element, eachrow(data.edge_to_cell_table)))] # We exclude the -1 + expected_output = a[Integer.(map(mim_positive_element, eachrow(offset_provider["E2C"].data)))] # We exclude the -1 @field_operator function fo_min_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return min_over(a(E2C), axis=E2CDim) end - fo_min_over(a, offset_provider=data.offset_provider, backend=backend, out=out) + fo_min_over(a, offset_provider=offset_provider, backend=backend, out=out) @test out == expected_output end From 9e7bd1c95242ecbe177f7652974777d894aa89b5 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Thu, 11 Jul 2024 10:26:33 +0200 Subject: [PATCH 30/40] Fix fo_neighbor_sum test (now it works for generic inputs) --- test/gt2py_fo_exec.jl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 3df2814..1af0c4f 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -175,14 +175,20 @@ function test_fo_remapping(offset_provider::Dict{String,Connectivity}, backend:: end function test_fo_neighbor_sum(offset_provider::Dict{String,Connectivity}, backend::String) - a = Field(Cell, collect(1.0:15.0)) + a = Field(Cell, collect(5.0:17.0)*3) out = Field(Edge, zeros(Float64, 12)) - + # Function to sum only the positive elements of each inner vector (to exclude the -1 in the connectivity) - sum_positive_elements(v) = sum(x -> x > 0 ? x : 0, v) + function sum_positive_elements(v, field_data) + return sum(x -> x > 0 ? field_data[Int(x)] : 0, v) + end # Compute the ground truth manually computing the sum on that dimension - expected_output = a[Integer.(map(sum_positive_elements, eachrow(offset_provider["E2C"].data)))] + edge_to_cell_data = offset_provider["E2C"].data + expected_output = Float64[] + for i in 1:size(edge_to_cell_data, 1) + push!(expected_output, sum_positive_elements(edge_to_cell_data[i, :], a)) + end @field_operator function fo_neighbor_sum(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return neighbor_sum(a(E2C), axis=E2CDim) From 9f7540c91defffdde1db977c43d80f4ba476f274 Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Thu, 11 Jul 2024 10:30:23 +0200 Subject: [PATCH 31/40] Fix usage of deprecated `size` function --- test/gt2py_fo_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 1af0c4f..cce2aaa 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -186,7 +186,7 @@ function test_fo_neighbor_sum(offset_provider::Dict{String,Connectivity}, backen # Compute the ground truth manually computing the sum on that dimension edge_to_cell_data = offset_provider["E2C"].data expected_output = Float64[] - for i in 1:size(edge_to_cell_data, 1) + for i in axes(edge_to_cell_data, 1) push!(expected_output, sum_positive_elements(edge_to_cell_data[i, :], a)) end From 476d3b996677a8671694549a15b7d4114ab65d21 Mon Sep 17 00:00:00 2001 From: Lorenzo Varese <55581163+lorenzovarese@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:32:55 +0200 Subject: [PATCH 32/40] Update src/gt2py/gt2py.jl Co-authored-by: Till Ehrengruber --- src/gt2py/gt2py.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gt2py/gt2py.jl b/src/gt2py/gt2py.jl index 9e44765..666aeb5 100644 --- a/src/gt2py/gt2py.jl +++ b/src/gt2py/gt2py.jl @@ -283,7 +283,7 @@ function convert_type(a::Connectivity) # account for different indexing in python return gtx.NeighborTableOffsetProvider( - #TODO(lorenzovarese): fix performance (conversion from 0-index to 1-index) (caching or directly store the 0-index version of the connectivity) + # TODO(lorenzovarese): fix performance (conversion from 0-index to 1-index) (caching or directly store the 0-index version of the connectivity) ifelse.(a.data .!= -1, a.data .- 1, a.data), target_dim, source_dim, From 1496c2eabe51db63d1f69d537bf0ea6de331127a Mon Sep 17 00:00:00 2001 From: Lorenzo Varese <55581163+lorenzovarese@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:33:39 +0200 Subject: [PATCH 33/40] Add comment on cast in test/gt2py_fo_exec.jl Co-authored-by: Till Ehrengruber --- test/gt2py_fo_exec.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index cce2aaa..4a2789a 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -137,6 +137,8 @@ function test_fo_nested_if_else(backend::String) @field_operator function fo_nested_if_else(f::Field{Tuple{Cell_},Int32})::Field{Tuple{Cell_},Int32} tmp = f if 1.0 < 10.0 + # TODO: The Int32 cast is ugly, but required to have consistent behaviour between embedded and GT4Py. + # We should fix the design. tmp = f .+ Int32(1) if 30 > 5 tmp = tmp .+ Int32(20) From c29ae2c34265d4ae2c972aebeeaca85bdc2fc045 Mon Sep 17 00:00:00 2001 From: Lorenzo Varese <55581163+lorenzovarese@users.noreply.github.com> Date: Thu, 11 Jul 2024 13:34:04 +0200 Subject: [PATCH 34/40] Update test/gt2py_fo_exec.jl Co-authored-by: Till Ehrengruber --- test/gt2py_fo_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 4a2789a..3ec86a8 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -109,7 +109,7 @@ function setup_simple_connectivity()::Dict{String,Connectivity} offset_provider = Dict{String,Connectivity}( "E2C" => E2C_offset_provider, "C2E" => C2E_offset_provider, - "E2CDim" => E2C_offset_provider #TODO(lorenzovarese) this is required for the embedded backend (note: python already uses E2C) + "E2CDim" => E2C_offset_provider # TODO(lorenzovarese): this is required for the embedded backend (note: python already uses E2C) ) return offset_provider From 5e74851a7ae3deee8cd53deaea45859f545117c2 Mon Sep 17 00:00:00 2001 From: Till Ehrengruber Date: Tue, 16 Jul 2024 17:03:17 +0200 Subject: [PATCH 35/40] Fix broken cartesian offset, origin, new tests for cartesian --- src/GridTools.jl | 8 +++++--- src/gt2py/jast_to_foast.jl | 13 +++++++++---- src/gt2py/preprocessing.jl | 13 +++++-------- test/gt2py_fo_exec.jl | 31 +++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/GridTools.jl b/src/GridTools.jl index ae56fe4..e5e7e52 100644 --- a/src/GridTools.jl +++ b/src/GridTools.jl @@ -207,6 +207,7 @@ function Field( return Field(Tuple(dim), data, Tuple(broadcast_dims), origin = origin) end +# TODO(tehrengruber): There is no need to have FieldOffset and FieldOffsetTS, remove FieldOffset struct FieldOffsetTS{ Name, Source <: Dimension, @@ -367,10 +368,11 @@ function remap_ts( offset::FieldOffsetTS{OffsetName, SourceDim, Tuple{TargetDim}}, nb_ind::Int64)::Field where {OffsetName, SourceDim <: Dimension, TargetDim <:Dimension} conn = OFFSET_PROVIDER[string(OffsetName)] + @assert conn isa Dimension - new_offsets = Dict(field.dims[i] => field.origin[i] for i in 1:length(field.dims)) - new_offsets[conn] = nb_ind - return Field(field.dims, field.data, field.broadcast_dims, origin = new_offsets) + new_origin = Dict(field.dims[i] => field.origin[i] for i in 1:length(field.dims)) + new_origin[conn] -= nb_ind + return Field(field.dims, field.data, field.broadcast_dims, origin = new_origin) end function remap_ts( diff --git a/src/gt2py/jast_to_foast.jl b/src/gt2py/jast_to_foast.jl index 06fe933..ef6f0d2 100644 --- a/src/gt2py/jast_to_foast.jl +++ b/src/gt2py/jast_to_foast.jl @@ -297,11 +297,11 @@ function visit_(sym::Val{:call}, args::Array, outer_loc) func = visit(args[1], outer_loc), args = [ visit(x, outer_loc) for - x in Base.tail(Tuple(args)) if (typeof(x) != Expr || x.head != :(kw)) + x in args[2:end] if (typeof(x) != Expr || x.head != :(kw)) ], kwargs = Dict( x.args[1] => visit(x.args[2], outer_loc) for - x in Base.tail(Tuple(args)) if (typeof(x) == Expr && x.head == :kw) + x in args[2:end] if (typeof(x) == Expr && x.head == :kw) ), location = outer_loc, ) @@ -345,10 +345,15 @@ function visit_(sym::Val{:(/=)}, args::Array, outer_loc) end function visit_(sym::Val{:ref}, args::Array, outer_loc) - if typeof(args[2]) <: Integer + if typeof(args[2]) <: Integer # TODO: also check that args[1] is an offset + # TODO(tehrengruber): This is an extremely dirty hack, we need to get + # the information from the offset provider or similar, but it is not + # available here. + is_cartesian = string(args[1])[1] in ['I', 'J', 'K'] + index = is_cartesian ? args[2] : args[2]-1 return foast.Subscript( value = visit(args[1], outer_loc), - index = args[2] - 1, # Due to different indexing in python + index = index, location = outer_loc, ) else diff --git a/src/gt2py/preprocessing.jl b/src/gt2py/preprocessing.jl index 2c0a81c..97ca6cc 100644 --- a/src/gt2py/preprocessing.jl +++ b/src/gt2py/preprocessing.jl @@ -57,12 +57,9 @@ function translate_closure_vars(j_closure_vars::Dict)::Dict new_value = nothing if typeof(value) <: FieldOffset - py_source = map( - dim -> gtx.Dimension( - get_dim_name(dim), - kind = py_dim_kind[get_dim_kind(dim)] - ), - value.source + py_source = gtx.Dimension( + get_dim_name(value.source), + kind = py_dim_kind[get_dim_kind(value.source)] ) py_target = map( dim -> gtx.Dimension( @@ -73,8 +70,8 @@ function translate_closure_vars(j_closure_vars::Dict)::Dict ) new_value = gtx.FieldOffset( value.name, - source = length(py_source) == 1 ? py_source[1] : py_source, - target = length(py_target) == 1 ? py_target[1] : py_target + source = py_source, + target = py_target ) elseif typeof(value) <: Function new_value = builtin_py_op[key] diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 3ec86a8..c8a7da4 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -130,6 +130,31 @@ function test_fo_addition(backend::String) @test all(out.data .== 0) end +function test_fo_cartesian_offset(backend::String) + inp = Field(K, collect(1.0:15.0)) + out = Field(K, zeros(Float64, 14)) # field is one smaller since we shift by one + + @field_operator function fo_cartesian_offset(inp::Field{Tuple{K_},Float64})::Field{Tuple{K_},Float64} + return inp(Koff[1]) + end + + fo_cartesian_offset(inp, backend=backend, out=out, offset_provider=Dict("Koff" => K)) + @test all(out.data .== 2.0:15.0) +end + +function test_fo_cartesian_offset_composed(backend::String) + inp = Field(K, collect(1.0:15.0)) + out = Field(K, zeros(Float64, 12)) # field is one smaller since we shift by one + + @field_operator function fo_cartesian_offset_composed(inp::Field{Tuple{K_},Float64})::Field{Tuple{K_},Float64} + tmp = inp(Koff[1]) + return tmp(Koff[2]) + end + + fo_cartesian_offset_composed(inp, backend=backend, out=out, offset_provider=Dict("Koff" => K)) + @test all(out.data .== 4.0:15.0) +end + function test_fo_nested_if_else(backend::String) a = Field(Cell, collect(Int32, 1:15)) # TODO(tehrengruber): if we don't use the right dtype here we get a horrible error in python out = Field(Cell, zeros(Int32, 15)) @@ -381,6 +406,12 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_addition, "embedded") testwrapper(nothing, test_fo_addition, "py") + testwrapper(nothing, test_fo_cartesian_offset, "embedded") + testwrapper(nothing, test_fo_cartesian_offset, "py") + + testwrapper(nothing, test_fo_cartesian_offset_composed, "embedded") + testwrapper(nothing, test_fo_cartesian_offset_composed, "py") + testwrapper(nothing, test_fo_nested_if_else, "embedded") testwrapper(nothing, test_fo_nested_if_else, "py") From 01ffe453ee484001e351d9675d1ff8d9143db825 Mon Sep 17 00:00:00 2001 From: Till Ehrengruber Date: Tue, 16 Jul 2024 17:08:05 +0200 Subject: [PATCH 36/40] Disabled test_fo_offset_array test again --- test/gt2py_fo_exec.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index c8a7da4..907b6c9 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -360,8 +360,6 @@ function test_fo_asinh(backend::String) end function test_fo_offset_array(backend::String) - # TODO OffsetArray is ignored and will be removed in the future - A = Field((Vertex, K), reshape(collect(1.0:15.0), 3, 5), origin=Dict(Vertex => -2, K => -1)) B = Field((K, Edge), reshape(ones(6), 3, 2)) @@ -445,8 +443,9 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_asinh, "embedded") testwrapper(nothing, test_fo_asinh, "py") - testwrapper(nothing, test_fo_offset_array, "embedded") - testwrapper(nothing, test_fo_offset_array, "py") + # TODO(tehrengruber): disabled for now until we understand what it is supposed to do + #testwrapper(nothing, test_fo_offset_array, "embedded") + #testwrapper(nothing, test_fo_offset_array, "py") testwrapper(nothing, test_nested_fo, "embedded") testwrapper(nothing, test_nested_fo, "py") From 495c7478de0399ec050a57bc1aa6ed93f3acc0a7 Mon Sep 17 00:00:00 2001 From: Till Ehrengruber Date: Wed, 17 Jul 2024 10:18:50 +0200 Subject: [PATCH 37/40] Fix multiplication of fields with scalars --- src/embedded/cust_broadcast.jl | 8 +++++--- src/gt2py/gt2py.jl | 18 ++++++++---------- src/gt2py/jast_to_foast.jl | 8 ++++---- test/gt2py_fo_exec.jl | 17 +++++++++++++++++ 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/embedded/cust_broadcast.jl b/src/embedded/cust_broadcast.jl index 27d9011..0b0ad16 100644 --- a/src/embedded/cust_broadcast.jl +++ b/src/embedded/cust_broadcast.jl @@ -1,5 +1,7 @@ Base.BroadcastStyle(::Type{<:Field}) = Broadcast.ArrayStyle{Field}() +# TODO(tehrengruber): Implement a range with an attached dimension instead of this single object +# for an entire domain. Invesitage what broadcast_dims is needed for here. struct FieldShape{ N, Dim <: NTuple{N, Dimension}, @@ -11,9 +13,9 @@ struct FieldShape{ broadcast_dims::B_Dim end -function shape(f::Field) - return FieldShape(f.dims, axes(f), f.broadcast_dims) -end +shape(f::Field) = FieldShape(f.dims, axes(f), f.broadcast_dims) + +Base.length(bc::FieldShape{N}) where N = N # Only called for assign broadcasting (.=) @inline function Base.Broadcast.materialize!(dest, bc::Broadcasted{ArrayStyle{Field}}) diff --git a/src/gt2py/gt2py.jl b/src/gt2py/gt2py.jl index 666aeb5..9ed12fd 100644 --- a/src/gt2py/gt2py.jl +++ b/src/gt2py/gt2py.jl @@ -208,20 +208,20 @@ function py_field_operator( end function jast_to_foast(expr::Expr, closure_vars::Dict) - expr, closure_vars, annotations = preprocess_definiton(expr, closure_vars) - expr, closure_vars = remove_function_aliases(expr, closure_vars) # TODO Can be ommited once gt4py allows aliases - foast_node = visit_jast(expr, closure_vars) - foast_node = postprocess_definition(foast_node, closure_vars, annotations) - return foast_node, closure_vars + expr, py_closure_vars, annotations = preprocess_definiton(expr, closure_vars) + expr, py_closure_vars = remove_function_aliases(expr, py_closure_vars) # TODO Can be ommited once gt4py allows aliases + foast_node = visit_jast(expr, py_closure_vars) + foast_node = postprocess_definition(foast_node, py_closure_vars, annotations) + return foast_node, py_closure_vars end function preprocess_definiton(expr::Expr, closure_vars::Dict) sat = single_assign_target_pass(expr) ucc = unchain_compairs_pass(sat) ssa = single_static_assign_pass(ucc) - closure_vars = translate_closure_vars(closure_vars) - annotations = get_annotation(ssa, closure_vars) - return (ssa, closure_vars, annotations) + py_closure_vars = translate_closure_vars(closure_vars) + annotations = get_annotation(ssa, py_closure_vars) + return (ssa, py_closure_vars, annotations) end function postprocess_definition(foast_node, closure_vars, annotations) @@ -261,8 +261,6 @@ function convert_type(a::Field) new_data = np.asarray(a.data) @warn "Dtype of the Field: $a is not concrete. Data must be copied to Python which may affect performance. Try using dtypes <: Array." end - println(new_data) - println(typeof(new_data)) offset = Dict(convert_type(dim) => a.origin[i] for (i, dim) in enumerate(a.dims)) diff --git a/src/gt2py/jast_to_foast.jl b/src/gt2py/jast_to_foast.jl index ef6f0d2..ec75203 100644 --- a/src/gt2py/jast_to_foast.jl +++ b/src/gt2py/jast_to_foast.jl @@ -442,18 +442,18 @@ function from_type_hint(expr::Expr, closure_vars::Dict) param_type = expr.args if param_type[1] == :Tuple return ts.TupleType( - types = [recursive_make_symbol(arg) for arg in Base.tail(param_type)] + types = [recursive_make_symbol(arg) for arg in param_type[]] ) elseif param_type[1] == :Field @assert length(param_type) == 3 ( "Field type requires two arguments, got $(length(param_type)-1) in $(param_type)." ) - dim = [] (dims, dtype) = param_type[2:end] - + # TODO: do some sanity checks here for example Field{Int64, Dims} will fail terribly - + + dim = [] for d in dims.args[2:end] @assert string(d) in keys(closure_vars) push!(dim, closure_vars[string(d)]) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 907b6c9..90bca5e 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -68,6 +68,8 @@ testwrapper(setup, mytest, 1, 2, 3) ``` """ function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) + args_str = join(map(string, args), ", ") + println("Executing test '$(nameof(testfunc))' with args $args_str") if setupfunc === nothing testfunc(args...) else @@ -142,6 +144,18 @@ function test_fo_cartesian_offset(backend::String) @test all(out.data .== 2.0:15.0) end +function test_fo_scalar_multiplication(backend::String) + inp = Field(Cell, collect(1.0:15.0)) + out = Field(Cell, zeros(Float64, 15)) + + @field_operator function fo_scalar_mult(inp::Field{Tuple{Cell_},Float64})::Field{Tuple{Cell_},Float64} + return 4.0*inp + end + + fo_scalar_mult(inp, backend=backend, out=out, offset_provider=Dict("Koff" => K)) + @test all(out.data .== 4*(1.0:15.0)) +end + function test_fo_cartesian_offset_composed(backend::String) inp = Field(K, collect(1.0:15.0)) out = Field(K, zeros(Float64, 12)) # field is one smaller since we shift by one @@ -404,6 +418,9 @@ function test_gt4py_fo_exec() testwrapper(nothing, test_fo_addition, "embedded") testwrapper(nothing, test_fo_addition, "py") + testwrapper(nothing, test_fo_scalar_multiplication, "embedded") + testwrapper(nothing, test_fo_scalar_multiplication, "py") + testwrapper(nothing, test_fo_cartesian_offset, "embedded") testwrapper(nothing, test_fo_cartesian_offset, "py") From 5f7ac95a4fb6ca1bec16f01444eea61fa15dc202 Mon Sep 17 00:00:00 2001 From: Lorenzo Varese <55581163+lorenzovarese@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:05:31 +0200 Subject: [PATCH 38/40] Remove generated function in getIndex of connectivity Co-authored-by: Till Ehrengruber --- src/GridTools.jl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/GridTools.jl b/src/GridTools.jl index e5e7e52..9127f7e 100644 --- a/src/GridTools.jl +++ b/src/GridTools.jl @@ -468,15 +468,7 @@ struct Connectivity dims::Integer end -@generated function Base.getindex(conn::Connectivity, row::Union{Integer, Colon}, col::Integer) - if row <: Integer - return :(conn.data[row, col]) - elseif row <: Colon - return :(conn.data[:, col]) - else - throw(ArgumentError("Unsupported index type")) - end -end +Base.getindex(conn::Connectivity, row::Union{Integer, Colon}, col::Integer) = conn.data[row, col] # Field operator ---------------------------------------------------------------------- From 449b4701a85ea48f6968e95e096ecd88eb053f45 Mon Sep 17 00:00:00 2001 From: Lorenzo Varese <55581163+lorenzovarese@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:08:23 +0200 Subject: [PATCH 39/40] Ignore only the element flagged with -1 in the connectivity table Co-authored-by: Till Ehrengruber --- test/gt2py_fo_exec.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 90bca5e..91deb37 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -221,7 +221,7 @@ function test_fo_neighbor_sum(offset_provider::Dict{String,Connectivity}, backen # Function to sum only the positive elements of each inner vector (to exclude the -1 in the connectivity) function sum_positive_elements(v, field_data) - return sum(x -> x > 0 ? field_data[Int(x)] : 0, v) + return sum(idx -> idx != -1 ? field_data[idx] : 0, v) end # Compute the ground truth manually computing the sum on that dimension From 512dd4577ade3ec738da0a163f0d38080f52523d Mon Sep 17 00:00:00 2001 From: LorenzoVarese Date: Wed, 17 Jul 2024 12:49:59 +0200 Subject: [PATCH 40/40] Fix field values comparison in the max and min over dimensions for gt2py_fo_exec tests --- test/gt2py_fo_exec.jl | 53 +++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/test/gt2py_fo_exec.jl b/test/gt2py_fo_exec.jl index 91deb37..16352de 100644 --- a/test/gt2py_fo_exec.jl +++ b/test/gt2py_fo_exec.jl @@ -69,7 +69,7 @@ testwrapper(setup, mytest, 1, 2, 3) """ function testwrapper(setupfunc::Union{Function,Nothing}, testfunc::Function, args...) args_str = join(map(string, args), ", ") - println("Executing test '$(nameof(testfunc))' with args $args_str") + println("Executing '$(nameof(testfunc))' with args: $args_str") if setupfunc === nothing testfunc(args...) else @@ -239,12 +239,30 @@ function test_fo_neighbor_sum(offset_provider::Dict{String,Connectivity}, backen @test out == expected_output end -function test_fo_max_over(offset_provider::Dict{String,Connectivity}, backend::String) +function compute_expected_output_comparing_values(offset_provider::Dict{String, Connectivity}, a::Field{Tuple{Cell_}, Float64}, operation::Function) + expected_output = Field(Edge, zeros(Float64, 12)) + E2C = offset_provider["E2C"] + for edge in 1:length(expected_output.data) + # Extract the neighboring cell indices for the current edge + neighbor_cells = E2C.data[edge, :] + # Filter out the -1 indices + valid_neighbors = filter(x -> x != -1, neighbor_cells) + # Compute the maximum/minimum value among the valid neighbors + if !isempty(valid_neighbors) + expected_output.data[edge] = operation(a[valid_neighbors]) + else + throw("E2C Connectivity is not defined correctly. An edge cannot have no neighbors.") + end + end + return expected_output +end + +function test_fo_max_over(offset_provider::Dict{String,Connectivity}, backend::String, a::Field) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - # Compute the ground truth manually computing max on that dimension - expected_output = a[Integer.(map(maximum, eachrow(offset_provider["E2C"].data)))] + # Compute the ground truth manually computing the maximum of the value of each neighbor + expected_output = expected_output = compute_expected_output_comparing_values(offset_provider, a, maximum) @field_operator function fo_max_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return max_over(a(E2C), axis=E2CDim) @@ -254,15 +272,12 @@ function test_fo_max_over(offset_provider::Dict{String,Connectivity}, backend::S @test out == expected_output end -function test_fo_min_over(offset_provider::Dict{String,Connectivity}, backend::String) +function test_fo_min_over(offset_provider::Dict{String,Connectivity}, backend::String, a::Field) a = Field(Cell, collect(1.0:15.0)) out = Field(Edge, zeros(Float64, 12)) - # Function to return the minimum positive element of each inner vector - mim_positive_element(v) = minimum(filter(x -> x > 0, v)) - - # Compute the ground truth manually computing min on that dimension - expected_output = a[Integer.(map(mim_positive_element, eachrow(offset_provider["E2C"].data)))] # We exclude the -1 + # Compute the ground truth manually computing the minimum of the value of each neighbor + expected_output = compute_expected_output_comparing_values(offset_provider, a, minimum) @field_operator function fo_min_over(a::Field{Tuple{Cell_},Float64})::Field{Tuple{Edge_},Float64} return min_over(a(E2C), axis=E2CDim) @@ -436,11 +451,21 @@ function test_gt4py_fo_exec() testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "embedded") testwrapper(setup_simple_connectivity, test_fo_neighbor_sum, "py") - testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded") - testwrapper(setup_simple_connectivity, test_fo_max_over, "py") + # Test the comparison of values in the Fields + field_values_increasing = Field(Cell, collect(1.0:15.0)) + field_values_decreasing = Field(Cell, reverse(collect(1.0:15.0))) + + testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded", field_values_increasing) + testwrapper(setup_simple_connectivity, test_fo_max_over, "py", field_values_increasing) + + testwrapper(setup_simple_connectivity, test_fo_max_over, "embedded", field_values_decreasing) + testwrapper(setup_simple_connectivity, test_fo_max_over, "py", field_values_decreasing) + + testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded", field_values_increasing) + testwrapper(setup_simple_connectivity, test_fo_min_over, "py", field_values_increasing) - testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded") - testwrapper(setup_simple_connectivity, test_fo_min_over, "py") + testwrapper(setup_simple_connectivity, test_fo_min_over, "embedded", field_values_decreasing) + testwrapper(setup_simple_connectivity, test_fo_min_over, "py", field_values_decreasing) testwrapper(nothing, test_fo_simple_broadcast, "embedded") testwrapper(nothing, test_fo_simple_broadcast, "py")