From 500102edc65d5071b0a13ca925410e01c94eae09 Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Mon, 29 Apr 2024 09:54:15 +0200 Subject: [PATCH] Deepcopy extras in benchmarking --- .../docs/src/tutorial.md | 4 +- .../src/scenarios/component.jl | 8 +- .../src/scenarios/default.jl | 114 ++++---- .../src/scenarios/scenario.jl | 67 ++--- .../src/scenarios/sparse.jl | 32 +-- .../src/test_differentiation.jl | 6 +- .../src/tests/benchmark.jl | 270 +++++++++++++----- .../test/zero_backends.jl | 2 +- 8 files changed, 309 insertions(+), 194 deletions(-) diff --git a/DifferentiationInterfaceTest/docs/src/tutorial.md b/DifferentiationInterfaceTest/docs/src/tutorial.md index 3ba0aebb7..193e586c0 100644 --- a/DifferentiationInterfaceTest/docs/src/tutorial.md +++ b/DifferentiationInterfaceTest/docs/src/tutorial.md @@ -84,9 +84,9 @@ Note that we only compare (possibly) in-place operators, because they are always ```@example tuto function formatter(v, i, j) - if j in (14, 15) # time, bytes + if j in (15, 16) # time, bytes return Printf.@sprintf("%.1e", v) - elseif j == 16 # allocs + elseif j == 17 # allocs return Printf.@sprintf("%.1f", v) else return v diff --git a/DifferentiationInterfaceTest/src/scenarios/component.jl b/DifferentiationInterfaceTest/src/scenarios/component.jl index 34dde350e..20e04b1f0 100644 --- a/DifferentiationInterfaceTest/src/scenarios/component.jl +++ b/DifferentiationInterfaceTest/src/scenarios/component.jl @@ -18,12 +18,12 @@ end function comp_to_num_scenarios_onearg(x::ComponentVector) # pushforward stays out of place scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ - PullbackScenario(comp_to_num; x=x, ref=comp_to_num_pullback, operator=op), - GradientScenario(comp_to_num; x=x, ref=comp_to_num_gradient, operator=op), + PullbackScenario(comp_to_num; x=x, ref=comp_to_num_pullback, place=place), + GradientScenario(comp_to_num; x=x, ref=comp_to_num_gradient, place=place), ], ) end @@ -32,7 +32,7 @@ function comp_to_num_scenarios_onearg(x::ComponentVector) scens, [ PushforwardScenario( - comp_to_num; x=x, ref=comp_to_num_pushforward, operator=op + comp_to_num; x=x, ref=comp_to_num_pushforward, place=place ), ], ) diff --git a/DifferentiationInterfaceTest/src/scenarios/default.jl b/DifferentiationInterfaceTest/src/scenarios/default.jl index bfc237bec..86a686e58 100644 --- a/DifferentiationInterfaceTest/src/scenarios/default.jl +++ b/DifferentiationInterfaceTest/src/scenarios/default.jl @@ -21,15 +21,11 @@ num_to_num_pullback(x, dy) = num_to_num_derivative(x) * dy function num_to_num_scenarios_onearg(x::Number) # everyone out of place return [ - PushforwardScenario( - num_to_num; x=x, ref=num_to_num_pushforward, operator=:outofplace - ), - PullbackScenario(num_to_num; x=x, ref=num_to_num_pullback, operator=:outofplace), - DerivativeScenario( - num_to_num; x=x, ref=num_to_num_derivative, operator=:outofplace - ), + PushforwardScenario(num_to_num; x=x, ref=num_to_num_pushforward, place=:outofplace), + PullbackScenario(num_to_num; x=x, ref=num_to_num_pullback, place=:outofplace), + DerivativeScenario(num_to_num; x=x, ref=num_to_num_derivative, place=:outofplace), SecondDerivativeScenario( - num_to_num; x=x, ref=num_to_num_second_derivative, operator=:outofplace + num_to_num; x=x, ref=num_to_num_second_derivative, place=:outofplace ), ] end @@ -72,28 +68,28 @@ end function num_to_arr_scenarios_onearg(x::Number, a::AbstractArray) # pullback stays out of place scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ PushforwardScenario( - _num_to_arr(a); x=x, ref=_num_to_arr_pushforward(a), operator=op + _num_to_arr(a); x=x, ref=_num_to_arr_pushforward(a), place=place ), DerivativeScenario( - _num_to_arr(a); x=x, ref=_num_to_arr_derivative(a), operator=op + _num_to_arr(a); x=x, ref=_num_to_arr_derivative(a), place=place ), SecondDerivativeScenario( - _num_to_arr(a); x=x, ref=_num_to_arr_second_derivative(a), operator=op + _num_to_arr(a); x=x, ref=_num_to_arr_second_derivative(a), place=place ), ], ) end - for op in (:outofplace,) + for place in (:outofplace,) append!( scens, [ PullbackScenario( - _num_to_arr(a); x=x, ref=_num_to_arr_pullback(a), operator=op + _num_to_arr(a); x=x, ref=_num_to_arr_pullback(a), place=place ), ], ) @@ -104,7 +100,7 @@ end function num_to_arr_scenarios_twoarg(x::Number, a::AbstractArray) # pullback stays out of place scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -113,19 +109,19 @@ function num_to_arr_scenarios_twoarg(x::Number, a::AbstractArray) x=x, y=similar(float.(a)), ref=_num_to_arr_pushforward(a), - operator=op, + place=place, ), DerivativeScenario( _num_to_arr!(a); x=x, y=similar(float.(a)), ref=_num_to_arr_derivative(a), - operator=op, + place=place, ), ], ) end - for op in (:outofplace,) + for place in (:outofplace,) append!( scens, [ @@ -134,7 +130,7 @@ function num_to_arr_scenarios_twoarg(x::Number, a::AbstractArray) x=x, y=similar(float.(a)), ref=_num_to_arr_pullback(a), - operator=op, + place=place, ), ], ) @@ -155,22 +151,22 @@ arr_to_num_hessian(x) = Matrix(Diagonal(-sin.(vec(x)))) function arr_to_num_scenarios_onearg(x::AbstractArray) # pushforward stays out of place scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ - PullbackScenario(arr_to_num; x=x, ref=arr_to_num_pullback, operator=op), - GradientScenario(arr_to_num; x=x, ref=arr_to_num_gradient, operator=op), - GradientScenario(arr_to_num; x=x, ref=arr_to_num_gradient, operator=op), - HVPScenario(arr_to_num; x=x, ref=arr_to_num_hvp, operator=op), - HessianScenario(arr_to_num; x=x, ref=arr_to_num_hessian, operator=op), + PullbackScenario(arr_to_num; x=x, ref=arr_to_num_pullback, place=place), + GradientScenario(arr_to_num; x=x, ref=arr_to_num_gradient, place=place), + GradientScenario(arr_to_num; x=x, ref=arr_to_num_gradient, place=place), + HVPScenario(arr_to_num; x=x, ref=arr_to_num_hvp, place=place), + HessianScenario(arr_to_num; x=x, ref=arr_to_num_hessian, place=place), ], ) end - for op in (:outofplace,) + for place in (:outofplace,) append!( scens, - [PushforwardScenario(arr_to_num; x=x, ref=arr_to_num_pushforward, operator=op)], + [PushforwardScenario(arr_to_num; x=x, ref=arr_to_num_pushforward, place=place)], ) end return scens @@ -192,15 +188,15 @@ vec_to_vec_jacobian(x) = vcat(Diagonal(cos.(x)), Diagonal(-sin.(x))) function vec_to_vec_scenarios_onearg(x::AbstractVector) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ PushforwardScenario( - vec_to_vec; x=x, ref=vec_to_vec_pushforward, operator=op + vec_to_vec; x=x, ref=vec_to_vec_pushforward, place=place ), - PullbackScenario(vec_to_vec; x=x, ref=vec_to_vec_pullback, operator=op), - JacobianScenario(vec_to_vec; x=x, ref=vec_to_vec_jacobian, operator=op), + PullbackScenario(vec_to_vec; x=x, ref=vec_to_vec_pullback, place=place), + JacobianScenario(vec_to_vec; x=x, ref=vec_to_vec_jacobian, place=place), ], ) end @@ -210,7 +206,7 @@ end function vec_to_vec_scenarios_twoarg(x::AbstractVector) n = length(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -219,13 +215,13 @@ function vec_to_vec_scenarios_twoarg(x::AbstractVector) x=x, y=similar(x, 2n), ref=vec_to_vec_pushforward, - operator=op, + place=place, ), PullbackScenario( - vec_to_vec!; x=x, y=similar(x, 2n), ref=vec_to_vec_pullback, operator=op + vec_to_vec!; x=x, y=similar(x, 2n), ref=vec_to_vec_pullback, place=place ), JacobianScenario( - vec_to_vec!; x=x, y=similar(x, 2n), ref=vec_to_vec_jacobian, operator=op + vec_to_vec!; x=x, y=similar(x, 2n), ref=vec_to_vec_jacobian, place=place ), ], ) @@ -247,15 +243,15 @@ vec_to_mat_jacobian(x) = vcat(Diagonal(cos.(x)), Diagonal(-sin.(x))) function vec_to_mat_scenarios_onearg(x::AbstractVector) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ PushforwardScenario( - vec_to_mat; x=x, ref=vec_to_mat_pushforward, operator=op + vec_to_mat; x=x, ref=vec_to_mat_pushforward, place=place ), - PullbackScenario(vec_to_mat; x=x, ref=vec_to_mat_pullback, operator=op), - JacobianScenario(vec_to_mat; x=x, ref=vec_to_mat_jacobian, operator=op), + PullbackScenario(vec_to_mat; x=x, ref=vec_to_mat_pullback, place=place), + JacobianScenario(vec_to_mat; x=x, ref=vec_to_mat_jacobian, place=place), ], ) end @@ -265,7 +261,7 @@ end function vec_to_mat_scenarios_twoarg(x::AbstractVector) n = length(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -274,21 +270,21 @@ function vec_to_mat_scenarios_twoarg(x::AbstractVector) x=x, y=similar(x, n, 2), ref=vec_to_mat_pushforward, - operator=op, + place=place, ), PullbackScenario( vec_to_mat!; x=x, y=similar(x, n, 2), ref=vec_to_mat_pullback, - operator=op, + place=place, ), JacobianScenario( vec_to_mat!; x=x, y=similar(x, n, 2), ref=vec_to_mat_jacobian, - operator=op, + place=place, ), ], ) @@ -319,18 +315,18 @@ mat_to_vec_jacobian(x) = vcat(Diagonal(vec(cos.(x))), Diagonal(vec(-sin.(x)))) function mat_to_vec_scenarios_onearg(x::AbstractMatrix) m, n = size(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ PushforwardScenario( - mat_to_vec; x=x, ref=mat_to_vec_pushforward, operator=op + mat_to_vec; x=x, ref=mat_to_vec_pushforward, place=place ), PullbackScenario( - mat_to_vec; x=randn(m, n), ref=mat_to_vec_pullback, operator=op + mat_to_vec; x=randn(m, n), ref=mat_to_vec_pullback, place=place ), JacobianScenario( - mat_to_vec; x=randn(m, n), ref=mat_to_vec_jacobian, operator=op + mat_to_vec; x=randn(m, n), ref=mat_to_vec_jacobian, place=place ), ], ) @@ -341,7 +337,7 @@ end function mat_to_vec_scenarios_twoarg(x::AbstractMatrix) m, n = size(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -350,21 +346,21 @@ function mat_to_vec_scenarios_twoarg(x::AbstractMatrix) x=x, y=similar(x, m * n * 2), ref=mat_to_vec_pushforward, - operator=op, + place=place, ), PullbackScenario( mat_to_vec!; x=x, y=similar(x, m * n * 2), ref=mat_to_vec_pullback, - operator=op, + place=place, ), JacobianScenario( mat_to_vec!; x=x, y=similar(x, m * n * 2), ref=mat_to_vec_jacobian, - operator=op, + place=place, ), ], ) @@ -393,15 +389,15 @@ mat_to_mat_jacobian(x) = vcat(Diagonal(vec(cos.(x))), Diagonal(vec(-sin.(x)))) function mat_to_mat_scenarios_onearg(x::AbstractMatrix) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ PushforwardScenario( - mat_to_mat; x=x, ref=mat_to_mat_pushforward, operator=op + mat_to_mat; x=x, ref=mat_to_mat_pushforward, place=place ), - PullbackScenario(mat_to_mat; x=x, ref=mat_to_mat_pullback, operator=op), - JacobianScenario(mat_to_mat; x=x, ref=mat_to_mat_jacobian, operator=op), + PullbackScenario(mat_to_mat; x=x, ref=mat_to_mat_pullback, place=place), + JacobianScenario(mat_to_mat; x=x, ref=mat_to_mat_jacobian, place=place), ], ) end @@ -411,7 +407,7 @@ end function mat_to_mat_scenarios_twoarg(x::AbstractMatrix) m, n = size(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -420,21 +416,21 @@ function mat_to_mat_scenarios_twoarg(x::AbstractMatrix) x=x, y=similar(x, m * n, 2), ref=mat_to_mat_pushforward, - operator=op, + place=place, ), PullbackScenario( mat_to_mat!; x=x, y=similar(x, m * n, 2), ref=mat_to_mat_pullback, - operator=op, + place=place, ), JacobianScenario( mat_to_mat!; x=x, y=similar(x, m * n, 2), ref=mat_to_mat_jacobian, - operator=op, + place=place, ), ], ) diff --git a/DifferentiationInterfaceTest/src/scenarios/scenario.jl b/DifferentiationInterfaceTest/src/scenarios/scenario.jl index 330027154..e0008efcd 100644 --- a/DifferentiationInterfaceTest/src/scenarios/scenario.jl +++ b/DifferentiationInterfaceTest/src/scenarios/scenario.jl @@ -34,18 +34,18 @@ Otherwise, `f` is interpreted as an 1-argument function `f(x) = y`. The reference keyword `ref` should be a function that takes `x` (and a potential seed `dx` or `dy`) to return the correct object. -The operator behavior keyword `operator` should be either `:inplace` or `:outofplace` depending on what must be tested. +The keyword `place` should be either `:inplace` or `:outofplace` depending on what kind of operator must be tested. """ -abstract type AbstractScenario{args,op,F,X,Y,R} end +abstract type AbstractScenario{args,place,F,X,Y,R} end -abstract type AbstractFirstOrderScenario{args,op,F,X,Y,R} <: - AbstractScenario{args,op,F,X,Y,R} end -abstract type AbstractSecondOrderScenario{args,op,F,X,Y,R} <: - AbstractScenario{args,op,F,X,Y,R} end +abstract type AbstractFirstOrderScenario{args,place,F,X,Y,R} <: + AbstractScenario{args,place,F,X,Y,R} end +abstract type AbstractSecondOrderScenario{args,place,F,X,Y,R} <: + AbstractScenario{args,place,F,X,Y,R} end scen_type(scenario::AbstractScenario) = nameof(typeof(scenario)) nb_args(::AbstractScenario{args}) where {args} = args -operator_place(::AbstractScenario{args,op}) where {args,op} = op +operator_place(::AbstractScenario{args,place}) where {args,place} = place function compatible(backend::AbstractADType, scen::AbstractScenario) if nb_args(scen) == 2 @@ -61,8 +61,8 @@ function group_by_scen_type(scenarios) ) end -function Base.string(scen::S) where {args,op,F,X,Y,S<:AbstractScenario{args,op,F,X,Y}} - return "$(S.name.name){$args,$op} $(string(scen.f)) : $X -> $Y" +function Base.string(scen::S) where {args,place,F,X,Y,S<:AbstractScenario{args,place,F,X,Y}} + return "$(S.name.name){$args,$place} $(string(scen.f)) : $X -> $Y" end ## Struct definitions @@ -72,8 +72,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct PushforwardScenario{args,op,F,X,Y,DX,R} <: - AbstractFirstOrderScenario{args,op,F,X,Y,R} +struct PushforwardScenario{args,place,F,X,Y,DX,R} <: + AbstractFirstOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -91,7 +91,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct PullbackScenario{args,op,F,X,Y,DY,R} <: AbstractFirstOrderScenario{args,op,F,X,Y,R} +struct PullbackScenario{args,place,F,X,Y,DY,R} <: + AbstractFirstOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -109,8 +110,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct DerivativeScenario{args,op,F,X<:Number,Y,R} <: - AbstractFirstOrderScenario{args,op,F,X,Y,R} +struct DerivativeScenario{args,place,F,X<:Number,Y,R} <: + AbstractFirstOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -126,8 +127,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct GradientScenario{args,op,F,X,Y<:Number,R} <: - AbstractFirstOrderScenario{args,op,F,X,Y,R} +struct GradientScenario{args,place,F,X,Y<:Number,R} <: + AbstractFirstOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -143,8 +144,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct JacobianScenario{args,op,F,X<:AbstractArray,Y<:AbstractArray,R} <: - AbstractFirstOrderScenario{args,op,F,X,Y,R} +struct JacobianScenario{args,place,F,X<:AbstractArray,Y<:AbstractArray,R} <: + AbstractFirstOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -160,8 +161,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct SecondDerivativeScenario{args,op,F,X<:Number,Y,R} <: - AbstractSecondOrderScenario{args,op,F,X,Y,R} +struct SecondDerivativeScenario{args,place,F,X<:Number,Y,R} <: + AbstractSecondOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -177,8 +178,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct HVPScenario{args,op,F,X,Y<:Number,DX,R} <: - AbstractSecondOrderScenario{args,op,F,X,Y,R} +struct HVPScenario{args,place,F,X,Y<:Number,DX,R} <: + AbstractSecondOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -196,8 +197,8 @@ end See [`AbstractScenario`](@ref) for details. """ -struct HessianScenario{args,op,F,X<:AbstractArray,Y<:Number,R} <: - AbstractSecondOrderScenario{args,op,F,X,Y,R} +struct HessianScenario{args,place,F,X<:AbstractArray,Y<:Number,R} <: + AbstractSecondOrderScenario{args,place,F,X,Y,R} "function" f::F "input" @@ -218,14 +219,14 @@ for S in ( :HessianScenario, ) @eval begin - function $S(f::F; x::X, y=nothing, ref::R=nothing, operator=:inplace) where {F,X,R} + function $S(f::F; x::X, y=nothing, ref::R=nothing, place=:inplace) where {F,X,R} args = isnothing(y) ? 1 : 2 if args == 2 f(y, x) else y = f(x) end - return ($S){args,operator,F,X,typeof(y),R}(f, x, y, ref) + return ($S){args,place,F,X,typeof(y),R}(f, x, y, ref) end function change_function(s::($S), f) @@ -234,7 +235,7 @@ for S in ( x=s.x, y=(nb_args(s) == 1 ? nothing : s.y), ref=s.ref, - operator=operator_place(s), + place=operator_place(s), ) end end @@ -243,7 +244,7 @@ end for S in (:PushforwardScenario, :HVPScenario) @eval begin function $S( - f::F; x::X, y=nothing, ref::R=nothing, dx=nothing, operator=:inplace + f::F; x::X, y=nothing, ref::R=nothing, dx=nothing, place=:inplace ) where {F,X,R} args = isnothing(y) ? 1 : 2 if args == 2 @@ -254,7 +255,7 @@ for S in (:PushforwardScenario, :HVPScenario) if isnothing(dx) dx = mysimilar_random(x) end - return ($S){args,operator,F,X,typeof(y),typeof(dx),R}(f, x, y, dx, ref) + return ($S){args,place,F,X,typeof(y),typeof(dx),R}(f, x, y, dx, ref) end function change_function(s::($S), f) @@ -263,7 +264,7 @@ for S in (:PushforwardScenario, :HVPScenario) x=s.x, y=(nb_args(s) == 1 ? nothing : s.y), ref=s.ref, - operator=operator_place(s), + place=operator_place(s), ) end end @@ -272,7 +273,7 @@ end for S in (:PullbackScenario,) @eval begin function $S( - f::F; x::X, y=nothing, ref::R=nothing, dy=nothing, operator=:inplace + f::F; x::X, y=nothing, ref::R=nothing, dy=nothing, place=:inplace ) where {F,X,R} args = isnothing(y) ? 1 : 2 if args == 2 @@ -283,7 +284,7 @@ for S in (:PullbackScenario,) if isnothing(dy) dy = mysimilar_random(y) end - return ($S){args,operator,F,X,typeof(y),typeof(dy),R}(f, x, y, dy, ref) + return ($S){args,place,F,X,typeof(y),typeof(dy),R}(f, x, y, dy, ref) end function change_function(s::($S), f) @@ -292,7 +293,7 @@ for S in (:PullbackScenario,) x=s.x, y=(nb_args(s) == 1 ? nothing : s.y), ref=s.ref, - operator=operator_place(s), + place=operator_place(s), ) end end diff --git a/DifferentiationInterfaceTest/src/scenarios/sparse.jl b/DifferentiationInterfaceTest/src/scenarios/sparse.jl index 6db2e1fd2..d52204213 100644 --- a/DifferentiationInterfaceTest/src/scenarios/sparse.jl +++ b/DifferentiationInterfaceTest/src/scenarios/sparse.jl @@ -32,17 +32,17 @@ end function sparse_vec_to_vec_scenarios(x::AbstractVector) n = length(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ - JacobianScenario(diffsquare; x=x, ref=diffsquare_jacobian, operator=op), + JacobianScenario(diffsquare; x=x, ref=diffsquare_jacobian, place=place), JacobianScenario( diffsquare!; x=x, y=similar(x, n - 1), ref=diffsquare_jacobian, - operator=op, + place=place, ), ], ) @@ -70,7 +70,7 @@ end function sparse_mat_to_vec_scenarios(x::AbstractMatrix) m, n = size(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -78,14 +78,14 @@ function sparse_mat_to_vec_scenarios(x::AbstractMatrix) diffsquarecube_matvec; x=x, ref=diffsquarecube_matvec_jacobian, - operator=op, + place=place, ), JacobianScenario( diffsquarecube_matvec!; x=x, y=similar(x, 2(m * n) - 2), ref=diffsquarecube_matvec_jacobian, - operator=op, + place=place, ), ], ) @@ -110,7 +110,7 @@ end function sparse_vec_to_mat_scenarios(x::AbstractVector) n = length(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -118,14 +118,14 @@ function sparse_vec_to_mat_scenarios(x::AbstractVector) diffsquarecube_vecmat; x=x, ref=diffsquarecube_vecmat_jacobian, - operator=op, + place=place, ), JacobianScenario( diffsquarecube_vecmat!; x=x, y=similar(x, n - 1, 2), ref=diffsquarecube_vecmat_jacobian, - operator=op, + place=place, ), ], ) @@ -152,7 +152,7 @@ end function sparse_mat_to_mat_scenarios(x::AbstractMatrix) m, n = size(x) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ @@ -160,14 +160,14 @@ function sparse_mat_to_mat_scenarios(x::AbstractMatrix) diffsquarecube_matmat; x=x, ref=diffsquarecube_matmat_jacobian, - operator=op, + place=place, ), JacobianScenario( diffsquarecube_matmat!; x=x, y=similar(x, m * n - 1, 2), ref=diffsquarecube_matmat_jacobian, - operator=op, + place=place, ), ], ) @@ -187,9 +187,9 @@ end function sparse_vec_to_num_scenarios(x::AbstractVector) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( - scens, [HessianScenario(sumdiffcube; x=x, ref=sumdiffcube_hessian, operator=op)] + scens, [HessianScenario(sumdiffcube; x=x, ref=sumdiffcube_hessian, place=place)] ) end return scens @@ -207,12 +207,12 @@ end function sparse_mat_to_num_scenarios(x::AbstractMatrix) scens = AbstractScenario[] - for op in (:outofplace, :inplace) + for place in (:outofplace, :inplace) append!( scens, [ HessianScenario( - sumdiffcube_mat; x=x, ref=sumdiffcube_mat_hessian, operator=op + sumdiffcube_mat; x=x, ref=sumdiffcube_mat_hessian, place=place ), ], ) diff --git a/DifferentiationInterfaceTest/src/test_differentiation.jl b/DifferentiationInterfaceTest/src/test_differentiation.jl index 8d7f9684d..2fbbe55cc 100644 --- a/DifferentiationInterfaceTest/src/test_differentiation.jl +++ b/DifferentiationInterfaceTest/src/test_differentiation.jl @@ -94,7 +94,7 @@ function test_differentiation( (:scenario_type, "$st - $j/$(length(grouped_scenarios))"), (:scenario, "$k/$(length(st_group))"), (:arguments, nb_args(scen)), - (:operator, operator_place(scen)), + (:place, operator_place(scen)), (:function, scen.f), (:input_type, typeof(scen.x)), (:input_size, size(scen.x)), @@ -180,7 +180,7 @@ function benchmark_differentiation( (:scenario_type, "$st - $j/$(length(grouped_scenarios))"), (:scenario, "$k/$(length(st_group))"), (:arguments, nb_args(scen)), - (:operator, operator_place(scen)), + (:place, operator_place(scen)), (:function, scen.f), (:input_type, typeof(scen.x)), (:input_size, size(scen.x)), @@ -188,7 +188,7 @@ function benchmark_differentiation( (:output_size, size(scen.y)), ], ) - run_benchmark!(benchmark_data, backend, scen) + run_benchmark!(benchmark_data, backend, scen; logging) end end end diff --git a/DifferentiationInterfaceTest/src/tests/benchmark.jl b/DifferentiationInterfaceTest/src/tests/benchmark.jl index 4ec4c0b8c..f1e5ebc16 100644 --- a/DifferentiationInterfaceTest/src/tests/benchmark.jl +++ b/DifferentiationInterfaceTest/src/tests/benchmark.jl @@ -70,8 +70,9 @@ $(TYPEDFIELDS) mode::AbstractMode scenario::Symbol operator::Symbol - func::Symbol arguments::Int + place::Symbol + func::Symbol input_type::Type output_type::Type input_size::Tuple @@ -100,8 +101,9 @@ function record!( mode=mode(backend), scenario=typeof(scenario).name.name, operator=Symbol(operator), - func=Symbol(scenario.f), arguments=nb_args(scenario), + place=operator_place(scenario), + func=Symbol(scenario.f), input_type=typeof(scenario.x), output_type=typeof(scenario.y), input_size=size(scenario.x), @@ -124,14 +126,15 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PushforwardScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_pushforward(f, ba, x, dx) bench0 = @be prepare_pushforward(f, ba, x, dx) samples = 1 evals = 1 - bench1 = @be value_and_pushforward(f, ba, x, dx, extras) - bench2 = @be pushforward(f, ba, x, dx, extras) + bench1 = @be deepcopy(extras) value_and_pushforward(f, ba, x, dx, _) evals = 1 + bench2 = @be deepcopy(extras) pushforward(f, ba, x, dx, _) evals = 1 # count cc = CallCounter(f) extras = prepare_pushforward(cc, ba, x, dx) @@ -142,6 +145,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -157,15 +161,19 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PushforwardScenario{1,:inplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_pushforward(f, ba, x, dx) bench0 = @be prepare_pushforward(f, ba, x, dx) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_pushforward!(f, _, ba, x, dx, extras) evals = - 1 - bench2 = @be mysimilar(y) pushforward!(f, _, ba, x, dx, extras) evals = 1 + bench1 = @be (dy=mysimilar(y), ext=deepcopy(extras)) value_and_pushforward!( + f, _.dy, ba, x, dx, _.ext + ) evals = 1 + bench2 = @be (dy=mysimilar(y), ext=deepcopy(extras)) pushforward!( + f, _.dy, ba, x, dx, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_pushforward(cc, ba, x, dx) @@ -176,6 +184,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -191,6 +200,7 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PushforwardScenario{2,:outofplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) f! = f @@ -199,9 +209,12 @@ function run_benchmark!( extras = prepare_pushforward(f!, mysimilar(y), ba, x, dx) bench0 = @be mysimilar(y) prepare_pushforward(f!, _, ba, x, dx) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_pushforward(f!, _, ba, x, dx, extras) evals = - 1 - bench2 = @be mysimilar(y) pushforward(f!, _, ba, x, dx, extras) evals = 1 + bench1 = @be (y=mysimilar(y), ext=deepcopy(extras)) value_and_pushforward( + f!, _.y, ba, x, dx, _.ext + ) evals = 1 + bench2 = @be (y=mysimilar(y), ext=deepcopy(extras)) pushforward( + f!, _.y, ba, x, dx, _.ext + ) evals = 1 # count cc! = CallCounter(f!) extras = prepare_pushforward(cc!, mysimilar(y), ba, x, dx) @@ -212,6 +225,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -227,19 +241,20 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PushforwardScenario{2,:inplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) f! = f (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark - extras = prepare_pushforward(f!, mysimilar(y), ba, x, dx) + extras = prepare_pushforward(f!, y, ba, x, dx) bench0 = @be mysimilar(y) prepare_pushforward(f!, _, ba, x, dx) evals = 1 samples = 1 - bench1 = @be (mysimilar(y), mysimilar(y)) value_and_pushforward!( - f!, _[1], _[2], ba, x, dx, extras + bench1 = @be (y=mysimilar(y), dy=mysimilar(y), ext=deepcopy(extras)) value_and_pushforward!( + f!, _.y, _.dy, ba, x, dx, _.ext ) evals = 1 - bench2 = @be (mysimilar(y), mysimilar(y)) pushforward!( - f!, _[1], _[2], ba, x, dx, extras + bench2 = @be (y=mysimilar(y), dy=mysimilar(y), ext=deepcopy(extras)) pushforward!( + f!, _.y, _.dy, ba, x, dx, _.ext ) evals = 1 # count cc! = CallCounter(f!) @@ -251,6 +266,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -268,16 +284,17 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PullbackScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y, dy) = deepcopy(scen) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) = try # benchmark extras = prepare_pullback(f, ba, x, dy) bench0 = @be prepare_pullback(f, ba, x, dy) samples = 1 evals = 1 - bench1 = @be value_and_pullback(f, ba, x, dy, extras) - bench2 = @be pullback(f, ba, x, dy, extras) - bench3 = @be value_and_pullback_split(f, ba, x, extras) - bench4 = @be last(value_and_pullback_split(f, ba, x, extras)) _(dy) + bench1 = @be deepcopy(extras) value_and_pullback(f, ba, x, dy, _) + bench2 = @be deepcopy(extras) pullback(f, ba, x, dy, _) + bench3 = @be deepcopy(extras) value_and_pullback_split(f, ba, x, _) + bench4 = @be last(value_and_pullback_split(f, ba, x, deepcopy(extras))) _(dy) # count cc = CallCounter(f) extras = prepare_pullback(cc, ba, x, dy) @@ -292,6 +309,7 @@ function run_benchmark!( calls4 = reset_count!(cc) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2, bench3, bench4 = failed_benchs(5) calls0, calls1, calls2, calls3, calls4 = -1, -1, -1, -1, -1 (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) @@ -306,19 +324,27 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PullbackScenario{1,:inplace}; + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::PullbackScenario{1,:inplace}; + logging::Bool, ) (; f, x, y, dy) = deepcopy(scen) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) = try # benchmark extras = prepare_pullback(f, ba, x, dy) bench0 = @be prepare_pullback(f, ba, x, dy) samples = 1 evals = 1 - bench1 = @be mysimilar(x) value_and_pullback!(f, _, ba, x, dy, extras) evals = 1 - bench2 = @be mysimilar(x) pullback!(f, _, ba, x, dy, extras) evals = 1 - bench3 = @be value_and_pullback!_split(f, ba, x, extras) - bench4 = @be (mysimilar(x), last(value_and_pullback!_split(f, ba, x, extras))) _[2]( - _[1], dy + bench1 = @be (dx=mysimilar(x), ext=deepcopy(extras)) value_and_pullback!( + f, _.dx, ba, x, dy, _.ext + ) evals = 1 + bench2 = @be (dx=mysimilar(x), ext=deepcopy(extras)) pullback!( + f, _.dx, ba, x, dy, _.ext ) evals = 1 + bench3 = @be deepcopy(extras) value_and_pullback!_split(f, ba, x, _) + bench4 = @be ( + dx=mysimilar(x), + (pullbackfunc!)=last(value_and_pullback!_split(f, ba, x, deepcopy(extras))), + ) _.pullbackfunc!(_.dx, dy) evals = 1 # count cc = CallCounter(f) extras = prepare_pullback(cc, ba, x, dy) @@ -333,6 +359,7 @@ function run_benchmark!( calls4 = reset_count!(cc) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2, bench3, bench4 = failed_benchs(5) calls0, calls1, calls2, calls3, calls4 = -1, -1, -1, -1, -1 (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) @@ -349,7 +376,8 @@ end function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, - scen::PullbackScenario{2,:outofplace}, + scen::PullbackScenario{2,:outofplace}; + logging::Bool, ) (; f, x, y, dy) = deepcopy(scen) f! = f @@ -358,11 +386,19 @@ function run_benchmark!( extras = prepare_pullback(f!, mysimilar(y), ba, x, dy) bench0 = @be mysimilar(y) prepare_pullback(f!, _, ba, x, dy) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_pullback(f!, _, ba, x, dy, extras) evals = 1 - bench2 = @be mysimilar(y) pullback(f!, _, ba, x, dy, extras) evals = 1 - bench3 = @be value_and_pullback_split(f!, y, ba, x, extras) - bench4 = @be last(value_and_pullback_split(f!, y, ba, x, extras)) _(y, dy) evals = - 1 + bench1 = @be (y=mysimilar(y), ext=deepcopy(extras)) value_and_pullback( + f!, _.y, ba, x, dy, _.ext + ) evals = 1 + bench2 = @be (y=mysimilar(y), ext=deepcopy(extras)) pullback( + f!, _.y, ba, x, dy, _.ext + ) evals = 1 + bench3 = @be deepcopy(extras) value_and_pullback_split(f!, y, ba, x, _) + bench4 = @be ( + y=mysimilar(y), + pullbackfunc=last( + value_and_pullback_split(f!, mysimilar(y), ba, x, deepcopy(extras)) + ), + ) _.pullbackfunc(_.y, dy) evals = 1 # count cc! = CallCounter(f!) extras = prepare_pullback(cc!, mysimilar(y), ba, x, dy) @@ -377,6 +413,7 @@ function run_benchmark!( calls4 = reset_count!(cc!) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2, bench3, bench4 = failed_benchs(5) calls0, calls1, calls2, calls3, calls4 = -1, -1, -1, -1, -1 (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) @@ -391,7 +428,10 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::PullbackScenario{2,:inplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::PullbackScenario{2,:inplace}; + logging::Bool, ) (; f, x, y, dy) = deepcopy(scen) f! = f @@ -400,14 +440,22 @@ function run_benchmark!( extras = prepare_pullback(f!, mysimilar(y), ba, x, dy) bench0 = @be mysimilar(y) prepare_pullback(f!, _, ba, x, dy) samples = 1 evals = 1 - bench1 = @be (mysimilar(y), mysimilar(x)) value_and_pullback!( - f!, _[1], _[2], ba, x, dy, extras + bench1 = @be (y=mysimilar(y), dx=mysimilar(x), ext=deepcopy(extras)) value_and_pullback!( + f!, _.y, _.dx, ba, x, dy, _.ext ) evals = 1 - bench2 = @be (mysimilar(y), mysimilar(x)) pullback!( - f!, _[1], _[2], ba, x, dy, extras + bench2 = @be (y=mysimilar(y), dx=mysimilar(x), ext=deepcopy(extras)) pullback!( + f!, _.y, _.dx, ba, x, dy, _.ext ) evals = 1 - bench3 = @be value_and_pullback!_split(f!, y, ba, x, extras) - bench4 = @be (mysimilar(x), last(value_and_pullback!_split(f!, y, ba, x, extras))) _[2](y, _[1], dy) evals = 1 + bench3 = @be (y=mysimilar(y), ext=deepcopy(extras)) value_and_pullback!_split( + f!, _.y, ba, x, _.ext + ) + bench4 = @be ( + y=mysimilar(y), + dx=mysimilar(x), + (pullbackfunc!)=last( + value_and_pullback!_split(f!, mysimilar(y), ba, x, deepcopy(extras)) + ), + ) _.pullbackfunc!(_.y, _.dx, dy) evals = 1 # count cc! = CallCounter(f!) extras = prepare_pullback(cc!, mysimilar(y), ba, x, dy) @@ -422,6 +470,7 @@ function run_benchmark!( calls4 = reset_count!(cc!) (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2, bench3, bench4 = failed_benchs(5) calls0, calls1, calls2, calls3, calls4 = -1, -1, -1, -1, -1 (; bench0, bench1, bench2, bench3, bench4, calls0, calls1, calls2, calls3, calls4) @@ -441,14 +490,15 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::DerivativeScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_derivative(f, ba, x) bench0 = @be prepare_derivative(f, ba, x) samples = 1 evals = 1 - bench1 = @be value_and_derivative(f, ba, x, extras) - bench2 = @be derivative(f, ba, x, extras) + bench1 = @be deepcopy(extras) value_and_derivative(f, ba, x, _) + bench2 = @be deepcopy(extras) derivative(f, ba, x, _) # count cc = CallCounter(f) extras = prepare_derivative(cc, ba, x) @@ -459,6 +509,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -474,14 +525,19 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::DerivativeScenario{1,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_derivative(f, ba, x) bench0 = @be prepare_derivative(f, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_derivative!(f, _, ba, x, extras) evals = 1 - bench2 = @be mysimilar(y) derivative!(f, _, ba, x, extras) evals = 1 + bench1 = @be (der=mysimilar(y), ext=deepcopy(extras)) value_and_derivative!( + f, _.der, ba, x, _.ext + ) evals = 1 + bench2 = @be (der=mysimilar(y), ext=deepcopy(extras)) derivative!( + f, _.der, ba, x, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_derivative(cc, ba, x) @@ -492,6 +548,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -507,6 +564,7 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::DerivativeScenario{2,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) f! = f @@ -514,8 +572,12 @@ function run_benchmark!( # benchmark extras = prepare_derivative(f!, mysimilar(y), ba, x) bench0 = @be mysimilar(y) prepare_derivative(f!, _, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_derivative(f!, _, ba, x, extras) evals = 1 - bench2 = @be mysimilar(y) derivative(f!, _, ba, x, extras) evals = 1 + bench1 = @be (y=mysimilar(y), ext=deepcopy(extras)) value_and_derivative( + f!, _.y, ba, x, _.ext + ) evals = 1 + bench2 = @be (y=mysimilar(y), ext=deepcopy(extras)) derivative( + f!, _.y, ba, x, _.ext + ) evals = 1 # count cc! = CallCounter(f!) extras = prepare_derivative(cc!, mysimilar(y), ba, x) @@ -526,6 +588,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -541,6 +604,7 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::DerivativeScenario{2,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) f! = f @@ -548,10 +612,12 @@ function run_benchmark!( # benchmark extras = prepare_derivative(f!, mysimilar(y), ba, x) bench0 = @be mysimilar(y) prepare_derivative(f!, _, ba, x) samples = 1 evals = 1 - bench1 = @be (mysimilar(y), mysimilar(y)) value_and_derivative!( - f!, _[1], _[2], ba, x, extras + bench1 = @be (y=mysimilar(y), der=mysimilar(y), ext=deepcopy(extras)) value_and_derivative!( + f!, _.y, _.der, ba, x, _.ext + ) evals = 1 + bench2 = @be (y=mysimilar(y), der=mysimilar(y), ext=deepcopy(extras)) derivative!( + f!, _.y, _.der, ba, x, _.ext ) evals = 1 - bench2 = @be (mysimilar(y), mysimilar(y)) derivative!(f!, _[1], _[2], ba, x, extras) evals = 1 # count cc! = CallCounter(f!) extras = prepare_derivative(cc!, mysimilar(y), ba, x) @@ -562,6 +628,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -579,14 +646,15 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::GradientScenario{1,:outofplace}; + logging::Bool, ) (; f, x) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_gradient(f, ba, x) bench0 = @be prepare_gradient(f, ba, x) samples = 1 evals = 1 - bench1 = @be value_and_gradient(f, ba, x, extras) - bench2 = @be gradient(f, ba, x, extras) + bench1 = @be deepcopy(extras) value_and_gradient(f, ba, x, _) + bench2 = @be deepcopy(extras) gradient(f, ba, x, _) # count cc = CallCounter(f) extras = prepare_gradient(cc, ba, x) @@ -597,6 +665,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -609,15 +678,22 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::GradientScenario{1,:inplace}; + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::GradientScenario{1,:inplace}; + logging::Bool, ) (; f, x) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_gradient(f, ba, x) bench0 = @be prepare_gradient(f, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(x) value_and_gradient!(f, _, ba, x, extras) evals = 1 - bench2 = @be mysimilar(x) gradient!(f, _, ba, x, extras) evals = 1 + bench1 = @be (grad=mysimilar(x), ext=deepcopy(extras)) value_and_gradient!( + f, _.grad, ba, x, _.ext + ) evals = 1 + bench2 = @be (grad=mysimilar(x), ext=deepcopy(extras)) gradient!( + f, _.grad, ba, x, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_gradient(cc, ba, x) @@ -628,6 +704,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -645,14 +722,15 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::JacobianScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try # benchmark extras = prepare_jacobian(f, ba, x) bench0 = @be prepare_jacobian(f, ba, x) samples = 1 evals = 1 - bench1 = @be value_and_jacobian(f, ba, x, extras) - bench2 = @be jacobian(f, ba, x, extras) + bench1 = @be deepcopy(extras) value_and_jacobian(f, ba, x, _) + bench2 = @be deepcopy(extras) jacobian(f, ba, x, _) # count cc = CallCounter(f) extras = prepare_jacobian(cc, ba, x) @@ -663,6 +741,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -675,7 +754,10 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::JacobianScenario{1,:inplace}; + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::JacobianScenario{1,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, bench2, calls0, calls1, calls2) = try @@ -683,9 +765,12 @@ function run_benchmark!( # benchmark extras = prepare_jacobian(f, ba, x) bench0 = @be prepare_jacobian(f, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(jac_template) value_and_jacobian!(f, _, ba, x, extras) evals = - 1 - bench2 = @be mysimilar(jac_template) jacobian!(f, _, ba, x, extras) evals = 1 + bench1 = @be (jac=mysimilar(jac_template), ext=deepcopy(extras)) value_and_jacobian!( + f, _.jac, ba, x, _.ext + ) evals = 1 + bench2 = @be (jac=mysimilar(jac_template), ext=deepcopy(extras)) jacobian!( + f, _.jac, ba, x, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_jacobian(cc, ba, x) @@ -696,6 +781,7 @@ function run_benchmark!( calls2 = reset_count!(cc) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -710,7 +796,8 @@ end function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, - scen::JacobianScenario{2,:outofplace}, + scen::JacobianScenario{2,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) f! = f @@ -718,8 +805,10 @@ function run_benchmark!( # benchmark extras = prepare_jacobian(f!, mysimilar(y), ba, x) bench0 = @be mysimilar(y) prepare_jacobian(f!, _, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(y) value_and_jacobian(f!, _, ba, x, extras) evals = 1 - bench2 = @be mysimilar(y) jacobian(f!, _, ba, x, extras) evals = 1 + bench1 = @be (y=mysimilar(y), ext=deepcopy(extras)) value_and_jacobian( + f!, _.y, ba, x, _.ext + ) evals = 1 + bench2 = @be (y=mysimilar(y), ext=deepcopy(extras)) jacobian(f!, _.y, ba, x, _.ext) evals = 1 # count cc! = CallCounter(f!) extras = prepare_jacobian(cc!, mysimilar(y), ba, x) @@ -730,6 +819,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -742,7 +832,10 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::JacobianScenario{2,:inplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::JacobianScenario{2,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) f! = f @@ -751,11 +844,11 @@ function run_benchmark!( # benchmark extras = prepare_jacobian(f!, mysimilar(y), ba, x) bench0 = @be mysimilar(y) prepare_jacobian(f!, _, ba, x) samples = 1 evals = 1 - bench1 = @be (mysimilar(y), mysimilar(jac_template)) value_and_jacobian!( - f!, _[1], _[2], ba, x, extras + bench1 = @be (y=mysimilar(y), jac=mysimilar(jac_template), ext=deepcopy(extras)) value_and_jacobian!( + f!, _.y, _.jac, ba, x, _.ext ) evals = 1 - bench2 = @be (mysimilar(y), mysimilar(jac_template)) jacobian!( - f!, _[1], _[2], ba, x, extras + bench2 = @be (y=mysimilar(y), jac=mysimilar(jac_template), ext=deepcopy(extras)) jacobian!( + f!, _.y, _.jac, ba, x, _.ext ) evals = 1 # count cc! = CallCounter(f!) @@ -767,6 +860,7 @@ function run_benchmark!( calls2 = reset_count!(cc!) (; bench0, bench1, bench2, calls0, calls1, calls2) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1, bench2 = failed_benchs(3) calls0, calls1, calls2 = -1, -1, -1 (; bench0, bench1, bench2, calls0, calls1, calls2) @@ -784,13 +878,14 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::SecondDerivativeScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try # benchmark extras = prepare_second_derivative(f, ba, x) bench0 = @be prepare_second_derivative(f, ba, x) samples = 1 evals = 1 - bench1 = @be second_derivative(f, ba, x, extras) + bench1 = @be deepcopy(extras) second_derivative(f, ba, x, _) # count cc = CallCounter(f) extras = prepare_second_derivative(cc, ba, x) @@ -799,6 +894,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) @@ -813,13 +909,16 @@ function run_benchmark!( data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::SecondDerivativeScenario{1,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try # benchmark extras = prepare_second_derivative(f, ba, x) bench0 = @be prepare_second_derivative(f, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(y) second_derivative!(f, _, ba, x, extras) evals = 1 + bench1 = @be (der=mysimilar(y), ext=deepcopy(extras)) second_derivative!( + f, _.der, ba, x, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_second_derivative(cc, ba, x) @@ -828,6 +927,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) @@ -841,14 +941,17 @@ end ## Hessian-vector product function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::HVPScenario{1,:outofplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::HVPScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try # benchmark extras = prepare_hvp(f, ba, x, dx) bench0 = @be prepare_hvp(f, ba, x, dx) samples = 1 evals = 1 - bench1 = @be hvp(f, ba, x, dx, extras) + bench1 = @be deepcopy(extras) hvp(f, ba, x, dx, _) # count cc = CallCounter(f) extras = prepare_hvp(cc, ba, x, dx) @@ -857,6 +960,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) @@ -868,14 +972,17 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::HVPScenario{1,:inplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::HVPScenario{1,:inplace}; + logging::Bool, ) (; f, x, y, dx) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try # benchmark extras = prepare_hvp(f, ba, x, dx) bench0 = @be prepare_hvp(f, ba, x, dx) samples = 1 evals = 1 - bench1 = @be mysimilar(x) hvp!(f, _, ba, x, dx, extras) evals = 1 + bench1 = @be (p=mysimilar(x), ext=deepcopy(extras)) hvp!(f, _.p, ba, x, dx, _.ext) evals = 1 # count cc = CallCounter(f) extras = prepare_hvp(cc, ba, x, dx) @@ -884,6 +991,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) @@ -897,14 +1005,17 @@ end ## Hessian function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::HessianScenario{1,:outofplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::HessianScenario{1,:outofplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try # benchmark extras = prepare_hessian(f, ba, x) bench0 = @be prepare_hessian(f, ba, x) samples = 1 evals = 1 - bench1 = @be hessian(f, ba, x, extras) + bench1 = @be deepcopy(extras) hessian(f, ba, x, _) # count cc = CallCounter(f) extras = prepare_hessian(cc, ba, x) @@ -913,6 +1024,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) @@ -924,7 +1036,10 @@ function run_benchmark!( end function run_benchmark!( - data::Vector{BenchmarkDataRow}, ba::AbstractADType, scen::HessianScenario{1,:inplace} + data::Vector{BenchmarkDataRow}, + ba::AbstractADType, + scen::HessianScenario{1,:inplace}; + logging::Bool, ) (; f, x, y) = deepcopy(scen) (; bench0, bench1, calls0, calls1) = try @@ -932,7 +1047,9 @@ function run_benchmark!( # benchmark extras = prepare_hessian(f, ba, x) bench0 = @be prepare_hessian(f, ba, x) samples = 1 evals = 1 - bench1 = @be mysimilar(hess_template) hessian!(f, _, ba, x, extras) evals = 1 + bench1 = @be (hess=mysimilar(hess_template), ext=deepcopy(extras)) hessian!( + f, _.hess, ba, x, _.ext + ) evals = 1 # count cc = CallCounter(f) extras = prepare_hessian(cc, ba, x) @@ -941,6 +1058,7 @@ function run_benchmark!( calls1 = reset_count!(cc) (; bench0, bench1, calls0, calls1) catch e + logging && @warn "Error during benchmarking" ba scen e bench0, bench1 = failed_benchs(2) calls0, calls1 = -1, -1 (; bench0, bench1, calls0, calls1) diff --git a/DifferentiationInterfaceTest/test/zero_backends.jl b/DifferentiationInterfaceTest/test/zero_backends.jl index 8e092cad5..4b850d038 100644 --- a/DifferentiationInterfaceTest/test/zero_backends.jl +++ b/DifferentiationInterfaceTest/test/zero_backends.jl @@ -76,7 +76,7 @@ ADTypes.mode(::FakeBackend) = ADTypes.ForwardMode() data3 = benchmark_differentiation( [FakeBackend()]; logging=get(ENV, "CI", "false") == "false" -); +); # this gives lots of warnings when logging is on, no worries df3 = DataFrames.DataFrame(data3)