drop_expr hangs on RuntimeGeneratedFunctions v0.5.11

[YingboMa]

See https://github.com/SciML/ModelingToolkit.jl/actions/runs/5159631398/jobs/9294793159?pr=2179#step:6:1048

One can reproduce it by running the test/bigsystem.jl test in ModelingToolkit.jl

julia> include("test/bigsystem.jl")
WARNING: redefinition of constant Main.X. This may fail, cause incorrect answers, or produce other errors.
WARNING: redefinition of constant Main.Y. This may fail, cause incorrect answers, or produce other errors.
WARNING: redefinition of constant Main.α₁. This may fail, cause incorrect answers, or produce other errors.
WARNING: redefinition of constant Main.Mx. This may fail, cause incorrect answers, or produce other errors.
WARNING: redefinition of constant Main.My. This may fail, cause incorrect answers, or produce other errors.
typeof.((du, u, MyA, AMx, DA)) = (Array{Num, 3}, Array{Num, 3}, Matrix{Num}, Matrix{Num}, Matrix{Num})
# ^C after 20 mins
^CERROR: LoadError: InterruptException:
Stacktrace:
  [1] drop_expr(::RuntimeGeneratedFunc)
    @ RuntimeGeneratedFunctions ~/.julia/packages/RuntimeGeneratedFunctions/Avj6S/src/RuntimeGeneratedFunctions.jl:83
  [2] (::Symbolics.var"#372#373"{SymbolicUtils.Code.LazyState})(thunk::SymbolicUtils.Code.F, a::Vector{Any})
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:360
  [3] (::Base.var"#4#5")(a::Any)
    @ Base ./generator.jl:36 [inlined]
  [4] iterate(g::Base.Generator, s::Vararg{Any})
    @ Base ./generator.jl:47 [inlined]
  [5] collect_to!(dest::AbstractArray{T}, itr::Any, offs::Any, st::Any) where T
    @ Base ./array.jl:885 [inlined]
  [6] collect_to_with_first!(dest::AbstractArray, v1::Any, itr::Any, st::Any)
    @ Base ./array.jl:863 [inlined]
  [7] collect(itr::Base.Generator{Base.)
    @ Base ./array.jl:837
  [8] map(::Function, ::Vector{SymbolicUtils, ::Vector{Vector{Any}})
    @ Base ./abstractarray.jl:3389
  [9] toexpr(p::SymbolicUtils.Code.S, st::SymbolicUtils.Code.L)
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:359
 [10] toexpr(l::SymbolicUtils.Code.L, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:269
 [11] toexpr(l::SymbolicUtils.Code.L, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:262
 [12] toexpr(f::SymbolicUtils.Code.F, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:350
 [13] macro expansion
    @ Symbolics ~/.julia/packages/RuntimeGeneratedFunctions/Avj6S/src/RuntimeGeneratedFunctions.jl:119 [inlined]
--- the last 12 lines are repeated 3 more times ---
 [50] (::Symbolics.var"#372#373"{SymbolicUtils.Code.LazyState})(thunk::SymbolicUtils.Code.F, a::Vector{Any})
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:360
 [51] (::Base.var"#4#5")(a::Any)
    @ Base ./generator.jl:36 [inlined]
 [52] iterate(g::Base.Generator, s::Vararg{Any})
    @ Base ./generator.jl:47 [inlined]
 [53] collect_to!(dest::AbstractArray{T}, itr::Any, offs::Any, st::Any) where T
    @ Base ./array.jl:885 [inlined]
 [54] collect_to_with_first!(dest::AbstractArray, v1::Any, itr::Any, st::Any)
    @ Base ./array.jl:863 [inlined]
 [55] collect(itr::Base.Generator{Base.)
    @ Base ./array.jl:837
 [56] map(::Function, ::Vector{SymbolicUtils, ::Vector{Vector{Any}})
    @ Base ./abstractarray.jl:3389
 [57] toexpr(p::SymbolicUtils.Code.S, st::SymbolicUtils.Code.L)
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:359
 [58] recurse_expr(ex::SymbolicUtils.Code.S, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:657
 [59] _broadcast_getindex_evalf(f::Tf, args::Vararg{Any, N}) where {Tf, N}
    @ Base.Broadcast ./broadcast.jl:709 [inlined]
 [60] _broadcast_getindex
    @ Base.Broadcast ./broadcast.jl:682 [inlined]
 [61] getindex(bc::Base.Broadcast.Broad, I::Union{Integer, Carte)
    @ Base.Broadcast ./broadcast.jl:636 [inlined]
 [62] copyto_nonleaf!(dest::Vector{Any}, bc::Base.Broadcast.Broad, iter::Base.OneTo{Int64}, state::Int64, count::Int64)
    @ Base.Broadcast ./broadcast.jl:1098
 [63] restart_copyto_nonleaf!(newdest::Vector{Any}, dest::Vector{LineNumberNod, bc::Base.Broadcast.Broad, val::Expr, I::Int64, iter::Base.OneTo{Int64}, state::Int64, count::Int64)
    @ Base.Broadcast ./broadcast.jl:1089
 [64] copyto_nonleaf!(dest::Vector{LineNumberNod, bc::Base.Broadcast.Broad, iter::Base.OneTo{Int64}, state::Int64, count::Int64)
    @ Base.Broadcast ./broadcast.jl:1105
 [65] copy(bc::Base.Broadcast.Broad)
    @ Base.Broadcast ./broadcast.jl:950 [inlined]
 [66] materialize(bc::Base.Broadcast.Broad)
    @ Base.Broadcast ./broadcast.jl:903 [inlined]
 [67] recurse_expr(ex::Expr, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:656
 [68] toexpr(exp::SymbolicUtils.Code.L, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:660
 [69] toexpr(l::SymbolicUtils.Code.L, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:269
 [70] toexpr(l::SymbolicUtils.Code.L, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:262
 [71] toexpr(f::SymbolicUtils.Code.F, st::SymbolicUtils.Code.L)
    @ SymbolicUtils.Code ~/src/julia/SymbolicUtils/src/code.jl:350
 [72] _build_function(target::Symbolics.JuliaTarge, rhss::Base.ReshapedArray{N, args::Array{Num, 3}; expression::Type, expression_module::Module, checkbounds::Bool, postprocess_fbody::Symbolics.var"#365#3, linenumbers::Bool, outputidxs::Nothing, skipzeros::Bool, force_SA::Bool, wrap_code::Tuple{Nothing, Nothi, fillzeros::Bool, states::SymbolicUtils.Code.L, iip_config::Tuple{Bool, Bool}, parallel::Symbolics.ShardedFor, cse::Bool, kwargs::Base.Pairs{Symbol, U)
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:307
 [73] build_function(::Base.ReshapedArray{N, ::Vararg{Any}; target::Symbolics.JuliaTarge, kwargs::Base.Pairs{Symbol, S)
    @ Symbolics ~/.julia/packages/Symbolics/k84uz/src/build_function.jl:87
in expression starting at /Users/scheme/src/julia/ModelingToolkit/test/bigsystem.jl:91

(mtkdae_10) pkg> st RuntimeGeneratedFunctions
Status `~/src/julia/env/mtkdae_10/Project.toml`
  [7e49a35a] RuntimeGeneratedFunctions v0.5.11

@c42f

[YingboMa]

RGF hangs on 0.5.11 and 0.5.10 but not on 0.5.9.

[c42f]

It's like we've deadlocked on the cache lock? But it's a SpinLock and we strictly use the scoped version of lock() and we don't do any IO inside the lock. I assume the Julia process is using CPU?

[YingboMa]

Are you able to reproduce it locally? For now, I just bounded RGF to be 0.5.9 for MTK.

[YingboMa]

It turns out to be Symbolics problem. JuliaSymbolics/Symbolics.jl#914

[c42f]

Glad you found it! I did look at the code but I couldn't see anything wrong at this end.

Does this mean RuntimeGeneratedFunctions is somehow incompatible with distributed code (which seems weird/broken). Or was it purely a problem downstream?