From caedbd6cb57ecccac8a5cec0e4fb7fb4ecb3fde6 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 31 Oct 2023 16:45:02 +0100 Subject: [PATCH 01/10] Turn ExpressionExplorer.jl into its own package --- Project.toml | 1 + src/Pluto.jl | 1 - src/analysis/ExpressionExplorer.jl | 1193 +--------------------------- src/analysis/ReactiveNode.jl | 78 -- src/analysis/TopologyUpdate.jl | 4 +- src/evaluation/MacroAnalysis.jl | 6 +- 6 files changed, 35 insertions(+), 1248 deletions(-) delete mode 100644 src/analysis/ReactiveNode.jl diff --git a/Project.toml b/Project.toml index a7a3ff9cef..263ce05ce3 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +ExpressionExplorer = "21656369-7473-754a-2065-74616d696c43" FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" FuzzyCompletions = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" diff --git a/src/Pluto.jl b/src/Pluto.jl index 8898d058f2..07ed7cd285 100644 --- a/src/Pluto.jl +++ b/src/Pluto.jl @@ -49,7 +49,6 @@ include("./evaluation/Throttled.jl") include("./runner/PlutoRunner.jl") include("./analysis/ExpressionExplorer.jl") include("./analysis/FunctionDependencies.jl") -include("./analysis/ReactiveNode.jl") include("./packages/PkgCompat.jl") include("./webserver/Status.jl") diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index 9b2153ccf3..00952cf9f5 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -1,292 +1,46 @@ -module ExpressionExplorer +using ExpressionExplorer -export compute_symbolreferences, try_compute_symbolreferences, compute_usings_imports, SymbolsState, FunctionName, join_funcname_parts +module ExpressionExplorerExtras +using ExpressionExplorer -import ..PlutoRunner -import Markdown -import Base: union, union!, ==, push! - -### -# TWO STATE OBJECTS -### - -const FunctionName = Vector{Symbol} - -""" -For an expression like `function Base.sqrt(x::Int)::Int x; end`, it has the following fields: -- `name::FunctionName`: the name, `[:Base, :sqrt]` -- `signature_hash::UInt`: a `UInt` that is unique for the type signature of the method declaration, ignoring argument names. In the example, this is equals `hash(ExpressionExplorer.canonalize( :(Base.sqrt(x::Int)::Int) ))`, see [`canonalize`](@ref) for more details. -""" -struct FunctionNameSignaturePair - name::FunctionName - signature_hash::UInt -end - -Base.:(==)(a::FunctionNameSignaturePair, b::FunctionNameSignaturePair) = a.name == b.name && a.signature_hash == b.signature_hash -Base.hash(a::FunctionNameSignaturePair, h::UInt) = hash(a.name, hash(a.signature_hash, h)) - -"SymbolsState trickles _down_ the ASTree: it carries referenced and defined variables from endpoints down to the root." -Base.@kwdef mutable struct SymbolsState - references::Set{Symbol} = Set{Symbol}() - assignments::Set{Symbol} = Set{Symbol}() - funccalls::Set{FunctionName} = Set{FunctionName}() - funcdefs::Dict{FunctionNameSignaturePair,SymbolsState} = Dict{FunctionNameSignaturePair,SymbolsState}() - macrocalls::Set{FunctionName} = Set{FunctionName}() -end - - -"ScopeState moves _up_ the ASTree: it carries scope information up towards the endpoints." -mutable struct ScopeState - inglobalscope::Bool - exposedglobals::Set{Symbol} - hiddenglobals::Set{Symbol} - definedfuncs::Set{Symbol} +struct PlutoConfiguration <: ExpressionExplorer.AbstractExpressionExplorerConfiguration end -ScopeState() = ScopeState(true, Set{Symbol}(), Set{Symbol}(), Set{Symbol}()) -# The `union` and `union!` overloads define how two `SymbolsState`s or two `ScopeState`s are combined. - -function union(a::Dict{FunctionNameSignaturePair,SymbolsState}, bs::Dict{FunctionNameSignaturePair,SymbolsState}...) - union!(Dict{FunctionNameSignaturePair,SymbolsState}(), a, bs...) -end - -function union!(a::Dict{FunctionNameSignaturePair,SymbolsState}, bs::Dict{FunctionNameSignaturePair,SymbolsState}...) - for b in bs - for (k, v) in b - if haskey(a, k) - a[k] = union!(a[k], v) - else - a[k] = v - end - end - a - end - return a -end - -function union(a::SymbolsState, b::SymbolsState) - SymbolsState(a.references ∪ b.references, a.assignments ∪ b.assignments, a.funccalls ∪ b.funccalls, a.funcdefs ∪ b.funcdefs, a.macrocalls ∪ b.macrocalls) -end - -function union!(a::SymbolsState, bs::SymbolsState...) - union!(a.references, (b.references for b in bs)...) - union!(a.assignments, (b.assignments for b in bs)...) - union!(a.funccalls, (b.funccalls for b in bs)...) - union!(a.funcdefs, (b.funcdefs for b in bs)...) - union!(a.macrocalls, (b.macrocalls for b in bs)...) - return a -end - -function union!(a::Tuple{FunctionName,SymbolsState}, bs::Tuple{FunctionName,SymbolsState}...) - a[1], union!(a[2], (b[2] for b in bs)...) -end -function union(a::ScopeState, b::ScopeState) - SymbolsState(a.inglobalscope && b.inglobalscope, a.exposedglobals ∪ b.exposedglobals, a.hiddenglobals ∪ b.hiddenglobals) -end - -function union!(a::ScopeState, bs::ScopeState...) - a.inglobalscope &= all((b.inglobalscope for b in bs)...) - union!(a.exposedglobals, (b.exposedglobals for b in bs)...) - union!(a.hiddenglobals, (b.hiddenglobals for b in bs)...) - union!(a.definedfuncs, (b.definedfuncs for b in bs)...) - return a -end - -function ==(a::SymbolsState, b::SymbolsState) - a.references == b.references && a.assignments == b.assignments && a.funccalls == b.funccalls && a.funcdefs == b.funcdefs && a.macrocalls == b.macrocalls -end - -Base.push!(x::Set) = x - -### -# HELPER FUNCTIONS -### - -# from the source code: https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm#L9 -const modifiers = [:(+=), :(-=), :(*=), :(/=), :(//=), :(^=), :(÷=), :(%=), :(<<=), :(>>=), :(>>>=), :(&=), :(⊻=), :(≔), :(⩴), :(≕)] -const modifiers_dotprefixed = [Symbol('.' * String(m)) for m in modifiers] - -function will_assign_global(assignee::Symbol, scopestate::ScopeState)::Bool - (scopestate.inglobalscope || assignee ∈ scopestate.exposedglobals) && (assignee ∉ scopestate.hiddenglobals || assignee ∈ scopestate.definedfuncs) -end - -function will_assign_global(assignee::Vector{Symbol}, scopestate::ScopeState)::Bool - if length(assignee) == 0 - false - elseif length(assignee) > 1 - scopestate.inglobalscope - else - will_assign_global(assignee[1], scopestate) - end -end +function ExpressionExplorer.explore_macrocall!(ex::Expr, scopestate::ScopeState{PlutoConfiguration}) + # Early stopping, this expression will have to be re-explored once + # the macro is expanded in the notebook process. + macro_name = ExpressionExplorer.split_funcname(ex.args[1]) + symstate = SymbolsState(macrocalls = Set{FunctionName}([macro_name])) -function get_global_assignees(assignee_exprs, scopestate::ScopeState)::Set{Symbol} - global_assignees = Set{Symbol}() - for ae in assignee_exprs - if isa(ae, Symbol) - will_assign_global(ae, scopestate) && push!(global_assignees, ae) - else - if ae.head == :(::) - will_assign_global(ae.args[1], scopestate) && push!(global_assignees, ae.args[1]) - else - @warn "Unknown assignee expression" ae - end - end - end - return global_assignees -end + # Because it sure wouldn't break anything, + # I'm also going to blatantly assume that any macros referenced in here... + # will end up in the code after the macroexpansion 🤷‍♀️ + # "You should make a new function for that" they said, knowing I would take the lazy route. + for arg in ex.args[begin+1:end] + macro_symstate = ExpressionExplorer.explore!(arg, ScopeState(scopestate.configuration)) -function get_assignees(ex::Expr)::FunctionName - if ex.head == :tuple - if length(ex.args) == 1 && Meta.isexpr(only(ex.args), :parameters) - # e.g. (x, y) in the ex (; x, y) = (x = 5, y = 6, z = 7) - args = only(ex.args).args + # Also, when this macro has something special inside like `Pkg.activate()`, + # we're going to treat it as normal code (so these heuristics trigger later) + # (Might want to also not let this to @eval macro, as an extra escape hatch if you + # really don't want pluto to see your Pkg.activate() call) + if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_symstate, expr = arg) + union!(symstate, macro_symstate) else - # e.g. (x, y) in the ex (x, y) = (1, 23) - args = ex.args - end - mapfoldl(get_assignees, union!, args; init=Symbol[]) - # filter(s->s isa Symbol, ex.args) - elseif ex.head == :(::) - # TODO: type is referenced - get_assignees(ex.args[1]) - elseif ex.head == :ref || ex.head == :(.) - Symbol[] - elseif ex.head == :... - # Handles splat assignments. e.g. _, y... = 1:5 - args = ex.args - mapfoldl(get_assignees, union!, args; init=Symbol[]) - else - @warn "unknown use of `=`. Assignee is unrecognised." ex - Symbol[] - end -end - -# e.g. x = 123, but ignore _ = 456 -get_assignees(ex::Symbol) = all_underscores(ex) ? Symbol[] : Symbol[ex] - -# When you assign to a datatype like Int, String, or anything bad like that -# e.g. 1 = 2 -# This is parsable code, so we have to treat it -get_assignees(::Any) = Symbol[] - -all_underscores(s::Symbol) = all(isequal('_'), string(s)) - -# TODO: this should return a FunctionName, and use `split_funcname`. -"Turn :(A{T}) into :A." -function uncurly!(ex::Expr, scopestate::ScopeState)::Tuple{Symbol,SymbolsState} - @assert ex.head == :curly - symstate = SymbolsState() - for curly_arg in ex.args[2:end] - arg_name, arg_symstate = explore_funcdef!(curly_arg, scopestate) - push!(scopestate.hiddenglobals, join_funcname_parts(arg_name)) - union!(symstate, arg_symstate) - end - Symbol(ex.args[1]), symstate -end - -uncurly!(ex::Expr)::Tuple{Symbol,SymbolsState} = ex.args[1], SymbolsState() - -uncurly!(s::Symbol, scopestate = nothing)::Tuple{Symbol,SymbolsState} = s, SymbolsState() - -"Turn `:(Base.Submodule.f)` into `[:Base, :Submodule, :f]` and `:f` into `[:f]`." -function split_funcname(funcname_ex::Expr)::FunctionName - if funcname_ex.head == :(.) - out = FunctionName() - args = funcname_ex.args - for arg in args - push!(out, split_funcname(arg)...) + union!(symstate, SymbolsState(macrocalls = macro_symstate.macrocalls)) end - return out - else - # a call to a function that's not a global, like calling an array element: `funcs[12]()` - # TODO: explore symstate! - return Symbol[] end -end - -function split_funcname(funcname_ex::QuoteNode)::FunctionName - split_funcname(funcname_ex.value) -end -function split_funcname(funcname_ex::GlobalRef)::FunctionName - split_funcname(funcname_ex.name) -end -function split_funcname(funcname_ex::Symbol)::FunctionName - Symbol[funcname_ex|>without_dotprefix|>without_dotsuffix] -end - -# this includes GlobalRef - it's fine that we don't recognise it, because you can't assign to a globalref? -function split_funcname(::Any)::FunctionName - Symbol[] -end - -"Allows comparing tuples to vectors since having constant vectors can be slower" -all_iters_eq(a, b) = length(a) == length(b) && all((aa == bb for (aa, bb) in zip(a, b))) - -function is_just_dots(ex::Expr) - ex.head == :(.) && all(is_just_dots, ex.args) -end -is_just_dots(::Union{QuoteNode,Symbol,GlobalRef}) = true -is_just_dots(::Any) = false - -"""Turn `Symbol(".+")` into `:(+)`""" -function without_dotprefix(funcname::Symbol)::Symbol - fn_str = String(funcname) - if length(fn_str) > 0 && fn_str[1] == '.' - Symbol(fn_str[2:end]) - else - funcname + # Some macros can be expanded on the server process + if ExpressionExplorer.join_funcname_parts(macro_name) ∈ can_macroexpand + new_ex = maybe_macroexpand_pluto(ex) + union!(symstate, ExpressionExplorer.explore!(new_ex, scopestate)) end -end -"""Turn `Symbol("sqrt.")` into `:sqrt`""" -function without_dotsuffix(funcname::Symbol)::Symbol - fn_str = String(funcname) - if length(fn_str) > 0 && fn_str[end] == '.' - Symbol(fn_str[1:end-1]) - else - funcname - end + return symstate end -"""Generates a vector of all possible variants from a function name - -``` -julia> generate_funcnames([:Base, :Foo, :bar]) -3-element Vector{Symbol}: - Symbol("Base.Foo.bar") - Symbol("Foo.bar") - :bar -``` - -""" -function generate_funcnames(funccall::FunctionName) - calls = Vector{FunctionName}(undef, length(funccall) - 1) - for i = length(funccall):-1:2 - calls[i-1] = funccall[i:end] - end - calls -end -""" -Turn `Symbol[:Module, :func]` into Symbol("Module.func"). - -This is **not** the same as the expression `:(Module.func)`, but is used to identify the function name using a single `Symbol` (like normal variables). -This means that it is only the inverse of `ExpressionExplorer.split_funcname` iff `length(parts) ≤ 1`. -""" -join_funcname_parts(parts::FunctionName) = Symbol(join(parts, '.')) - -# this is stupid -- désolé -function is_joined_funcname(joined::Symbol) - joined !== :.. #= .. is a valid identifier 😐 =# && occursin('.', String(joined)) -end - -"Module so I don't pollute the whole ExpressionExplorer scope" -module MacroHasSpecialHeuristicInside -import ...Pluto -import ..ExpressionExplorer, ..SymbolsState """ Uses `cell_precedence_heuristic` to determine if we need to include the contents of this macro in the symstate. @@ -309,729 +63,6 @@ function macro_has_special_heuristic_inside(; symstate::SymbolsState, expr::Expr return Pluto.cell_precedence_heuristic(fake_topology, fake_cell) < Pluto.DEFAULT_PRECEDENCE_HEURISTIC end -# Having written this... I know I said I was lazy... I was wrong -end - - -### -# MAIN RECURSIVE FUNCTION -### - -# Spaghetti code for a spaghetti problem 🍝 - -# Possible leaf: value -# Like: a = 1 -# 1 is a value (Int64) -function explore!(@nospecialize(value), scopestate::ScopeState)::SymbolsState - # includes: LineNumberNode, Int64, String, Markdown.LaTeX, DataType and more. - return SymbolsState() -end - -# Possible leaf: symbol -# Like a = x -# x is a symbol -# We handle the assignment separately, and explore!(:a, ...) will not be called. -# Therefore, this method only handles _references_, which are added to the symbolstate, depending on the scopestate. -function explore!(sym::Symbol, scopestate::ScopeState)::SymbolsState - if sym ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(references = Set([sym])) - end -end - -""" -Returns whether or not an assignment Expr(:(=),...) is assigning to a new function - * f(x) = ... - * f(x)::V = ... - * f(::T) where {T} = ... -""" -is_function_assignment(ex::Expr)::Bool = ex.args[1] isa Expr && (ex.args[1].head == :call || ex.args[1].head == :where || (ex.args[1].head == :(::) && ex.args[1].args[1] isa Expr && ex.args[1].args[1].head == :call)) - -anonymous_name() = Symbol("anon", rand(UInt64)) - -function explore_assignment!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - if is_function_assignment(ex) - # f(x, y) = x + y - # Rewrite to: - # function f(x, y) x + y end - return explore!(Expr(:function, ex.args...), scopestate) - end - - val = ex.args[2] - # Handle generic types assignments A{B} = C{B, Int} - if ex.args[1] isa Expr && ex.args[1].head::Symbol == :curly - assignees, symstate = explore_funcdef!(ex.args[1], scopestate)::Tuple{Vector{Symbol}, SymbolsState} - innersymstate = union!(symstate, explore!(val, scopestate)) - else - assignees = get_assignees(ex.args[1]) - symstate = innersymstate = explore!(val, scopestate) - end - - global_assignees = get_global_assignees(assignees, scopestate) - - # If we are _not_ assigning a global variable, then this symbol hides any global definition with that name - union!(scopestate.hiddenglobals, setdiff(assignees, global_assignees)) - assigneesymstate = explore!(ex.args[1], scopestate) - - union!(scopestate.hiddenglobals, global_assignees) - union!(symstate.assignments, global_assignees) - union!(symstate.references, setdiff(assigneesymstate.references, global_assignees)) - union!(symstate.funccalls, filter!(call -> length(call) != 1 || only(call) ∉ global_assignees, assigneesymstate.funccalls)) - filter!(!all_underscores, symstate.references) # Never record _ as a reference - - return symstate -end - -function explore_modifiers!(ex::Expr, scopestate::ScopeState) - # We change: a[1] += 123 - # to: a[1] = a[1] + 123 - # We transform the modifier back to its operator - # for when users redefine the + function - - operator = let - s = string(ex.head) - Symbol(s[1:prevind(s, lastindex(s))]) - end - expanded_expr = Expr(:(=), ex.args[1], Expr(:call, operator, ex.args[1], ex.args[2])) - return explore!(expanded_expr, scopestate) -end - -function explore_dotprefixed_modifiers!(ex::Expr, scopestate::ScopeState) - # We change: a[1] .+= 123 - # to: a[1] .= a[1] + 123 - - operator = Symbol(string(ex.head)[2:end-1]) - expanded_expr = Expr(:(.=), ex.args[1], Expr(:call, operator, ex.args[1], ex.args[2])) - return explore!(expanded_expr, scopestate) -end - -"Unspecialized mapfoldl." -function umapfoldl(@nospecialize(f::Function), itr::Vector; init=SymbolsState()) - if isempty(itr) - return init - else - out = init - for e in itr - union!(out, f(e)) - end - return out - end -end - -function explore_inner_scoped(ex::Expr, scopestate::ScopeState)::SymbolsState - # Because we are entering a new scope, we create a copy of the current scope state, and run it through the expressions. - innerscopestate = deepcopy(scopestate) - innerscopestate.inglobalscope = false - - return umapfoldl(a -> explore!(a, innerscopestate), ex.args) -end - -function explore_filter!(ex::Expr, scopestate::ScopeState) - # In a filter, the assignment is the second expression, the condition the first - args = collect(reverse(ex.args)) - umapfoldl(a -> explore!(a, scopestate), args)::SymbolsState -end - -function explore_generator!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - # In a `generator`, a single expression is followed by the iterator assignments. - # In a `for`, this expression comes at the end. - - # This is not strictly the normal form of a `for` but that's okay - return explore!(Expr(:for, Iterators.reverse(ex.args[2:end])..., ex.args[1]), scopestate) -end - -function explore_macrocall!(ex::Expr, scopestate::ScopeState) - # Early stopping, this expression will have to be re-explored once - # the macro is expanded in the notebook process. - macro_name = split_funcname(ex.args[1]) - symstate = SymbolsState(macrocalls = Set{FunctionName}([macro_name])) - - # Because it sure wouldn't break anything, - # I'm also going to blatantly assume that any macros referenced in here... - # will end up in the code after the macroexpansion 🤷‍♀️ - # "You should make a new function for that" they said, knowing I would take the lazy route. - for arg in ex.args[begin+1:end] - macro_symstate = explore!(arg, ScopeState()) - - # Also, when this macro has something special inside like `Pkg.activate()`, - # we're going to treat it as normal code (so these heuristics trigger later) - # (Might want to also not let this to @eval macro, as an extra escape hatch if you - # really don't want pluto to see your Pkg.activate() call) - if arg isa Expr && MacroHasSpecialHeuristicInside.macro_has_special_heuristic_inside(symstate = macro_symstate, expr = arg) - union!(symstate, macro_symstate) - else - union!(symstate, SymbolsState(macrocalls = macro_symstate.macrocalls)) - end - end - - # Some macros can be expanded on the server process - if join_funcname_parts(macro_name) ∈ can_macroexpand - new_ex = maybe_macroexpand(ex) - union!(symstate, explore!(new_ex, scopestate)) - end - - return symstate -end - -function funcname_symstate!(funcname::FunctionName, scopestate::ScopeState)::SymbolsState - if length(funcname) == 0 - explore!(ex.args[1], scopestate) - elseif length(funcname) == 1 - if funcname[1] ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(funccalls = Set{FunctionName}([funcname])) - end - elseif funcname[1] ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(references = Set{Symbol}([funcname[1]]), funccalls = Set{FunctionName}([funcname])) - end -end - -function explore_call!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - if is_just_dots(ex.args[1]) - funcname = split_funcname(ex.args[1])::FunctionName - symstate = funcname_symstate!(funcname, scopestate) - - # Explore code inside function arguments: - union!(symstate, explore!(Expr(:block, ex.args[2:end]...), scopestate)) - - # Make `@macroexpand` and `Base.macroexpand` reactive by referencing the first macro in the second - # argument to the call. - if (all_iters_eq((:Base, :macroexpand), funcname) || all_iters_eq((:macroexpand,), funcname)) && - length(ex.args) >= 3 && - ex.args[3] isa QuoteNode && - Meta.isexpr(ex.args[3].value, :macrocall) - expanded_macro = split_funcname(ex.args[3].value.args[1]) - union!(symstate, SymbolsState(macrocalls = Set{FunctionName}([expanded_macro]))) - elseif all_iters_eq((:BenchmarkTools, :generate_benchmark_definition), funcname) && - length(ex.args) == 10 - block = Expr(:block, - map(ex.args[[8,7,9]]) do child - if (Meta.isexpr(child, :copyast, 1) && child.args[1] isa QuoteNode && child.args[1].value isa Expr) - child.args[1].value - else - nothing - end - end... - ) - union!(symstate, explore_inner_scoped(block, scopestate)) - end - - return symstate - else - return explore!(Expr(:block, ex.args...), scopestate) - end -end - -function explore_struct!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - structnameexpr = ex.args[2] - structfields = ex.args[3].args - - equiv_func = Expr(:function, Expr(:call, structnameexpr, structfields...), Expr(:block, nothing)) - - # struct should always be in Global state - globalscopestate = deepcopy(scopestate) - globalscopestate.inglobalscope = true - - # we register struct definitions as both a variable and a function. This is because deleting a struct is trickier than just deleting its methods. - # Due to this, outer constructors have to be defined in the same cell where the struct is defined. - # See https://github.com/fonsp/Pluto.jl/issues/732 for more details - inner_symstate = explore!(equiv_func, globalscopestate) - - structname = first(keys(inner_symstate.funcdefs)).name |> join_funcname_parts - push!(inner_symstate.assignments, structname) - return inner_symstate -end - -function explore_abstract!(ex::Expr, scopestate::ScopeState) - explore_struct!(Expr(:struct, false, ex.args[1], Expr(:block, nothing)), scopestate) -end - -function explore_function_macro!(ex::Expr, scopestate::ScopeState) - symstate = SymbolsState() - # Creates local scope - - funcroot = ex.args[1] - - # Because we are entering a new scope, we create a copy of the current scope state, and run it through the expressions. - innerscopestate = deepcopy(scopestate) - innerscopestate.inglobalscope = false - - funcname, innersymstate = explore_funcdef!(funcroot, innerscopestate)::Tuple{FunctionName,SymbolsState} - - # Macro are called using @funcname, but defined with funcname. We need to change that in our scopestate - # (The `!= 0` is for when the function named couldn't be parsed) - if ex.head == :macro && length(funcname) != 0 - funcname = Symbol[Symbol('@', funcname[1])] - push!(innerscopestate.hiddenglobals, only(funcname)) - elseif length(funcname) == 1 - push!(scopestate.definedfuncs, funcname[end]) - push!(scopestate.hiddenglobals, funcname[end]) - elseif length(funcname) > 1 - push!(symstate.references, funcname[end-1]) # reference the module of the extended function - push!(scopestate.hiddenglobals, funcname[end-1]) - end - - union!(innersymstate, explore!(Expr(:block, ex.args[2:end]...), innerscopestate)) - funcnamesig = FunctionNameSignaturePair(funcname, hash(canonalize(funcroot))) - - if will_assign_global(funcname, scopestate) - symstate.funcdefs[funcnamesig] = innersymstate - else - # The function is not defined globally. However, the function can still modify the global scope or reference globals, e.g. - - # let - # function f(x) - # global z = x + a - # end - # f(2) - # end - - # so we insert the function's inner symbol state here, as if it was a `let` block. - symstate = innersymstate - end - - return symstate -end - -function explore_try!(ex::Expr, scopestate::ScopeState) - symstate = SymbolsState() - - # Handle catch first - if ex.args[3] != false - union!(symstate, explore_inner_scoped(ex.args[3], scopestate)) - # If we catch a symbol, it could shadow a global reference, remove it - if ex.args[2] != false - setdiff!(symstate.references, Symbol[ex.args[2]]) - end - end - - # Handle the try block - union!(symstate, explore_inner_scoped(ex.args[1], scopestate)) - - # Handle finally - if 4 <= length(ex.args) <= 5 && ex.args[4] isa Expr - union!(symstate, explore_inner_scoped(ex.args[4], scopestate)) - end - - # Finally, handle else - if length(ex.args) == 5 - union!(symstate, explore_inner_scoped(ex.args[5], scopestate)) - end - - return symstate -end - -function explore_anonymous_function!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - tempname = anonymous_name() - - # We will rewrite this to a normal function definition, with a temporary name - funcroot = ex.args[1] - args_ex = if funcroot isa Symbol || (funcroot isa Expr && funcroot.head == :(::)) - [funcroot] - elseif funcroot.head == :tuple || funcroot.head == :(...) || funcroot.head == :block - funcroot.args - else - @error "Unknown lambda type" - end - - equiv_func = Expr(:function, Expr(:call, tempname, args_ex...), ex.args[2]) - - return explore!(equiv_func, scopestate) -end - -function explore_global!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # global x, y, z - if length(ex.args) > 1 - return umapfoldl(arg -> explore!(Expr(:global, arg), scopestate), ex.args) - end - - # We have one of: - # global x; - # global x = 1; - # global x += 1; - - # where x can also be a tuple: - # global a,b = 1,2 - - globalisee = ex.args[1] - - if isa(globalisee, Symbol) - push!(scopestate.exposedglobals, globalisee) - return SymbolsState() - elseif isa(globalisee, Expr) - # temporarily set inglobalscope to true - old = scopestate.inglobalscope - scopestate.inglobalscope = true - result = explore!(globalisee, scopestate) - scopestate.inglobalscope = old - return result::SymbolsState - else - @error "unknown global use" ex - return explore!(globalisee, scopestate)::SymbolsState - end -end - -function explore_local!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # Turn `local x, y` in `local x; local y - if length(ex.args) > 1 - return umapfoldl(arg -> explore!(Expr(:local, arg), scopestate), ex.args) - end - - localisee = ex.args[1] - - if isa(localisee, Symbol) - push!(scopestate.hiddenglobals, localisee) - return SymbolsState() - elseif isa(localisee, Expr) && (localisee.head == :(=) || localisee.head in modifiers) - push!(scopestate.hiddenglobals, get_assignees(localisee.args[1])...) - return explore!(localisee, scopestate)::SymbolsState - else - @warn "unknown local use" ex - return explore!(localisee, scopestate)::SymbolsState - end -end - -function explore_tuple!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # There are two (legal) cases: - # 1. Creating a tuple: - # (a, b, c, 1, f()...) - # 2. Creating a named tuple (contains at least one Expr(:(=))): - # (a=1, b=2, c=3, d, f()...) - - # !!! Note that :(a, b = 1, 2) is the definition of a named tuple - # with fields :a, :b and :2 and not a multiple assignments to a and b which - # would always be a :(=) with tuples for the lhs and/or rhs. - # Using Meta.parse() (like Pluto does) or using a quote block - # returns the assignment version. - # - # julia> eval(:(a, b = 1, 2)) # Named tuple - # ERROR: syntax: invalid named tuple element "2" - # - # julia> eval(Meta.parse("a, b = 1, 2")) # Assignment to a and b - # (1, 2) - # - # julia> Meta.parse("a, b = 1, 2").head, :(a, b = 1, 2).head - # (:(=), :tuple) - - return umapfoldl(a -> explore!(to_kw(a), scopestate), ex.args) -end - -function explore_broadcast!(ex::Expr, scopestate::ScopeState) - # pointwise function call, e.g. sqrt.(nums) - # we rewrite to a regular call - - return explore!(Expr(:call, ex.args[1], ex.args[2].args...), scopestate) -end - -function explore_load!(ex::Expr, scopestate::ScopeState) - imports = if ex.args[1].head == :(:) - ex.args[1].args[2:end] - else - ex.args - end - - packagenames = map(e -> e.args[end], imports) - - return SymbolsState(assignments = Set{Symbol}(packagenames))::SymbolsState -end - -function explore_quote!(ex::Expr, scopestate::ScopeState) - # Look through the quote and only returns explore! deeper into :$'s - # I thought we need to handle strings in the same way, - # but strings do just fine with the catch all at the end - # and actually strings don't always have a :$ expression, sometimes just - # plain Symbols (which we should then be interpreted as variables, - # which is different to how we handle Symbols in quote'd expressions) - return explore_interpolations!(ex.args[1], scopestate)::SymbolsState -end - -function explore_module!(ex::Expr, scopestate::ScopeState) - # Does create it's own scope, but can import from outer scope, that's what `explore_module_definition!` is for - symstate = explore_module_definition!(ex, scopestate) - return union(symstate, SymbolsState(assignments = Set{Symbol}([ex.args[2]])))::SymbolsState -end - -function explore_fallback!(ex::Expr, scopestate::ScopeState) - # fallback, includes: - # begin, block, do, toplevel, const - # (and hopefully much more!) - - # Does not create scope (probably) - - return umapfoldl(a -> explore!(a, scopestate), ex.args) -end - -# General recursive method. Is never a leaf. -# Modifies the `scopestate`. -function explore!(ex::Expr, scopestate::ScopeState)::SymbolsState - if ex.head == :(=) - return explore_assignment!(ex, scopestate) - elseif ex.head in modifiers - return explore_modifiers!(ex, scopestate) - elseif ex.head in modifiers_dotprefixed - return explore_dotprefixed_modifiers!(ex, scopestate) - elseif ex.head == :let || ex.head == :for || ex.head == :while - # Creates local scope - return explore_inner_scoped(ex, scopestate) - elseif ex.head == :filter - return explore_filter!(ex, scopestate) - elseif ex.head == :generator - return explore_generator!(ex, scopestate) - elseif ex.head == :macrocall - return explore_macrocall!(ex, scopestate) - elseif ex.head == :call - return explore_call!(ex, scopestate) - elseif Meta.isexpr(ex, :parameters) - return umapfoldl(a -> explore!(to_kw(a), scopestate), ex.args) - elseif ex.head == :kw - return explore!(ex.args[2], scopestate) - elseif ex.head == :struct - return explore_struct!(ex, scopestate) - elseif ex.head == :abstract - return explore_abstract!(ex, scopestate) - elseif ex.head == :function || ex.head == :macro - return explore_function_macro!(ex, scopestate) - elseif ex.head == :try - return explore_try!(ex, scopestate) - elseif ex.head == :(->) - return explore_anonymous_function!(ex, scopestate) - elseif ex.head == :global - return explore_global!(ex, scopestate) - elseif ex.head == :local - return explore_local!(ex, scopestate) - elseif ex.head == :tuple - return explore_tuple!(ex, scopestate) - elseif Meta.isexpr(ex, :(.), 2) && ex.args[2] isa Expr && ex.args[2].head == :tuple - return explore_broadcast!(ex, scopestate) - elseif ex.head == :using || ex.head == :import - return explore_load!(ex, scopestate) - elseif ex.head == :quote - return explore_quote!(ex, scopestate) - elseif ex.head == :module - return explore_module!(ex, scopestate) - elseif Meta.isexpr(ex, Symbol("'"), 1) - # a' corresponds to adjoint(a) - return explore!(Expr(:call, :adjoint, ex.args[1]), scopestate) - elseif ex.head == :meta - return SymbolsState() - else - return explore_fallback!(ex, scopestate) - end -end - -""" -Goes through a module definition, and picks out `import ..x`'s, which are references to the outer module. -We need `module_depth + 1` dots before the specifier, so nested modules can still access Pluto. -""" -function explore_module_definition!(ex::Expr, scopestate; module_depth::Number = 0) - if ex.head == :using || ex.head == :import - # We don't care about anything after the `:` here - import_names = if ex.args[1].head == :(:) - [ex.args[1].args[1]] - else - ex.args - end - - - symstate = SymbolsState() - for import_name_expr in import_names - if ( - Meta.isexpr(import_name_expr, :., module_depth + 2) && - all(x -> x == :., import_name_expr.args[begin:end-1]) && - import_name_expr.args[end] isa Symbol - ) - # Theoretically it could still use an assigment from the same cell, if it weren't - # for the fact that modules need to be top level, and we don't support multiple (toplevel) expressions in a cell yet :D - push!(symstate.references, import_name_expr.args[end]) - end - - end - - return symstate - elseif ex.head == :module - # Explorer the block inside with one more depth added - return explore_module_definition!(ex.args[3], scopestate, module_depth = module_depth + 1) - elseif ex.head == :quote - # TODO? Explore interpolations, modules can't be in interpolations, but `import`'s can >_> - return SymbolsState() - else - # Go deeper - return umapfoldl(a -> explore_module_definition!(a, scopestate, module_depth = module_depth), ex.args) - end -end -explore_module_definition!(expr, scopestate; module_depth::Number = 1) = SymbolsState() - - -"Go through a quoted expression and use explore! for :\$ expressions" -function explore_interpolations!(ex::Expr, scopestate) - if ex.head == :$ - return explore!(ex.args[1], scopestate)::SymbolsState - else - # We are still in a quote, so we do go deeper, but we keep ignoring everything except :$'s - return umapfoldl(a -> explore_interpolations!(a, scopestate), ex.args) - end -end -explore_interpolations!(anything_else, scopestate) = SymbolsState() - -function to_kw(ex::Expr) - if Meta.isexpr(ex, :(=)) - Expr(:kw, ex.args...) - else - ex - end -end -to_kw(x) = x - -""" -Return the function name and the SymbolsState from argument defaults. Add arguments as hidden globals to the `scopestate`. - -Is also used for `struct` and `abstract`. -""" -function explore_funcdef!(ex::Expr, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - if ex.head == :call - params_to_explore = ex.args[2:end] - # Using the keyword args syntax f(;y) the :parameters node is the first arg in the AST when it should - # be explored last. We change from (parameters, ...) to (..., parameters) - if length(params_to_explore) >= 2 && params_to_explore[1] isa Expr && params_to_explore[1].head == :parameters - params_to_explore = [params_to_explore[2:end]..., params_to_explore[1]] - end - - # Handle struct as callables, `(obj::MyType)(a, b) = ...` - # or `function (obj::MyType)(a, b) ...; end` by rewriting it as: - # function MyType(obj, a, b) ...; end - funcroot = ex.args[1] - if Meta.isexpr(funcroot, :(::)) - if last(funcroot.args) isa Symbol - return explore_funcdef!(Expr(:call, reverse(funcroot.args)..., params_to_explore...), scopestate) - else - # Function call as type: (obj::typeof(myotherobject))() - symstate = explore!(last(funcroot.args), scopestate) - name, declaration_symstate = if length(funcroot.args) == 1 - explore_funcdef!(Expr(:call, anonymous_name(), params_to_explore...), scopestate) - else - explore_funcdef!(Expr(:call, anonymous_name(), first(funcroot.args), params_to_explore...), scopestate) - end - return name, union!(symstate, declaration_symstate) - end - end - - # get the function name - name, symstate = explore_funcdef!(funcroot, scopestate) - # and explore the function arguments - return umapfoldl(a -> explore_funcdef!(a, scopestate), params_to_explore; init=(name, symstate)) - elseif ex.head == :(::) || ex.head == :kw || ex.head == :(=) - # Treat custom struct constructors as a local scope function - if ex.head == :(=) && is_function_assignment(ex) - symstate = explore!(ex, scopestate) - return Symbol[], symstate - end - - # account for unnamed params, like in f(::Example) = 1 - if ex.head == :(::) && length(ex.args) == 1 - symstate = explore!(ex.args[1], scopestate) - - return Symbol[], symstate - end - - # For a() = ... in a struct definition - if Meta.isexpr(ex, :(=), 2) && Meta.isexpr(ex.args[1], :call) - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, explore!(ex.args[2], scopestate)) - return name, symstate - end - - # recurse by starting by the right hand side because f(x=x) references the global variable x - rhs_symstate = if length(ex.args) > 1 - # use `explore!` (not `explore_funcdef!`) to explore the argument's default value - these can contain arbitrary expressions - explore!(ex.args[2], scopestate) - else - SymbolsState() - end - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, rhs_symstate) - - return name, symstate - - elseif ex.head == :where - # function(...) where {T, S <: R, U <: A.B} - # supertypes `R` and `A.B` are referenced - supertypes_symstate = SymbolsState() - for a in ex.args[2:end] - name, inner_symstate = explore_funcdef!(a, scopestate) - if length(name) == 1 - push!(scopestate.hiddenglobals, name[1]) - end - union!(supertypes_symstate, inner_symstate) - end - # recurse - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, supertypes_symstate) - return name, symstate - - elseif ex.head == :(<:) - # for use in `struct` and `abstract` - name, symstate = uncurly!(ex.args[1], scopestate) - if length(ex.args) != 1 - union!(symstate, explore!(ex.args[2], scopestate)) - end - return Symbol[name], symstate - - elseif ex.head == :curly - name, symstate = uncurly!(ex, scopestate) - return Symbol[name], symstate - - elseif Meta.isexpr(ex, :parameters) - init = (Symbol[], SymbolsState()) - return umapfoldl(a -> explore_funcdef!(to_kw(a), scopestate), ex.args; init) - - elseif ex.head == :tuple - init = (Symbol[], SymbolsState()) - return umapfoldl(a -> explore_funcdef!(a, scopestate), ex.args; init) - - elseif ex.head == :(.) - return split_funcname(ex), SymbolsState() - - elseif ex.head == :(...) - return explore_funcdef!(ex.args[1], scopestate) - else - return Symbol[], explore!(ex, scopestate) - end -end - -function explore_funcdef!(ex::QuoteNode, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - explore_funcdef!(ex.value, scopestate) -end - -function explore_funcdef!(ex::Symbol, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - push!(scopestate.hiddenglobals, ex) - Symbol[ex|>without_dotprefix|>without_dotsuffix], SymbolsState() -end - -function explore_funcdef!(::Any, ::ScopeState)::Tuple{FunctionName,SymbolsState} - Symbol[], SymbolsState() -end - - const can_macroexpand_no_bind = Set(Symbol.(["@md_str", "Markdown.@md_str", "@gensym", "Base.@gensym", "@enum", "Base.@enum", "@assert", "Base.@assert", "@cmd"])) const can_macroexpand = can_macroexpand_no_bind ∪ Set(Symbol.(["@bind", "PlutoRunner.@bind"])) @@ -1039,7 +70,7 @@ const can_macroexpand = can_macroexpand_no_bind ∪ Set(Symbol.(["@bind", "Pluto """ If the macro is **known to Pluto**, expand or 'mock expand' it, if not, return the expression. Macros from external packages are not expanded, this is done later in the pipeline. See https://github.com/fonsp/Pluto.jl/pull/1032 """ -function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) +function maybe_macroexpand_pluto(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) result::Expr = if ex.head === :macrocall funcname = split_funcname(ex.args[1]) funcname_joined = join_funcname_parts(funcname) @@ -1057,7 +88,7 @@ function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=tr # Not using broadcasting because that is expensive compilation-wise for `result.args::Any`. expanded = Any[] for arg in result.args - ex = maybe_macroexpand(arg; recursive, expand_bind) + ex = maybe_macroexpand_pluto(arg; recursive, expand_bind) push!(expanded, ex) end return Expr(result.head, expanded...) @@ -1066,180 +97,16 @@ function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=tr end end -maybe_macroexpand(ex::Any; kwargs...) = ex - - -### -# CANONICALIZE FUNCTION DEFINITIONS -### - -""" -Turn a function definition expression (`Expr`) into a "canonical" form, in the sense that two methods that would evaluate to the same method signature have the same canonical form. Part of a solution to https://github.com/fonsp/Pluto.jl/issues/177. Such a canonical form cannot be achieved statically with 100% correctness (impossible), but we can make it good enough to be practical. - - -# Wait, "evaluate to the same method signature"? - -In Pluto, you cannot do definitions of **the same global variable** in different cells. This is needed for reactivity to work, and it avoid ambiguous notebooks and stateful stuff. This rule used to also apply to functions: you had to place all methods of a function in one cell. (Go and read more about methods in Julia if you haven't already.) But this is quite annoying, especially because multiple dispatch is so important in Julia code. So we allow methods of the same function to be defined across multiple cells, but we still want to throw errors when you define **multiple methods with the same signature**, because one overrides the other. For example: -```julia -julia> f(x) = 1 -f (generic function with 1 method) - -julia> f(x) = 2 -f (generic function with 1 method) -`` - -After adding the second method, the function still has only 1 method. This is because the second definition overrides the first one, instead of being added to the method table. This example should be illegal in Julia, for the same reason that `f = 1` and `f = 2` is illegal. So our problem is: how do we know that two cells will define overlapping methods? - -Ideally, we would just evaluate the user's code and **count methods** afterwards, letting Julia do the work. Unfortunately, we need to know this info _before_ we run cells, otherwise we don't know in which order to run a notebook! There are ways to break this circle, but it would complicate our process quite a bit. - -Instead, we will do _static analysis_ on the function definition expressions to determine whether they overlap. This is non-trivial. For example, `f(x)` and `f(y::Any)` define the same method. Trickier examples are here: https://github.com/fonsp/Pluto.jl/issues/177#issuecomment-645039993 - -# Wait, "function definition expressions"? -For example: - -```julia -e = :(function f(x::Int, y::String) - x + y - end) - -dump(e, maxdepth=2) - -#= -gives: - -Expr - head: Symbol function - args: Array{Any}((2,)) - 1: Expr - 2: Expr -=# -``` - -This first arg is the function head: - -```julia -e.args[1] == :(f(x::Int, y::String)) -``` - -# Mathematics -Our problem is to find a way to compute the equivalence relation ~ on `H × H`, with `H` the set of function head expressions, defined as: - -`a ~ b` iff evaluating both expressions results in a function with exactly one method. - -_(More precisely, evaluating `Expr(:function, x, Expr(:block))` with `x ∈ {a, b}`.)_ - -The equivalence sets are isomorphic to the set of possible Julia methods. - -Instead of finding a closed form algorithm for `~`, we search for a _canonical form_: a function `canonical: H -> H` that chooses one canonical expression per equivalence class. It has the property - -`canonical(a) = canonical(b)` implies `a ~ b`. - -We use this **canonical form** of the function's definition expression as its "signature". We compare these canonical forms when determining whether two function expressions will result in overlapping methods. - -# Example -```julia -e1 = :(f(x, z::Any)) -e2 = :(g(x, y)) - -canonalize(e1) == canonalize(e2) -``` - -```julia -e1 = :(f(x)) -e2 = :(g(x, y)) - -canonalize(e1) != canonalize(e2) -``` +maybe_macroexpand_pluto(ex::Any; kwargs...) = ex -```julia -e1 = :(f(a::X, b::wow(ie), c, d...; e=f) where T) -e2 = :(g(z::X, z::wow(ie), z::Any, z... ) where T) -canonalize(e1) == canonalize(e2) -``` -""" -function canonalize(ex::Expr) - if ex.head == :where - Expr(:where, canonalize(ex.args[1]), ex.args[2:end]...) - elseif ex.head == :call || ex.head == :tuple - skip_index = ex.head == :call ? 2 : 1 - # ex.args[1], if ex.head == :call this is the function name, we dont want it - - interesting = filter(ex.args[skip_index:end]) do arg - !(arg isa Expr && arg.head == :parameters) - end - - hide_argument_name.(interesting) - elseif ex.head == :(::) - canonalize(ex.args[1]) - elseif ex.head == :curly || ex.head == :(<:) - # for struct definitions, which we hackily treat as functions - nothing - else - @error "Failed to canonalize this strange looking function" ex - nothing - end -end - -# for `function g end` -canonalize(::Symbol) = nothing - -function hide_argument_name(ex::Expr) - if ex.head == :(::) && length(ex.args) > 1 - Expr(:(::), nothing, ex.args[2:end]...) - elseif ex.head == :(...) - Expr(:(...), hide_argument_name(ex.args[1])) - elseif ex.head == :kw - Expr(:kw, hide_argument_name(ex.args[1]), nothing) - else - ex - end -end -hide_argument_name(::Symbol) = Expr(:(::), nothing, :Any) -hide_argument_name(x::Any) = x ### # UTILITY FUNCTIONS ### -function handle_recursive_functions!(symstate::SymbolsState) - # We do something special to account for recursive functions: - # If a function `f` calls a function `g`, and both are defined inside this cell, the reference to `g` inside the symstate of `f` will be deleted. - # The motivitation is that normally, an assignment (or function definition) will add that symbol to a list of 'hidden globals' - any future references to that symbol will be ignored. i.e. the _local definition hides a global_. - # In the case of functions, you can reference functions and variables that do not yet exist, and so they won't be in the list of hidden symbols when the function definition is analysed. - # Of course, our method will fail if a referenced function is defined both inside the cell **and** in another cell. However, this will lead to a MultipleDefinitionError before anything bad happens. - K = keys(symstate.funcdefs) - for (func, inner_symstate) in symstate.funcdefs - inner_symstate.references = setdiff(inner_symstate.references, K) - inner_symstate.funccalls = setdiff(inner_symstate.funccalls, K) - end - return nothing -end - -""" - compute_symbolreferences(ex::Any)::SymbolsState -Return the global references, assignment, function calls and function definitions inside an arbitrary expression. -Inside Pluto, `ex` is always an `Expr`. However, we still accept `Any` to allow people outside Pluto to use this to do syntax analysis. -""" -function compute_symbolreferences(ex::Any)::SymbolsState - symstate = explore!(ex, ScopeState()) - handle_recursive_functions!(symstate) - return symstate -end -function try_compute_symbolreferences(ex::Any)::SymbolsState - try - compute_symbolreferences(ex) - catch e - if e isa InterruptException - rethrow(e) - end - @error "Expression explorer failed on: " ex - showerror(stderr, e, stacktrace(catch_backtrace())) - SymbolsState(references = Set{Symbol}([:fake_reference_to_prevent_it_from_looking_like_a_text_only_cell])) - end -end Base.@kwdef struct UsingsImports usings::Set{Expr} = Set{Expr}() diff --git a/src/analysis/ReactiveNode.jl b/src/analysis/ReactiveNode.jl deleted file mode 100644 index 8fbf6dda99..0000000000 --- a/src/analysis/ReactiveNode.jl +++ /dev/null @@ -1,78 +0,0 @@ -import .ExpressionExplorer: SymbolsState, FunctionName, FunctionNameSignaturePair, try_compute_symbolreferences, generate_funcnames - -"Every cell is a node in the reactive graph. The nodes/point/vertices are the _cells_, and the edges/lines/arrows are the _dependencies between cells_. In a reactive notebook, these dependencies are the **global variable references and definitions**. (For the mathies: a reactive notebook is represented by a _directed multigraph_. A notebook without reactivity errors is an _acyclic directed multigraph_.) This struct contains the back edges (`references`) and forward edges (`definitions`, `soft_definitions`, `funcdefs_with_signatures`, `funcdefs_without_signatures`) of a single node. - -Before 0.12.0, we could have written this struct with just two fields: `references` and `definitions` (both of type `Set{Symbol}`) because we used variable names to form the reactive links. However, to support defining _multiple methods of the same function in different cells_ (https://github.com/fonsp/Pluto.jl/issues/177), we needed to change this. You might want to think about this old behavior first (try it on paper) before reading on. - -The essential idea is that edges are still formed by variable names. Simple global variables (`x = 1`) are registered by their name as `Symbol`, but _function definitions_ `f(x::Int) = 5` are sometimes stored in two ways: -- by their name (`f`) as `Symbol`, in `funcdefs_without_signatures`, and -- by their name with its method signature as `FunctionNameSignaturePair`, in `funcdefs_with_signatures`. - -The name _without_ signature is most important: it is used to find the reactive dependencies between cells. The name _with_ signature is needed to detect multiple cells that define methods with the _same_ signature (`f(x) = 1` and `f(x) = 2`) - this is illegal. This is why we do not collect `definitions`, `funcdefs_with_signatures` and `funcdefs_without_signatures` onto a single pile: we need them separately for different searches. -" -Base.@kwdef struct ReactiveNode - references::Set{Symbol} = Set{Symbol}() - definitions::Set{Symbol} = Set{Symbol}() - soft_definitions::Set{Symbol} = Set{Symbol}() - funcdefs_with_signatures::Set{FunctionNameSignaturePair} = Set{FunctionNameSignaturePair}() - funcdefs_without_signatures::Set{Symbol} = Set{Symbol}() - macrocalls::Set{Symbol} = Set{Symbol}() -end - -function Base.union!(a::ReactiveNode, bs::ReactiveNode...) - union!(a.references, (b.references for b in bs)...) - union!(a.definitions, (b.definitions for b in bs)...) - union!(a.soft_definitions, (b.soft_definitions for b in bs)...) - union!(a.funcdefs_with_signatures, (b.funcdefs_with_signatures for b in bs)...) - union!(a.funcdefs_without_signatures, (b.funcdefs_without_signatures for b in bs)...) - union!(a.macrocalls, (b.macrocalls for b in bs)...) - return a -end - -"Turn a `SymbolsState` into a `ReactiveNode`. The main differences are: -- A `SymbolsState` is a nested structure of function definitions inside function definitions inside... This conversion flattens this structure by merging `SymbolsState`s from defined functions. -- `ReactiveNode` functions as a cache to improve efficienty, by turning the nested structures into multiple `Set{Symbol}`s with fast lookups." -function ReactiveNode(symstate::SymbolsState) - macrocalls = Iterators.map(join_funcname_parts, symstate.macrocalls) |> Set{Symbol} - result = ReactiveNode(; - references=Set{Symbol}(symstate.references), - definitions=Set{Symbol}(symstate.assignments), - macrocalls=macrocalls, - ) - - # defined functions are 'exploded' into the cell's reactive node - for (_, body_symstate) in symstate.funcdefs - union!(result, ReactiveNode(body_symstate)) - end - # union!(result, (ReactiveNode(body_symstate) for (_, body_symstate) in symstate.funcdefs)...) - - # now we will add the function names to our edges: - funccalls = Set{Symbol}(symstate.funccalls .|> join_funcname_parts) - FunctionDependencies.maybe_add_dependent_funccalls!(funccalls) - union!(result.references, funccalls) - - union!(result.references, macrocalls) - - for (namesig, body_symstate) in symstate.funcdefs - push!(result.funcdefs_with_signatures, namesig) - push!(result.funcdefs_without_signatures, join_funcname_parts(namesig.name)) - - generated_names = generate_funcnames(namesig.name) - generated_names_syms = Iterators.map(join_funcname_parts, generated_names) |> Set{Symbol} - - # add the generated names so that they are added as soft definitions - # this means that they will not be used if a cycle is created - union!(result.soft_definitions, generated_names_syms) - - filter!(!∈(generated_names_syms), result.references) # don't reference defined functions (simulated recursive calls) - end - - return result -end - -# Convenience functions -ReactiveNode(code::String) = ReactiveNode(try_compute_symbolreferences(Meta.parse(code))) -ReactiveNode(code::Expr) = error("Use ReactiveNode_from_expr(code) instead.") - -# Mot just a method of ReactiveNode because an expression is not necessarily a `Expr`, e.g. `Meta.parse("\"hello!\"") isa String`. -ReactiveNode_from_expr(expr::Any) = ReactiveNode(try_compute_symbolreferences(expr)) diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 0f81070725..40680c6593 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -11,9 +11,7 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce old_code = old_topology.codes[cell] if old_code.code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) - new_symstate = new_code.parsedcode |> - ExpressionExplorer.try_compute_symbolreferences - new_reactive_node = ReactiveNode(new_symstate) + new_reactive_node = compute_reactive_node(new_code.parsedcode) updated_nodes[cell] = new_reactive_node elseif old_code.forced_expr_id !== nothing diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index 3043e3284b..ff7cbbba34 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -112,8 +112,8 @@ function resolve_topology( result = macroexpand_cell(cell) if result isa Success (expr, computer_id) = result.result - expanded_node = ExpressionExplorer.try_compute_symbolreferences(expr) |> ReactiveNode - function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) + expanded_node = ExpressionExplorer.compute_reactive_node(expr) + function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else result @@ -185,7 +185,7 @@ So, the resulting reactive nodes may not be absolutely accurate. If you can run """ function static_macroexpand(topology::NotebookTopology, cell::Cell) new_node = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> - ExpressionExplorer.try_compute_symbolreferences |> ReactiveNode + ExpressionExplorer.compute_reactive_node union!(new_node.macrocalls, topology.nodes[cell].macrocalls) new_node From e9e237a6d40f2017873493183fca2fbff2c9b908 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 31 Oct 2023 16:59:57 +0100 Subject: [PATCH 02/10] fixefix --- src/analysis/ExpressionExplorer.jl | 13 +++++++++---- src/analysis/MoreAnalysis.jl | 4 ++-- src/analysis/TopologyUpdate.jl | 3 ++- src/analysis/is_just_text.jl | 2 +- src/evaluation/MacroAnalysis.jl | 8 +++++--- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index 00952cf9f5..dc46d5fb13 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -1,7 +1,12 @@ using ExpressionExplorer +const ReactiveNode_from_expr = ExpressionExplorer.compute_reactive_node + module ExpressionExplorerExtras +import ..Pluto +import ..PlutoRunner using ExpressionExplorer +using ExpressionExplorer: ScopeState struct PlutoConfiguration <: ExpressionExplorer.AbstractExpressionExplorerConfiguration end @@ -50,13 +55,13 @@ function macro_has_special_heuristic_inside(; symstate::SymbolsState, expr::Expr # Also, because I'm lazy and don't want to copy any code, imma use cell_precedence_heuristic here. # Sad part is, that this will also include other symbols used in this macro... but come'on local fake_cell = Pluto.Cell() - local fake_reactive_node = Pluto.ReactiveNode(symstate) + local fake_reactive_node = ReactiveNode(symstate) local fake_expranalysiscache = Pluto.ExprAnalysisCache( parsedcode = expr, module_usings_imports = ExpressionExplorer.compute_usings_imports(expr), ) local fake_topology = Pluto.NotebookTopology( - nodes = Pluto.ImmutableDefaultDict(Pluto.ReactiveNode, Dict(fake_cell => fake_reactive_node)), + nodes = Pluto.ImmutableDefaultDict(ReactiveNode, Dict(fake_cell => fake_reactive_node)), codes = Pluto.ImmutableDefaultDict(Pluto.ExprAnalysisCache, Dict(fake_cell => fake_expranalysiscache)), cell_order = Pluto.ImmutableVector([fake_cell]), ) @@ -72,8 +77,8 @@ If the macro is **known to Pluto**, expand or 'mock expand' it, if not, return t """ function maybe_macroexpand_pluto(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) result::Expr = if ex.head === :macrocall - funcname = split_funcname(ex.args[1]) - funcname_joined = join_funcname_parts(funcname) + funcname = ExpressionExplorer.split_funcname(ex.args[1]) + funcname_joined = ExpressionExplorer.join_funcname_parts(funcname) if funcname_joined ∈ (expand_bind ? can_macroexpand : can_macroexpand_no_bind) macroexpand(PlutoRunner, ex; recursive=false)::Expr diff --git a/src/analysis/MoreAnalysis.jl b/src/analysis/MoreAnalysis.jl index 7db3f28b53..1e6dfe989d 100644 --- a/src/analysis/MoreAnalysis.jl +++ b/src/analysis/MoreAnalysis.jl @@ -3,12 +3,12 @@ module MoreAnalysis export bound_variable_connections_graph import ..Pluto -import ..Pluto: Cell, Notebook, NotebookTopology, ExpressionExplorer +import ..Pluto: Cell, Notebook, NotebookTopology, ExpressionExplorer, ExpressionExplorerExtras "Find all subexpressions of the form `@bind symbol something`, and extract the `symbol`s." function find_bound_variables(expr) found = Set{Symbol}() - _find_bound_variables!(found, ExpressionExplorer.maybe_macroexpand(expr; recursive=true, expand_bind=false)) + _find_bound_variables!(found, ExpressionExplorerExtras.maybe_macroexpand_pluto(expr; recursive=true, expand_bind=false)) found end diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 40680c6593..98fa42403b 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -1,4 +1,5 @@ import .ExpressionExplorer +import .ExpressionExplorerExtras import .ExpressionExplorer: join_funcname_parts, SymbolsState, FunctionNameSignaturePair "Return a copy of `old_topology`, but with recomputed results from `cells` taken into account." @@ -11,7 +12,7 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce old_code = old_topology.codes[cell] if old_code.code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) - new_reactive_node = compute_reactive_node(new_code.parsedcode) + new_reactive_node = compute_reactive_node(new_code.parsedcode; configuration=ExpressionExplorerExtras.PlutoConfiguration()) updated_nodes[cell] = new_reactive_node elseif old_code.forced_expr_id !== nothing diff --git a/src/analysis/is_just_text.jl b/src/analysis/is_just_text.jl index 38339f68af..525cf27270 100644 --- a/src/analysis/is_just_text.jl +++ b/src/analysis/is_just_text.jl @@ -17,7 +17,7 @@ function is_just_text(topology::NotebookTopology, cell::Cell)::Bool (length(node.references) == 2 && :PlutoRunner in node.references && Symbol("PlutoRunner.throw_syntax_error") in node.references)) && - no_loops(ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true)) + no_loops(ExpressionExplorerExtras.maybe_macroexpand_pluto(topology.codes[cell].parsedcode; recursive=true)) end function no_loops(ex::Expr) diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index ff7cbbba34..f13694cb4c 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -112,7 +112,7 @@ function resolve_topology( result = macroexpand_cell(cell) if result isa Success (expr, computer_id) = result.result - expanded_node = ExpressionExplorer.compute_reactive_node(expr) + expanded_node = ExpressionExplorer.compute_reactive_node(expr; configuration=ExpressionExplorerExtras.PlutoConfiguration()) function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else @@ -184,8 +184,10 @@ end So, the resulting reactive nodes may not be absolutely accurate. If you can run code in a session, use `resolve_topology` instead. """ function static_macroexpand(topology::NotebookTopology, cell::Cell) - new_node = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> - ExpressionExplorer.compute_reactive_node + new_node = ExpressionExplorer.compute_reactive_node( + ExpressionExplorerExtras.maybe_macroexpand_pluto(topology.codes[cell].parsedcode; recursive=true); + configuration=ExpressionExplorerExtras.PlutoConfiguration() + ) union!(new_node.macrocalls, topology.nodes[cell].macrocalls) new_node From 70617abf042c704ab5dd2f9e22c06b6565a0ef6f Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Tue, 31 Oct 2023 17:02:38 +0100 Subject: [PATCH 03/10] remove EE tests --- test/ExpressionExplorer.jl | 818 ------------------------------------- test/helpers.jl | 110 +---- test/runtests.jl | 1 - test/webserver.jl | 13 + 4 files changed, 14 insertions(+), 928 deletions(-) delete mode 100644 test/ExpressionExplorer.jl diff --git a/test/ExpressionExplorer.jl b/test/ExpressionExplorer.jl deleted file mode 100644 index f0e85bfb3d..0000000000 --- a/test/ExpressionExplorer.jl +++ /dev/null @@ -1,818 +0,0 @@ -using Test -import Pluto: PlutoRunner - -#= -`@test_broken` means that the test doesn't pass right now, but we want it to pass. Feel free to try to fix it and open a PR! -Some of these @test_broken lines are commented out to prevent printing to the terminal, but we still want them fixed. - -# When working on ExpressionExplorer: - -- Go to runtests.jl and move `include("ExpressionExplorer.jl")` to the second line, so that they run instantly (after loading the helper functions). Be careful not to commit this change. -- If you are fixing a `@test_broken`: - - uncomment that line if needed - - change `@test_broken` to `@test` - - remove `verbose=false` at the end of the line -- If you are fixing something else: - - you can add lots of tests! They run super fast, don't worry about duplicates too much - --fons =# - -@testset "Explore Expressions" begin - let - EE = Pluto.ExpressionExplorer - scopestate = EE.ScopeState() - - @inferred EE.explore_assignment!(:(f(x) = x), scopestate) - @inferred EE.explore_modifiers!(:(1 + 1), scopestate) - @inferred EE.explore_dotprefixed_modifiers!(:([1] .+ [1]), scopestate) - @inferred EE.explore_inner_scoped(:(let x = 1 end), scopestate) - @inferred EE.explore_filter!(:(filter(true, a)), scopestate) - @inferred EE.explore_generator!(:((x for x in a)), scopestate) - @inferred EE.explore_macrocall!(:(@time 1), scopestate) - @inferred EE.explore_call!(:(f(x)), scopestate) - @inferred EE.explore_struct!(:(struct A end), scopestate) - @inferred EE.explore_abstract!(:(abstract type A end), scopestate) - @inferred EE.explore_function_macro!(:(function f(x); x; end), scopestate) - @inferred EE.explore_try!(:(try nothing catch end), scopestate) - @inferred EE.explore_anonymous_function!(:(x -> x), scopestate) - @inferred EE.explore_global!(:(global x = 1), scopestate) - @inferred EE.explore_local!(:(local x = 1), scopestate) - @inferred EE.explore_tuple!(:((a, b)), scopestate) - @inferred EE.explore_broadcast!(:(func.(a)), scopestate) - @inferred EE.explore_load!(:(using Foo), scopestate) - let - @inferred EE.explore_interpolations!(:(quote 1 end), scopestate) - @inferred EE.explore_quote!(:(quote 1 end), scopestate) - end - @inferred EE.explore_module!(:(module A end), scopestate) - @inferred EE.explore_fallback!(:(1 + 1), scopestate) - @inferred EE.explore!(:(1 + 1), scopestate) - - @inferred EE.split_funcname(:(Base.Submodule.f)) - @inferred EE.maybe_macroexpand(:(@time 1)) - end - - @testset "Basics" begin - # Note that Meta.parse(x) is not always an Expr. - @test testee(:(a), [:a], [], [], []) - @test testee(Expr(:toplevel, :a), [:a], [], [], []) - @test testee(:(1 + 1), [], [], [:+], []) - @test testee(:(sqrt(1)), [], [], [:sqrt], []) - @test testee(:(x = 3), [], [:x], [], []) - @test testee(:(x = x), [:x], [:x], [], []) - @test testee(:(x = 1 + y), [:y], [:x], [:+], []) - @test testee(:(x = +(a...)), [:a], [:x], [:+], []) - @test testee(:(1:3), [], [], [:(:)], []) - end - @testset "Bad code" begin - # @test_nowarn testee(:(begin end = 2), [:+], [], [:+], [], verbose=false) - @test testee(:(123 = x), [:x], [], [], []) - @test_nowarn testee(:((a = b, c, d = 123,)), [:b], [], [], [], verbose=false) - @test_nowarn testee(:((a = b, c[r] = 2, d = 123,)), [:b], [], [], [], verbose=false) - - @test_nowarn testee(:(function f(function g() end) end), [], [], [:+], [], verbose=false) - @test_nowarn testee(:(function f() Base.sqrt(x::String) = 2; end), [], [], [:+], [], verbose=false) - @test_nowarn testee(:(function f() global g(x) = x; end), [], [], [], [], verbose=false) - end - @testset "Lists and structs" begin - @test testee(:(1:3), [], [], [:(:)], []) - @test testee(:(a[1:3,4]), [:a], [], [:(:)], []) - @test testee(:(a[b]), [:a, :b], [], [], []) - @test testee(:([a[1:3,4]; b[5]]), [:b, :a], [], [:(:)], []) - @test testee(:(a.someproperty), [:a], [], [], []) # `a` can also be a module - @test testee(:([a..., b]), [:a, :b], [], [], []) - @test testee(:(struct a; b; c; end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(abstract type a end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(let struct a; b; c; end end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(let abstract type a end end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - - @test testee(:(module a; f(x) = x; z = r end), [], [:a], [], []) - end - @testset "Types" begin - @test testee(:(x::Foo = 3), [:Foo], [:x], [], []) - @test testee(:(x::Foo), [:x, :Foo], [], [], []) - @test testee(quote - a::Foo, b::String = 1, "2" - end, [:Foo, :String], [:a, :b], [], []) - @test testee(:(Foo[]), [:Foo], [], [], []) - @test testee(:(x isa Foo), [:x, :Foo], [], [:isa], []) - - @test testee(quote - (x[])::Int = 1 - end, [:Int, :x], [], [], []) - @test testee(quote - (x[])::Int, y = 1, 2 - end, [:Int, :x], [:y], [], []) - - @test testee(:(A{B} = B), [], [:A], [], []) - @test testee(:(A{T} = Union{T,Int}), [:Int, :Union], [:A], [], []) - - @test testee(:(abstract type a end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a <: b{C} end), [], [:a], [], [:a => ([:b, :C], [], [], [])]) - @test testee(:(abstract type a{T} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T,S} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T} <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T} <: b{T} end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test_nowarn testee(macroexpand(Main, :(@enum a b c)), [], [], [], []; verbose=false) - - e = :(struct a end) # needs to be on its own line to create LineNumberNode - @test testee(e, [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(struct a <: b; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a{T,S}; c::T; d::Foo; end), [], [:a], [], [:a => ([:Foo], [], [], [])]) - @test testee(:(struct a{T} <: b; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a{T} <: b{T}; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a; c; a(x=y) = new(x, z); end), [], [:a], [], [:a => ([:y, :z], [], [:new], [])]) - @test testee(:(struct a{A,B<:C{A}}; i::A; j::B end), [], [:a], [], [:a => ([:C], [], [], [])]) - @test testee(:(struct a{A,B<:C{<:A}} <: D{A,B}; i::A; j::B end), [], [:a], [], [:a => ([:C, :D], [], [], [])]) - @test testee(:(struct a{A,DD<:B.C{D.E{A}}} <: K.A{A} i::A; j::DD; k::C end), [], [:a], [], [:a => ([:B, :C, :D, :K], [], [], [])]) - @test testee(:(struct a; x; a(t::T) where {T} = new(t); end), [], [:a], [], [:a => ([], [], [[:new]], [])]) - @test testee(:(struct a; x; y; a(t::T) where {T} = new(t, T); end), [], [:a], [], [:a => ([], [], [[:new]], [])]) - @test testee(:(struct a; f() = a() end), [], [:a], [], [:a => ([], [], [], [])]) - - @test testee(:(abstract type a <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T,S} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T} <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T} <: b{T} end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{A,B<:C{A}} end), [], [:a], [], [:a => ([:C], [], [], [])]) - @test testee(:(abstract type a{A,B<:C{<:A}} <: D{A,B} end), [], [:a], [], [:a => ([:C, :D], [], [], [])]) - @test testee(:(abstract type a{A,DD<:B.C{D.E{A}}} <: K.A{A} end), [], [:a], [], [:a => ([:B, :D, :K], [], [], [])]) - # @test_broken testee(:(struct a; c; a(x=y) = new(x,z); end), [], [:a], [], [:a => ([:y, :z], [], [], [])], verbose=false) - end - @testset "Assignment operator & modifiers" begin - # https://github.com/JuliaLang/julia/blob/f449765943ba414bd57c3d1a44a73e5a0bb27534/base/docs/basedocs.jl#L239-L244 - @test testee(:(a = a), [:a], [:a], [], []) - @test testee(:(a = a + 1), [:a], [:a], [:+], []) - @test testee(:(x = a = a + 1), [:a], [:a, :x], [:+], []) - @test testee(:(const a = b), [:b], [:a], [], []) - @test testee(:(f(x) = x), [], [], [], [:f => ([], [], [], [])]) - @test testee(:(a[b,c,:] = d), [:a, :b, :c, :d, :(:)], [], [], []) - @test testee(:(a.b = c), [:a, :c], [], [], []) - @test testee(:(f(a, b=c, d=e; f=g)), [:a, :c, :e, :g], [], [:f], []) - - @test testee(:(a += 1), [:a], [:a], [:+], []) - @test testee(:(a >>>= 1), [:a], [:a], [:>>>], []) - @test testee(:(a ⊻= 1), [:a], [:a], [:⊻], []) - @test testee(:(a[1] += 1), [:a], [], [:+], []) - @test testee(:(x = let a = 1; a += b end), [:b], [:x], [:+], []) - @test testee(:(_ = a + 1), [:a], [], [:+], []) - @test testee(:(a = _ + 1), [], [:a], [:+], []) - - @test testee(:(f()[] = 1), [], [], [:f], []) - @test testee(:(x[f()] = 1), [:x], [], [:f], []) - end - @testset "Multiple assignments" begin - # Note that using the shorthand syntax :(a = 1, b = 2) to create an expression - # will automatically return a :tuple Expr and not a multiple assignment - # we use quotes instead of this syntax to be sure of what is tested since quotes - # would behave the same way as Meta.parse() which Pluto uses to evaluate cell code. - ex = quote - a, b = 1, 2 - end - @test Meta.isexpr(ex.args[2], :(=)) - - @test testee(quote - a, b = 1, 2 - end, [], [:a, :b], [], []) - @test testee(quote - a, _, c, __ = 1, 2, 3, _d - end, [:_d], [:a, :c], [], []) - @test testee(quote - (a, b) = 1, 2 - end, [], [:a, :b], [], []) - @test testee(quote - a = (b, c) - end, [:b, :c], [:a], [], []) - @test testee(quote - a, (b, c) = [e,[f,g]] - end, [:e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - a, (b, c) = [e,[f,g]] - end, [:e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - (x, y), a, (b, c) = z, e, (f, g) - end, [:z, :e, :f, :g], [:x, :y, :a, :b, :c], [], []) - @test testee(quote - (x[i], y.r), a, (b, c) = z, e, (f, g) - end, [:x, :i, :y, :z, :e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - (a[i], b.r) = (c.d, 2) - end, [:a, :b, :i, :c], [], [], []) - @test testee(quote - a, b... = 0:5 - end, [],[:a, :b], [[:(:)]], []) - @test testee(quote - a[x], x = 1, 2 - end, [:a], [:x], [], []) - @test testee(quote - x, a[x] = 1, 2 - end, [:a], [:x], [], []) - @test testee(quote - f, a[f()] = g - end, [:g, :a], [:f], [], []) - @test testee(quote - a[f()], f = g - end, [:g, :a], [:f], [], []) - @test testee(quote (; a, b) = x end, [:x], [:a, :b], [], []) - @test testee(quote a = (b, c) end, [:b, :c], [:a], [], []) - - @test testee(:(const a, b = 1, 2), [], [:a, :b], [], []) - end - @testset "Tuples" begin - ex = :(1, 2, a, b, c) - @test Meta.isexpr(ex, :tuple) - - @test testee(:((a, b,)), [:a,:b], [], [], []) - @test testee(:((a, b, c, 1, 2, 3, :d, f()..., let y = 3 end)), [:a, :b, :c], [], [:f], []) - - @test testee(:((a = b, c = 2, d = 123,)), [:b], [], [], []) - @test testee(:((a = b, c, d, f()..., let x = (;a = e) end...)), [:b, :c, :d, :e], [], [:f], []) - @test testee(:((a = b,)), [:b], [], [], []) - @test testee(:(a = b, c), [:b, :c], [], [], []) - @test testee(:(a, b = c), [:a, :c], [], [], []) - - # Invalid named tuples but still parses just fine - @test testee(:((a, b = 1, 2)), [:a], [], [], []) - @test testee(:((a, b) = 1, 2), [], [], [], []) - end - @testset "Broadcasting" begin - @test testee(:(a .= b), [:b, :a], [], [], []) # modifies elements, doesn't set `a` - @test testee(:(a .+= b), [:b, :a], [], [:+], []) - @test testee(:(a[i] .+= b), [:b, :a, :i], [], [:+], []) - @test testee(:(a .+ b ./ sqrt.(c, d)), [:a, :b, :c, :d], [], [:+, :/, :sqrt], []) - - # in 1.5 :(.+) is a symbol, in 1.6 its Expr:(:(.), :+) - broadcasted_add = :(.+) isa Symbol ? :(.+) : :+ - @test testee(:(f = .+), [broadcasted_add], [:f], [], []) - @test testee(:(reduce(.+, foo)), [broadcasted_add, :foo], [], [:reduce], []) - end - @testset "`for` & `while`" begin - @test testee(:(for k in 1:n; k + s; end), [:n, :s], [], [:+, :(:)], []) - @test testee(:(for k in 1:2, r in 3:4; global z = k + r; end), [], [:z], [:+, :(:)], []) - @test testee(:(while k < 2; r = w; global z = k + r; end), [:k, :w], [:z], [:+, :(<)], []) - end - @testset "`try` & `catch` & `else` & `finally`" begin - @test testee(:(try a = b + 1 catch; end), [:b], [], [:+], []) - @test testee(:(try a() catch e; e end), [], [], [:a], []) - @test testee(:(try a() catch; e end), [:e], [], [:a], []) - @test testee(:(try a + 1 catch a; a end), [:a], [], [:+], []) - @test testee(:(try 1 catch e; e finally a end), [:a], [], [], []) - @test testee(:(try 1 finally a end), [:a], [], [], []) - - # try catch else was introduced in 1.8 - @static if VERSION >= v"1.8.0" - @test testee(Meta.parse("try 1 catch else x = 1; x finally a; end"), [:a], [], [], []) - @test testee(Meta.parse("try 1 catch else x = j; x finally a; end"), [:a, :j], [], [], []) - @test testee(Meta.parse("try x = 2 catch else x finally a; end"), [:a, :x], [], [], []) - @test testee(Meta.parse("try x = 2 catch else x end"), [:x], [], [], []) - end - end - @testset "Comprehensions" begin - @test testee(:([sqrt(s) for s in 1:n]), [:n], [], [:sqrt, :(:)], []) - @test testee(:([sqrt(s + r) for s in 1:n, r in k]), [:n, :k], [], [:sqrt, :(:), :+], []) - @test testee(:([s + j + r + m for s in 1:3 for j in 4:5 for (r, l) in [(1, 2)]]), [:m], [], [:+, :(:)], []) - @test testee(:([a for a in b if a != 2]), [:b], [], [:(!=)], []) - @test testee(:([a for a in f() if g(a)]), [], [], [:f, :g], []) - @test testee(:([c(a) for a in f() if g(a)]), [], [], [:c, :f, :g], []) - @test testee(:([k for k in P, j in 1:k]), [:k, :P], [], [:(:)], []) - - @test testee(:([a for a in a]), [:a], [], [], []) - @test testee(:(for a in a; a; end), [:a], [], [], []) - @test testee(:(let a = a; a; end), [:a], [], [], []) - @test testee(:(let a = a end), [:a], [], [], []) - @test testee(:(let a = b end), [:b], [], [], []) - @test testee(:(a = a), [:a], [:a], [], []) - @test testee(:(a = [a for a in a]), [:a], [:a], [], []) - end - @testset "Multiple expressions" begin - @test testee(:(x = let r = 1; r + r end), [], [:x], [:+], []) - @test testee(:(begin let r = 1; r + r end; r = 2 end), [], [:r], [:+], []) - @test testee(:((k = 2; 123)), [], [:k], [], []) - @test testee(:((a = 1; b = a + 1)), [], [:a, :b], [:+], []) - @test testee(Meta.parse("a = 1; b = a + 1"), [], [:a, :b], [:+], []) - @test testee(:((a = b = 1)), [], [:a, :b], [], []) - @test testee(:(let k = 2; 123 end), [], [], [], []) - @test testee(:(let k() = 2 end), [], [], [], []) - end - @testset "Functions" begin - @test testee(:(function g() r = 2; r end), [], [], [], [ - :g => ([], [], [], []) - ]) - @test testee(:(function g end), [], [], [], [ - :g => ([], [], [], []) - ]) - @test testee(:(function f() g(x) = x; end), [], [], [], [ - :f => ([], [], [], []) # g is not a global def - ]) - @test testee(:(function f(z) g(x) = x; g(z) end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(function f(x, y=1; r, s=3 + 3) r + s + x * y * z end), [], [], [], [ - :f => ([:z], [], [:+, :*], []) - ]) - @test testee(:(function f(x) x * y * z end), [], [], [], [ - :f => ([:y, :z], [], [:*], []) - ]) - @test testee(:(function f(x) x = x / 3; x end), [], [], [], [ - :f => ([], [], [:/], []) - ]) - @test testee(:(function f(x) a end; function f(x, y) b end), [], [], [], [ - :f => ([:a, :b], [], [], []) - ]) - @test testee(:(function f(x, args...; kwargs...) return [x, y, args..., kwargs...] end), [], [], [], [ - :f => ([:y], [], [], []) - ]) - @test testee(:(function f(x; y=x) y + x end), [], [], [], [ - :f => ([], [], [:+], []) - ]) - @test testee(:(function (A::MyType)(x; y=x) y + x end), [], [], [], [ - :MyType => ([], [], [:+], []) - ]) - @test testee(:(f(x, y=a + 1) = x * y * z), [], [], [], [ - :f => ([:z, :a], [], [:*, :+], []) - ]) - @test testee(:(f(x, y...) = y),[],[],[],[ - :f => ([], [], [], []) - ]) - @test testee(:(f((x, y...), z) = y),[],[],[],[ - :f => ([], [], [], []) - ]) - @test testee(:(begin f() = 1; f end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(begin f() = 1; f() end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(begin - f(x) = (global a = √b) - f(x, y) = (global c = -d) - end), [], [], [], [ - :f => ([:b, :d], [:a, :c], [:√, :-], []) - ]) - @test testee(:(Base.show() = 0), [:Base], [], [], [ - [:Base, :show] => ([], [], [], []) - ]) - @test testee(:((x;p) -> f(x+p)), [], [], [], [ - :anon => ([], [], [:f, :+], []) - ]) - @test testee(:(() -> Date), [], [], [], [ - :anon => ([:Date], [], [], []) - ]) - @test testee(:(begin x; p end -> f(x+p)), [], [], [], [ - :anon => ([], [], [:f, :+], []) - ]) - @test testee(:(minimum(x) do (a, b); a + b end), [:x], [], [:minimum], [ - :anon => ([], [], [:+], []) - ]) - @test testee(:(f = x -> x * y), [], [:f], [], [ - :anon => ([:y], [], [:*], []) - ]) - @test testee(:(f = (x, y) -> x * y), [], [:f], [], [ - :anon => ([], [], [:*], []) - ]) - @test testee(:(f = (x, y = a + 1) -> x * y), [], [:f], [], [ - :anon => ([:a], [], [:*, :+], []) - ]) - @test testee(:((((a, b), c), (d, e)) -> a * b * c * d * e * f), [], [], [], [ - :anon => ([:f], [], [:*], []) - ]) - @test testee(:((a...) -> f(a...)), [], [], [], [ - :anon => ([], [], [:f], []) - ]) - @test testee(:(f = (args...) -> [args..., y]), [], [:f], [], [ - :anon => ([:y], [], [], []) - ]) - @test testee(:(f = (x, args...; kwargs...) -> [x, y, args..., kwargs...]), [], [:f], [], [ - :anon => ([:y], [], [], []) - ]) - @test testee(:(f = function (a, b) a + b * n end), [:n], [:f], [:+, :*], []) - @test testee(:(f = function () a + b end), [:a, :b], [:f], [:+], []) - - @test testee(:(g(; b=b) = b), [], [], [], [:g => ([:b], [], [], [])]) - @test testee(:(g(b=b) = b), [], [], [], [:g => ([:b], [], [], [])]) - @test testee(:(f(x = y) = x), [], [], [], [:f => ([:y], [], [], [])]) - @test testee(:(f(x, g=function(y=x) x + y + z end) = x * g(x)), [], [], [], [ - :f => ([:z], [], [:+, :*], []) - ]) - - @test testee(:(func(a)), [:a], [], [:func], []) - @test testee(:(func(a; b=c)), [:a, :c], [], [:func], []) - @test testee(:(func(a, b=c)), [:a, :c], [], [:func], []) - @test testee(:(√ b), [:b], [], [:√], []) - @test testee(:(funcs[i](b)), [:funcs, :i, :b], [], [], []) - @test testee(:(f(a)(b)), [:a, :b], [], [:f], []) - @test testee(:(f(a).b()), [:a], [], [:f], []) - @test testee(:(f(a...)),[:a],[],[:f],[]) - @test testee(:(f(a, b...)),[:a, :b],[],[:f],[]) - - @test testee(:(a.b(c)), [:a, :c], [], [[:a,:b]], []) - @test testee(:(a.b.c(d)), [:a, :d], [], [[:a,:b,:c]], []) - @test testee(:(a.b(c)(d)), [:a, :c, :d], [], [[:a,:b]], []) - @test testee(:(a.b(c).d(e)), [:a, :c, :e], [], [[:a,:b]], []) - @test testee(:(a.b[c].d(e)), [:a, :c, :e], [], [], []) - @test testee(:(let aa = blah; aa.f() end), [:blah], [], [], []) - @test testee(:(let aa = blah; aa.f(a, b, c) end), [:blah, :a, :b, :c], [], [], []) - @test testee(:(f(a) = a.b()), [], [], [], [:f => ([], [], [], [])]) - - @test testee(:(function f() - function hello() - end - hello() - end), [], [], [], [:f => ([], [], [], [])]) - @test testee(:(function a() - b() = Test() - b() - end), [], [], [], [:a => ([], [], [:Test], [])]) - @test testee(:(begin - function f() - g() = z - g() - end - g() - end), [], [], [:g], [:f => ([:z], [], [], [])]) - end - @testset "Julia lowering" begin - @test test_expression_explorer(expr=:(a'b), references=[:a, :b], funccalls=[:*, :adjoint]) - end - @testset "Functions & types" begin - @test testee(:(function f(y::Int64=a)::String string(y) end), [], [], [], [ - :f => ([:String, :Int64, :a], [], [:string], []) - ]) - @test testee(:(f(a::A)::C = a.a;), [], [], [], [ - :f => ([:A, :C], [], [], []) - ]) - @test testee(:(function f(x::T; k=1) where T return x + 1 end), [], [], [], [ - :f => ([], [], [:+], []) - ]) - @test testee(:(function f(x::T; k=1) where {T,S <: R} return x + 1 end), [], [], [], [ - :f => ([:R], [], [:+], []) - ]) - @test testee(:(f(x)::String = x), [], [], [], [ - :f => ([:String], [], [], []) - ]) - @test testee(:(MIME"text/html"), [], [], [], [], [Symbol("@MIME_str")]) - @test testee(:(function f(::MIME"text/html") 1 end), [], [], [], [ - :f => ([], [], [], [], [Symbol("@MIME_str")]) - ]) - @test testee(:(a(a::AbstractArray{T}) where T = 5), [], [], [], [ - :a => ([:AbstractArray], [], [], []) - ]) - @test testee(:(a(a::AbstractArray{T,R}) where {T,S} = a + b), [], [], [], [ - :a => ([:AbstractArray, :b, :R], [], [:+], []) - ]) - @test testee(:(f(::A) = 1), [], [], [], [ - :f => ([:A], [], [], []) - ]) - @test testee(:(f(::A, ::B) = 1), [], [], [], [ - :f => ([:A, :B], [], [], []) - ]) - @test testee(:(f(a::A, ::B, c::C...) = a + c), [], [], [], [ - :f => ([:A, :B, :C], [], [:+], []) - ]) - - @test testee(:((obj::MyType)(x,y) = x + z), [], [], [], [ - :MyType => ([:z], [], [:+], []) - ]) - @test testee(:((obj::MyType)() = 1), [], [], [], [ - :MyType => ([], [], [], []) - ]) - @test testee(:((obj::MyType)(x, args...; kwargs...) = [x, y, args..., kwargs...]), [], [], [], [ - :MyType => ([:y], [], [], []) - ]) - @test testee(:(function (obj::MyType)(x, y) x + z end), [], [], [], [ - :MyType => ([:z], [], [:+], []) - ]) - @test testee(:(begin struct MyType x::String end; (obj::MyType)(y) = obj.x + y; end), [], [:MyType], [], [ - :MyType => ([:String], [], [:+], []) - ]) - @test testee(:(begin struct MyType x::String end; function(obj::MyType)(y) obj.x + y; end; end), [], [:MyType], [], [ - :MyType => ([:String], [], [:+], []) - ]) - @test testee(:((::MyType)(x,y) = x + y), [], [], [], [ - :MyType => ([], [], [:+], []) - ]) - @test testee(:((obj::typeof(Int64[]))(x, y::Float64) = obj + x + y), [], [], [], [ - :anon => ([:Int64, :Float64], [], [:+, :typeof], []) - ]) - @test testee(:((::Get(MyType))(x, y::OtherType) = y * x + z), [], [], [], [ - :anon => ([:MyType, :z, :OtherType], [], [:Get, :*, :+], []) - ]) - end - @testset "Scope modifiers" begin - @test testee(:(let; global a, b = 1, 2 end), [], [:a, :b], [], []) - @test_broken testee(:(let; global a = b = 1 end), [], [:a], [], []; verbose=false) - @test testee(:(let; global k = 3 end), [], [:k], [], []) - @test_broken testee(:(let; global k = r end), [], [:k], [], []; verbose=false) - @test testee(:(let; global k = 3; k end), [], [:k], [], []) - @test testee(:(let; global k += 3 end), [:k], [:k], [:+], []) - @test testee(:(let; global k; k = 4 end), [], [:k], [], []) - @test testee(:(let; global k; b = 5 end), [], [], [], []) - @test testee(:(let; global x, y, z; b = 5; x = 1; (y,z) = 3 end), [], [:x, :y, :z], [], []) - @test testee(:(let; global x, z; b = 5; x = 1; end), [], [:x], [], []) - @test testee(:(let a = 1, b = 2; show(a + b) end), [], [], [:show, :+], []) - @test_broken testee(:(let a = 1; global a = 2; end), [], [:a], [], []; verbose=false) - - @test testee(:(begin local a, b = 1, 2 end), [], [], [], []) - @test testee(:(begin local a = b = 1 end), [], [:b], [], []) - @test testee(:(begin local k = 3 end), [], [], [], []) - @test testee(:(begin local k = r end), [:r], [], [], []) - @test testee(:(begin local k = 3; k; b = 4 end), [], [:b], [], []) - @test testee(:(begin local k += 3 end), [], [], [:+], []) # does not reference global k - @test testee(:(begin local k; k = 4 end), [], [], [], []) - @test testee(:(begin local k; b = 5 end), [], [:b], [], []) - @test testee(:(begin local r[1] = 5 end), [:r], [], [], []) - @test testee(:(begin local a, b; a = 1; b = 2 end), [], [], [], []) - @test testee(:(begin a; local a, b; a = 1; b = 2 end), [:a], [], [], []) - @test_broken testee(:(begin begin local a = 2 end; a end), [:a], [], [], []; verbose=false) - - @test testee(:(function f(x) global k = x end), [], [], [], [ - :f => ([], [:k], [], []) - ]) - @test testee(:((begin x = 1 end, y)), [:y], [:x], [], []) - @test testee(:(x = let; global a += 1 end), [:a], [:x, :a], [:+], []) - end - @testset "`import` & `using`" begin - @test testee(:(using Plots), [], [:Plots], [], []) - @test testee(:(using Plots.ExpressionExplorer), [], [:ExpressionExplorer], [], []) - @test testee(:(using JSON, UUIDs), [], [:JSON, :UUIDs], [], []) - @test testee(:(import Pluto), [], [:Pluto], [], []) - @test testee(:(import Pluto: wow, wowie), [], [:wow, :wowie], [], []) - @test testee(:(import Pluto.ExpressionExplorer.wow, Plutowie), [], [:wow, :Plutowie], [], []) - @test testee(:(import .Pluto: wow), [], [:wow], [], []) - @test testee(:(import ..Pluto: wow), [], [:wow], [], []) - @test testee(:(let; import Pluto.wow, Dates; end), [], [:wow, :Dates], [], []) - @test testee(:(while false; import Pluto.wow, Dates; end), [], [:wow, :Dates], [], []) - @test testee(:(try; using Pluto.wow, Dates; catch; end), [], [:wow, :Dates], [], []) - @test testee(:(module A; import B end), [], [:A], [], []) - end - @testset "Foreign macros" begin - # parameterizedfunctions - @test testee(quote - f = @ode_def LotkaVolterra begin - dx = a*x - b*x*y - dy = -c*y + d*x*y - end a b c d - end, [], [:f], [], [], [Symbol("@ode_def")]) - @test testee(quote - f = @ode_def begin - dx = a*x - b*x*y - dy = -c*y + d*x*y - end a b c d - end, [], [:f], [], [], [Symbol("@ode_def")]) - # flux - @test testee(:(@functor Asdf), [], [], [], [], [Symbol("@functor")]) - # symbolics - @test testee(:(@variables a b c), [], [], [], [], [Symbol("@variables")]) - @test testee(:(@variables a b[1:2] c(t) d(..)), [], [], [], [], [Symbol("@variables")]) - @test testee(:(@variables a b[1:x] c[1:10](t) d(..)), [], [], [], [], [Symbol("@variables")]) - @test_nowarn testee(:(@variables(m, begin - x - y[i=1:2] >= i, (start = i, base_name = "Y_$i") - z, Bin - end)), [:m, :Bin], [:x, :y, :z], [Symbol("@variables")], [], verbose=false) - # jump - # @test testee(:(@variable(m, x)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, 1<=x)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, 1<=x<=2)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, r <= x[i=keys(asdf)] <= ub[i])), [:m, :r, :asdf, :ub], [:x], [:keys, Symbol("@variable")], []) - # @test testee(:(@variable(m, x, lower_bound=0)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, base_name="x", lower_bound=0)), [:m], [], [Symbol("@variable")], []) - # @test testee(:(@variables(m, begin - # x - # y[i=1:2] >= i, (start = i, base_name = "Y_$i") - # z, Bin - # end)), [:m, :Bin], [:x, :y, :z], [Symbol("@variables")], []) - end - @testset "Macros" begin - # Macros tests are not just in ExpressionExplorer now - - @test testee(:(@time a = 2), [], [], [], [], [Symbol("@time")]) - @test testee(:(@f(x; y=z)), [], [], [], [], [Symbol("@f")]) - @test testee(:(@f(x, y = z)), [], [], [], [], [Symbol("@f")]) # https://github.com/fonsp/Pluto.jl/issues/252 - @test testee(:(Base.@time a = 2), [], [], [], [], [[:Base, Symbol("@time")]]) - # @test_nowarn testee(:(@enum a b = d c), [:d], [:a, :b, :c], [Symbol("@enum")], []) - # @enum is tested in test/React.jl instead - @test testee(:(@gensym a b c), [], [:a, :b, :c], [:gensym], [], [Symbol("@gensym")]) - @test testee(:(Base.@gensym a b c), [], [:a, :b, :c], [:gensym], [], [[:Base, Symbol("@gensym")]]) - @test testee(:(Base.@kwdef struct A; x = 1; y::Int = two; z end), [], [], [], [], [[:Base, Symbol("@kwdef")]]) - @test testee(quote "asdf" f(x) = x end, [], [], [], [], [Symbol("@doc")]) - - @test testee(:(@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) - @test testee(:(PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [[:PlutoRunner, Symbol("@bind")]]) - @test_broken testee(:(Main.PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) - @test testee(:(let @bind a b end), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) - - @test testee(:(@asdf a = x1 b = x2 c = x3), [], [], [], [], [Symbol("@asdf")]) # https://github.com/fonsp/Pluto.jl/issues/670 - - @test testee(:(@einsum a[i,j] := x[i]*y[j]), [], [], [], [], [Symbol("@einsum")]) - @test testee(:(@tullio a := f(x)[i+2j, k[j]] init=z), [], [], [], [], [Symbol("@tullio")]) - @test testee(:(Pack.@asdf a[1,k[j]] := log(x[i]/y[j])), [], [], [], [], [[:Pack, Symbol("@asdf")]]) - - @test testee(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) - @test testee(:(md"hey $(@bind a b) $(a)"), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) - @test testee(:(md"hey $(a) $(@bind a b)"), [:a, :b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) - @test testee(:(html"a $(b = c)"), [], [], [], [], [Symbol("@html_str")]) - @test testee(:(md"a $(b = c) $(b)"), [:c], [:b], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(md"\* $r"), [:r], [], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(md"a \$(b = c)"), [], [], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(macro a() end), [], [], [], [ - Symbol("@a") => ([], [], [], []) - ]) - @test testee(:(macro a(b::Int); b end), [], [], [], [ - Symbol("@a") => ([:Int], [], [], []) - ]) - @test testee(:(macro a(b::Int=c) end), [], [], [], [ - Symbol("@a") => ([:Int, :c], [], [], []) - ]) - @test testee(:(macro a(); b = c; return b end), [], [], [], [ - Symbol("@a") => ([:c], [], [], []) - ]) - @test test_expression_explorer( - expr=:(@parent @child 10), - macrocalls=[Symbol("@parent"), Symbol("@child")], - ) - @test test_expression_explorer( - expr=:(@parent begin @child 1 + @grandchild 10 end), - macrocalls=[Symbol("@parent"), Symbol("@child"), Symbol("@grandchild")], - ) - @test testee(macroexpand(Main, :(@noinline f(x) = x)), [], [], [], [ - Symbol("f") => ([], [], [], []) - ]) - end - @testset "Macros and heuristics" begin - @test test_expression_explorer( - expr=:(@macro import Pkg), - macrocalls=[Symbol("@macro")], - definitions=[:Pkg], - ) - @test test_expression_explorer( - expr=:(@macro Pkg.activate("..")), - macrocalls=[Symbol("@macro")], - references=[:Pkg], - funccalls=[[:Pkg, :activate]], - ) - @test test_expression_explorer( - expr=:(@macro Pkg.add("Pluto.jl")), - macrocalls=[Symbol("@macro")], - references=[:Pkg], - funccalls=[[:Pkg, :add]], - ) - @test test_expression_explorer( - expr=:(@macro include("Firebasey.jl")), - macrocalls=[Symbol("@macro")], - funccalls=[[:include]], - ) - end - @testset "Module imports" begin - @test test_expression_explorer( - expr=quote - module X - import ..imported_from_outside - end - end, - references=[:imported_from_outside], - definitions=[:X], - ) - @test test_expression_explorer( - expr=quote - module X - import ..imported_from_outside - import Y - import ...where_would_this_even_come_from - import .not_defined_but_sure - end - end, - references=[:imported_from_outside], - definitions=[:X], - ) - # More advanced, might not be possible easily - @test test_expression_explorer( - expr=quote - module X - module Y - import ...imported_from_outside - end - end - end, - references=[:imported_from_outside], - definitions=[:X] - ) - end - @testset "String interpolation & expressions" begin - @test testee(:("a $b"), [:b], [], [], []) - @test testee(:("a $(b = c)"), [:c], [:b], [], []) - # @test_broken testee(:(`a $b`), [:b], [], [], []) - # @test_broken testee(:(`a $(b = c)`), [:c], [:b], [], []) - @test testee(:(ex = :(yayo)), [], [:ex], [], []) - @test testee(:(ex = :(yayo + $r)), [:r], [:ex], [], []) - @test test_expression_explorer( - expr=:(quote $(x) end), - references=[:x], - ) - @test test_expression_explorer( - expr=:(quote z = a + $(x) + b() end), - references=[:x], - ) - @test test_expression_explorer( - expr=:(:($(x))), - references=[:x], - ) - @test test_expression_explorer( - expr=:(:(z = a + $(x) + b())), - references=[:x], - ) - end - @testset "Special reactivity rules" begin - @test testee( - :(BenchmarkTools.generate_benchmark_definition(Main, Symbol[], Any[], Symbol[], (), $(Expr(:copyast, QuoteNode(:(f(x, y, z))))), $(Expr(:copyast, QuoteNode(:()))), $(Expr(:copyast, QuoteNode(nothing))), BenchmarkTools.parameters())), - [:Main, :BenchmarkTools, :Any, :Symbol, :x, :y, :z], [], [[:BenchmarkTools, :generate_benchmark_definition], [:BenchmarkTools, :parameters], :f], [] - ) - @test testee( - :(BenchmarkTools.generate_benchmark_definition(Main, Symbol[], Any[], Symbol[], (), $(Expr(:copyast, QuoteNode(:(f(x, y, z))))), $(Expr(:copyast, QuoteNode(:(x = A + B)))), $(Expr(:copyast, QuoteNode(nothing))), BenchmarkTools.parameters())), - [:Main, :BenchmarkTools, :Any, :Symbol, :y, :z, :A, :B], [], [[:BenchmarkTools, :generate_benchmark_definition], [:BenchmarkTools, :parameters], :f, :+], [] - ) - @test testee( - :(Base.macroexpand(Main, $(QuoteNode(:(@enum a b c))))), - [:Main, :Base], [], [[:Base, :macroexpand]], [], [Symbol("@enum")] - ) - end - @testset "Invalid code sometimes generated by macros" begin - @test testee( - :(f(; $(:(x = true)))), - [], [], [:f], [] - ) - @test testee( - :(f(a, b, c; y, z = a, $(:(x = true)))), - [:a, :b, :c, :y], [], [:f], [] - ) - @test testee( - :(f(a, b, c; y, z = a, $(:(x = true))) = nothing), - [], [], [], [ - :f => ([:nothing], [], [], []) - ] - ) - end - @testset "Extracting `using` and `import`" begin - expr = quote - using A - import B - if x - using .C: r - import ..D.E: f, g - else - import H.I, J, K.L - end - - quote - using Nonono - end - end - result = ExpressionExplorer.compute_usings_imports(expr) - @test result.usings == Set{Expr}([ - :(using A), - :(using .C: r), - ]) - @test result.imports == Set{Expr}([ - :(import B), - :(import ..D.E: f, g), - :(import H.I, J, K.L), - ]) - - @test ExpressionExplorer.external_package_names(result) == Set{Symbol}([ - :A, :B, :H, :J, :K - ]) - - @test ExpressionExplorer.external_package_names(:(using Plots, Something.Else, .LocalModule)) == Set([:Plots, :Something]) - @test ExpressionExplorer.external_package_names(:(import Plots.A: b, c)) == Set([:Plots]) - - @test ExpressionExplorer.external_package_names(Meta.parse("import Foo as Bar, Baz.Naz as Jazz")) == Set([:Foo, :Baz]) - end - - @testset "ReactiveNode" begin - rn = Pluto.ReactiveNode_from_expr(quote - () -> Date - end) - @test :Date ∈ rn.references - end -end - -@testset "UTF-8 to Codemirror UTF-16 byte mapping" begin - # range ends are non inclusives - tests = [ - (" aaaa", (2, 4), (1, 3)), # cm is zero based - (" 🍕🍕", (2, 6), (1, 3)), # a 🍕 is two UTF16 codeunits - (" 🍕🍕", (6, 10), (3, 5)), # a 🍕 is two UTF16 codeunits - ] - for (s, (start_byte, end_byte), (from, to)) in tests - @test PlutoRunner.map_byte_range_to_utf16_codepoints(s, start_byte, end_byte) == (from, to) - end -end diff --git a/test/helpers.jl b/test/helpers.jl index 3b745c25fc..ff7406b42d 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -11,8 +11,7 @@ function print_timeroutput() end @timeit TOUT "import Pluto" import Pluto -import Pluto.ExpressionExplorer -import Pluto.ExpressionExplorer: SymbolsState, compute_symbolreferences, FunctionNameSignaturePair, UsingsImports, compute_usings_imports +using ExpressionExplorer using Sockets using Test using HTTP @@ -20,114 +19,7 @@ import Pkg import Malt import Malt.Distributed -function Base.show(io::IO, s::SymbolsState) - print(io, "SymbolsState([") - join(io, s.references, ", ") - print(io, "], [") - join(io, s.assignments, ", ") - print(io, "], [") - join(io, s.funccalls, ", ") - print(io, "], [") - if isempty(s.funcdefs) - print(io, "]") - else - println(io) - for (k, v) in s.funcdefs - print(io, " ", k, ": ", v) - println(io) - end - print(io, "]") - end - if !isempty(s.macrocalls) - print(io, "], [") - print(io, s.macrocalls) - print(io, "])") - else - print(io, ")") - end -end - -"Calls `ExpressionExplorer.compute_symbolreferences` on the given `expr` and test the found SymbolsState against a given one, with convient syntax. -# Example - -```jldoctest -julia> @test testee(:( - begin - a = b + 1 - f(x) = x / z - end), - [:b, :+], # 1st: expected references - [:a, :f], # 2nd: expected definitions - [:+], # 3rd: expected function calls - [ - :f => ([:z, :/], [], [:/], []) - ]) # 4th: expected function definitions, with inner symstate using the same syntax -true -``` -" -function testee(expr::Any, expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []; verbose::Bool=true) - expected = easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls) - - original_hash = Pluto.PlutoRunner.expr_hash(expr) - result = compute_symbolreferences(expr) - new_hash = Pluto.PlutoRunner.expr_hash(expr) - if original_hash != new_hash - error("\n== The expression explorer modified the expression. Don't do that! ==\n") - end - - # Anonymous function are given a random name, which looks like anon67387237861123 - # To make testing easier, we rename all such functions to anon - new_name(sym) = startswith(string(sym), "anon") ? :anon : sym - - result.assignments = Set(new_name.(result.assignments)) - result.funcdefs = let - newfuncdefs = Dict{FunctionNameSignaturePair,SymbolsState}() - for (k, v) in result.funcdefs - union!(newfuncdefs, Dict(FunctionNameSignaturePair(new_name.(k.name), hash("hello")) => v)) - end - newfuncdefs - end - - if verbose && expected != result - println() - println("FAILED TEST") - println(expr) - println() - dump(expr, maxdepth=20) - println() - @show expected - resulted = result - @show resulted - println() - end - return expected == result -end - -""" -Like `testee` but actually a convenient syntax -""" -function test_expression_explorer(; expr, references=[], definitions=[], funccalls=[], funcdefs=[], macrocalls=[]) - testee(expr, references, definitions, funccalls, funcdefs, macrocalls) -end - -function easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []) - array_to_set(array) = map(array) do k - new_k = k isa Symbol ? [k] : k - return new_k - end |> Set - new_expected_funccalls = array_to_set(expected_funccalls) - - new_expected_funcdefs = map(expected_funcdefs) do (k, v) - new_k = k isa Symbol ? [k] : k - new_v = v isa SymbolsState ? v : easy_symstate(v...) - return FunctionNameSignaturePair(new_k, hash("hello")) => new_v - end |> Dict - - new_expected_macrocalls = array_to_set(expected_macrocalls) - - SymbolsState(Set(expected_references), Set(expected_definitions), new_expected_funccalls, new_expected_funcdefs, new_expected_macrocalls) -end function insert_cell!(notebook, cell) notebook.cells_dict[cell.cell_id] = cell diff --git a/test/runtests.jl b/test/runtests.jl index 88901d4ae3..d280a597f5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,7 +36,6 @@ verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @timeit_include("packages/PkgCompat.jl") -@timeit_include("ExpressionExplorer.jl") @timeit_include("MethodSignatures.jl") @timeit_include("MoreAnalysis.jl") @timeit_include("Analysis.jl") diff --git a/test/webserver.jl b/test/webserver.jl index 009ff34631..401df2ace0 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -49,6 +49,19 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll close(server) end +@testset "UTF-8 to Codemirror UTF-16 byte mapping" begin + # range ends are non inclusives + tests = [ + (" aaaa", (2, 4), (1, 3)), # cm is zero based + (" 🍕🍕", (2, 6), (1, 3)), # a 🍕 is two UTF16 codeunits + (" 🍕🍕", (6, 10), (3, 5)), # a 🍕 is two UTF16 codeunits + ] + for (s, (start_byte, end_byte), (from, to)) in tests + @test PlutoRunner.map_byte_range_to_utf16_codepoints(s, start_byte, end_byte) == (from, to) + end +end + + @testset "Exports" begin port, socket = @inferred Pluto.port_serversocket(Sockets.ip"0.0.0.0", nothing, 5543) From 4870b8bfd2e78d1a0dac197e91e7d24b25da93d8 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 6 Nov 2023 14:46:51 +0100 Subject: [PATCH 04/10] updates --- Project.toml | 1 + src/analysis/ExpressionExplorer.jl | 128 ++--------------------------- src/evaluation/MacroAnalysis.jl | 4 +- src/notebook/Cell.jl | 1 - src/runner/PlutoRunner.jl | 2 +- 5 files changed, 12 insertions(+), 124 deletions(-) diff --git a/Project.toml b/Project.toml index 263ce05ce3..83b3e70c70 100644 --- a/Project.toml +++ b/Project.toml @@ -39,6 +39,7 @@ Base64 = "1" Configurations = "0.15, 0.16, 0.17" Dates = "1" Downloads = "1" +ExpressionExplorer = "0.3" FileWatching = "1" FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.5.2" diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index dc46d5fb13..ae4f10b5aa 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -1,6 +1,6 @@ using ExpressionExplorer -const ReactiveNode_from_expr = ExpressionExplorer.compute_reactive_node +@deprecate ReactiveNode_from_expr(args...; kwargs...) ExpressionExplorer.compute_reactive_node(args...; kwargs...) module ExpressionExplorerExtras import ..Pluto @@ -106,27 +106,10 @@ maybe_macroexpand_pluto(ex::Any; kwargs...) = ex -### -# UTILITY FUNCTIONS -### +############### - -Base.@kwdef struct UsingsImports - usings::Set{Expr} = Set{Expr}() - imports::Set{Expr} = Set{Expr}() -end - -is_implicit_using(ex::Expr) = Meta.isexpr(ex, :using) && length(ex.args) >= 1 && !Meta.isexpr(ex.args[1], :(:)) -function transform_dot_notation(ex::Expr) - if Meta.isexpr(ex, :(.)) - Expr(:block, ex.args[end]) - else - ex - end -end - function collect_implicit_usings(ex::Expr) if is_implicit_using(ex) Set{Expr}(Iterators.map(transform_dot_notation, ex.args)) @@ -136,113 +119,18 @@ function collect_implicit_usings(ex::Expr) end collect_implicit_usings(usings::Set{Expr}) = mapreduce(collect_implicit_usings, union!, usings; init = Set{Expr}()) -collect_implicit_usings(usings_imports::UsingsImports) = collect_implicit_usings(usings_imports.usings) - -# Performance analysis: https://gist.github.com/fonsp/280f6e883f419fb3a59231b2b1b95cab -"Preallocated version of [`compute_usings_imports`](@ref)." -function compute_usings_imports!(out::UsingsImports, ex::Any) - if isa(ex, Expr) - if ex.head == :using - push!(out.usings, ex) - elseif ex.head == :import - push!(out.imports, ex) - elseif ex.head != :quote - for a in ex.args - compute_usings_imports!(out, a) - end - end - end - out -end +collect_implicit_usings(usings_imports::ExpressionExplorer.UsingsImports) = collect_implicit_usings(usings_imports.usings) -""" -Given `:(using Plots, Something.Else, .LocalModule)`, return `Set([:Plots, :Something])`. -""" -function external_package_names(ex::Expr)::Set{Symbol} - @assert ex.head == :import || ex.head == :using - if Meta.isexpr(ex.args[1], :(:)) - external_package_names(Expr(ex.head, ex.args[1].args[1])) - else - out = Set{Symbol}() - for a in ex.args - if Meta.isexpr(a, :as) - a = a.args[1] - end - if Meta.isexpr(a, :(.)) - if a.args[1] != :(.) - push!(out, a.args[1]) - end - end - end - out - end -end - -function external_package_names(x::UsingsImports)::Set{Symbol} - union!(Set{Symbol}(), Iterators.map(external_package_names, x.usings)..., Iterators.map(external_package_names, x.imports)...) -end - -"Get the sets of `using Module` and `import Module` subexpressions that are contained in this expression." -compute_usings_imports(ex) = compute_usings_imports!(UsingsImports(), ex) - -"Return whether the expression is of the form `Expr(:toplevel, LineNumberNode(..), any)`." -function is_toplevel_expr(ex::Expr)::Bool - Meta.isexpr(ex, :toplevel, 2) && (ex.args[1] isa LineNumberNode) -end -is_toplevel_expr(::Any)::Bool = false +is_implicit_using(ex::Expr) = Meta.isexpr(ex, :using) && length(ex.args) >= 1 && !Meta.isexpr(ex.args[1], :(:)) -"If the expression is a (simple) assignemnt at its root, return the assignee as `Symbol`, return `nothing` otherwise." -function get_rootassignee(ex::Expr, recurse::Bool = true)::Union{Symbol,Nothing} - if is_toplevel_expr(ex) && recurse - get_rootassignee(ex.args[2], false) - elseif Meta.isexpr(ex, :macrocall, 3) - rooter_assignee = get_rootassignee(ex.args[3], true) - if rooter_assignee !== nothing - Symbol(string(ex.args[1]) * " " * string(rooter_assignee)) - else - nothing - end - elseif Meta.isexpr(ex, :const, 1) - rooter_assignee = get_rootassignee(ex.args[1], false) - if rooter_assignee !== nothing - Symbol("const " * string(rooter_assignee)) - else - nothing - end - elseif ex.head == :(=) && ex.args[1] isa Symbol - ex.args[1] +function transform_dot_notation(ex::Expr) + if Meta.isexpr(ex, :(.)) + Expr(:block, ex.args[end]) else - nothing + ex end end -get_rootassignee(ex::Any, recuse::Bool = true)::Union{Symbol,Nothing} = nothing - -"Is this code simple enough that we can wrap it inside a function to boost performance? Look for [`PlutoRunner.Computer`](@ref) to learn more." -function can_be_function_wrapped(x::Expr) - if x.head === :global || # better safe than sorry - x.head === :using || - x.head === :import || - x.head === :module || - x.head === :incomplete || - # Only bail on named functions, but anonymous functions (args[1].head == :tuple) are fine. - # TODO Named functions INSIDE other functions should be fine too - (x.head === :function && !Meta.isexpr(x.args[1], :tuple)) || - x.head === :macro || - # Cells containing macrocalls will actually be function wrapped using the expanded version of the expression - # See https://github.com/fonsp/Pluto.jl/pull/1597 - x.head === :macrocall || - x.head === :struct || - x.head === :abstract || - (x.head === :(=) && is_function_assignment(x)) || # f(x) = ... - (x.head === :call && (x.args[1] === :eval || x.args[1] === :include)) - false - else - all(can_be_function_wrapped, x.args) - end - -end -can_be_function_wrapped(x::Any) = true end diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index f13694cb4c..a3ebb36409 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -30,7 +30,7 @@ function with_new_soft_definitions(topology::NotebookTopology, cell::Cell, soft_ ) end -collect_implicit_usings(topology::NotebookTopology, cell::Cell) = ExpressionExplorer.collect_implicit_usings(topology.codes[cell].module_usings_imports) +collect_implicit_usings(topology::NotebookTopology, cell::Cell) = ExpressionExplorerExtras.collect_implicit_usings(topology.codes[cell].module_usings_imports) function cells_with_deleted_macros(old_topology::NotebookTopology, new_topology::NotebookTopology) old_macros = mapreduce(c -> defined_macros(old_topology, c), union!, all_cells(old_topology); init=Set{Symbol}()) @@ -113,7 +113,7 @@ function resolve_topology( if result isa Success (expr, computer_id) = result.result expanded_node = ExpressionExplorer.compute_reactive_node(expr; configuration=ExpressionExplorerExtras.PlutoConfiguration()) - function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) + function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else result diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 48f4862bad..c04732764d 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -1,5 +1,4 @@ import UUIDs: UUID, uuid1 -import .ExpressionExplorer: SymbolsState, UsingsImports const METADATA_DISABLED_KEY = "disabled" const METADATA_SHOW_LOGS_KEY = "show_logs" diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 6cce83afa7..28eb0decb1 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -341,7 +341,7 @@ function get_module_names(workspace_module, module_ex::Expr) end function collect_soft_definitions(workspace_module, modules::Set{Expr}) - mapreduce(module_ex -> get_module_names(workspace_module, module_ex), union!, modules; init=Set{Symbol}()) + mapreduce(module_ex -> get_module_names(workspace_module, module_ex), union!, modules; init=Set{Symbol}()) end From d90813827108460c53ceb3d6deee707e16ee3296 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 13 Nov 2023 11:56:56 +0100 Subject: [PATCH 05/10] match EE updates (which are not yet released so CI will fail) --- src/analysis/Errors.jl | 2 +- src/analysis/ExpressionExplorer.jl | 43 +++++++++++++++++++++++++++--- src/analysis/Topology.jl | 2 +- src/analysis/TopologyUpdate.jl | 2 +- src/evaluation/MacroAnalysis.jl | 2 +- src/evaluation/Run.jl | 3 ++- src/evaluation/RunBonds.jl | 2 +- src/evaluation/WorkspaceManager.jl | 4 +-- src/runner/PlutoRunner.jl | 2 +- test/MoreAnalysis.jl | 40 +++++++++++++++++++++++++++ 10 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/analysis/Errors.jl b/src/analysis/Errors.jl index eb63f550f0..d849160f8b 100644 --- a/src/analysis/Errors.jl +++ b/src/analysis/Errors.jl @@ -1,5 +1,5 @@ import Base: showerror -import .ExpressionExplorer: FunctionName, join_funcname_parts +import .ExpressionExplorer: FunctionName abstract type ReactivityError <: Exception end diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index ae4f10b5aa..6c37dd9b3f 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -37,7 +37,7 @@ function ExpressionExplorer.explore_macrocall!(ex::Expr, scopestate::ScopeState{ end # Some macros can be expanded on the server process - if ExpressionExplorer.join_funcname_parts(macro_name) ∈ can_macroexpand + if macro_name.joined ∈ can_macroexpand new_ex = maybe_macroexpand_pluto(ex) union!(symstate, ExpressionExplorer.explore!(new_ex, scopestate)) end @@ -78,9 +78,8 @@ If the macro is **known to Pluto**, expand or 'mock expand' it, if not, return t function maybe_macroexpand_pluto(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) result::Expr = if ex.head === :macrocall funcname = ExpressionExplorer.split_funcname(ex.args[1]) - funcname_joined = ExpressionExplorer.join_funcname_parts(funcname) - if funcname_joined ∈ (expand_bind ? can_macroexpand : can_macroexpand_no_bind) + if funcname.joined ∈ (expand_bind ? can_macroexpand : can_macroexpand_no_bind) macroexpand(PlutoRunner, ex; recursive=false)::Expr else ex @@ -133,4 +132,42 @@ function transform_dot_notation(ex::Expr) end + +############### + + +""" +```julia +can_be_function_wrapped(ex)::Bool +``` + +Is this code simple enough that we can wrap it inside a function, and run the function in global scope instead of running the code directly? Look for `Pluto.PlutoRunner.Computer` to learn more. +""" +function can_be_function_wrapped(x::Expr) + if x.head === :global || # better safe than sorry + x.head === :using || + x.head === :import || + x.head === :export || + x.head === :public || # Julia 1.11 + x.head === :module || + x.head === :incomplete || + # Only bail on named functions, but anonymous functions (args[1].head == :tuple) are fine. + # TODO Named functions INSIDE other functions should be fine too + (x.head === :function && !Meta.isexpr(x.args[1], :tuple)) || + x.head === :macro || + # Cells containing macrocalls will actually be function wrapped using the expanded version of the expression + # See https://github.com/fonsp/Pluto.jl/pull/1597 + x.head === :macrocall || + x.head === :struct || + x.head === :abstract || + (x.head === :(=) && ExpressionExplorer.is_function_assignment(x)) || # f(x) = ... + (x.head === :call && (x.args[1] === :eval || x.args[1] === :include)) + false + else + all(can_be_function_wrapped, x.args) + end +end + +can_be_function_wrapped(x::Any) = true + end diff --git a/src/analysis/Topology.jl b/src/analysis/Topology.jl index d08ee7d5c0..6a4ab6ade0 100644 --- a/src/analysis/Topology.jl +++ b/src/analysis/Topology.jl @@ -15,7 +15,7 @@ ExprAnalysisCache(notebook, cell::Cell) = let code=cell.code, parsedcode=parsedcode, module_usings_imports=ExpressionExplorer.compute_usings_imports(parsedcode), - function_wrapped=ExpressionExplorer.can_be_function_wrapped(parsedcode), + function_wrapped=ExpressionExplorerExtras.can_be_function_wrapped(parsedcode), ) end diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 98fa42403b..19e77dc0c1 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -1,6 +1,6 @@ import .ExpressionExplorer import .ExpressionExplorerExtras -import .ExpressionExplorer: join_funcname_parts, SymbolsState, FunctionNameSignaturePair +import .ExpressionExplorer: SymbolsState, FunctionNameSignaturePair "Return a copy of `old_topology`, but with recomputed results from `cells` taken into account." function updated_topology(old_topology::NotebookTopology, notebook::Notebook, cells) diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index a3ebb36409..ab4e799b52 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -113,7 +113,7 @@ function resolve_topology( if result isa Success (expr, computer_id) = result.result expanded_node = ExpressionExplorer.compute_reactive_node(expr; configuration=ExpressionExplorerExtras.PlutoConfiguration()) - function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) + function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else result diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index e74f0d8c4d..071a414d10 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -164,7 +164,8 @@ function run_reactive_core!( )...) if will_run_code(notebook) - deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs, to_reimport, cells_to_macro_invalidate; to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars` + to_delete_funcs_simple = Set{Tuple{Vararg{Symbol}}}((id, name.parts) for (id,name) in to_delete_funcs) + deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs_simple, to_reimport, cells_to_macro_invalidate; to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars` end foreach(v -> delete!(notebook.bonds, v), to_delete_vars) diff --git a/src/evaluation/RunBonds.jl b/src/evaluation/RunBonds.jl index 291b6eef85..3371f938ee 100644 --- a/src/evaluation/RunBonds.jl +++ b/src/evaluation/RunBonds.jl @@ -41,7 +41,7 @@ function set_bond_values_reactive(; bond_value_pairs = zip(syms_to_set, new_values) syms_to_set_set = Set{Symbol}(syms_to_set) - function custom_deletion_hook((session, notebook)::Tuple{ServerSession,Notebook}, old_workspace_name, new_workspace_name, to_delete_vars::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, to_reimport::Set{Expr}, invalidated_cell_uuids::Set{UUID}; to_run::AbstractVector{Cell}) + function custom_deletion_hook((session, notebook)::Tuple{ServerSession,Notebook}, old_workspace_name, new_workspace_name, to_delete_vars::Set{Symbol}, methods_to_delete, to_reimport, invalidated_cell_uuids; to_run) to_delete_vars = union(to_delete_vars, syms_to_set_set) # also delete the bound symbols WorkspaceManager.move_vars( (session, notebook), diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 12fbee2dd9..1fa1cf89e3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -551,7 +551,7 @@ function move_vars( old_workspace_name::Symbol, new_workspace_name::Union{Nothing,Symbol}, to_delete::Set{Symbol}, - methods_to_delete::Set{Tuple{UUID,FunctionName}}, + methods_to_delete::Set{Tuple{UUID,Vararg{Symbol}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}, keep_registered::Set{Symbol}=Set{Symbol}(); @@ -574,7 +574,7 @@ function move_vars( end) end -function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) +function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,Vararg{Symbol}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) move_vars(session_notebook, bump_workspace_module(session_notebook)..., to_delete, methods_to_delete, module_imports_to_move, invalidated_cell_uuids; kwargs...) end diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 28eb0decb1..2bcc9ecc78 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -688,7 +688,7 @@ function move_vars( old_workspace_name::Symbol, new_workspace_name::Symbol, vars_to_delete::Set{Symbol}, - methods_to_delete::Set{Tuple{UUID,Vector{Symbol}}}, + methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}, keep_registered::Set{Symbol}, diff --git a/test/MoreAnalysis.jl b/test/MoreAnalysis.jl index 76d0da4f2e..71646afb40 100644 --- a/test/MoreAnalysis.jl +++ b/test/MoreAnalysis.jl @@ -74,4 +74,44 @@ using Test @test transform(connections) == transform(wanted_connections) end + + + + @testset "can_be_function_wrapped" begin + + c = ExpressionExplorerExtras.can_be_function_wrapped + + + @test c(quote + a = b + C + if d + for i = 1:10 + while Y + end + end + end + end) + + + @test c(quote + map(1:10) do i + i + 1 + end + end) + + + @test !c(quote + function x(x) + X + end + end) + + @test !c(quote + if false + using Asdf + end + end) + + + end end From 29c2ac3bbf5ddca316471d10b7b4cb55975dcdf8 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 13 Nov 2023 12:34:31 +0100 Subject: [PATCH 06/10] more fixes --- src/analysis/ExpressionExplorer.jl | 2 +- src/evaluation/Run.jl | 2 +- src/evaluation/WorkspaceManager.jl | 4 ++-- src/runner/PlutoRunner.jl | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index 6c37dd9b3f..c37c730ed9 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -117,7 +117,7 @@ function collect_implicit_usings(ex::Expr) end end -collect_implicit_usings(usings::Set{Expr}) = mapreduce(collect_implicit_usings, union!, usings; init = Set{Expr}()) +collect_implicit_usings(usings::Union{AbstractSet{Expr},AbstractVector{Expr}}) = mapreduce(collect_implicit_usings, union!, usings; init = Set{Expr}()) collect_implicit_usings(usings_imports::ExpressionExplorer.UsingsImports) = collect_implicit_usings(usings_imports.usings) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 071a414d10..32cfa7c9fa 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -164,7 +164,7 @@ function run_reactive_core!( )...) if will_run_code(notebook) - to_delete_funcs_simple = Set{Tuple{Vararg{Symbol}}}((id, name.parts) for (id,name) in to_delete_funcs) + to_delete_funcs_simple = Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}((id, name.parts) for (id,name) in to_delete_funcs) deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs_simple, to_reimport, cells_to_macro_invalidate; to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars` end diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 1fa1cf89e3..e670d841d3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -551,7 +551,7 @@ function move_vars( old_workspace_name::Symbol, new_workspace_name::Union{Nothing,Symbol}, to_delete::Set{Symbol}, - methods_to_delete::Set{Tuple{UUID,Vararg{Symbol}}}, + methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}, keep_registered::Set{Symbol}=Set{Symbol}(); @@ -574,7 +574,7 @@ function move_vars( end) end -function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,Vararg{Symbol}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) +function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) move_vars(session_notebook, bump_workspace_module(session_notebook)..., to_delete, methods_to_delete, module_imports_to_move, invalidated_cell_uuids; kwargs...) end diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 2bcc9ecc78..43b60af19c 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -829,7 +829,7 @@ end # try_delete_toplevel_methods(workspace, [name]) # end -function try_delete_toplevel_methods(workspace::Module, (cell_id, name_parts)::Tuple{UUID,Vector{Symbol}})::Bool +function try_delete_toplevel_methods(workspace::Module, (cell_id, name_parts)::Tuple{UUID,Tuple{Vararg{Symbol}}})::Bool try val = workspace for name in name_parts From a4e29d85a234296b04d598d0fecbef6d3726ed01 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Mon, 13 Nov 2023 14:19:48 +0100 Subject: [PATCH 07/10] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 83b3e70c70..19e7f6810e 100644 --- a/Project.toml +++ b/Project.toml @@ -39,7 +39,7 @@ Base64 = "1" Configurations = "0.15, 0.16, 0.17" Dates = "1" Downloads = "1" -ExpressionExplorer = "0.3" +ExpressionExplorer = "0.4" FileWatching = "1" FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.5.2" From 2aacc500dd9a947f129f53a2335aba00a5a7ad4a Mon Sep 17 00:00:00 2001 From: Paul Berg Date: Mon, 13 Nov 2023 17:24:44 +0100 Subject: [PATCH 08/10] import ExpressionExplorerExtras in MoreAnalysis.jl --- test/MoreAnalysis.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MoreAnalysis.jl b/test/MoreAnalysis.jl index 71646afb40..a08516f299 100644 --- a/test/MoreAnalysis.jl +++ b/test/MoreAnalysis.jl @@ -1,4 +1,4 @@ -import Pluto: Pluto, Cell +import Pluto: Pluto, Cell, ExpressionExplorerExtras import Pluto.MoreAnalysis using Test From 734c447902c1d8e53a3f091244c9314ae5f1fad0 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 16 Nov 2023 12:51:48 +0100 Subject: [PATCH 09/10] Use ExpressionExplorer without config, but with a pretransformer (#2715) --- Project.toml | 2 +- src/analysis/ExpressionExplorer.jl | 84 ++++++---- src/analysis/TopologyUpdate.jl | 2 +- src/evaluation/MacroAnalysis.jl | 11 +- test/ExpressionExplorer.jl | 246 +++++++++++++++++++++++++++++ test/runtests.jl | 1 + 6 files changed, 310 insertions(+), 36 deletions(-) create mode 100644 test/ExpressionExplorer.jl diff --git a/Project.toml b/Project.toml index 19e7f6810e..81efaaed48 100644 --- a/Project.toml +++ b/Project.toml @@ -39,7 +39,7 @@ Base64 = "1" Configurations = "0.15, 0.16, 0.17" Dates = "1" Downloads = "1" -ExpressionExplorer = "0.4" +ExpressionExplorer = "0.5, 0.6" FileWatching = "1" FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.5.2" diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index c37c730ed9..f2a89d930c 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -8,41 +8,65 @@ import ..PlutoRunner using ExpressionExplorer using ExpressionExplorer: ScopeState -struct PlutoConfiguration <: ExpressionExplorer.AbstractExpressionExplorerConfiguration -end +""" +ExpressionExplorer does not explore inside macro calls, i.e. the arguments of a macrocall (like `a+b` in `@time a+b`) are ignored. +Normally, you would macroexpand an expression before giving it to ExpressionExplorer, but in Pluto we sometimes need to explore expressions *before* executing code. -function ExpressionExplorer.explore_macrocall!(ex::Expr, scopestate::ScopeState{PlutoConfiguration}) - # Early stopping, this expression will have to be re-explored once - # the macro is expanded in the notebook process. - macro_name = ExpressionExplorer.split_funcname(ex.args[1]) - symstate = SymbolsState(macrocalls = Set{FunctionName}([macro_name])) - - # Because it sure wouldn't break anything, - # I'm also going to blatantly assume that any macros referenced in here... - # will end up in the code after the macroexpansion 🤷‍♀️ - # "You should make a new function for that" they said, knowing I would take the lazy route. - for arg in ex.args[begin+1:end] - macro_symstate = ExpressionExplorer.explore!(arg, ScopeState(scopestate.configuration)) - - # Also, when this macro has something special inside like `Pkg.activate()`, - # we're going to treat it as normal code (so these heuristics trigger later) - # (Might want to also not let this to @eval macro, as an extra escape hatch if you - # really don't want pluto to see your Pkg.activate() call) - if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_symstate, expr = arg) - union!(symstate, macro_symstate) +In those cases, we want most accurate result possible. Our extra needs are: +1. Macros included in Julia base, Markdown and `@bind` can be expanded statically. (See `maybe_macroexpand_pluto`.) +2. If a macrocall argument contains a "special heuristic" like `Pkg.activate()` or `using Something`, we need to surface this to be visible to ExpressionExplorer and Pluto. We do this by placing the macrocall in a block, and copying the argument after to the macrocall. +3. If a macrocall argument contains other macrocalls, we need these nested macrocalls to be visible. We do this by placing the macrocall in a block, and creating new macrocall expressions with the nested macrocall names, but without arguments. +""" +function pretransform_pluto(ex) + if Meta.isexpr(ex, :macrocall) + to_add = Expr[] + + maybe_expanded = maybe_macroexpand_pluto(ex) + if maybe_expanded === ex + # we were not able to expand statically + for arg in ex.args[begin+1:end] + # TODO: test nested macrocalls + arg_transformed = pretransform_pluto(arg) + macro_arg_symstate = ExpressionExplorer.compute_symbols_state(arg_transformed) + + # When this macro has something special inside like `Pkg.activate()`, we're going to make sure that ExpressionExplorer treats it as normal code, not inside a macrocall. (so these heuristics trigger later) + if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_arg_symstate, expr = arg_transformed) + # then the whole argument expression should be added + push!(to_add, arg_transformed) + else + for fn in macro_arg_symstate.macrocalls + push!(to_add, Expr(:macrocall, fn)) + # fn is a FunctionName + # normally this would not be a legal expression, but ExpressionExplorer handles it correctly so it's all cool + end + end + end + + Expr( + :block, + # the original expression, not expanded. ExpressionExplorer will just explore the name of the macro, and nothing else. + ex, + # any expressions that we need to sneakily add + to_add... + ) else - union!(symstate, SymbolsState(macrocalls = macro_symstate.macrocalls)) + Expr( + :block, + # We were able to expand the macro, so let's recurse on the result. + pretransform_pluto(maybe_expanded), + # the name of the macro that got expanded + Expr(:macrocall, ex.args[1]), + ) end + elseif Meta.isexpr(ex, :module) + ex + elseif ex isa Expr + # recurse + Expr(ex.head, (pretransform_pluto(a) for a in ex.args)...) + else + ex end - - # Some macros can be expanded on the server process - if macro_name.joined ∈ can_macroexpand - new_ex = maybe_macroexpand_pluto(ex) - union!(symstate, ExpressionExplorer.explore!(new_ex, scopestate)) - end - - return symstate end diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 19e77dc0c1..6723087eb5 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -12,7 +12,7 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce old_code = old_topology.codes[cell] if old_code.code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) - new_reactive_node = compute_reactive_node(new_code.parsedcode; configuration=ExpressionExplorerExtras.PlutoConfiguration()) + new_reactive_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(new_code.parsedcode)) updated_nodes[cell] = new_reactive_node elseif old_code.forced_expr_id !== nothing diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index ab4e799b52..addccd557f 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -105,14 +105,14 @@ function resolve_topology( end function analyze_macrocell(cell::Cell) - if unresolved_topology.nodes[cell].macrocalls ⊆ ExpressionExplorer.can_macroexpand + if unresolved_topology.nodes[cell].macrocalls ⊆ ExpressionExplorerExtras.can_macroexpand return Skipped() end result = macroexpand_cell(cell) if result isa Success (expr, computer_id) = result.result - expanded_node = ExpressionExplorer.compute_reactive_node(expr; configuration=ExpressionExplorerExtras.PlutoConfiguration()) + expanded_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(expr)) function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else @@ -185,9 +185,12 @@ So, the resulting reactive nodes may not be absolutely accurate. If you can run """ function static_macroexpand(topology::NotebookTopology, cell::Cell) new_node = ExpressionExplorer.compute_reactive_node( - ExpressionExplorerExtras.maybe_macroexpand_pluto(topology.codes[cell].parsedcode; recursive=true); - configuration=ExpressionExplorerExtras.PlutoConfiguration() + ExpressionExplorerExtras.pretransform_pluto( + ExpressionExplorerExtras.maybe_macroexpand_pluto( + topology.codes[cell].parsedcode; recursive=true + ) ) + ) union!(new_node.macrocalls, topology.nodes[cell].macrocalls) new_node diff --git a/test/ExpressionExplorer.jl b/test/ExpressionExplorer.jl new file mode 100644 index 0000000000..7835ba19a7 --- /dev/null +++ b/test/ExpressionExplorer.jl @@ -0,0 +1,246 @@ + + +const ObjectID = typeof(objectid("hello computer")) + +function Base.show(io::IO, s::SymbolsState) + print(io, "SymbolsState([") + join(io, s.references, ", ") + print(io, "], [") + join(io, s.assignments, ", ") + print(io, "], [") + join(io, s.funccalls, ", ") + print(io, "], [") + if isempty(s.funcdefs) + print(io, "]") + else + println(io) + for (k, v) in s.funcdefs + print(io, " ", k, ": ", v) + println(io) + end + print(io, "]") + end + if !isempty(s.macrocalls) + print(io, "], [") + print(io, s.macrocalls) + print(io, "])") + else + print(io, ")") + end +end + +"Calls `ExpressionExplorer.compute_symbolreferences` on the given `expr` and test the found SymbolsState against a given one, with convient syntax. + +# Example + +```jldoctest +julia> @test testee(:( + begin + a = b + 1 + f(x) = x / z + end), + [:b, :+], # 1st: expected references + [:a, :f], # 2nd: expected definitions + [:+], # 3rd: expected function calls + [ + :f => ([:z, :/], [], [:/], []) + ]) # 4th: expected function definitions, with inner symstate using the same syntax +true +``` +" +function testee(expr::Any, expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []; verbose::Bool=true, transformer::Function=identify) + expected = easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls) + + expr_transformed = transformer(expr) + + original_hash = expr_hash(expr_transformed) + result = ExpressionExplorer.compute_symbolreferences(expr_transformed) + # should not throw: + ReactiveNode(result) + + new_hash = expr_hash(expr_transformed) + if original_hash != new_hash + error("\n== The expression explorer modified the expression. Don't do that! ==\n") + end + + # Anonymous function are given a random name, which looks like anon67387237861123 + # To make testing easier, we rename all such functions to anon + new_name(fn::FunctionName) = FunctionName(map(new_name, fn.parts)...) + new_name(sym::Symbol) = startswith(string(sym), "anon") ? :anon : sym + + result.assignments = Set(new_name.(result.assignments)) + result.funcdefs = let + newfuncdefs = Dict{FunctionNameSignaturePair,SymbolsState}() + for (k, v) in result.funcdefs + union!(newfuncdefs, Dict(FunctionNameSignaturePair(new_name(k.name), hash("hello")) => v)) + end + newfuncdefs + end + + if verbose && expected != result + println() + println("FAILED TEST") + println(expr) + println() + dump(expr, maxdepth=20) + println() + dump(expr_transformed, maxdepth=20) + println() + @show expected + resulted = result + @show resulted + println() + end + return expected == result +end + + + + +expr_hash(e::Expr) = objectid(e.head) + mapreduce(p -> objectid((p[1], expr_hash(p[2]))), +, enumerate(e.args); init=zero(ObjectID)) +expr_hash(x) = objectid(x) + + + + + +function easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []) + array_to_set(array) = map(array) do k + new_k = FunctionName(k) + return new_k + end |> Set + new_expected_funccalls = array_to_set(expected_funccalls) + + new_expected_funcdefs = map(expected_funcdefs) do (k, v) + new_k = FunctionName(k) + new_v = v isa SymbolsState ? v : easy_symstate(v...) + return FunctionNameSignaturePair(new_k, hash("hello")) => new_v + end |> Dict + + new_expected_macrocalls = array_to_set(expected_macrocalls) + + SymbolsState(Set(expected_references), Set(expected_definitions), new_expected_funccalls, new_expected_funcdefs, new_expected_macrocalls) +end + + + + + +t(args...; kwargs...) = testee(args...; transformer=Pluto.ExpressionExplorerExtras.pretransform_pluto, kwargs...) + + +""" +Like `t` but actually a convenient syntax +""" +function test_expression_explorer(; expr, references=[], definitions=[], funccalls=[], funcdefs=[], macrocalls=[], kwargs...) + t(expr, references, definitions, funccalls, funcdefs, macrocalls; kwargs...) +end + +@testset "Macros w/ Pluto 1" begin + # Macros tests are not just in ExpressionExplorer now + + @test t(:(@time a = 2), [], [], [], [], [Symbol("@time")]) + @test t(:(@f(x; y=z)), [], [], [], [], [Symbol("@f")]) + @test t(:(@f(x, y = z)), [], [], [], [], [Symbol("@f")]) # https://github.com/fonsp/Pluto.jl/issues/252 + @test t(:(Base.@time a = 2), [], [], [], [], [[:Base, Symbol("@time")]]) + # @test_nowarn t(:(@enum a b = d c), [:d], [:a, :b, :c], [Symbol("@enum")], []) + # @enum is tested in test/React.jl instead + @test t(:(@gensym a b c), [], [:a, :b, :c], [:gensym], [], [Symbol("@gensym")]) + @test t(:(Base.@gensym a b c), [], [:a, :b, :c], [:gensym], [], [[:Base, Symbol("@gensym")]]) + @test t(:(Base.@kwdef struct A; x = 1; y::Int = two; z end), [], [], [], [], [[:Base, Symbol("@kwdef")]]) + @test t(quote "asdf" f(x) = x end, [], [], [], [], [Symbol("@doc")]) + + # @test t(:(@bind a b), [], [], [], [], [Symbol("@bind")]) + # @test t(:(PlutoRunner.@bind a b), [], [], [], [], [[:PlutoRunner, Symbol("@bind")]]) + # @test_broken t(:(Main.PlutoRunner.@bind a b), [:b], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) + # @test t(:(let @bind a b end), [], [], [], [], [Symbol("@bind")]) + + @test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) + # @test t(:(md"hey $(@bind a b) $(a)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")]) + # @test t(:(md"hey $(a) $(@bind a b)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")]) + + @test t(:(@asdf a = x1 b = x2 c = x3), [], [], [], [], [Symbol("@asdf")]) # https://github.com/fonsp/Pluto.jl/issues/670 + + @test t(:(@aa @bb xxx), [], [], [], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa @bb(xxx) @cc(yyy)), [], [], [], [], [Symbol("@aa"), Symbol("@bb"), Symbol("@cc")]) + + @test t(:(Pkg.activate()), [:Pkg], [], [[:Pkg,:activate]], [], []) + @test t(:(@aa(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa")]) + @test t(:(@aa @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa @assert @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate], [:throw], [:AssertionError]], [], [Symbol("@aa"), Symbol("@assert"), Symbol("@bb")]) + @test t(:(@aa @bb(Xxx.xxxxxxxx())), [], [], [], [], [Symbol("@aa"), Symbol("@bb")]) + + @test t(:(include()), [], [], [[:include]], [], []) + @test t(:(:(include())), [], [], [], [], []) + @test t(:(:($(include()))), [], [], [[:include]], [], []) + @test t(:(@xx include()), [], [], [[:include]], [], [Symbol("@xx")]) + @test t(quote + module A + include() + Pkg.activate() + @xoxo asdf + end + end, [], [:A], [], [], []) + + + @test t(:(@aa @bb(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa")]) + @test t(:(using Zozo), [], [:Zozo], [], [], []) + + e = :(using Zozo) + @test ExpressionExplorer.compute_usings_imports( + e + ).usings == [e] + @test ExpressionExplorer.compute_usings_imports( + :(@aa @bb($e)) + ).usings == [e] + + + @test t(:(@einsum a[i,j] := x[i]*y[j]), [], [], [], [], [Symbol("@einsum")]) + @test t(:(@tullio a := f(x)[i+2j, k[j]] init=z), [], [], [], [], [Symbol("@tullio")]) + @test t(:(Pack.@asdf a[1,k[j]] := log(x[i]/y[j])), [], [], [], [], [[:Pack, Symbol("@asdf")]]) + + + @test t(:(html"a $(b = c)"), [], [], [], [], [Symbol("@html_str")]) + @test t(:(md"a $(b = c) $(b)"), [:c], [:b], [:getindex], [], [Symbol("@md_str")]) + @test t(:(md"\* $r"), [:r], [], [:getindex], [], [Symbol("@md_str")]) + @test t(:(md"a \$(b = c)"), [], [], [:getindex], [], [Symbol("@md_str")]) + @test t(:(macro a() end), [], [], [], [ + Symbol("@a") => ([], [], [], []) + ]) + @test t(:(macro a(b::Int); b end), [], [], [], [ + Symbol("@a") => ([:Int], [], [], []) + ]) + @test t(:(macro a(b::Int=c) end), [], [], [], [ + Symbol("@a") => ([:Int, :c], [], [], []) + ]) + @test t(:(macro a(); b = c; return b end), [], [], [], [ + Symbol("@a") => ([:c], [], [], []) + ]) + @test test_expression_explorer(; + expr=:(@parent @child 10), + macrocalls=[Symbol("@parent"), Symbol("@child")], + ) + @test test_expression_explorer(; + expr=:(@parent begin @child 1 + @grandchild 10 end), + macrocalls=[Symbol("@parent"), Symbol("@child"), Symbol("@grandchild")], + ) + @test t(macroexpand(Main, :(@noinline f(x) = x)), [], [], [], [ + Symbol("f") => ([], [], [], []) + ]) +end + + +@testset "Macros w/ Pluto 2" begin + + @test t(:(@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) + @test t(:(PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [[:PlutoRunner, Symbol("@bind")]]) + @test_broken t(:(Main.PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) + @test t(:(let @bind a b end), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) + + @test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) + @test t(:(md"hey $(@bind a b) $(a)"), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) + @test t(:(md"hey $(a) $(@bind a b)"), [:a, :b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) + + +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index d280a597f5..bbd8215e86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -48,6 +48,7 @@ verify_no_running_processes() verify_no_running_processes() print_timeroutput() +@timeit_include("ExpressionExplorer.jl") # TODO: test PlutoRunner functions like: # - from_this_notebook From eac8fc90f88c0b9fda9862b0b9d37f7e616fb0d2 Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Thu, 16 Nov 2023 13:08:03 +0100 Subject: [PATCH 10/10] =?UTF-8?q?ExpressionExplorer=201.0=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project.toml | 2 +- src/analysis/ExpressionExplorer.jl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 81efaaed48..483bf57ced 100644 --- a/Project.toml +++ b/Project.toml @@ -39,7 +39,7 @@ Base64 = "1" Configurations = "0.15, 0.16, 0.17" Dates = "1" Downloads = "1" -ExpressionExplorer = "0.5, 0.6" +ExpressionExplorer = "0.5, 0.6, 1" FileWatching = "1" FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.5.2" diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index f2a89d930c..e4d64526a6 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -26,7 +26,6 @@ function pretransform_pluto(ex) if maybe_expanded === ex # we were not able to expand statically for arg in ex.args[begin+1:end] - # TODO: test nested macrocalls arg_transformed = pretransform_pluto(arg) macro_arg_symstate = ExpressionExplorer.compute_symbols_state(arg_transformed)