Merge branch 'master' into issue-#432

Author: KristofferC

Project.toml

@@ -1,6 +1,6 @@
 name = "JuliaInterpreter"
 uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
-version = "0.9.0"
+version = "0.9.10"
 
 [deps]
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
@@ -16,8 +16,10 @@ julia = "1.6"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+FunctionWrappers = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
@@ -26,4 +28,4 @@ Tensors = "48a634ad-e948-5137-8d70-aa71f2a747f4"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["DataFrames", "Dates", "Distributed", "HTTP", "LinearAlgebra", "Mmap", "PyCall", "SHA", "SparseArrays", "Tensors", "Test"]
+test = ["DataFrames", "Dates", "Distributed", "FunctionWrappers", "HTTP", "LinearAlgebra", "Logging", "Mmap", "PyCall", "SHA", "SparseArrays", "Tensors", "Test"]

docs/src/dev_reference.md

@@ -105,7 +105,6 @@ JuliaInterpreter.is_doc_expr
 JuliaInterpreter.is_global_ref
 CodeTracking.whereis
 JuliaInterpreter.linenumber
-JuliaInterpreter.statementnumber
 JuliaInterpreter.Variable
 JuliaInterpreter.locals
 JuliaInterpreter.whichtt

src/breakpoints.jl

@@ -1,3 +1,5 @@
+using Base: Callable
+
 const _breakpoints = AbstractBreakpoint[]
 
 """
@@ -60,11 +62,20 @@ function add_to_existing_framecodes(bp::AbstractBreakpoint)
     end
 end
 
-function add_breakpoint_if_match!(framecode::FrameCode, bp::AbstractBreakpoint)
+function add_breakpoint_if_match!(framecode::FrameCode, bp::BreakpointSignature)
     if framecode_matches_breakpoint(framecode, bp)
-        stmtidx = bp.line === 0 ? 1 : statementnumber(framecode, bp.line)
-        breakpoint!(framecode, stmtidx, bp.condition, bp.enabled[])
-        push!(bp.instances, BreakpointRef(framecode, stmtidx))
+        scope = framecode.scope
+        matching_file = if scope isa Method
+            scope.file
+        else
+            # TODO: make more precise?
+            first(framecode.src.linetable).file
+        end
+        stmtidxs = bp.line === 0 ? [1] : statementnumbers(framecode, bp.line, matching_file::Symbol)
+        stmtidxs === nothing && return
+        breakpoint!(framecode, stmtidxs, bp.condition, bp.enabled[])
+        foreach(stmtidx -> push!(bp.instances, BreakpointRef(framecode, stmtidx)), stmtidxs)
+        return
     end
 end
 
@@ -111,10 +122,10 @@ end
 breakpoint(radius2, Tuple{Int,Int}, :(y > x))
 ```
 """
-function breakpoint(f::Union{Method, Function}, sig=nothing, line::Integer=0, condition::Condition=nothing)
-    if sig !== nothing && f isa Function
+function breakpoint(f::Union{Method, Callable}, sig=nothing, line::Integer=0, condition::Condition=nothing)
+    if sig !== nothing && f isa Callable
         sig = Base.to_tuple_type(sig)
-        sig = Tuple{typeof(f), sig.parameters...}
+        sig = Tuple{_Typeof(f), sig.parameters...}
     end
     bp = BreakpointSignature(f, sig, line, condition, Ref(true), BreakpointRef[])
     add_to_existing_framecodes(bp)
@@ -129,9 +140,9 @@ function breakpoint(f::Union{Method, Function}, sig=nothing, line::Integer=0, co
     firehooks(breakpoint, bp)
     return bp
 end
-breakpoint(f::Union{Method, Function}, sig, condition::Condition) = breakpoint(f, sig, 0, condition)
-breakpoint(f::Union{Method, Function}, line::Integer, condition::Condition=nothing) = breakpoint(f, nothing, line, condition)
-breakpoint(f::Union{Method, Function}, condition::Condition) = breakpoint(f, nothing, 0, condition)
+breakpoint(f::Union{Method, Callable}, sig, condition::Condition) = breakpoint(f, sig, 0, condition)
+breakpoint(f::Union{Method, Callable}, line::Integer, condition::Condition=nothing) = breakpoint(f, nothing, line, condition)
+breakpoint(f::Union{Method, Callable}, condition::Condition) = breakpoint(f, nothing, 0, condition)
 
 
 """
@@ -154,29 +165,24 @@ function breakpoint(file::AbstractString, line::Integer, condition::Condition=no
     return bp
 end
 
-function framecode_matches_breakpoint(framecode::FrameCode, bp::BreakpointFileLocation)
-    if framecode.scope isa Method
-        meth = framecode.scope::Method
-        methpath = CodeTracking.maybe_fix_path(String(meth.file))
-        ispath(methpath) && (methpath = realpath(methpath))
-        if bp.abspath == methpath || endswith(methpath, bp.path)
-            return method_contains_line(meth, bp.line)
-        else
-            return false
-        end
-    else
-        w = whereis(framecode, 1)
-        w === nothing && return false
-        path, _ = w
-        path = CodeTracking.maybe_fix_path(path)
-        ispath(path) && (path = realpath(path))
-
-        if bp.abspath == path || endswith(path, bp.path)
-            return toplevel_code_contains_line(framecode, bp.line)
-        else
-            return false
+function add_breakpoint_if_match!(framecode::FrameCode, bp::BreakpointFileLocation)
+    framecode_contains_file = false
+    matching_file = nothing
+    for file in framecode.unique_files
+        filepath = CodeTracking.maybe_fix_path(String(file))
+        if Base.samefile(bp.abspath, filepath) || endswith(filepath, bp.path)
+            framecode_contains_file = true
+            matching_file = file
+            break
         end
     end
+    framecode_contains_file || return nothing
+
+    stmtidxs = bp.line === 0 ? [1] : statementnumbers(framecode, bp.line, matching_file::Symbol)
+    stmtidxs === nothing && return
+    breakpoint!(framecode, stmtidxs, bp.condition, bp.enabled[])
+    foreach(stmtidx -> push!(bp.instances, BreakpointRef(framecode, stmtidx)), stmtidxs)
+    return
 end
 
 function shouldbreak(frame::Frame, pc::Int)
@@ -238,6 +244,8 @@ function breakpoint!(framecode::FrameCode, pc, condition::Condition=nothing, ena
         framecode.breakpoints[stmtidx] = BreakpointState(enabled, Core.eval(mod, fex))
     end
 end
+breakpoint!(framecode::FrameCode, pcs::AbstractArray, condition::Condition=nothing, enabled=true) = 
+    foreach(pc -> breakpoint!(framecode, pc, condition, enabled), pcs)
 breakpoint!(frame::Frame, pc=frame.pc, condition::Condition=nothing) =
     breakpoint!(frame.framecode, pc, condition)
 

src/builtins.jl

@@ -13,6 +13,16 @@ end
 const kwinvoke_name = isdefined(Core, Symbol("#kw##invoke")) ? Symbol("#kw##invoke") : Symbol("##invoke")
 const kwinvoke_instance = getfield(Core, kwinvoke_name).instance
 
+
+function maybe_recurse_expanded_builtin(frame, new_expr)
+    f = new_expr.args[1]
+    if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction)
+        return maybe_evaluate_builtin(frame, new_expr, true)
+    else
+        return new_expr
+    end
+end
+
 """
     ret = maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 
@@ -22,8 +32,6 @@ Otherwise, return `call_expr`.
 If `expand` is true, `Core._apply` calls will be resolved as a call to the applied function.
 """
 function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
-    # By having each call appearing statically in the "switch" block below,
-    # each gets call-site optimized.
     args = call_expr.args
     nargs = length(args) - 1
     fex = args[1]
@@ -32,13 +40,11 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
     else
         f = @lookup(frame, fex)
     end
-    # Builtins and intrinsics have empty method tables. We can circumvent
-    # a long "switch" check by looking for this.
-    mt = typeof(f).name.mt
-    if isa(mt, Core.MethodTable)
-        isempty(mt) || return call_expr
+    if !(isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction))
+        return call_expr
     end
-    # Builtins
+    # By having each call appearing statically in the "switch" block below,
+    # each gets call-site optimized.
     if f === <:
         if nargs == 2
             return Some{Any}(<:(@lookup(frame, args[2]), @lookup(frame, args[3])))
@@ -62,7 +68,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in argsflat
             push!(new_expr.args, (isa(x, Symbol) || isa(x, Expr) || isa(x, QuoteNode)) ? QuoteNode(x) : x)
         end
-        return new_expr
+        return maybe_recurse_expanded_builtin(frame, new_expr)
     elseif @static isdefined(Core, :_call_latest) ? f === Core._call_latest : false
         args = getargs(args, frame)
         if !expand
@@ -73,7 +79,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in args
             push!(new_expr.args, (isa(x, Symbol) || isa(x, Expr) || isa(x, QuoteNode)) ? QuoteNode(x) : x)
         end
-        return new_expr
+        return maybe_recurse_expanded_builtin(frame, new_expr)
     elseif @static isdefined(Core, :_apply_latest) ? f === Core._apply_latest : false
         args = getargs(args, frame)
         if !expand
@@ -84,7 +90,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in args
             push!(new_expr.args, (isa(x, Symbol) || isa(x, Expr) || isa(x, QuoteNode)) ? QuoteNode(x) : x)
         end
-        return new_expr
+        return maybe_recurse_expanded_builtin(frame, new_expr)
     elseif @static isdefined(Core, :_apply_iterate) ? f === Core._apply_iterate : false
         argswrapped = getargs(args, frame)
         if !expand
@@ -99,7 +105,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         for x in argsflat
             push!(new_expr.args, (isa(x, Symbol) || isa(x, Expr) || isa(x, QuoteNode)) ? QuoteNode(x) : x)
         end
-        return new_expr
+        return maybe_recurse_expanded_builtin(frame, new_expr)
     elseif f === Core._apply_pure
         return Some{Any}(Core._apply_pure(getargs(args, frame)...))
     elseif f === Core._expr

src/commands.jl

@@ -218,7 +218,18 @@ function maybe_step_through_wrapper!(@nospecialize(recurse), frame::Frame)
     length(stmts) < 2 && return frame
     last = stmts[end-1]
     isexpr(last, :(=)) && (last = last.args[2])
-    is_kw = isa(scope, Method) && endswith(String(Base.unwrap_unionall(Base.unwrap_unionall(scope.sig).parameters[1]).name.name), "#kw")
+
+    is_kw = false
+    if isa(scope, Method)
+        unwrap1 = Base.unwrap_unionall(scope.sig)
+        if unwrap1 isa DataType
+            param1 = Base.unwrap_unionall(unwrap1.parameters[1])
+            if param1 isa DataType
+                is_kw = endswith(String(param1.name.name), "#kw")
+            end
+        end
+    end
+
     has_selfarg = isexpr(last, :call) && any(isequal(SlotNumber(1)), last.args)
     issplatcall, _callee = unpack_splatcall(last)
     if is_kw || has_selfarg || (issplatcall && is_bodyfunc(_callee))
@@ -389,6 +400,18 @@ function maybe_step_through_nkw_meta!(frame)
 end
 
 
+function more_calls_on_current_line(frame)
+    _, curr_line = whereis(frame)
+    curr_pc = frame.pc + 1
+    while curr_pc <= length(frame.framecode.src.code)
+        _, new_line = whereis(frame, curr_pc)
+        new_line == curr_line || return false
+        is_call(pc_expr(frame, curr_pc)) && return true
+        curr_pc += 1
+    end
+    return false
+end
+
 """
     ret = debug_command(recurse, frame, cmd, rootistoplevel=false; line=nothing)
     ret = debug_command(frame, cmd, rootistoplevel=false; line=nothing)
@@ -398,6 +421,7 @@ Perform one "debugger" command. The keyword arguments are not used for all debug
 
 - `:n`: advance to the next line
 - `:s`: step into the next call
+- `:sl` step into the last call on the current line (e.g. steps into `f` if the line is `f(g(h(x)))`).
 - `:until`: advance the frame to line `line` if given, otherwise advance to the line after the current line
 - `:c`: continue execution until termination or reaching a breakpoint
 - `:finish`: finish the current frame and return to the parent
@@ -433,7 +457,12 @@ function debug_command(@nospecialize(recurse), frame::Frame, cmd::Symbol, rootis
         cmd === :n && return maybe_reset_frame!(recurse, frame, next_line!(recurse, frame, istoplevel), rootistoplevel)
         cmd === :se && return maybe_reset_frame!(recurse, frame, step_expr!(recurse, frame, istoplevel), rootistoplevel)
         cmd === :until && return maybe_reset_frame!(recurse, frame, until_line!(recurse, frame, line, istoplevel), rootistoplevel)
-
+        if cmd === :sl
+            while more_calls_on_current_line(frame)
+                next_call!(recurse, frame, istoplevel)
+            end
+            return debug_command(recurse, frame, :s, rootistoplevel; line)
+        end
         enter_generated = false
         if cmd === :sg
             enter_generated = true

src/construct.jl

@@ -35,14 +35,14 @@ const compiled_modules = Set{Module}()
 
 const junk_framedata = FrameData[] # to allow re-use of allocated memory (this is otherwise a bottleneck)
 const junk_frames = Frame[]
-debug_recycle() = false
+debug_mode() = false
 @noinline function _check_frame_not_in_junk(frame)
     @assert frame.framedata ∉ junk_framedata
     @assert frame ∉ junk_frames
 end
 
 @inline function recycle(frame)
-    debug_recycle() && _check_frame_not_in_junk(frame)
+    debug_mode() && _check_frame_not_in_junk(frame)
     push!(junk_framedata, frame.framedata)
     push!(junk_frames, frame)
 end
@@ -435,10 +435,12 @@ function push_modex!(iter::ExprSplitter, mod::Module, ex::Expr)
     if ex.head === :toplevel || ex.head === :block
         # Issue #427
         modifies_scope = false
-        for a in ex.args
-            if isa(a, Expr) && a.head ∈ (:local, :global)
-                modifies_scope = true
-                break
+        if ex.head === :block
+            for a in ex.args
+                if isa(a, Expr) && a.head ∈ (:local, :global)
+                    modifies_scope = true
+                    break
+                end
             end
         end
         push!(iter.index, modifies_scope ? 0 : 1)

src/interpret.jl

@@ -165,9 +165,24 @@ function evaluate_foreigncall(@nospecialize(recurse), frame::Frame, call_expr::E
     if !isempty(data.sparams) && scope isa Method
         sig = scope.sig
         args[2] = instantiate_type_in_env(args[2], sig, data.sparams)
-        args[3] = Core.svec(map(args[3]) do arg
-            instantiate_type_in_env(arg, sig, data.sparams)
-        end...)
+        @static if VERSION < v"1.7.0"
+            arg3 = args[3]
+            if arg3 isa Core.SimpleVector
+                args[3] = Core.svec(map(arg3) do arg
+                    instantiate_type_in_env(arg, sig, data.sparams)
+                end...)
+            else
+                args[3] = instantiate_type_in_env(arg3, sig, data.sparams)
+                args[4] = Core.svec(map(args[4]::Core.SimpleVector) do arg
+                    instantiate_type_in_env(arg, sig, data.sparams)
+                end...)
+            end
+        else
+            args[3] = instantiate_type_in_env(args[3], sig, data.sparams)
+            args[4] = Core.svec(map(args[4]::Core.SimpleVector) do arg
+                instantiate_type_in_env(arg, sig, data.sparams)
+            end...)
+        end
     end
     return Core.eval(moduleof(frame), Expr(head, args...))
 end
@@ -462,14 +477,14 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev
                     return (frame.pc = node.args[2]::Int)
                 end
             elseif node.head === :enter
-                rhs = node.args[1]
+                rhs = node.args[1]::Int
                 push!(data.exception_frames, rhs)
             elseif node.head === :leave
                 for _ = 1:node.args[1]::Int
                     pop!(data.exception_frames)
                 end
             elseif node.head === :pop_exception
-                n = lookup_var(frame, node.args[1])
+                n = lookup_var(frame, node.args[1]::SSAValue)::Int
                 deleteat!(data.exception_frames, n+1:length(data.exception_frames))
             elseif node.head === :return
                 return nothing
@@ -489,7 +504,7 @@ function step_expr!(@nospecialize(recurse), frame, @nospecialize(node), istoplev
                     end
                     Core.eval(mod, Expr(:const, name))
                 elseif node.head === :thunk
-                    newframe = Frame(moduleof(frame), node.args[1])
+                    newframe = Frame(moduleof(frame), node.args[1]::CodeInfo)
                     if isa(recurse, Compiled)
                         finish!(recurse, newframe, true)
                     else