Skip to content

Commit 7635190

Browse files
vtjnashaviatesk
andauthored
Add edges vector to CodeInstance/CodeInfo to keep backedges as edges (#54894)
Appears to add about 11MB (128MB to 139MB) to the system image, and to decrease the stdlib size by 55 MB (325MB to 270MB), so seems overall favorable right now. The edges are computed following the encoding <https://hackmd.io/sjPig55kS4a5XNWC6HmKSg?both#Edges-Encoding> to correctly reflect the backedges. Co-authored-by: Shuhei Kadowaki <[email protected]>
1 parent 706a4f6 commit 7635190

38 files changed

+1159
-1222
lines changed

base/Base.jl

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,6 @@ function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has
204204
return Core._hasmethod(tt)
205205
end
206206

207-
208207
# core operations & types
209208
include("promotion.jl")
210209
include("tuple.jl")

base/boot.jl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -535,11 +535,11 @@ function CodeInstance(
535535
mi::MethodInstance, owner, @nospecialize(rettype), @nospecialize(exctype), @nospecialize(inferred_const),
536536
@nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt,
537537
effects::UInt32, @nospecialize(analysis_results),
538-
relocatability::UInt8, edges::Union{DebugInfo,Nothing})
538+
relocatability::UInt8, di::Union{DebugInfo,Nothing}, edges::SimpleVector)
539539
return ccall(:jl_new_codeinst, Ref{CodeInstance},
540-
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any),
540+
(Any, Any, Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, UInt8, Any, Any),
541541
mi, owner, rettype, exctype, inferred_const, inferred, const_flags, min_world, max_world,
542-
effects, analysis_results, relocatability, edges)
542+
effects, analysis_results, relocatability, di, edges)
543543
end
544544
GlobalRef(m::Module, s::Symbol) = ccall(:jl_module_globalref, Ref{GlobalRef}, (Any, Any), m, s)
545545
Module(name::Symbol=:anonymous, std_imports::Bool=true, default_names::Bool=true) = ccall(:jl_f_new_module, Ref{Module}, (Any, Bool, Bool), name, std_imports, default_names)

base/compiler/abstractinterpretation.jl

Lines changed: 96 additions & 119 deletions
Large diffs are not rendered by default.

base/compiler/inferencestate.jl

Lines changed: 7 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,7 @@ mutable struct InferenceState
247247
# TODO: Could keep this sparsely by doing structural liveness analysis ahead of time.
248248
bb_vartables::Vector{Union{Nothing,VarTable}} # nothing if not analyzed yet
249249
ssavaluetypes::Vector{Any}
250-
stmt_edges::Vector{Vector{Any}}
250+
edges::Vector{Any}
251251
stmt_info::Vector{CallInfo}
252252

253253
#= intermediate states for interprocedural abstract interpretation =#
@@ -302,7 +302,7 @@ mutable struct InferenceState
302302
nssavalues = src.ssavaluetypes::Int
303303
ssavalue_uses = find_ssavalue_uses(code, nssavalues)
304304
nstmts = length(code)
305-
stmt_edges = Vector{Vector{Any}}(undef, nstmts)
305+
edges = []
306306
stmt_info = CallInfo[ NoCallInfo() for i = 1:nstmts ]
307307

308308
nslots = length(src.slotflags)
@@ -327,7 +327,7 @@ mutable struct InferenceState
327327
unreachable = BitSet()
328328
pclimitations = IdSet{InferenceState}()
329329
limitations = IdSet{InferenceState}()
330-
cycle_backedges = Vector{Tuple{InferenceState,Int}}()
330+
cycle_backedges = Tuple{InferenceState,Int}[]
331331
callstack = AbsIntState[]
332332
tasks = WorkThunk[]
333333

@@ -350,10 +350,12 @@ mutable struct InferenceState
350350

351351
restrict_abstract_call_sites = isa(def, Module)
352352

353+
parentid = frameid = cycleid = 0
354+
353355
this = new(
354356
mi, world, mod, sptypes, slottypes, src, cfg, spec_info,
355-
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, stmt_edges, stmt_info,
356-
tasks, pclimitations, limitations, cycle_backedges, callstack, 0, 0, 0,
357+
currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, ssavaluetypes, edges, stmt_info,
358+
tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid,
357359
result, unreachable, valid_worlds, bestguess, exc_bestguess, ipo_effects,
358360
restrict_abstract_call_sites, cache_mode, insert_coverage,
359361
interp)
@@ -754,30 +756,6 @@ function record_ssa_assign!(𝕃ᵢ::AbstractLattice, ssa_id::Int, @nospecialize
754756
return nothing
755757
end
756758

757-
function add_cycle_backedge!(caller::InferenceState, frame::InferenceState)
758-
update_valid_age!(caller, frame.valid_worlds)
759-
backedge = (caller, caller.currpc)
760-
contains_is(frame.cycle_backedges, backedge) || push!(frame.cycle_backedges, backedge)
761-
add_backedge!(caller, frame.linfo)
762-
return frame
763-
end
764-
765-
function get_stmt_edges!(caller::InferenceState, currpc::Int=caller.currpc)
766-
stmt_edges = caller.stmt_edges
767-
if !isassigned(stmt_edges, currpc)
768-
return stmt_edges[currpc] = Any[]
769-
else
770-
return stmt_edges[currpc]
771-
end
772-
end
773-
774-
function empty_backedges!(frame::InferenceState, currpc::Int=frame.currpc)
775-
if isassigned(frame.stmt_edges, currpc)
776-
empty!(frame.stmt_edges[currpc])
777-
end
778-
return nothing
779-
end
780-
781759
function narguments(sv::InferenceState, include_va::Bool=true)
782760
nargs = Int(sv.src.nargs)
783761
if !include_va
@@ -1008,32 +986,6 @@ function callers_in_cycle(sv::InferenceState)
1008986
end
1009987
callers_in_cycle(sv::IRInterpretationState) = AbsIntCycle(sv.callstack::Vector{AbsIntState}, 0, 0)
1010988

1011-
# temporarily accumulate our edges to later add as backedges in the callee
1012-
function add_backedge!(caller::InferenceState, mi::MethodInstance)
1013-
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
1014-
return push!(get_stmt_edges!(caller), mi)
1015-
end
1016-
function add_backedge!(irsv::IRInterpretationState, mi::MethodInstance)
1017-
return push!(irsv.edges, mi)
1018-
end
1019-
1020-
function add_invoke_backedge!(caller::InferenceState, @nospecialize(invokesig::Type), mi::MethodInstance)
1021-
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
1022-
return push!(get_stmt_edges!(caller), invokesig, mi)
1023-
end
1024-
function add_invoke_backedge!(irsv::IRInterpretationState, @nospecialize(invokesig::Type), mi::MethodInstance)
1025-
return push!(irsv.edges, invokesig, mi)
1026-
end
1027-
1028-
# used to temporarily accumulate our no method errors to later add as backedges in the callee method table
1029-
function add_mt_backedge!(caller::InferenceState, mt::MethodTable, @nospecialize(typ))
1030-
isa(caller.linfo.def, Method) || return nothing # don't add backedges to toplevel method instance
1031-
return push!(get_stmt_edges!(caller), mt, typ)
1032-
end
1033-
function add_mt_backedge!(irsv::IRInterpretationState, mt::MethodTable, @nospecialize(typ))
1034-
return push!(irsv.edges, mt, typ)
1035-
end
1036-
1037989
get_curr_ssaflag(sv::InferenceState) = sv.src.ssaflags[sv.currpc]
1038990
get_curr_ssaflag(sv::IRInterpretationState) = sv.ir.stmts[sv.curridx][:flag]
1039991

base/compiler/optimize.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,7 @@ struct InliningState{Interp<:AbstractInterpreter}
141141
interp::Interp
142142
end
143143
function InliningState(sv::InferenceState, interp::AbstractInterpreter)
144-
edges = sv.stmt_edges[1]
145-
return InliningState(edges, sv.world, interp)
144+
return InliningState(sv.edges, sv.world, interp)
146145
end
147146
function InliningState(interp::AbstractInterpreter)
148147
return InliningState(Any[], get_inference_world(interp), interp)
@@ -225,6 +224,7 @@ include("compiler/ssair/irinterp.jl")
225224
function ir_to_codeinf!(opt::OptimizationState)
226225
(; linfo, src) = opt
227226
src = ir_to_codeinf!(src, opt.ir::IRCode)
227+
src.edges = opt.inlining.edges
228228
opt.ir = nothing
229229
maybe_validate_code(linfo, src, "optimized")
230230
return src

base/compiler/ssair/inlining.jl

Lines changed: 45 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ end
2828

2929
struct ConstantCase
3030
val::Any
31-
ConstantCase(@nospecialize val) = new(val)
31+
edge::CodeInstance
32+
ConstantCase(@nospecialize(val), edge::CodeInstance) = new(val, edge)
3233
end
3334

3435
struct SomeCase
@@ -68,11 +69,12 @@ struct InliningEdgeTracker
6869
new(state.edges, invokesig)
6970
end
7071

71-
function add_inlining_backedge!((; edges, invokesig)::InliningEdgeTracker, mi::MethodInstance)
72+
function add_inlining_edge!(et::InliningEdgeTracker, edge::Union{CodeInstance,MethodInstance})
73+
(; edges, invokesig) = et
7274
if invokesig === nothing
73-
push!(edges, mi)
75+
add_one_edge!(edges, edge)
7476
else # invoke backedge
75-
push!(edges, invoke_signature(invokesig), mi)
77+
add_invoke_edge!(edges, invoke_signature(invokesig), edge)
7678
end
7779
return nothing
7880
end
@@ -784,10 +786,10 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}},
784786
end
785787

786788
function compileable_specialization(mi::MethodInstance, effects::Effects,
787-
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
789+
et::InliningEdgeTracker, @nospecialize(info::CallInfo), state::InliningState)
788790
mi_invoke = mi
789791
method, atype, sparams = mi.def::Method, mi.specTypes, mi.sparam_vals
790-
if compilesig_invokes
792+
if OptimizationParams(state.interp).compilesig_invokes
791793
new_atype = get_compileable_sig(method, atype, sparams)
792794
new_atype === nothing && return nothing
793795
if atype !== new_atype
@@ -805,43 +807,41 @@ function compileable_specialization(mi::MethodInstance, effects::Effects,
805807
return nothing
806808
end
807809
end
808-
add_inlining_backedge!(et, mi) # to the dispatch lookup
809-
mi_invoke !== mi && push!(et.edges, method.sig, mi_invoke) # add_inlining_backedge to the invoke call, if that is different
810+
add_inlining_edge!(et, mi) # to the dispatch lookup
811+
if mi_invoke !== mi
812+
add_invoke_edge!(et.edges, method.sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different
813+
end
810814
return InvokeCase(mi_invoke, effects, info)
811815
end
812816

813-
function compileable_specialization(match::MethodMatch, effects::Effects,
814-
et::InliningEdgeTracker, @nospecialize(info::CallInfo); compilesig_invokes::Bool=true)
815-
mi = specialize_method(match)
816-
return compileable_specialization(mi, effects, et, info; compilesig_invokes)
817-
end
818-
819817
struct InferredResult
820818
src::Any # CodeInfo or IRCode
821819
effects::Effects
822-
InferredResult(@nospecialize(src), effects::Effects) = new(src, effects)
820+
edge::CodeInstance
821+
InferredResult(@nospecialize(src), effects::Effects, edge::CodeInstance) = new(src, effects, edge)
823822
end
824823
@inline function get_cached_result(state::InliningState, mi::MethodInstance)
825824
code = get(code_cache(state), mi, nothing)
826825
if code isa CodeInstance
827826
if use_const_api(code)
828827
# in this case function can be inlined to a constant
829-
return ConstantCase(quoted(code.rettype_const))
828+
return ConstantCase(quoted(code.rettype_const), code)
830829
end
831830
return code
832831
end
833832
return nothing
834833
end
835834
@inline function get_local_result(inf_result::InferenceResult)
835+
@assert isdefined(inf_result, :ci_as_edge) "InferenceResult without ci_as_edge"
836836
effects = inf_result.ipo_effects
837837
if is_foldable_nothrow(effects)
838838
res = inf_result.result
839839
if isa(res, Const) && is_inlineable_constant(res.val)
840840
# use constant calling convention
841-
return ConstantCase(quoted(res.val))
841+
return ConstantCase(quoted(res.val), inf_result.ci_as_edge)
842842
end
843843
end
844-
return InferredResult(inf_result.src, effects)
844+
return InferredResult(inf_result.src, effects, inf_result.ci_as_edge)
845845
end
846846

847847
# the general resolver for usual and const-prop'ed calls
@@ -861,30 +861,28 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult,
861861
inferred_result = get_cached_result(state, mi)
862862
end
863863
if inferred_result isa ConstantCase
864-
add_inlining_backedge!(et, mi)
864+
add_inlining_edge!(et, inferred_result.edge)
865865
return inferred_result
866866
elseif inferred_result isa InferredResult
867-
(; src, effects) = inferred_result
867+
(; src, effects, edge) = inferred_result
868868
elseif inferred_result isa CodeInstance
869869
src = @atomic :monotonic inferred_result.inferred
870870
effects = decode_effects(inferred_result.ipo_purity_bits)
871+
edge = inferred_result
871872
else # there is no cached source available, bail out
872-
return compileable_specialization(mi, Effects(), et, info;
873-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
873+
return compileable_specialization(mi, Effects(), et, info, state)
874874
end
875875

876876
# the duplicated check might have been done already within `analyze_method!`, but still
877877
# we need it here too since we may come here directly using a constant-prop' result
878878
if !OptimizationParams(state.interp).inlining || is_stmt_noinline(flag)
879-
return compileable_specialization(mi, effects, et, info;
880-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
879+
return compileable_specialization(edge.def, effects, et, info, state)
881880
end
882881

883882
src_inlining_policy(state.interp, src, info, flag) ||
884-
return compileable_specialization(mi, effects, et, info;
885-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
883+
return compileable_specialization(edge.def, effects, et, info, state)
886884

887-
add_inlining_backedge!(et, mi)
885+
add_inlining_edge!(et, edge)
888886
if inferred_result isa CodeInstance
889887
ir, spec_info, debuginfo = retrieve_ir_for_inlining(inferred_result, src)
890888
else
@@ -904,7 +902,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U
904902

905903
cached_result = get_cached_result(state, mi)
906904
if cached_result isa ConstantCase
907-
add_inlining_backedge!(et, mi)
905+
add_inlining_edge!(et, cached_result.edge)
908906
return cached_result
909907
elseif cached_result isa CodeInstance
910908
src = @atomic :monotonic cached_result.inferred
@@ -915,7 +913,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U
915913

916914
src_inlining_policy(state.interp, src, info, flag) || return nothing
917915
ir, spec_info, debuginfo = retrieve_ir_for_inlining(cached_result, src)
918-
add_inlining_backedge!(et, mi)
916+
add_inlining_edge!(et, cached_result)
919917
return InliningTodo(mi, ir, spec_info, debuginfo, effects)
920918
end
921919

@@ -1119,7 +1117,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
11191117
# e.g. rewrite `((t::Tuple)...,)` to `t`
11201118
nonempty_idx = 0
11211119
𝕃ₒ = optimizer_lattice(state.interp)
1122-
for i = (arg_start + 1):length(argtypes)
1120+
for i = (arg_start+1):length(argtypes)
11231121
ti = argtypes[i]
11241122
(𝕃ₒ, ti, Tuple{}) && continue
11251123
if (𝕃ₒ, ti, Tuple) && nonempty_idx == 0
@@ -1137,7 +1135,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
11371135
# Try to figure out the signature of the function being called
11381136
# and if rewrite_apply_exprargs can deal with this form
11391137
arginfos = MaybeAbstractIterationInfo[]
1140-
for i = (arg_start + 1):length(argtypes)
1138+
for i = (arg_start+1):length(argtypes)
11411139
thisarginfo = nothing
11421140
if !is_valid_type_for_apply_rewrite(argtypes[i], OptimizationParams(state.interp))
11431141
isa(info, ApplyCallInfo) || return nothing
@@ -1403,9 +1401,7 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig
14031401
result, match, argtypes, info, flag, state; allow_typevars=true)
14041402
end
14051403
if !fully_covered
1406-
atype = argtypes_to_type(sig.argtypes)
1407-
# We will emit an inline MethodError so we need a backedge to the MethodTable
1408-
add_uncovered_edges!(state.edges, info, atype)
1404+
# We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it
14091405
end
14101406
elseif !isempty(cases)
14111407
# if we've not seen all candidates, union split is valid only for dispatch tuples
@@ -1453,30 +1449,28 @@ end
14531449

14541450
function semiconcrete_result_item(result::SemiConcreteResult,
14551451
@nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
1456-
mi = result.mi
1452+
mi = result.edge.def
14571453
et = InliningEdgeTracker(state)
14581454

14591455
if (!OptimizationParams(state.interp).inlining || is_stmt_noinline(flag) ||
14601456
# For `NativeInterpreter`, `SemiConcreteResult` may be produced for
14611457
# a `@noinline`-declared method when it's marked as `@constprop :aggressive`.
14621458
# Suppress the inlining here (unless inlining is requested at the callsite).
14631459
(is_declared_noinline(mi.def::Method) && !is_stmt_inline(flag)))
1464-
return compileable_specialization(mi, result.effects, et, info;
1465-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
1460+
return compileable_specialization(mi, result.effects, et, info, state)
14661461
end
14671462
src_inlining_policy(state.interp, result.ir, info, flag) ||
1468-
return compileable_specialization(mi, result.effects, et, info;
1469-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
1463+
return compileable_specialization(mi, result.effects, et, info, state)
14701464

1471-
add_inlining_backedge!(et, mi)
1465+
add_inlining_edge!(et, result.edge)
14721466
preserve_local_sources = OptimizationParams(state.interp).preserve_local_sources
14731467
ir, _, debuginfo = retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources)
14741468
return InliningTodo(mi, ir, result.spec_info, debuginfo, result.effects)
14751469
end
14761470

14771471
function handle_semi_concrete_result!(cases::Vector{InliningCase}, result::SemiConcreteResult,
14781472
match::MethodMatch, @nospecialize(info::CallInfo), flag::UInt32, state::InliningState)
1479-
mi = result.mi
1473+
mi = result.edge.def
14801474
validate_sparams(mi.sparam_vals) || return false
14811475
item = semiconcrete_result_item(result, info, flag, state)
14821476
item === nothing && return false
@@ -1499,11 +1493,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn
14991493
invokesig::Union{Nothing,Vector{Any}}=nothing)
15001494
if !may_inline_concrete_result(result)
15011495
et = InliningEdgeTracker(state, invokesig)
1502-
return compileable_specialization(result.mi, result.effects, et, info;
1503-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
1496+
return compileable_specialization(result.edge.def, result.effects, et, info, state)
15041497
end
15051498
@assert result.effects === EFFECTS_TOTAL
1506-
return ConstantCase(quoted(result.result))
1499+
return ConstantCase(quoted(result.result), result.edge)
15071500
end
15081501

15091502
function handle_cases!(todo::Vector{Pair{Int,Any}}, ir::IRCode, idx::Int, stmt::Expr,
@@ -1552,11 +1545,16 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp
15521545
info isa MethodResultPure && (info = info.info)
15531546
info isa ConstCallInfo && (info = info.call)
15541547
info isa MethodMatchInfo || return nothing
1555-
length(info.results) == 1 || return nothing
1548+
length(info.edges) == length(info.results) == 1 || return nothing
15561549
match = info.results[1]::MethodMatch
15571550
match.fully_covers || return nothing
1558-
case = compileable_specialization(match, Effects(), InliningEdgeTracker(state), info;
1559-
compilesig_invokes=OptimizationParams(state.interp).compilesig_invokes)
1551+
edge = info.edges[1]
1552+
if edge === nothing
1553+
edge = specialize_method(match)
1554+
else
1555+
edge = edge.def
1556+
end
1557+
case = compileable_specialization(edge, Effects(), InliningEdgeTracker(state), info, state)
15601558
case === nothing && return nothing
15611559
stmt.head = :invoke_modify
15621560
pushfirst!(stmt.args, case.invoke)

0 commit comments

Comments
 (0)