28
28
29
29
struct ConstantCase
30
30
val:: Any
31
- ConstantCase (@nospecialize val) = new (val)
31
+ edge:: CodeInstance
32
+ ConstantCase (@nospecialize (val), edge:: CodeInstance ) = new (val, edge)
32
33
end
33
34
34
35
struct SomeCase
@@ -68,11 +69,12 @@ struct InliningEdgeTracker
68
69
new (state. edges, invokesig)
69
70
end
70
71
71
- function add_inlining_backedge! ((; edges, invokesig):: InliningEdgeTracker , mi:: MethodInstance )
72
+ function add_inlining_edge! (et:: InliningEdgeTracker , edge:: Union{CodeInstance,MethodInstance} )
73
+ (; edges, invokesig) = et
72
74
if invokesig === nothing
73
- push ! (edges, mi )
75
+ add_one_edge ! (edges, edge )
74
76
else # invoke backedge
75
- push ! (edges, invoke_signature (invokesig), mi )
77
+ add_invoke_edge ! (edges, invoke_signature (invokesig), edge )
76
78
end
77
79
return nothing
78
80
end
@@ -784,10 +786,10 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}},
784
786
end
785
787
786
788
function compileable_specialization (mi:: MethodInstance , effects:: Effects ,
787
- et:: InliningEdgeTracker , @nospecialize (info:: CallInfo ); compilesig_invokes :: Bool = true )
789
+ et:: InliningEdgeTracker , @nospecialize (info:: CallInfo ), state :: InliningState )
788
790
mi_invoke = mi
789
791
method, atype, sparams = mi. def:: Method , mi. specTypes, mi. sparam_vals
790
- if compilesig_invokes
792
+ if OptimizationParams (state . interp) . compilesig_invokes
791
793
new_atype = get_compileable_sig (method, atype, sparams)
792
794
new_atype === nothing && return nothing
793
795
if atype != = new_atype
@@ -805,43 +807,41 @@ function compileable_specialization(mi::MethodInstance, effects::Effects,
805
807
return nothing
806
808
end
807
809
end
808
- add_inlining_backedge! (et, mi) # to the dispatch lookup
809
- mi_invoke != = mi && push! (et. edges, method. sig, mi_invoke) # add_inlining_backedge to the invoke call, if that is different
810
+ add_inlining_edge! (et, mi) # to the dispatch lookup
811
+ if mi_invoke != = mi
812
+ add_invoke_edge! (et. edges, method. sig, mi_invoke) # add_inlining_edge to the invoke call, if that is different
813
+ end
810
814
return InvokeCase (mi_invoke, effects, info)
811
815
end
812
816
813
- function compileable_specialization (match:: MethodMatch , effects:: Effects ,
814
- et:: InliningEdgeTracker , @nospecialize (info:: CallInfo ); compilesig_invokes:: Bool = true )
815
- mi = specialize_method (match)
816
- return compileable_specialization (mi, effects, et, info; compilesig_invokes)
817
- end
818
-
819
817
struct InferredResult
820
818
src:: Any # CodeInfo or IRCode
821
819
effects:: Effects
822
- InferredResult (@nospecialize (src), effects:: Effects ) = new (src, effects)
820
+ edge:: CodeInstance
821
+ InferredResult (@nospecialize (src), effects:: Effects , edge:: CodeInstance ) = new (src, effects, edge)
823
822
end
824
823
@inline function get_cached_result (state:: InliningState , mi:: MethodInstance )
825
824
code = get (code_cache (state), mi, nothing )
826
825
if code isa CodeInstance
827
826
if use_const_api (code)
828
827
# in this case function can be inlined to a constant
829
- return ConstantCase (quoted (code. rettype_const))
828
+ return ConstantCase (quoted (code. rettype_const), code )
830
829
end
831
830
return code
832
831
end
833
832
return nothing
834
833
end
835
834
@inline function get_local_result (inf_result:: InferenceResult )
835
+ @assert isdefined (inf_result, :ci_as_edge ) " InferenceResult without ci_as_edge"
836
836
effects = inf_result. ipo_effects
837
837
if is_foldable_nothrow (effects)
838
838
res = inf_result. result
839
839
if isa (res, Const) && is_inlineable_constant (res. val)
840
840
# use constant calling convention
841
- return ConstantCase (quoted (res. val))
841
+ return ConstantCase (quoted (res. val), inf_result . ci_as_edge )
842
842
end
843
843
end
844
- return InferredResult (inf_result. src, effects)
844
+ return InferredResult (inf_result. src, effects, inf_result . ci_as_edge )
845
845
end
846
846
847
847
# the general resolver for usual and const-prop'ed calls
@@ -861,30 +861,28 @@ function resolve_todo(mi::MethodInstance, result::Union{Nothing,InferenceResult,
861
861
inferred_result = get_cached_result (state, mi)
862
862
end
863
863
if inferred_result isa ConstantCase
864
- add_inlining_backedge ! (et, mi )
864
+ add_inlining_edge ! (et, inferred_result . edge )
865
865
return inferred_result
866
866
elseif inferred_result isa InferredResult
867
- (; src, effects) = inferred_result
867
+ (; src, effects, edge ) = inferred_result
868
868
elseif inferred_result isa CodeInstance
869
869
src = @atomic :monotonic inferred_result. inferred
870
870
effects = decode_effects (inferred_result. ipo_purity_bits)
871
+ edge = inferred_result
871
872
else # there is no cached source available, bail out
872
- return compileable_specialization (mi, Effects (), et, info;
873
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
873
+ return compileable_specialization (mi, Effects (), et, info, state)
874
874
end
875
875
876
876
# the duplicated check might have been done already within `analyze_method!`, but still
877
877
# we need it here too since we may come here directly using a constant-prop' result
878
878
if ! OptimizationParams (state. interp). inlining || is_stmt_noinline (flag)
879
- return compileable_specialization (mi, effects, et, info;
880
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
879
+ return compileable_specialization (edge. def, effects, et, info, state)
881
880
end
882
881
883
882
src_inlining_policy (state. interp, src, info, flag) ||
884
- return compileable_specialization (mi, effects, et, info;
885
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
883
+ return compileable_specialization (edge. def, effects, et, info, state)
886
884
887
- add_inlining_backedge ! (et, mi )
885
+ add_inlining_edge ! (et, edge )
888
886
if inferred_result isa CodeInstance
889
887
ir, spec_info, debuginfo = retrieve_ir_for_inlining (inferred_result, src)
890
888
else
@@ -904,7 +902,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U
904
902
905
903
cached_result = get_cached_result (state, mi)
906
904
if cached_result isa ConstantCase
907
- add_inlining_backedge ! (et, mi )
905
+ add_inlining_edge ! (et, cached_result . edge )
908
906
return cached_result
909
907
elseif cached_result isa CodeInstance
910
908
src = @atomic :monotonic cached_result. inferred
@@ -915,7 +913,7 @@ function resolve_todo(mi::MethodInstance, @nospecialize(info::CallInfo), flag::U
915
913
916
914
src_inlining_policy (state. interp, src, info, flag) || return nothing
917
915
ir, spec_info, debuginfo = retrieve_ir_for_inlining (cached_result, src)
918
- add_inlining_backedge ! (et, mi )
916
+ add_inlining_edge ! (et, cached_result )
919
917
return InliningTodo (mi, ir, spec_info, debuginfo, effects)
920
918
end
921
919
@@ -1119,7 +1117,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
1119
1117
# e.g. rewrite `((t::Tuple)...,)` to `t`
1120
1118
nonempty_idx = 0
1121
1119
𝕃ₒ = optimizer_lattice (state. interp)
1122
- for i = (arg_start + 1 ): length (argtypes)
1120
+ for i = (arg_start+ 1 ): length (argtypes)
1123
1121
ti = argtypes[i]
1124
1122
⊑ (𝕃ₒ, ti, Tuple{}) && continue
1125
1123
if ⊑ (𝕃ₒ, ti, Tuple) && nonempty_idx == 0
@@ -1137,7 +1135,7 @@ function inline_apply!(todo::Vector{Pair{Int,Any}},
1137
1135
# Try to figure out the signature of the function being called
1138
1136
# and if rewrite_apply_exprargs can deal with this form
1139
1137
arginfos = MaybeAbstractIterationInfo[]
1140
- for i = (arg_start + 1 ): length (argtypes)
1138
+ for i = (arg_start+ 1 ): length (argtypes)
1141
1139
thisarginfo = nothing
1142
1140
if ! is_valid_type_for_apply_rewrite (argtypes[i], OptimizationParams (state. interp))
1143
1141
isa (info, ApplyCallInfo) || return nothing
@@ -1403,9 +1401,7 @@ function compute_inlining_cases(@nospecialize(info::CallInfo), flag::UInt32, sig
1403
1401
result, match, argtypes, info, flag, state; allow_typevars= true )
1404
1402
end
1405
1403
if ! fully_covered
1406
- atype = argtypes_to_type (sig. argtypes)
1407
- # We will emit an inline MethodError so we need a backedge to the MethodTable
1408
- add_uncovered_edges! (state. edges, info, atype)
1404
+ # We will emit an inline MethodError in this case, but that info already came inference, so we must already have the uncovered edge for it
1409
1405
end
1410
1406
elseif ! isempty (cases)
1411
1407
# if we've not seen all candidates, union split is valid only for dispatch tuples
@@ -1453,30 +1449,28 @@ end
1453
1449
1454
1450
function semiconcrete_result_item (result:: SemiConcreteResult ,
1455
1451
@nospecialize (info:: CallInfo ), flag:: UInt32 , state:: InliningState )
1456
- mi = result. mi
1452
+ mi = result. edge . def
1457
1453
et = InliningEdgeTracker (state)
1458
1454
1459
1455
if (! OptimizationParams (state. interp). inlining || is_stmt_noinline (flag) ||
1460
1456
# For `NativeInterpreter`, `SemiConcreteResult` may be produced for
1461
1457
# a `@noinline`-declared method when it's marked as `@constprop :aggressive`.
1462
1458
# Suppress the inlining here (unless inlining is requested at the callsite).
1463
1459
(is_declared_noinline (mi. def:: Method ) && ! is_stmt_inline (flag)))
1464
- return compileable_specialization (mi, result. effects, et, info;
1465
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
1460
+ return compileable_specialization (mi, result. effects, et, info, state)
1466
1461
end
1467
1462
src_inlining_policy (state. interp, result. ir, info, flag) ||
1468
- return compileable_specialization (mi, result. effects, et, info;
1469
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
1463
+ return compileable_specialization (mi, result. effects, et, info, state)
1470
1464
1471
- add_inlining_backedge ! (et, mi )
1465
+ add_inlining_edge ! (et, result . edge )
1472
1466
preserve_local_sources = OptimizationParams (state. interp). preserve_local_sources
1473
1467
ir, _, debuginfo = retrieve_ir_for_inlining (mi, result. ir, preserve_local_sources)
1474
1468
return InliningTodo (mi, ir, result. spec_info, debuginfo, result. effects)
1475
1469
end
1476
1470
1477
1471
function handle_semi_concrete_result! (cases:: Vector{InliningCase} , result:: SemiConcreteResult ,
1478
1472
match:: MethodMatch , @nospecialize (info:: CallInfo ), flag:: UInt32 , state:: InliningState )
1479
- mi = result. mi
1473
+ mi = result. edge . def
1480
1474
validate_sparams (mi. sparam_vals) || return false
1481
1475
item = semiconcrete_result_item (result, info, flag, state)
1482
1476
item === nothing && return false
@@ -1499,11 +1493,10 @@ function concrete_result_item(result::ConcreteResult, @nospecialize(info::CallIn
1499
1493
invokesig:: Union{Nothing,Vector{Any}} = nothing )
1500
1494
if ! may_inline_concrete_result (result)
1501
1495
et = InliningEdgeTracker (state, invokesig)
1502
- return compileable_specialization (result. mi, result. effects, et, info;
1503
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
1496
+ return compileable_specialization (result. edge. def, result. effects, et, info, state)
1504
1497
end
1505
1498
@assert result. effects === EFFECTS_TOTAL
1506
- return ConstantCase (quoted (result. result))
1499
+ return ConstantCase (quoted (result. result), result . edge )
1507
1500
end
1508
1501
1509
1502
function handle_cases! (todo:: Vector{Pair{Int,Any}} , ir:: IRCode , idx:: Int , stmt:: Expr ,
@@ -1552,11 +1545,16 @@ function handle_modifyop!_call!(ir::IRCode, idx::Int, stmt::Expr, info::ModifyOp
1552
1545
info isa MethodResultPure && (info = info. info)
1553
1546
info isa ConstCallInfo && (info = info. call)
1554
1547
info isa MethodMatchInfo || return nothing
1555
- length (info. results) == 1 || return nothing
1548
+ length (info. edges) == length (info . results) == 1 || return nothing
1556
1549
match = info. results[1 ]:: MethodMatch
1557
1550
match. fully_covers || return nothing
1558
- case = compileable_specialization (match, Effects (), InliningEdgeTracker (state), info;
1559
- compilesig_invokes= OptimizationParams (state. interp). compilesig_invokes)
1551
+ edge = info. edges[1 ]
1552
+ if edge === nothing
1553
+ edge = specialize_method (match)
1554
+ else
1555
+ edge = edge. def
1556
+ end
1557
+ case = compileable_specialization (edge, Effects (), InliningEdgeTracker (state), info, state)
1560
1558
case === nothing && return nothing
1561
1559
stmt. head = :invoke_modify
1562
1560
pushfirst! (stmt. args, case. invoke)
0 commit comments