gf.c: Fix backedge de-duplication bug (#58106)

This code was checking for the old edge type (`MethodInstance`) instead
of the new one (`CodeInstance`), causing duplicate non-invoke edges to
accumulate in our `backedges`.

Author: topolarity

Compiler/src/typeinfer.jl

@@ -706,6 +706,8 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector)
             # `invoke` edge
             elseif isa(callee, CodeInstance)
                 callee = get_ci_mi(callee)
+            else
+                callee = callee::MethodInstance
             end
             ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller)
             i += 2

Compiler/test/invalidation.jl

@@ -162,6 +162,43 @@ begin
     @test "42" == String(take!(GLOBAL_BUFFER))
 end
 
+begin
+    deduped_callee(x::Int) = @noinline rand(Int)
+    deduped_caller1(x::Int) = @noinline deduped_callee(x)
+    deduped_caller2(x::Int) = @noinline deduped_callee(x)
+
+    # run inference on both `deduped_callerx` and `deduped_callee`
+    let (src, rt) = code_typed((Int,); interp=InvalidationTester()) do x
+            @inline deduped_caller1(x)
+            @inline deduped_caller2(x)
+        end |> only
+        @test rt === Int
+        @test any(isinvoke(:deduped_callee), src.code)
+    end
+
+    # Verify that adding the backedge again does not actually add a new backedge
+    let mi1 = Base.method_instance(deduped_caller1, (Int,)),
+        mi2 = Base.method_instance(deduped_caller2, (Int,)),
+        ci1 = mi1.cache
+        ci2 = mi2.cache
+
+        callee_mi = Base.method_instance(deduped_callee, (Int,))
+
+        # Inference should have added the callers to the callee's backedges
+        @test ci1 in callee_mi.backedges
+        @test ci2 in callee_mi.backedges
+
+        N = length(callee_mi.backedges)
+        Core.Compiler.store_backedges(ci1, Core.svec(callee_mi))
+        Core.Compiler.store_backedges(ci2, Core.svec(callee_mi))
+        N′ = length(callee_mi.backedges)
+
+        # The number of backedges should not be affected by an additional store,
+        # since de-duplication should have noticed the edge is already tracked
+        @test N == N′
+    end
+end
+
 # we can avoid adding backedge even if the callee's return type is not the top
 # when the return value is not used within the caller
 begin take!(GLOBAL_BUFFER)

src/gf.c

@@ -2004,7 +2004,7 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee,
                 if (ciedge != (jl_value_t*)caller)
                     continue;
                 jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(backedges, i - 1) : NULL;
-                if (invokeTypes && jl_is_method_instance(invokeTypes))
+                if (invokeTypes && jl_is_code_instance(invokeTypes))
                     invokeTypes = NULL;
                 if ((invokesig == NULL && invokeTypes == NULL) ||
                     (invokesig && invokeTypes && jl_types_equal(invokesig, invokeTypes))) {