fix #30335, regression in intersection of unions of typevars

Author: JeffBezanson

src/subtype.c

@@ -1333,12 +1333,17 @@ static jl_value_t *intersect_all(jl_value_t *x, jl_value_t *y, jl_stenv_t *e);
 // intersect in nested union environment, similar to subtype_ccheck
 static jl_value_t *intersect_aside(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int depth)
 {
-    jl_value_t *res;
+    // band-aid for #30335
+    if (x == (jl_value_t*)jl_any_type && !jl_is_typevar(y))
+        return y;
+    if (y == (jl_value_t*)jl_any_type && !jl_is_typevar(x))
+        return x;
+
     int savedepth = e->invdepth;
     jl_unionstate_t oldRunions = e->Runions;
     e->invdepth = depth;
 
-    res = intersect_all(x, y, e);
+    jl_value_t *res = intersect_all(x, y, e);
 
     e->Runions = oldRunions;
     e->invdepth = savedepth;
@@ -1466,6 +1471,8 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int
         return (jl_value_t*)b;
     }
     else if (bb->constraintkind == 2) {
+        // TODO: removing this case fixes many test_brokens in test/subtype.jl
+        // but breaks other tests.
         if (!subtype_in_env(a, bb->ub, e))
             return jl_bottom_type;
         jl_value_t *lb = simple_join(bb->lb, a);
@@ -1624,6 +1631,10 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
                 // you can construct `T{x} where x` even if T's parameter is actually
                 // limited. in that case we might get an invalid instantiation here.
                 res = jl_substitute_var(res, vb->var, varval);
+                // simplify chains of UnionAlls where bounds become equal
+                while (jl_is_unionall(res) && obviously_egal(((jl_unionall_t*)res)->var->lb,
+                                                             ((jl_unionall_t*)res)->var->ub))
+                    res = jl_instantiate_unionall((jl_unionall_t*)res, ((jl_unionall_t*)res)->var->lb);
             }
             JL_CATCH {
                 res = jl_bottom_type;

test/subtype.jl

@@ -1399,3 +1399,24 @@ end
 @testintersect(Tuple{Pair{Int64,2}, NTuple},
                Tuple{Pair{F,N},Tuple{Vararg{F,N}}} where N where F,
                Tuple{Pair{Int64,2}, Tuple{Int64,Int64}})
+
+# issue #30335
+@testintersect(Tuple{Any,Rational{Int},Int},
+               Tuple{LT,R,I} where LT<:Union{I, R} where R<:Rational{I} where I<:Integer,
+               Tuple{LT,Rational{Int},Int} where LT<:Union{Rational{Int},Int})
+
+#@testintersect(Tuple{Any,Tuple{Int},Int},
+#               Tuple{LT,R,I} where LT<:Union{I, R} where R<:Tuple{I} where I<:Integer,
+#               Tuple{LT,Tuple{Int},Int} where LT<:Union{Tuple{Int},Int})
+# fails due to this:
+let U = Tuple{Union{LT, LT1},Union{R, R1},Int64} where LT1<:R1 where R1<:Tuple{Int64} where LT<:Int64 where R<:Tuple{Int64},
+    U2 = Union{Tuple{LT,R,Int64} where LT<:Int64 where R<:Tuple{Int64}, Tuple{LT,R,Int64} where LT<:R where R<:Tuple{Int64}},
+    V = Tuple{Union{Tuple{Int},Int},Tuple{Int},Int},
+    V2 = Tuple{L,Tuple{Int},Int} where L<:Union{Tuple{Int},Int}
+    @test U == U2
+    @test U == V
+    @test U == V2
+    @test V == V2
+    @test U2 == V
+    @test_broken U2 == V2
+end