Rewrite subtype_tuple to fix extra cases

Author: Keno

base/compiler/typelimits.jl

@@ -92,10 +92,13 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
         return t # easy case
     elseif isa(t, DataType) && isempty(t.parameters)
         return t # fast path: unparameterized are always simple
-    elseif isa(unwrap_unionall(t), DataType) && isa(c, Type) && c !== Union{} && c <: t
-        return t # t is already wider than the comparison in the type lattice
-    elseif is_derived_type_from_any(unwrap_unionall(t), sources, depth)
-        return t # t isn't something new
+    else
+        ut = unwrap_unionall(t)
+        if isa(ut, DataType) && ut.name !== unwrap_unionall(Vararg).name && isa(c, Type) && c !== Union{} && c <: t
+            return t # t is already wider than the comparison in the type lattice
+        elseif is_derived_type_from_any(ut, sources, depth)
+            return t # t isn't something new
+        end
     end
     # peel off (and ignore) wrappers - they contribute no useful information, so we don't need to consider their size
     # first attempt to turn `c` into a type that contributes meaningful information

src/jltypes.c

@@ -1085,10 +1085,19 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value
                 continue;
             if (jl_is_datatype(pi))
                 continue;
+            if (jl_is_vararg_type(pi)) {
+                pi = jl_unwrap_vararg(pi);
+                if (jl_has_free_typevars(pi))
+                    continue;
+            }
             // normalize types equal to wrappers (prepare for wrapper_id)
             jl_value_t *tw = extract_wrapper(pi);
             if (tw && tw != pi && (tn != jl_type_typename || jl_typeof(pi) == jl_typeof(tw)) &&
                     jl_types_equal(pi, tw)) {
+                if (jl_is_vararg_type(iparams[i])) {
+                    tw = jl_wrap_vararg(tw, jl_tparam1(jl_unwrap_unionall(iparams[i])));
+                    tw = jl_rewrap_unionall(tw, iparams[i]);
+                }
                 iparams[i] = tw;
                 if (p) jl_gc_wb(p, tw);
             }