fix #33954, recursion through field types in is_derived_type

base/compiler/typelimits.jl

@@ -29,17 +29,20 @@ end
 
 # try to find `type` somewhere in `comparison` type
 # at a minimum nesting depth of `mindepth`
-function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
+function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int, depthlimit::Int = 10)
+    if depthlimit <= 0
+        return false
+    end
     if t === c
         return mindepth <= 1
     end
     if isa(c, Union)
         # see if it is one of the elements of the union
-        return is_derived_type(t, c.a, mindepth) || is_derived_type(t, c.b, mindepth)
+        return is_derived_type(t, c.a, mindepth, depthlimit) || is_derived_type(t, c.b, mindepth, depthlimit)
     elseif isa(c, UnionAll)
         # see if it is derived from the body
         # also handle the var here, since this construct bounds the mindepth to the smallest possible value
-        return is_derived_type(t, c.var.ub, mindepth) || is_derived_type(t, c.body, mindepth)
+        return is_derived_type(t, c.var.ub, mindepth, depthlimit) || is_derived_type(t, c.body, mindepth, depthlimit)
     elseif isa(c, DataType)
         if mindepth > 0
             mindepth -= 1
@@ -55,9 +58,10 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
         # see if it was extracted from a type parameter
         cP = c.parameters
         for p in cP
-            is_derived_type(t, p, mindepth) && return true
+            is_derived_type(t, p, mindepth, depthlimit) && return true
         end
-        if isconcretetype(c) && isbitstype(c)
+        # for tuples, parameters === fieldtypes
+        if c.name !== Tuple.name && isconcretetype(c) && isbitstype(c)
             # see if it was extracted from a fieldtype
             # however, only look through types that can be inlined
             # to ensure monotonicity of derivation
@@ -68,7 +72,7 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
             for f in cF
                 # often a parameter is also a field type; avoid searching twice
                 if !contains_is(c.parameters, f)
-                    is_derived_type(t, f, mindepth) && return true
+                    is_derived_type(t, f, mindepth, depthlimit - 1) && return true
                 end
             end
         end

src/datatype.c

@@ -306,6 +306,8 @@ static int references_name(jl_value_t *p, jl_typename_t *name) JL_NOTSAFEPOINT
     if (jl_is_datatype(p)) {
         if (((jl_datatype_t*)p)->name == name)
             return 1;
+        if (jl_is_primitivetype(p))
+            return 0;
         size_t i, l = jl_nparams(p);
         for (i = 0; i < l; i++) {
             if (references_name(jl_tparam(p, i), name))
@@ -523,6 +525,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
         //else if (st->layout->fielddesc_type != 0) // GC only implements support for this
         //    isinlinealloc = 0;
         isinlinealloc = 0;
+        isbitstype = 0;
     }
     st->isbitstype = isbitstype;
     st->isinlinealloc = isinlinealloc;

test/compiler/inference.jl

@@ -2485,3 +2485,10 @@ end
 # constant prop of `Symbol("")`
 f_getf_computed_symbol(p) = getfield(p, Symbol("first"))
 @test Base.return_types(f_getf_computed_symbol, Tuple{Pair{Int8,String}}) == [Int8]
+
+# issue #33954
+struct X33954
+    x::Ptr{X33954}
+end
+f33954(x) = rand(Bool) ? f33954((x,)) : x
+@test Base.return_types(f33954, Tuple{X33954})[1] >: X33954