fix #33954, recursion through field types in is_derived_type

base/compiler/typelimits.jl

@@ -57,21 +57,6 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
         for p in cP
             is_derived_type(t, p, mindepth) && return true
         end
-        if isconcretetype(c) && isbitstype(c)
-            # see if it was extracted from a fieldtype
-            # however, only look through types that can be inlined
-            # to ensure monotonicity of derivation
-            # since we know that for immutable, concrete, bits types,
-            # the field types must have been constructed prior to the type,
-            # it cannot have a reference cycle in the type graph
-            cF = c.types
-            for f in cF
-                # often a parameter is also a field type; avoid searching twice
-                if !contains_is(c.parameters, f)
-                    is_derived_type(t, f, mindepth) && return true
-                end
-            end
-        end
     end
     return false
 end

src/datatype.c

@@ -306,6 +306,8 @@ static int references_name(jl_value_t *p, jl_typename_t *name) JL_NOTSAFEPOINT
     if (jl_is_datatype(p)) {
         if (((jl_datatype_t*)p)->name == name)
             return 1;
+        if (((jl_datatype_t*)p)->layout && jl_datatype_nfields(p) == 0)
+            return 0;
         size_t i, l = jl_nparams(p);
         for (i = 0; i < l; i++) {
             if (references_name(jl_tparam(p, i), name))
@@ -401,6 +403,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
             jl_value_t *fld = jl_svecref(w->types, i);
             if (references_name(fld, w->name)) {
                 isinlinealloc = 0;
+                isbitstype = 0;
                 break;
             }
         }

test/compiler/inference.jl

@@ -2485,3 +2485,10 @@ end
 # constant prop of `Symbol("")`
 f_getf_computed_symbol(p) = getfield(p, Symbol("first"))
 @test Base.return_types(f_getf_computed_symbol, Tuple{Pair{Int8,String}}) == [Int8]
+
+# issue #33954
+struct X33954
+    x::Ptr{X33954}
+end
+f33954(x) = rand(Bool) ? f33954((x,)) : x
+@test Base.return_types(f33954, Tuple{X33954})[1] >: X33954

test/core.jl

@@ -7140,3 +7140,19 @@ end
 struct SplatBadIterate; end
 Base.iterate(s::SplatBadIterate, args...) = ()
 @test_throws BoundsError (SplatBadIterate()...,)
+
+# issue #33954, layout with circular type parameters but not fields
+struct P33954{T}
+end
+struct A33954
+    x::P33954{A33954}
+end
+@test isbitstype(Tuple{A33954})
+struct Q33954{T}
+    x::Int
+end
+struct B33954
+    x::Q33954{B33954}
+end
+@test_broken isbitstype(Tuple{B33954})
+@test_broken isbitstype(B33954)