fix: memory-leak due to unbound type-parameter recursion

FakeTypeName expanded a type's parameters recursively with no size bound.
A few types explode into enormous trees: type-domain packages whose types
recurse into themselves (e.g. TypeDomainNaturalNumbers — naturals as nested
types, rationals as continued fractions), and even some Base/LinearAlgebra
signatures whose `where`-bounded `Union`s reach tens of thousands of nodes.
This produced multi-gigabyte cache files (2.2 GB for TypeDomainNaturalNumbers)
that exhausted memory — OOMing the machine — when read back into the store.

Cap expansion with a per-type budget of expanded DataTypes
(MAX_EXPANDED_TYPES = 128): once it is spent, a type's remaining parameters
are dropped and only its name is kept. The budget bounds any shape of
explosion (depth, width, or indirect recursion through variable bounds),
while ordinary types stay far below it and are unaffected.

Author: Patrick Häcker

src/faketypes.jl

@@ -14,24 +14,44 @@ struct FakeTypeName
     parameters::Vector{Any}
 end
 
-function FakeTypeName(@nospecialize(x))
+# Type parameters are expanded recursively so cached signatures keep their
+# structure, but a few types expand into huge trees — type-domain packages
+# whose types recurse into themselves (naturals as nested `NonnegativeInteger`,
+# rationals as continued fractions), and even some Base/LinearAlgebra
+# signatures whose `where`-bounded `Union`s reach tens of thousands of nodes —
+# which unbounded would produce multi-gigabyte caches that exhaust memory on
+# read. To prevent that, `budget` caps how many `DataType`s are expanded while
+# building one type: once it is spent the remaining parameters are dropped,
+# keeping only their names. The limit sits far above any ordinary type, so only
+# these outliers are truncated; the serializer's own MAX_DEPTH guard is a
+# separate cycle check, not a size bound.
+const MAX_EXPANDED_TYPES = 128
+
+mutable struct ExpandBudget
+    remaining::Int
+end
+ExpandBudget() = ExpandBudget(MAX_EXPANDED_TYPES)
+
+function FakeTypeName(@nospecialize(x), budget::ExpandBudget=ExpandBudget())
     @static if !(Vararg isa Type)
         x isa typeof(Vararg) && return FakeTypeofVararg(x)
     end
     if x isa DataType
         xname = x.name
-        xnamename = xname.name
-        ft = FakeTypeName(VarRef(VarRef(x.name.module), xnamename), [])
-        for p in x.parameters
-            push!(ft.parameters, _parameter(p))
+        ft = FakeTypeName(VarRef(VarRef(xname.module), xname.name), [])
+        if budget.remaining > 0
+            budget.remaining -= 1
+            for p in x.parameters
+                push!(ft.parameters, _parameter(p, budget))
+            end
         end
         ft
     elseif x isa Union
-        FakeUnion(x)
+        FakeUnion(x, budget)
     elseif x isa UnionAll
-        FakeUnionAll(x)
+        FakeUnionAll(x, budget)
     elseif x isa TypeVar
-        FakeTypeVar(x)
+        FakeTypeVar(x, budget)
     elseif x isa Core.TypeofBottom
         FakeTypeofBottom()
     elseif x isa Module
@@ -46,28 +66,28 @@ struct FakeUnion
     a
     b
 end
-FakeUnion(u::Union) = FakeUnion(FakeTypeName(u.a), FakeTypeName(u.b))
+FakeUnion(u::Union, budget::ExpandBudget=ExpandBudget()) = FakeUnion(FakeTypeName(u.a, budget), FakeTypeName(u.b, budget))
 struct FakeTypeVar
     name::Symbol
     lb
     ub
 end
-FakeTypeVar(tv::TypeVar) = FakeTypeVar(tv.name, FakeTypeName(tv.lb), FakeTypeName(tv.ub))
+FakeTypeVar(tv::TypeVar, budget::ExpandBudget=ExpandBudget()) = FakeTypeVar(tv.name, FakeTypeName(tv.lb, budget), FakeTypeName(tv.ub, budget))
 struct FakeUnionAll
     var::FakeTypeVar
     body::Any
 end
-FakeUnionAll(ua::UnionAll) = FakeUnionAll(FakeTypeVar(ua.var), FakeTypeName(ua.body))
+FakeUnionAll(ua::UnionAll, budget::ExpandBudget=ExpandBudget()) = FakeUnionAll(FakeTypeVar(ua.var, budget), FakeTypeName(ua.body, budget))
 
-function _parameter(@nospecialize(p))
+function _parameter(@nospecialize(p), budget::ExpandBudget=ExpandBudget())
     if p isa Union{Int,Symbol,Bool,Char}
         p
     elseif !(p isa Type) && isbitstype(typeof(p))
         0
     elseif p isa Tuple
-        _parameter.(p)
+        _parameter.(p, Ref(budget))
     else
-        FakeTypeName(p)
+        FakeTypeName(p, budget)
     end
 end
 

test/runtests.jl

@@ -280,6 +280,26 @@ end
     end
 end
 
+@testitem "recursive type parameters are bounded" begin
+    using SymbolServer: FakeTypeName, ExpandBudget
+
+    # Number of DataTypes that were expanded (i.e. kept their parameters).
+    expanded(x) = x isa FakeTypeName && !isempty(x.parameters) ? 1 + sum(expanded, x.parameters) : 0
+
+    limit = 8
+    deep = foldl((T, _) -> Tuple{T}, 1:100; init = Int)   # Tuple{Tuple{…{Int}}}
+    wide = Tuple{ntuple(i -> Val{i}, 100)...}              # Tuple{Val{1},…,Val{100}}
+
+    # However a type explodes — by depth or by width — expansion stops at the budget.
+    for T in (deep, wide)
+        @test expanded(FakeTypeName(T, ExpandBudget(limit))) <= limit
+    end
+
+    # An ordinary type has far fewer nodes than the budget, so nothing is dropped.
+    ordinary = Dict{String,Vector{Tuple{Int,Float64}}}
+    @test FakeTypeName(ordinary, ExpandBudget(limit)) == FakeTypeName(ordinary)
+end
+
 @testitem "Pipe names" begin
     import UUIDs