Fix Lua GC reentrancy leak: eagerly release C++ refs via tracking lists

Lua userdata (ScriptedArtboard, ScriptedViewModel) hold rcp<> refs to
C++ objects (DataContext, ViewModelInstance, File). When a parent artboard
is destroyed, nested GC calls from within finalizers are no-ops due to
Luau's non-reentrant GC, leaving these refs leaked. Fix by tracking all
instances in linked lists and eagerly nullifying their C++ refs before
relying on GC.

Author: mfazekas

include/rive/lua/rive_lua_libs.hpp

@@ -572,7 +572,8 @@ class ScriptedRenderer
 class ScriptReffedArtboard : public RefCnt<ScriptReffedArtboard>
 {
 public:
-    ScriptReffedArtboard(rcp<File> file,
+    ScriptReffedArtboard(lua_State* L,
+                         rcp<File> file,
                          std::unique_ptr<ArtboardInstance>&& artboardInstance,
                          rcp<ViewModelInstance> viewModelInstance,
                          rcp<DataContext> parentDataContext);
@@ -582,12 +583,18 @@ class ScriptReffedArtboard : public RefCnt<ScriptReffedArtboard>
     Artboard* artboard();
     StateMachineInstance* stateMachine();
     rcp<ViewModelInstance> viewModelInstance() { return m_viewModelInstance; }
+    void releaseReferences();
+    static void releaseAll(lua_State* state);
 
 private:
     rcp<File> m_file;
     std::unique_ptr<ArtboardInstance> m_artboard;
     std::unique_ptr<StateMachineInstance> m_stateMachine;
     rcp<ViewModelInstance> m_viewModelInstance;
+    lua_State* m_luaState = nullptr;
+    ScriptReffedArtboard* m_trackNext = nullptr;
+    ScriptReffedArtboard* m_trackPrev = nullptr;
+    static ScriptReffedArtboard* s_head;
 };
 
 class ScriptedArtboard
@@ -630,7 +637,6 @@ class ScriptedArtboard
 private:
     lua_State* m_state = nullptr;
     rcp<ScriptReffedArtboard> m_scriptReffedArtboard = nullptr;
-    rcp<DataContext> m_dataContext = nullptr;
     int m_dataRef = 0;
 };
 
@@ -704,11 +710,17 @@ class ScriptedViewModel
         return m_viewModelInstance;
     }
 
+    void releaseReferences();
+    static void releaseAll(lua_State* state);
+
 private:
     lua_State* m_state;
     rcp<ViewModel> m_viewModel;
     rcp<ViewModelInstance> m_viewModelInstance;
     std::unordered_map<std::string, int> m_propertyRefs;
+    ScriptedViewModel* m_trackNext = nullptr;
+    ScriptedViewModel* m_trackPrev = nullptr;
+    static ScriptedViewModel* s_head;
 };
 
 class ScriptedPropertyViewModel : public ScriptedProperty

include/rive/scripted/scripted_object.hpp

@@ -59,6 +59,9 @@ class ScriptedObject : public FileAssetReferencer,
     lua_State* state() const { return m_vm ? m_vm->state() : nullptr; }
 #endif
     void scriptDispose();
+#ifdef WITH_RIVE_SCRIPTING
+    static void collectLuaGarbage(lua_State* state);
+#endif
     virtual bool addScriptedDirt(ComponentDirt value, bool recurse = false) = 0;
     void setAsset(rcp<FileAsset> asset) override;
     static ScriptedObject* from(Core* object);

src/artboard.cpp

@@ -141,6 +141,13 @@ Artboard::~Artboard()
         return vmObjects.count(object) != 0;
     };
 
+#ifdef WITH_RIVE_SCRIPTING
+    lua_State* luaState = nullptr;
+    if (!m_ScriptedObjects.empty())
+    {
+        luaState = m_ScriptedObjects[0]->state();
+    }
+#endif
     // Second pass: delete non-VM objects.
     for (auto object : m_Objects)
     {
@@ -154,6 +161,9 @@ Artboard::~Artboard()
         }
         delete object;
     }
+#ifdef WITH_RIVE_SCRIPTING
+    ScriptedObject::collectLuaGarbage(luaState);
+#endif
     for (auto object : m_invalidObjects)
     {
         if (object == nullptr)

src/lua/lua_artboards.cpp

@@ -15,14 +15,18 @@
 
 using namespace rive;
 
+ScriptReffedArtboard* ScriptReffedArtboard::s_head = nullptr;
+
 ScriptReffedArtboard::ScriptReffedArtboard(
+    lua_State* L,
     rcp<File> file,
     std::unique_ptr<ArtboardInstance>&& artboardInstance,
     rcp<ViewModelInstance> viewModelInstance,
     rcp<DataContext> parentDataContext) :
     m_file(file),
     m_artboard(std::move(artboardInstance)),
-    m_stateMachine(m_artboard->defaultStateMachine())
+    m_stateMachine(m_artboard->defaultStateMachine()),
+    m_luaState(L)
 {
     if (viewModelInstance)
     {
@@ -45,18 +49,58 @@ ScriptReffedArtboard::ScriptReffedArtboard(
             m_stateMachine->bindViewModelInstance(m_viewModelInstance);
         }
     }
+    // Track this instance for cleanup during artboard destruction
+    m_trackNext = s_head;
+    m_trackPrev = nullptr;
+    if (s_head)
+    {
+        s_head->m_trackPrev = this;
+    }
+    s_head = this;
 }
 
 ScriptReffedArtboard::~ScriptReffedArtboard()
 {
-    // Make sure state machine is deleted before artboard since
-    // StateMachineInstance destructor accesses the artboard.
+    // Remove from tracking list
+    if (m_trackPrev)
+    {
+        m_trackPrev->m_trackNext = m_trackNext;
+    }
+    else if (s_head == this)
+    {
+        s_head = m_trackNext;
+    }
+    if (m_trackNext)
+    {
+        m_trackNext->m_trackPrev = m_trackPrev;
+    }
+    releaseReferences();
+}
+
+void ScriptReffedArtboard::releaseReferences()
+{
+    // StateMachine must be deleted before artboard since its destructor
+    // accesses the artboard.
     m_stateMachine = nullptr;
-    // Make sure artboard is deleted before file.
     m_artboard = nullptr;
+    m_viewModelInstance = nullptr;
     m_file = nullptr;
 }
 
+void ScriptReffedArtboard::releaseAll(lua_State* state)
+{
+    ScriptReffedArtboard* current = s_head;
+    while (current)
+    {
+        ScriptReffedArtboard* next = current->m_trackNext;
+        if (current->m_luaState == state)
+        {
+            current->releaseReferences();
+        }
+        current = next;
+    }
+}
+
 rive::rcp<rive::File> ScriptReffedArtboard::file() { return m_file; }
 
 Artboard* ScriptReffedArtboard::artboard() { return m_artboard.get(); }
@@ -264,7 +308,7 @@ int ScriptedArtboard::instance(lua_State* L,
                                   m_scriptReffedArtboard->file(),
                                   std::move(artboardInstance),
                                   viewModelInstance,
-                                  m_dataContext);
+                                  artboard()->dataContext());
     return 1;
 }
 
@@ -384,11 +428,11 @@ ScriptedArtboard::ScriptedArtboard(
     rcp<DataContext> dataContext) :
     m_state(L),
     m_scriptReffedArtboard(
-        make_rcp<ScriptReffedArtboard>(file,
+        make_rcp<ScriptReffedArtboard>(L,
+                                       file,
                                        std::move(artboardInstance),
                                        viewModelInstance,
-                                       dataContext)),
-    m_dataContext(dataContext)
+                                       dataContext))
 {}
 
 ScriptedArtboard::~ScriptedArtboard()

src/lua/lua_properties.cpp

@@ -245,18 +245,61 @@ void ScriptedPropertyViewModel::setValue(ScriptedViewModel* scriptedViewModel)
     }
 }
 
+ScriptedViewModel* ScriptedViewModel::s_head = nullptr;
+
 ScriptedViewModel::ScriptedViewModel(lua_State* L,
                                      rcp<ViewModel> viewModel,
                                      rcp<ViewModelInstance> viewModelInstance) :
     m_state(L), m_viewModel(viewModel), m_viewModelInstance(viewModelInstance)
-{}
+{
+    m_trackNext = s_head;
+    m_trackPrev = nullptr;
+    if (s_head)
+    {
+        s_head->m_trackPrev = this;
+    }
+    s_head = this;
+}
 
 ScriptedViewModel::~ScriptedViewModel()
 {
+    if (m_trackPrev)
+    {
+        m_trackPrev->m_trackNext = m_trackNext;
+    }
+    else if (s_head == this)
+    {
+        s_head = m_trackNext;
+    }
+    if (m_trackNext)
+    {
+        m_trackNext->m_trackPrev = m_trackPrev;
+    }
     for (auto itr : m_propertyRefs)
     {
         lua_unref(m_state, itr.second);
     }
+    releaseReferences();
+}
+
+void ScriptedViewModel::releaseReferences()
+{
+    m_viewModel = nullptr;
+    m_viewModelInstance = nullptr;
+}
+
+void ScriptedViewModel::releaseAll(lua_State* state)
+{
+    ScriptedViewModel* current = s_head;
+    while (current)
+    {
+        ScriptedViewModel* next = current->m_trackNext;
+        if (current->m_state == state)
+        {
+            current->releaseReferences();
+        }
+        current = next;
+    }
 }
 
 int ScriptedViewModel::instance(lua_State* L)

src/scripted/scripted_object.cpp

@@ -355,14 +355,23 @@ void ScriptedObject::scriptDispose()
         lua_unref(L, m_self);
         disposeScriptedContext();
 #ifdef TESTING
-        // Force GC to collect any ScriptedArtboard instances created via
-        // instance()
         lua_gc(L, LUA_GCCOLLECT, 0);
 #endif
     }
     m_vm = nullptr;
     m_self = 0;
 }
+
+void ScriptedObject::collectLuaGarbage(lua_State* state)
+{
+    if (state != nullptr)
+    {
+        ScriptReffedArtboard::releaseAll(state);
+        ScriptedViewModel::releaseAll(state);
+        lua_gc(state, LUA_GCCOLLECT, 0);
+        lua_gc(state, LUA_GCCOLLECT, 0);
+    }
+}
 #else
 void ScriptedObject::setArtboardInput(std::string name, Artboard* artboard) {}