WIP: occlusion culling + MASSIVE memory leak (30GB after 30 minutes)

sheazywi · sheazywi · commit 20afc5b9393b · 2026-05-18T23:42:04.000-04:00
diff --git a/Core/Source/Lux/Project/Project.h b/Core/Source/Lux/Project/Project.h
@@ -53,6 +53,7 @@ namespace Lux
 	struct ProjectSceneRendererSettings
 	{
 		bool EnableFrustumCulling = true;
+		bool EnableOcclusionCulling = true;
 		bool EnableGPUDrivenRendering = true;
 		bool EnableGTAO = true;
 		bool GTAOBentNormals = false;
@@ -62,6 +63,7 @@ namespace Lux
 		bool EnableJumpFlood = true;
 
 		bool SoftShadows = true;
+		bool EnableShadowCulling = true;
 		float MaxShadowDistance = 200.0f;
 		float ShadowFade = 25.0f;
 		float ShadowCascadeSplitLambda = 0.92f;
diff --git a/Core/Source/Lux/Project/ProjectRuntimeFormat.h b/Core/Source/Lux/Project/ProjectRuntimeFormat.h
@@ -11,7 +11,7 @@ namespace Lux
 		struct FileHeader
 		{
 			char Header[4] = { 'L', 'P', 'R', 'J' };
-			uint32_t Version = 2;
+			uint32_t Version = 3;
 		};
 
 		struct Audio
diff --git a/Core/Source/Lux/Project/ProjectSerializer.cpp b/Core/Source/Lux/Project/ProjectSerializer.cpp
@@ -118,6 +118,7 @@ namespace Lux
 			out << YAML::Key << "Rendering" << YAML::Value;
 			out << YAML::BeginMap;
 			out << YAML::Key << "FrustumCulling" << YAML::Value << settings.EnableFrustumCulling;
+			out << YAML::Key << "OcclusionCulling" << YAML::Value << settings.EnableOcclusionCulling;
 			out << YAML::Key << "GPUDrivenRendering" << YAML::Value << settings.EnableGPUDrivenRendering;
 			out << YAML::Key << "GTAO" << YAML::Value << settings.EnableGTAO;
 			out << YAML::Key << "GTAOBentNormals" << YAML::Value << settings.GTAOBentNormals;
@@ -130,6 +131,7 @@ namespace Lux
 			out << YAML::Key << "Shadows" << YAML::Value;
 			out << YAML::BeginMap;
 			out << YAML::Key << "SoftShadows" << YAML::Value << settings.SoftShadows;
+			out << YAML::Key << "ShadowCulling" << YAML::Value << settings.EnableShadowCulling;
 			out << YAML::Key << "MaxDistance" << YAML::Value << settings.MaxShadowDistance;
 			out << YAML::Key << "DistanceFade" << YAML::Value << settings.ShadowFade;
 			out << YAML::Key << "SplitLambda" << YAML::Value << settings.ShadowCascadeSplitLambda;
@@ -177,6 +179,7 @@ namespace Lux
 			if (auto rendering = node["Rendering"])
 			{
 				settings.EnableFrustumCulling = rendering["FrustumCulling"].as<bool>(settings.EnableFrustumCulling);
+				settings.EnableOcclusionCulling = rendering["OcclusionCulling"].as<bool>(settings.EnableOcclusionCulling);
 				settings.EnableGPUDrivenRendering = rendering["GPUDrivenRendering"].as<bool>(settings.EnableGPUDrivenRendering);
 				settings.EnableGTAO = rendering["GTAO"].as<bool>(settings.EnableGTAO);
 				settings.GTAOBentNormals = rendering["GTAOBentNormals"].as<bool>(settings.GTAOBentNormals);
@@ -189,6 +192,7 @@ namespace Lux
 			if (auto shadows = node["Shadows"])
 			{
 				settings.SoftShadows = shadows["SoftShadows"].as<bool>(settings.SoftShadows);
+				settings.EnableShadowCulling = shadows["ShadowCulling"].as<bool>(settings.EnableShadowCulling);
 				settings.MaxShadowDistance = shadows["MaxDistance"].as<float>(settings.MaxShadowDistance);
 				settings.ShadowFade = shadows["DistanceFade"].as<float>(settings.ShadowFade);
 				settings.ShadowCascadeSplitLambda = shadows["SplitLambda"].as<float>(settings.ShadowCascadeSplitLambda);
@@ -229,6 +233,7 @@ namespace Lux
 		void WriteSceneRendererRuntimeSettings(FileStreamWriter& serializer, const ProjectSceneRendererSettings& settings)
 		{
 			serializer.WriteRaw(settings.EnableFrustumCulling);
+			serializer.WriteRaw(settings.EnableOcclusionCulling);
 			serializer.WriteRaw(settings.EnableGPUDrivenRendering);
 			serializer.WriteRaw(settings.EnableGTAO);
 			serializer.WriteRaw(settings.GTAOBentNormals);
@@ -238,6 +243,7 @@ namespace Lux
 			serializer.WriteRaw(settings.EnableJumpFlood);
 
 			serializer.WriteRaw(settings.SoftShadows);
+			serializer.WriteRaw(settings.EnableShadowCulling);
 			serializer.WriteRaw(settings.MaxShadowDistance);
 			serializer.WriteRaw(settings.ShadowFade);
 			serializer.WriteRaw(settings.ShadowCascadeSplitLambda);
@@ -262,9 +268,11 @@ namespace Lux
 			serializer.WriteRaw(settings.SSRDepthTolerance);
 		}
 
-		void ReadSceneRendererRuntimeSettings(FileStreamReader& stream, ProjectSceneRendererSettings& settings)
+		void ReadSceneRendererRuntimeSettings(FileStreamReader& stream, ProjectSceneRendererSettings& settings, uint32_t version)
 		{
 			stream.ReadRaw(settings.EnableFrustumCulling);
+			if (version >= 3)
+				stream.ReadRaw(settings.EnableOcclusionCulling);
 			stream.ReadRaw(settings.EnableGPUDrivenRendering);
 			stream.ReadRaw(settings.EnableGTAO);
 			stream.ReadRaw(settings.GTAOBentNormals);
@@ -274,6 +282,8 @@ namespace Lux
 			stream.ReadRaw(settings.EnableJumpFlood);
 
 			stream.ReadRaw(settings.SoftShadows);
+			if (version >= 3)
+				stream.ReadRaw(settings.EnableShadowCulling);
 			stream.ReadRaw(settings.MaxShadowDistance);
 			stream.ReadRaw(settings.ShadowFade);
 			stream.ReadRaw(settings.ShadowCascadeSplitLambda);
@@ -663,7 +673,7 @@ namespace Lux
 		}
 
 		if (projectInfo.HeaderData.Version >= 2)
-			ReadSceneRendererRuntimeSettings(stream, config.SceneRenderer);
+			ReadSceneRendererRuntimeSettings(stream, config.SceneRenderer, projectInfo.HeaderData.Version);
 
 		const std::filesystem::path overridesFile = filepath.parent_path() / "Project.yaml";
 		if (std::filesystem::exists(overridesFile))
diff --git a/Core/Source/Lux/Renderer/Renderer.cpp b/Core/Source/Lux/Renderer/Renderer.cpp
@@ -29,6 +29,7 @@
 
 #include <glm/gtc/type_ptr.hpp>
 
+#include <algorithm>
 #include <filesystem>
 #include <format>
 #include <shared_mutex>
@@ -199,13 +200,21 @@ namespace Lux {
 
 	struct ShaderDependencies
 	{
-		std::vector<Ref<PipelineCompute>> ComputePipelines;
-		std::vector<Ref<Pipeline>> Pipelines;
-		std::vector<Ref<Material>> Materials;
+		std::vector<WeakRef<PipelineCompute>> ComputePipelines;
+		std::vector<WeakRef<Pipeline>> Pipelines;
+		std::vector<WeakRef<Material>> Materials;
 	};
 	static std::unordered_map<size_t, ShaderDependencies> s_ShaderDependencies;
 	static std::shared_mutex s_ShaderDependenciesMutex; // ShaderDependencies can be accessed (and modified) from multiple threads, hence require synchronization
 
+	template<typename T>
+	static void PruneDeadDependencies(std::vector<WeakRef<T>>& dependencies)
+	{
+		dependencies.erase(
+			std::remove_if(dependencies.begin(), dependencies.end(), [](const WeakRef<T>& dependency) { return !dependency; }),
+			dependencies.end());
+	}
+
 
 	struct GlobalShaderInfo
 	{
@@ -260,19 +269,19 @@ namespace Lux {
 
 	static RendererData* s_RendererData = nullptr;
 
-	void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<PipelineCompute> computePipeline)
+	void Renderer::RegisterShaderDependency(Ref<Shader> shader, PipelineCompute* computePipeline)
 	{
 		std::scoped_lock lock(s_ShaderDependenciesMutex);
 		s_ShaderDependencies[shader->GetHash()].ComputePipelines.push_back(computePipeline);
 	}
 
-	void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<Pipeline> pipeline)
+	void Renderer::RegisterShaderDependency(Ref<Shader> shader, Pipeline* pipeline)
 	{
 		std::scoped_lock lock(s_ShaderDependenciesMutex);
 		s_ShaderDependencies[shader->GetHash()].Pipelines.push_back(pipeline);
 	}
 
-	void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<Material> material)
+	void Renderer::RegisterShaderDependency(Ref<Shader> shader, Material* material)
 	{
 		std::scoped_lock lock(s_ShaderDependenciesMutex);
 		s_ShaderDependencies[shader->GetHash()].Materials.push_back(material);
@@ -282,25 +291,31 @@ namespace Lux {
 	{
 		ShaderDependencies dependencies;
 		{
-			std::shared_lock lock(s_ShaderDependenciesMutex);
+			std::scoped_lock lock(s_ShaderDependenciesMutex);
 			if (auto it = s_ShaderDependencies.find(hash); it != s_ShaderDependencies.end())
 			{
-				dependencies = it->second; // expensive to copy, but we need to release the lock (in particular to avoid potential deadlock if things like material->OnShaderReloaded() happen to ask for the lock)
+				PruneDeadDependencies(it->second.Pipelines);
+				PruneDeadDependencies(it->second.ComputePipelines);
+				PruneDeadDependencies(it->second.Materials);
+				dependencies = it->second; // Copy weak refs so callbacks run outside the registry lock.
 			}
 		}
 		for (auto& pipeline : dependencies.Pipelines)
 		{
-			pipeline->Invalidate();
+			if (pipeline)
+				pipeline->Invalidate();
 		}
 
 		for (auto& computePipeline : dependencies.ComputePipelines)
 		{
-			computePipeline->CreatePipeline();
+			if (computePipeline)
+				computePipeline->CreatePipeline();
 		}
 
 		for (auto& material : dependencies.Materials)
 		{
-			material->OnShaderReloaded();
+			if (material)
+				material->OnShaderReloaded();
 		}
 	}
 
diff --git a/Core/Source/Lux/Renderer/Renderer.h b/Core/Source/Lux/Renderer/Renderer.h
@@ -180,9 +180,9 @@ namespace Lux {
 		static Ref<Environment> GetEmptyEnvironment();
 		static Ref<Environment> GetDefaultEnvironment();
 
-		static void RegisterShaderDependency(Ref<Shader> shader, Ref<PipelineCompute> computePipeline);
-		static void RegisterShaderDependency(Ref<Shader> shader, Ref<Pipeline> pipeline);
-		static void RegisterShaderDependency(Ref<Shader> shader, Ref<Material> material);
+		static void RegisterShaderDependency(Ref<Shader> shader, PipelineCompute* computePipeline);
+		static void RegisterShaderDependency(Ref<Shader> shader, Pipeline* pipeline);
+		static void RegisterShaderDependency(Ref<Shader> shader, Material* material);
 		static void OnShaderReloaded(size_t hash);
 
 		static uint32_t GetCurrentFrameIndex();
diff --git a/Core/Source/Lux/Renderer/Renderer2D.cpp b/Core/Source/Lux/Renderer/Renderer2D.cpp
@@ -305,6 +305,12 @@ namespace Lux {
 				ldelete[] buffer;
 		}
 
+		for (auto buffers : m_LineOnTopVertexBufferBases)
+		{
+			for (auto buffer : buffers)
+				ldelete[] buffer;
+		}
+
 		for (auto buffers : m_CircleVertexBufferBases)
 		{
 			for (auto buffer : buffers)
diff --git a/Core/Source/Lux/Renderer/SceneRenderer.cpp b/Core/Source/Lux/Renderer/SceneRenderer.cpp
diff --git a/Core/Source/Lux/Renderer/SceneRenderer.h b/Core/Source/Lux/Renderer/SceneRenderer.h
diff --git a/Editor/Source/Panels/SceneRendererPanel.cpp b/Editor/Source/Panels/SceneRendererPanel.cpp

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ namespace Lux`
`11`	`11`	`struct FileHeader`
`12`	`12`	`{`
`13`	`13`	`char Header[4] = { 'L', 'P', 'R', 'J' };`
`14`		`- uint32_t Version = 2;`
	`14`	`+ uint32_t Version = 3;`
`15`	`15`	`};`
`16`	`16`
`17`	`17`	`struct Audio`
Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@`
`29`	`29`
`30`	`30`	`#include <glm/gtc/type_ptr.hpp>`
`31`	`31`
	`32`	`+#include <algorithm>`
`32`	`33`	`#include <filesystem>`
`33`	`34`	`#include <format>`
`34`	`35`	`#include <shared_mutex>`
`@@ -199,13 +200,21 @@ namespace Lux {`
`199`	`200`
`200`	`201`	`struct ShaderDependencies`
`201`	`202`	`{`
`202`		`- std::vector<Ref<PipelineCompute>> ComputePipelines;`
`203`		`- std::vector<Ref<Pipeline>> Pipelines;`
`204`		`- std::vector<Ref<Material>> Materials;`
	`203`	`+ std::vector<WeakRef<PipelineCompute>> ComputePipelines;`
	`204`	`+ std::vector<WeakRef<Pipeline>> Pipelines;`
	`205`	`+ std::vector<WeakRef<Material>> Materials;`
`205`	`206`	`};`
`206`	`207`	`static std::unordered_map<size_t, ShaderDependencies> s_ShaderDependencies;`
`207`	`208`	`static std::shared_mutex s_ShaderDependenciesMutex; // ShaderDependencies can be accessed (and modified) from multiple threads, hence require synchronization`
`208`	`209`
	`210`	`+ template<typename T>`
	`211`	`+ static void PruneDeadDependencies(std::vector<WeakRef<T>>& dependencies)`
	`212`	`+ {`
	`213`	`+ dependencies.erase(`
	`214`	`+ std::remove_if(dependencies.begin(), dependencies.end(), [](const WeakRef<T>& dependency) { return !dependency; }),`
	`215`	`+ dependencies.end());`
	`216`	`+ }`
	`217`	`+`
`209`	`218`
`210`	`219`	`struct GlobalShaderInfo`
`211`	`220`	`{`
`@@ -260,19 +269,19 @@ namespace Lux {`
`260`	`269`
`261`	`270`	`static RendererData* s_RendererData = nullptr;`
`262`	`271`
`263`		`- void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<PipelineCompute> computePipeline)`
	`272`	`+ void Renderer::RegisterShaderDependency(Ref<Shader> shader, PipelineCompute* computePipeline)`
`264`	`273`	`{`
`265`	`274`	`std::scoped_lock lock(s_ShaderDependenciesMutex);`
`266`	`275`	`s_ShaderDependencies[shader->GetHash()].ComputePipelines.push_back(computePipeline);`
`267`	`276`	`}`
`268`	`277`
`269`		`- void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<Pipeline> pipeline)`
	`278`	`+ void Renderer::RegisterShaderDependency(Ref<Shader> shader, Pipeline* pipeline)`
`270`	`279`	`{`
`271`	`280`	`std::scoped_lock lock(s_ShaderDependenciesMutex);`
`272`	`281`	`s_ShaderDependencies[shader->GetHash()].Pipelines.push_back(pipeline);`
`273`	`282`	`}`
`274`	`283`
`275`		`- void Renderer::RegisterShaderDependency(Ref<Shader> shader, Ref<Material> material)`
	`284`	`+ void Renderer::RegisterShaderDependency(Ref<Shader> shader, Material* material)`
`276`	`285`	`{`
`277`	`286`	`std::scoped_lock lock(s_ShaderDependenciesMutex);`
`278`	`287`	`s_ShaderDependencies[shader->GetHash()].Materials.push_back(material);`
`@@ -282,25 +291,31 @@ namespace Lux {`
`282`	`291`	`{`
`283`	`292`	`ShaderDependencies dependencies;`
`284`	`293`	`{`
`285`		`- std::shared_lock lock(s_ShaderDependenciesMutex);`
	`294`	`+ std::scoped_lock lock(s_ShaderDependenciesMutex);`
`286`	`295`	`if (auto it = s_ShaderDependencies.find(hash); it != s_ShaderDependencies.end())`
`287`	`296`	`{`
`288`		`- dependencies = it->second; // expensive to copy, but we need to release the lock (in particular to avoid potential deadlock if things like material->OnShaderReloaded() happen to ask for the lock)`
	`297`	`+ PruneDeadDependencies(it->second.Pipelines);`
	`298`	`+ PruneDeadDependencies(it->second.ComputePipelines);`
	`299`	`+ PruneDeadDependencies(it->second.Materials);`
	`300`	`+ dependencies = it->second; // Copy weak refs so callbacks run outside the registry lock.`
`289`	`301`	`}`
`290`	`302`	`}`
`291`	`303`	`for (auto& pipeline : dependencies.Pipelines)`
`292`	`304`	`{`
`293`		`- pipeline->Invalidate();`
	`305`	`+ if (pipeline)`
	`306`	`+ pipeline->Invalidate();`
`294`	`307`	`}`
`295`	`308`
`296`	`309`	`for (auto& computePipeline : dependencies.ComputePipelines)`
`297`	`310`	`{`
`298`		`- computePipeline->CreatePipeline();`
	`311`	`+ if (computePipeline)`
	`312`	`+ computePipeline->CreatePipeline();`
`299`	`313`	`}`
`300`	`314`
`301`	`315`	`for (auto& material : dependencies.Materials)`
`302`	`316`	`{`
`303`		`- material->OnShaderReloaded();`
	`317`	`+ if (material)`
	`318`	`+ material->OnShaderReloaded();`
`304`	`319`	`}`
`305`	`320`	`}`
`306`	`321`
Original file line number	Diff line number	Diff line change
`@@ -305,6 +305,12 @@ namespace Lux {`
`305`	`305`	`ldelete[] buffer;`
`306`	`306`	`}`
`307`	`307`
	`308`	`+ for (auto buffers : m_LineOnTopVertexBufferBases)`
	`309`	`+ {`
	`310`	`+ for (auto buffer : buffers)`
	`311`	`+ ldelete[] buffer;`
	`312`	`+ }`
	`313`	`+`
`308`	`314`	`for (auto buffers : m_CircleVertexBufferBases)`
`309`	`315`	`{`
`310`	`316`	`for (auto buffer : buffers)`