Improve memory management in user and batch processing

Enhanced memory cleanup in Add-CIPPDbItem and Set-CIPPDBCacheUsers by explicitly clearing variables and invoking garbage collection with logging. Optimized hashtable property removal in Add-CIPPAzDataTableEntity for better performance and reliability.

Author: JohnDuprey

Modules/CIPPCore/Public/Add-CIPPAzDataTableEntity.ps1

@@ -65,11 +65,11 @@ function Add-CIPPAzDataTableEntity {
             try {
                 # Ensure all property values are not null for string properties
                 if ($SingleEnt -is [hashtable]) {
-                    foreach ($key in @($SingleEnt.Keys)) {
-                        if ($null -eq $SingleEnt[$key]) {
-                            $SingleEnt.Remove($key)
-                        }
+                    $keysToRemove = @($SingleEnt.Keys | Where-Object { $null -eq $SingleEnt[$_] })
+                    foreach ($key in $keysToRemove) {
+                        $SingleEnt.Remove($key)
                     }
+                    $keysToRemove = $null
                 } elseif ($SingleEnt -is [PSCustomObject]) {
                     $propsToRemove = [system.Collections.Generic.List[string]]::new()
                     foreach ($prop in $SingleEnt.PSObject.Properties) {

Modules/CIPPCore/Public/Add-CIPPDbItem.ps1

@@ -110,7 +110,20 @@ function Add-CIPPDbItem {
                 # Clear batch variables to free memory
                 $Entities = $null
                 $Batch = $null
-                [System.GC]::Collect()
+
+                # Capture memory before GC
+                $MemoryBeforeGC = [System.GC]::GetTotalMemory($false)
+
+                # Force GC between batches to prevent accumulation
+                [System.GC]::Collect([System.GC]::MaxGeneration, [System.GCCollectionMode]::Forced)
+                [System.GC]::WaitForPendingFinalizers()
+                [System.GC]::Collect([System.GC]::MaxGeneration, [System.GCCollectionMode]::Forced)
+
+                # Log memory usage after GC
+                $MemoryAfterGC = [System.GC]::GetTotalMemory($false)
+                $FreedMB = [math]::Round(($MemoryBeforeGC - $MemoryAfterGC) / 1MB, 2)
+                $CurrentMemoryMB = [math]::Round($MemoryAfterGC / 1MB, 2)
+                Write-Information "Batch $([Math]::Ceiling($ProcessedCount / $BatchSize))/$([Math]::Ceiling($TotalCount / $BatchSize)) complete. Processed: $ProcessedCount/$TotalCount | Memory: ${CurrentMemoryMB}MB | Freed: ${FreedMB}MB"
             }
 
         }

Modules/CIPPCore/Public/Set-CIPPDBCacheUsers.ps1

@@ -32,6 +32,12 @@ function Set-CIPPDBCacheUsers {
         Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Cached users successfully' -sev Debug
 
     } catch {
+        # Cleanup on error to prevent memory leak
+        if ($null -ne $Users) { 
+            $Users.Clear()
+            $Users = $null
+        }
+        $UsersResponse = $null
         Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Failed to cache users: $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
     }
 }