Merge branch 'dev' of https://github.com/KelvinTegelaar/CIPP-API into dev

Author: KelvinTegelaar

Modules/CIPPAlerts/Public/Alerts/Get-CIPPAlertQuotaUsed.ps1

@@ -8,34 +8,32 @@ function Get-CIPPAlertQuotaUsed {
         [Parameter(Mandatory = $false)]
         [Alias('input')]
         $InputValue,
+        [Parameter(Mandatory)]
         $TenantFilter
     )
 
+    $Threshold = if ($InputValue.QuotaUsedQuota) { [int]$InputValue.QuotaUsedQuota } else { 90 }
+    $ExcludedRaw = Get-CIPPTextReplacement -TenantFilter $TenantFilter -Text ([string]$InputValue.QuotaUsedExcludedMailboxes)
+    $Excluded = @($ExcludedRaw -split ',' | ForEach-Object { $_.Trim().ToLower() } | Where-Object { $_ })
+
     try {
-        $AlertData = New-GraphGetRequest -uri "https://graph.microsoft.com/beta/reports/getMailboxUsageDetail(period='D7')?`$format=application/json" -tenantid $TenantFilter
+        $AlertData = New-GraphGetRequest -uri "https://graph.microsoft.com/beta/reports/getMailboxUsageDetail(period='D7')?`$format=application/json&`$top=999" -tenantid $TenantFilter
     } catch {
+        $ErrorMessage = Get-CippException -Exception $_
+        Write-LogMessage -API 'Alerts' -tenant $TenantFilter -message "Mailbox quota Alert: Unable to get mailbox usage: $($ErrorMessage.NormalizedError)" -sev Error -LogData $ErrorMessage
         return
     }
+
     $OverQuota = $AlertData | ForEach-Object {
-        if ([string]::IsNullOrEmpty($_.StorageUsedInBytes) -or [string]::IsNullOrEmpty($_.prohibitSendReceiveQuotaInBytes) -or $_.StorageUsedInBytes -eq 0 -or $_.prohibitSendReceiveQuotaInBytes -eq 0) { return }
-        try {
-            $PercentLeft = [math]::round(($_.storageUsedInBytes / $_.prohibitSendReceiveQuotaInBytes) * 100)
-        } catch { $PercentLeft = 100 }
-        try {
-            if ([int]$InputValue -gt 0) {
-                $Value = [int]$InputValue
-            } else {
-                $Value = 90
-            }
-        } catch {
-            $Value = 90
-        }
-        if ($PercentLeft -gt $Value) {
+        if (!$_.StorageUsedInBytes -or !$_.prohibitSendReceiveQuotaInBytes) { return }
+        if ($Excluded -contains $_.userPrincipalName.ToLower()) { return }
+        $UsagePercent = [math]::Round(($_.storageUsedInBytes / $_.prohibitSendReceiveQuotaInBytes) * 100)
+        if ($UsagePercent -gt $Threshold) {
             [PSCustomObject]@{
-                Message                         = "$($_.userPrincipalName): Mailbox is more than $($value)% full. Mailbox is $PercentLeft% full"
+                Message                         = "$($_.userPrincipalName): Mailbox is more than $($Threshold)% full. Mailbox is $UsagePercent% full"
                 Owner                           = $_.userPrincipalName
                 RecipientType                   = $_.recipientType
-                UsagePercent                    = $PercentLeft
+                UsagePercent                    = $UsagePercent
                 StorageUsedInBytes              = $_.storageUsedInBytes
                 ProhibitSendReceiveQuotaInBytes = $_.prohibitSendReceiveQuotaInBytes
                 Tenant                          = $TenantFilter

Modules/CIPPCore/Public/Entrypoints/HTTP Functions/New-CippCoreRequest.ps1

@@ -3,7 +3,7 @@ using namespace Microsoft.Azure.Functions.PowerShellWorker
 function New-CippCoreRequest {
     <#
     .SYNOPSIS
-        Main entrypoint for all HTTP triggered functions in CIPP
+        Main entrypoint for all HTTP triggered functions in CIPP, this must live in the CIPPCore module
     .DESCRIPTION
         This function is the main entry point for all HTTP triggered functions in CIPP. It routes requests to the appropriate function based on the CIPPEndpoint parameter in the request.
     .FUNCTIONALITY

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-CIPPDBTestsRun.ps1

@@ -38,6 +38,8 @@ function Start-CIPPDBTestsRun {
     }
 
     try {
+        try { [CIPP.TestDataCache]::Clear() } catch { Write-Information "TestDataCache clear skipped: $($_.Exception.Message)" }
+
         $AllTenantsList = if ($TenantFilter -eq 'allTenants') {
             $DbCounts = Get-CIPPDbItem -CountsOnly -TenantFilter 'allTenants'
             $TenantsWithData = $DbCounts | Where-Object { (($_.DataCount ?? $_.Count) ?? 0) -gt 0 } | Select-Object -ExpandProperty PartitionKey -Unique

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-UserTasksOrchestrator.ps1

@@ -11,6 +11,8 @@ function Start-UserTasksOrchestrator {
         $TaskId = $null
     )
 
+    try { [CIPP.TestDataCache]::ClearExpired() } catch { Write-Information "TestDataCache clearexpired skipped: $($_.Exception.Message)" }
+
     $Table = Get-CippTable -tablename 'ScheduledTasks'
 
     if ($TaskId) {

Modules/CIPPCore/Public/Entrypoints/Timer Functions/Start-DurableCleanup.ps1

@@ -18,6 +18,10 @@ function Start-DurableCleanup {
         [int]$MaxDuration = 86400
     )
 
+    if ($env:CIPPNG -eq 'true') {
+        return
+    }
+
     $WarningPreference = 'SilentlyContinue'
     $TargetTime = (Get-Date).ToUniversalTime().AddSeconds(-$MaxDuration)
     $Context = New-AzDataTableContext -ConnectionString $env:AzureWebJobsStorage

Modules/CIPPTests/Public/Tests/GenericTests/Identity/Invoke-CippTestGenericTest002.ps1

@@ -54,11 +54,11 @@ function Invoke-CippTestGenericTest002 {
             $LicList = ($Entry.Value.Licenses | Sort-Object) -join ', '
             $null = $Result.Append("| $DisplayName | $LicList |`n")
             $DisplayCount++
-            if ($DisplayCount -ge 100) { break }
+            if ($DisplayCount -ge 500) { break }
         }
 
-        if ($UserLicenseMap.Count -gt 100) {
-            $null = $Result.Append("`n*Showing 100 of $($UserLicenseMap.Count) licensed users.*`n")
+        if ($UserLicenseMap.Count -gt 500) {
+            $null = $Result.Append("`n*Showing 500 of $($UserLicenseMap.Count) licensed users.*`n")
         }
 
         Add-CippTestResult -TenantFilter $Tenant -TestId 'GenericTest002' -TestType 'Identity' -Status 'Informational' -ResultMarkdown $Result -Risk 'Informational' -Name 'User License Overview' -UserImpact 'Low' -ImplementationEffort 'Low' -Category 'Tenant Overview'

Shared/CIPPSharp/CIPPTestDataCache.cs

@@ -6,15 +6,17 @@
 using System.Reflection;
 using System.Text.Json;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace CIPP
 {
     /// <summary>
-    /// Process-scoped, thread-safe LRU cache for test data lookups.
-    /// Shared across all PowerShell runspaces. Bounded by both a byte-size
-    /// cap (default 50 MB) and a short TTL (default 1 minute) so that test
-    /// suites running against a single tenant get fast cache hits without
-    /// accumulating stale Gen2 roots that cause GC thrashing.
+    /// Host-scoped, thread-safe LRU cache for test data lookups. The DLL is
+    /// loaded once per Azure Functions host, so every PowerShell worker
+    /// process on that host shares this exact instance. Bounded by both a
+    /// byte-size cap (default 100 MB) and a short TTL (default 5 minutes)
+    /// so that test suites running against a single tenant get fast cache
+    /// hits without accumulating stale Gen2 roots that cause GC thrashing.
     /// </summary>
     public static class TestDataCache
     {
@@ -28,10 +30,12 @@ public static class TestDataCache
         private static readonly Dictionary<string, LinkedListNode<string>> _lruIndex = new(); // key → node
         private static readonly object _lruLock = new();
         private static long _currentBytes;
-        private static int _accessCount;
+        private static long _accessCount;
         private static long _hits;
         private static long _misses;
         private static long _evictions;
+        private static long _oversized;
+        private static int _sweepInFlight; // 0 = idle, 1 = a background ClearExpired is running
 
         private sealed class CacheEntry
         {
@@ -60,7 +64,11 @@ public static void Configure(long maxBytes = 100L * 1024 * 1024, int ttlSeconds
 
         public static bool TryGet(string key, out object? value)
         {
-            Interlocked.Increment(ref _accessCount);
+            var count = Interlocked.Increment(ref _accessCount);
+            // Every ~1000 accesses, kick off a background sweep so TTL-expired
+            // entries that nobody re-reads still get evicted. CAS-guarded so
+            // overlapping triggers collapse to a single sweep.
+            if ((count % 1000) == 0) TryFireBackgroundSweep();
 
             if (_cache.TryGetValue(key, out var entry) && !entry.IsExpired)
             {
@@ -95,9 +103,14 @@ public static void Set(string key, object? value)
             int itemCount = value is ICollection col ? col.Count : 0;
             long sizeBytes = EstimateValueSize(value, itemCount);
 
-            // If a single entry exceeds the cap, don't cache it at all
+            // If a single entry exceeds the cap, don't cache it at all — bump
+            // _oversized so GetDiagnostics surfaces these silent drops instead
+            // of leaving callers to chase phantom misses.
             if (sizeBytes > _maxBytes)
+            {
+                Interlocked.Increment(ref _oversized);
                 return;
+            }
 
             // Remove existing entry for this key first
             if (_cache.ContainsKey(key))
@@ -170,6 +183,7 @@ public static void Clear()
             Interlocked.Exchange(ref _hits, 0);
             Interlocked.Exchange(ref _misses, 0);
             Interlocked.Exchange(ref _evictions, 0);
+            Interlocked.Exchange(ref _oversized, 0);
         }
 
         /// <summary>
@@ -191,6 +205,40 @@ public static int ClearTenant(string tenantFilter)
             return removed;
         }
 
+        /// <summary>
+        /// Remove every entry whose TTL has elapsed. Pair to the lazy per-key
+        /// eviction in TryGet — handles keys that nobody reads again. Safe to
+        /// call from anywhere; the background sweep triggered by TryGet uses
+        /// this method.
+        /// </summary>
+        public static int ClearExpired()
+        {
+            // Snapshot first; RemoveEntry mutates _cache and _lruIndex under _lruLock.
+            var expiredKeys = new List<string>();
+            foreach (var kvp in _cache)
+            {
+                if (kvp.Value.IsExpired) expiredKeys.Add(kvp.Key);
+            }
+            foreach (var key in expiredKeys) RemoveEntry(key);
+            return expiredKeys.Count;
+        }
+
+        /// <summary>
+        /// Fire-and-forget a single background ClearExpired sweep. The CAS guard
+        /// collapses overlapping triggers so we never have more than one sweep
+        /// running at a time, regardless of read pressure.
+        /// </summary>
+        private static void TryFireBackgroundSweep()
+        {
+            if (Interlocked.CompareExchange(ref _sweepInFlight, 1, 0) != 0) return;
+            Task.Run(() =>
+            {
+                try { ClearExpired(); }
+                catch { /* swallow — sweep is best-effort */ }
+                finally { Interlocked.Exchange(ref _sweepInFlight, 0); }
+            });
+        }
+
         public static int Count => _cache.Count;
         public static long CurrentBytes => Interlocked.Read(ref _currentBytes);
         public static double CurrentMB => Math.Round(Interlocked.Read(ref _currentBytes) / (1024.0 * 1024.0), 2);
@@ -199,6 +247,7 @@ public static int ClearTenant(string tenantFilter)
         public static long Hits => Interlocked.Read(ref _hits);
         public static long Misses => Interlocked.Read(ref _misses);
         public static long Evictions => Interlocked.Read(ref _evictions);
+        public static long Oversized => Interlocked.Read(ref _oversized);
         public static double HitRate => (_hits + _misses) > 0
             ? Math.Round(_hits * 100.0 / (_hits + _misses), 1) : 0;
 
@@ -326,12 +375,16 @@ private static long EstimateValueSize(object? value, int itemCount)
         /// </summary>
         public static CacheDiagnostics GetDiagnostics()
         {
-            var now = DateTime.UtcNow;
             var entries = _cache.ToArray(); // snapshot
 
             long totalBytes = 0;
             var byType = new Dictionary<string, TypeBucket>();
+            int active = 0, expired = 0;
+            DateTime? earliestExpiry = null, latestExpiry = null;
 
+            // Single pass — use the SizeBytes stored at insert instead of
+            // re-running EstimateValueSize (which would JSON-serialize every
+            // PSObject tree on every diagnostic poll and thrash the LOH).
             foreach (var kvp in entries)
             {
                 var parts = kvp.Key.Split('|', 2);
@@ -341,7 +394,7 @@ public static CacheDiagnostics GetDiagnostics()
                 if (kvp.Value.Value is ICollection col)
                     itemCount = col.Count;
 
-                long entryBytes = EstimateValueSize(kvp.Value.Value, itemCount);
+                long entryBytes = kvp.Value.SizeBytes;
                 totalBytes += entryBytes;
 
                 if (!byType.TryGetValue(dataType, out var bucket))
@@ -352,12 +405,7 @@ public static CacheDiagnostics GetDiagnostics()
                 bucket.EntryCount++;
                 bucket.TotalBytes += entryBytes;
                 bucket.TotalItems += itemCount;
-            }
 
-            int active = 0, expired = 0;
-            DateTime? earliestExpiry = null, latestExpiry = null;
-            foreach (var kvp in entries)
-            {
                 if (kvp.Value.IsExpired) { expired++; } else { active++; }
                 var exp = kvp.Value.ExpiresUtc;
                 if (earliestExpiry == null || exp < earliestExpiry) earliestExpiry = exp;
@@ -377,9 +425,10 @@ public static CacheDiagnostics GetDiagnostics()
                 Misses = Interlocked.Read(ref _misses),
                 HitRate = HitRate,
                 Evictions = Interlocked.Read(ref _evictions),
+                Oversized = Interlocked.Read(ref _oversized),
                 EarliestExpiryUtc = earliestExpiry,
                 LatestExpiryUtc = latestExpiry,
-                AccessCount = _accessCount,
+                AccessCount = Interlocked.Read(ref _accessCount),
                 TypeBreakdown = byType.Values
                     .OrderByDescending(b => b.TotalBytes)
                     .ToList(),
@@ -401,9 +450,10 @@ public class CacheDiagnostics
         public long Misses { get; set; }
         public double HitRate { get; set; }
         public long Evictions { get; set; }
+        public long Oversized { get; set; }
         public DateTime? EarliestExpiryUtc { get; set; }
         public DateTime? LatestExpiryUtc { get; set; }
-        public int AccessCount { get; set; }
+        public long AccessCount { get; set; }
         public List<TypeBucket> TypeBreakdown { get; set; } = new();
     }
 

Shared/CIPPSharp/bin/CIPPSharp.dll

@@ -5,8 +5,9 @@ $repoRoot = Split-Path -Parent $toolsRoot
 $modulesRoot = Join-Path $repoRoot 'Modules'
 $outputRoot = Join-Path $repoRoot 'Output'
 
-Install-Module -Name ModuleBuilder -MaximumVersion 3.1.9 -Scope CurrentUser -Force -AllowClobber
-Import-Module -Name ModuleBuilder -Force
+Import-Module -Name (Join-Path $toolsRoot 'Metadata\1.5.7\Metadata.psd1') -Force
+Import-Module -Name (Join-Path $toolsRoot 'Configuration\1.6.0\Configuration.psd1') -Force
+Import-Module -Name (Join-Path $toolsRoot 'ModuleBuilder\3.1.8\ModuleBuilder.psd1') -Force
 
 Write-Host "Repo root: $repoRoot"
 Set-Location -Path $repoRoot