Merge pull request #980 from KelvinTegelaar/dev

[pull] dev from KelvinTegelaar:dev

Author: pull[bot]

Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/CIPPDBCache/Push-ExecCIPPDBCache.ps1

@@ -45,8 +45,19 @@ function Push-ExecCIPPDBCache {
         # Build the full function name
         $FullFunctionName = "Set-CIPPDBCache$Name"
 
-        # Check if function exists
-        $Function = Get-Command -Name $FullFunctionName -ErrorAction SilentlyContinue
+        # Cache the resolved command per process so back-to-back HTTP-driven refreshes
+        # don't repeat the module command-table walk.
+        if (-not $script:CIPPDBCacheFunctionLookup) {
+            $script:CIPPDBCacheFunctionLookup = [System.Collections.Generic.Dictionary[string, object]]::new([System.StringComparer]::OrdinalIgnoreCase)
+            Write-Information "[CacheInit] CIPPDBCacheFunctionLookup initialized in PID $PID"
+        }
+        if ($script:CIPPDBCacheFunctionLookup.ContainsKey($FullFunctionName)) {
+            Write-Information "[CacheHit] CIPPDBCacheFunctionLookup PID=$PID Key=$FullFunctionName Size=$($script:CIPPDBCacheFunctionLookup.Count)"
+        } else {
+            Write-Information "[CacheMiss] CIPPDBCacheFunctionLookup PID=$PID Key=$FullFunctionName Size=$($script:CIPPDBCacheFunctionLookup.Count) - resolving via Get-Command"
+            $script:CIPPDBCacheFunctionLookup[$FullFunctionName] = Get-Command -Name $FullFunctionName -ErrorAction SilentlyContinue
+        }
+        $Function = $script:CIPPDBCacheFunctionLookup[$FullFunctionName]
         if (-not $Function) {
             throw "Function $FullFunctionName does not exist"
         }

Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Tests/Push-CIPPTest.ps1

@@ -12,6 +12,14 @@ function Push-CIPPTest {
 
     Write-Information "Running test $TestId for tenant $TenantFilter"
 
+    # Per-process cache of resolved test function commands so that a flat orchestrator
+    # firing thousands of activities doesn't repeat the module command-table walk
+    # for every task.
+    if (-not $script:CIPPTestFunctionLookup) {
+        $script:CIPPTestFunctionLookup = [System.Collections.Generic.Dictionary[string, object]]::new([System.StringComparer]::OrdinalIgnoreCase)
+        Write-Information "[CacheInit] CIPPTestFunctionLookup initialized in PID $PID"
+    }
+
     try {
         if ($TestId -like 'CustomScript-*') {
             $ScriptGuid = $TestId -replace '^CustomScript-', ''
@@ -23,13 +31,20 @@ function Push-CIPPTest {
 
         $FunctionName = "Invoke-CippTest$TestId"
 
-        if (-not (Get-Command $FunctionName -Module CIPPTests -ErrorAction SilentlyContinue)) {
+        if ($script:CIPPTestFunctionLookup.ContainsKey($FunctionName)) {
+            Write-Information "[CacheHit] CIPPTestFunctionLookup PID=$PID Key=$FunctionName Size=$($script:CIPPTestFunctionLookup.Count)"
+        } else {
+            Write-Information "[CacheMiss] CIPPTestFunctionLookup PID=$PID Key=$FunctionName Size=$($script:CIPPTestFunctionLookup.Count) - resolving via Get-Command"
+            $script:CIPPTestFunctionLookup[$FunctionName] = Get-Command $FunctionName -Module CIPPTests -ErrorAction SilentlyContinue
+        }
+        $TestCommand = $script:CIPPTestFunctionLookup[$FunctionName]
+        if (-not $TestCommand) {
             Write-LogMessage -API 'Tests' -tenant $TenantFilter -message "Test function not found: $FunctionName" -sev Error
             return @{ testRun = $false }
         }
 
         Write-Information "Executing $FunctionName for $TenantFilter"
-        & $FunctionName -Tenant $TenantFilter
+        & $TestCommand -Tenant $TenantFilter
         Write-Host "Returning true, test has run for $tenantFilter"
         return @{ testRun = $true }
 

Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Tests/Push-CIPPTestsList.ps1

@@ -22,11 +22,17 @@ function Push-CIPPTestsList {
     try {
         Write-Information "Building test suite list for tenant: $TenantFilter"
 
-        # Check if tenant has data before emitting any work
-        $DbCounts = Get-CIPPDbItem -TenantFilter $TenantFilter -CountsOnly
-        if (($DbCounts | Measure-Object -Property DataCount -Sum).Sum -eq 0) {
-            Write-Information "Tenant $TenantFilter has no data in database. Skipping tests."
-            return @()
+        # The orchestrator (Start-CIPPDBTestsRun) already filtered the tenant list to those
+        # with cached data, so the previous per-tenant `Get-CIPPDbItem -CountsOnly` recheck
+        # was a redundant Table query (one extra round-trip per tenant). The orchestrator
+        # may pass SkipDbCheck=$true when it has already verified data presence; otherwise
+        # we fall back to a check here for any direct invocations.
+        if (-not $Item.SkipDbCheck) {
+            $DbCounts = Get-CIPPDbItem -TenantFilter $TenantFilter -CountsOnly
+            if (($DbCounts | Measure-Object -Property DataCount -Sum).Sum -eq 0) {
+                Write-Information "Tenant $TenantFilter has no data in database. Skipping tests."
+                return @()
+            }
         }
 
         # Emit one task per suite — suite names must match the ValidateSet in Invoke-CIPPTestCollection.

Modules/CIPPCore/Public/Add-CIPPDbItem.ps1

@@ -29,10 +29,15 @@ function Add-CIPPDbItem {
         $Batch = [System.Collections.Generic.List[hashtable]]::new()
         $NewRowKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
         $TotalProcessed = 0
+        # Cache regex instances so each row pays only the match cost, not regex compilation.
+        # Two passes preserve the original semantics: path/wildcard chars → '_', control chars → stripped.
+        $RowKeyPathRegex = [regex]::new('[/\\#?]')
+        $RowKeyControlRegex = [regex]::new('[\u0000-\u001F\u007F-\u009F]')
 
         if ($TenantFilter -match '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$') {
             try {
-                $TenantFilter = (Get-Tenants -TenantFilter $TenantFilter -IncludeErrors | Select-Object -First 1).defaultDomainName
+                $TenantLookup = @(Get-Tenants -TenantFilter $TenantFilter -IncludeErrors)
+                if ($TenantLookup.Count -gt 0) { $TenantFilter = $TenantLookup[0].defaultDomainName }
             } catch {}
         }
     }
@@ -48,7 +53,7 @@ function Add-CIPPDbItem {
         foreach ($Item in @($InputObject)) {
             if ($null -eq $Item) { continue }
             $ItemId = $Item.ExternalDirectoryObjectId ?? $Item.id ?? $Item.Identity ?? $Item.skuId ?? $Item.userPrincipalName ?? [guid]::NewGuid().ToString()
-            $RowKey = "$Type-$ItemId" -replace '[/\\#?]', '_' -replace '[\u0000-\u001F\u007F-\u009F]', ''
+            $RowKey = $RowKeyControlRegex.Replace($RowKeyPathRegex.Replace("$Type-$ItemId", '_'), '')
             if ($NewRowKeys.Add($RowKey)) {
                 $Batch.Add(@{
                         PartitionKey = $TenantFilter

Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-CIPPDBTestsRun.ps1

@@ -58,11 +58,14 @@ function Start-CIPPDBTestsRun {
             return
         }
 
-        # Phase 1: Build per-tenant list activities (discover tests per tenant)
+        # Phase 1: Build per-tenant list activities (discover tests per tenant).
+        # The tenants below were already filtered by data presence above, so we pass
+        # SkipDbCheck=$true to avoid a redundant CountsOnly round-trip per tenant.
         $Batch = foreach ($Tenant in $AllTenantsList) {
             @{
                 FunctionName = 'CIPPTestsList'
                 TenantFilter = $Tenant
+                SkipDbCheck  = $true
             }
         }
 

Modules/CIPPCore/Public/Get-CIPPDbItem.ps1

@@ -62,8 +62,9 @@ function Get-CIPPDbItem {
                 $Conditions.Add('DataCount ge 0')
             }
             $Filter = [string]::Join(' and ', $Conditions)
+            # -Property does the projection server-side; the trailing Select-Object was
+            # redundant (and rebuilt every row as a NoteProperty bag, slowing later filters).
             $Results = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property 'PartitionKey', 'RowKey', 'DataCount', 'Timestamp'
-            $Results = $Results | Select-Object PartitionKey, RowKey, DataCount, Timestamp
         } else {
             if (-not $Type) {
                 throw 'Type parameter is required when CountsOnly is not specified'

Modules/CIPPCore/Public/Get-CIPPDomainAnalyser.ps1

@@ -14,6 +14,17 @@ function Get-CIPPDomainAnalyser {
     #>
     [CmdletBinding()]
     param([string]$TenantFilter)
+
+    if (-not $script:CIPPDomainAnalyserCache) {
+        $script:CIPPDomainAnalyserCache = @{}
+    }
+    $CacheKey = if ([string]::IsNullOrEmpty($TenantFilter)) { 'AllTenants' } else { $TenantFilter }
+    $CacheNow = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()
+    $CachedEntry = $script:CIPPDomainAnalyserCache[$CacheKey]
+    if ($CachedEntry -and ($CacheNow - $CachedEntry.Timestamp) -lt 300) {
+        return $CachedEntry.Results
+    }
+
     $DomainTable = Get-CIPPTable -Table 'Domains'
 
     # Get all the things
@@ -43,5 +54,6 @@ function Get-CIPPDomainAnalyser {
     } catch {
         $Results = @()
     }
+    $script:CIPPDomainAnalyserCache[$CacheKey] = @{ Results = $Results; Timestamp = $CacheNow }
     return $Results
 }

Modules/CIPPCore/Public/Get-CippDbRole.ps1

@@ -11,7 +11,7 @@ function Get-CippDbRole {
         [switch]$CisaHighlyPrivilegedRoles
     )
 
-    $Roles = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Roles'
+    $Roles = Get-CIPPTestData -TenantFilter $TenantFilter -Type 'Roles'
 
     if ($IncludePrivilegedRoles) {
         $PrivilegedRoleTemplateIds = @(

Modules/CIPPCore/Public/Get-CippDbRoleMembers.ps1

@@ -8,9 +8,9 @@ function Get-CippDbRoleMembers {
         [string]$RoleTemplateId
     )
 
-    $RoleAssignments = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'RoleAssignmentScheduleInstances'
-    $RoleEligibilities = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'RoleEligibilitySchedules'
-    $DirectRoleAssignments = New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Roles' | Where-Object { $_.roleTemplateId -eq $RoleTemplateId } | Select-Object -ExpandProperty members
+    $RoleAssignments = Get-CIPPTestData -TenantFilter $TenantFilter -Type 'RoleAssignmentScheduleInstances'
+    $RoleEligibilities = Get-CIPPTestData -TenantFilter $TenantFilter -Type 'RoleEligibilitySchedules'
+    $DirectRoleAssignments = Get-CIPPTestData -TenantFilter $TenantFilter -Type 'Roles' | Where-Object { $_.roleTemplateId -eq $RoleTemplateId } | Select-Object -ExpandProperty members
 
     $ActiveMembers = $RoleAssignments | Where-Object {
         $_.roleDefinitionId -eq $RoleTemplateId -and $_.assignmentType -eq 'Assigned'

Modules/CIPPCore/Public/Invoke-CIPPDBCacheCollection.ps1

@@ -181,13 +181,13 @@ function Invoke-CIPPDBCacheCollection {
             Write-Information "  [$CollectionType] Collecting $CacheType for $TenantFilter"
             & $FullFunctionName @Params
             $ItemStopwatch.Stop()
-            $ElapsedSeconds = [math]::Round($ItemStopwatch.Elapsed.TotalSeconds, 3)
+            $ElapsedSeconds = '{0:N3}' -f $ItemStopwatch.Elapsed.TotalSeconds
             $Timings.Add("$CacheType : ${ElapsedSeconds}s")
             Write-Information "  [$CollectionType] Completed $CacheType for $TenantFilter - Took ${ElapsedSeconds} seconds"
             $SuccessCount++
         } catch {
             $ItemStopwatch.Stop()
-            $ElapsedSeconds = [math]::Round($ItemStopwatch.Elapsed.TotalSeconds, 3)
+            $ElapsedSeconds = '{0:N3}' -f $ItemStopwatch.Elapsed.TotalSeconds
             $FailedCount++
             $Errors.Add("$CacheType : $($_.Exception.Message)")
             $Timings.Add("$CacheType : ${ElapsedSeconds}s (FAILED)")
@@ -196,7 +196,7 @@ function Invoke-CIPPDBCacheCollection {
     }
 
     $CollectionStopwatch.Stop()
-    $TotalElapsed = [math]::Round($CollectionStopwatch.Elapsed.TotalSeconds, 3)
+    $TotalElapsed = '{0:N3}' -f $CollectionStopwatch.Elapsed.TotalSeconds
     $Summary = "$CollectionType collection for $TenantFilter completed in ${TotalElapsed} seconds - $SuccessCount succeeded, $FailedCount failed out of $($CacheTypes.Count)"
     Write-Information $Summary
     Write-Information "  Timings: $($Timings -join ' | ')"