Skip to content

Commit d31dd4a

Browse files
JohnDupreyJohnDuprey
committed
feat: Enhance audit log processing with improved caching and state management
1 parent db56aab commit d31dd4a

3 files changed

Lines changed: 66 additions & 26 deletions

File tree

‎Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,12 @@ function New-CIPPAuditLogSearchResultsCache {
6262
$searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId
6363
foreach ($searchResult in $searchResults) {
6464
$cacheEntity = @{
65-
RowKey = $searchResult.id
66-
PartitionKey = $TenantFilter
67-
SearchId = $SearchId
68-
JSON = [string]($searchResult | ConvertTo-Json -Depth 10)
65+
RowKey = $searchResult.id
66+
PartitionKey = $TenantFilter
67+
SearchId = $SearchId
68+
JSON = [string]($searchResult | ConvertTo-Json -Depth 10)
69+
CippProcessing = $false
70+
CippProcessingStarted = ''
6971
}
7072
Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force
7173
}

‎Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogProcessingBatch.ps1‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ function Push-AuditLogProcessingBatch {
77
Loads CacheWebhooks in pages, groups by tenant, and returns batch items
88
for AuditLogTenantProcess activities. Running in an activity isolates
99
the memory usage from the timer function.
10+
11+
Rows are stamped with CippProcessing = true and CippProcessingStarted timestamp
12+
before being included in the batch, so that subsequent 15-minute timer runs skip
13+
them instead of spawning duplicate activities. Rows stuck in processing state for
14+
more than 4 hours (e.g. from a worker crash) are automatically recovered and
15+
re-queued on the next run.
1016
.FUNCTIONALITY
1117
Entrypoint
1218
#>
@@ -17,18 +23,42 @@ function Push-AuditLogProcessingBatch {
1723
$TotalRows = 0
1824
$PageSize = 20000
1925
$Skip = 0
26+
$NowUtc = (Get-Date).ToUniversalTime()
27+
$StaleThreshold = $NowUtc.AddHours(-4)
2028

2129
do {
22-
$WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable -First $PageSize -Skip $Skip
30+
# Fetch only the properties needed to determine claim status and build the batch
31+
$WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable -First $PageSize -Skip $Skip -Property @('PartitionKey', 'RowKey', 'ETag', 'CippProcessing', 'CippProcessingStarted')
2332
$PageCount = $WebhookCache.Count
24-
$TenantGroups = $WebhookCache | Group-Object -Property PartitionKey
33+
34+
# Filter client-side: skip rows actively claimed unless the claim is stale (> 4 hours old)
35+
$TenantGroups = $WebhookCache | Where-Object {
36+
-not $_.CippProcessing -or
37+
(-not $_.CippProcessingStarted) -or
38+
([datetime]$_.CippProcessingStarted -lt $StaleThreshold)
39+
} | Group-Object -Property PartitionKey
2540
$WebhookCache = $null
2641

2742
if ($TenantGroups) {
28-
$TotalRows += ($TenantGroups | Measure-Object -Property Count -Sum).Sum
43+
$ProcessingTimestamp = $NowUtc.ToString('yyyy-MM-ddTHH:mm:ssZ')
2944
foreach ($TenantGroup in $TenantGroups) {
3045
$TenantFilter = $TenantGroup.Name
31-
$RowIds = @($TenantGroup.Group.RowKey)
46+
$Rows = @($TenantGroup.Group)
47+
$RowIds = @($Rows.RowKey)
48+
49+
# Claim these rows so subsequent timer runs skip them (UpsertMerge preserves JSON and other fields)
50+
foreach ($Row in $Rows) {
51+
$ClaimEntity = [PSCustomObject]@{
52+
PartitionKey = $Row.PartitionKey
53+
RowKey = $Row.RowKey
54+
ETag = $Row.ETag
55+
CippProcessing = $true
56+
CippProcessingStarted = $ProcessingTimestamp
57+
}
58+
Add-CIPPAzDataTableEntity @WebhookCacheTable -Entity $ClaimEntity -OperationType UpsertMerge
59+
}
60+
61+
$TotalRows += $RowIds.Count
3262
for ($i = 0; $i -lt $RowIds.Count; $i += 500) {
3363
$BatchRowIds = $RowIds[$i..([Math]::Min($i + 499, $RowIds.Count - 1))]
3464
$AllBatchItems.Add([PSCustomObject]@{

‎Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -190,11 +190,16 @@ function Test-CIPPAuditLogRules {
190190
# Try CippReportingDB first (pre-populated by timer, same pattern as Add-CIPPApplicationPermission)
191191
Write-Information "Checking CippReportingDB for directory data for tenant $TenantFilter"
192192
try {
193-
$Users = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Users')
194-
$ServicePrincipals = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'ServicePrincipals')
193+
$Users = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Users') | Select-Object id, displayName, userPrincipalName, accountEnabled
194+
$Groups = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Groups') | Select-Object id, displayName, mailEnabled, securityEnabled
195+
$Devices = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Devices') | Select-Object id, displayName, deviceId
196+
$ServicePrincipals = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'ServicePrincipals') | Select-Object id, appId, displayName, appDisplayName, accountEnabled, servicePrincipalType, tags
197+
Write-Information "Loaded from CippReportingDB: $($Users.Count) users, $($Groups.Count) groups, $($Devices.Count) devices, $($ServicePrincipals.Count) service principals"
195198
} catch {
196199
Write-Information "CippReportingDB query failed for ${TenantFilter}: $($_.Exception.Message)"
197200
$Users = @()
201+
$Groups = @()
202+
$Devices = @()
198203
$ServicePrincipals = @()
199204
}
200205

@@ -227,18 +232,8 @@ function Test-CIPPAuditLogRules {
227232
$Users = ($Response | Where-Object { $_.id -eq 'users' }).body.value ?? @()
228233
$Groups = ($Response | Where-Object { $_.id -eq 'groups' }).body.value ?? @()
229234
$Devices = ($Response | Where-Object { $_.id -eq 'devices' }).body.value ?? @()
230-
$ServicePrincipals = ($Response | Where-Object { $_.id -eq 'servicePrincipals' }).body.value ?? @()
235+
$ServicePrincipals = @(($Response | Where-Object { $_.id -eq 'servicePrincipals' }).body.value) | Select-Object id, displayName
231236
$Response = $null
232-
} else {
233-
try {
234-
$Groups = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Groups')
235-
$Devices = @(New-CIPPDbRequest -TenantFilter $TenantFilter -Type 'Devices')
236-
} catch {
237-
Write-Information "CippReportingDB Groups/Devices query failed for ${TenantFilter}: $($_.Exception.Message)"
238-
$Groups = @()
239-
$Devices = @()
240-
}
241-
Write-Information "Loaded from CippReportingDB: $($Users.Count) users, $($Groups.Count) groups, $($Devices.Count) devices, $($ServicePrincipals.Count) service principals"
242237
}
243238

244239
# Build hashtables for O(1) GUID lookups
@@ -322,15 +317,28 @@ function Test-CIPPAuditLogRules {
322317
$UserLookup = ($UsersLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue -AsHashtable) ?? @{}
323318
$GroupLookup = ($GroupsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue -AsHashtable) ?? @{}
324319
$DeviceLookup = ($DevicesLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue -AsHashtable) ?? @{}
325-
$ServicePrincipalLookup = ($ServicePrincipalsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue -AsHashtable) ?? @{}
320+
$ServicePrincipalLookup = @{}
321+
$RawSPLookup = ($ServicePrincipalsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue -AsHashtable) ?? @{}
322+
foreach ($key in $RawSPLookup.Keys) {
323+
$sp = $RawSPLookup[$key]
324+
$ServicePrincipalLookup[$key] = [ordered]@{
325+
id = $sp.id
326+
appId = $sp.appId
327+
displayName = $sp.displayName
328+
appDisplayName = $sp.appDisplayName
329+
accountEnabled = $sp.accountEnabled
330+
servicePrincipalType = $sp.servicePrincipalType
331+
tags = $sp.tags
332+
}
333+
}
326334
Write-Information "Loaded hashtables: $($UserLookup.Count) users, $($GroupLookup.Count) groups, $($DeviceLookup.Count) devices, $($ServicePrincipalLookup.Count) service principals"
327335
} else {
328336
# Old format (array) - convert to hashtables
329337
Write-Information "Converting legacy array cache to hashtables for tenant $TenantFilter"
330-
$Users = ($UsersLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue) ?? @()
331-
$Groups = ($GroupsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue) ?? @()
332-
$Devices = ($DevicesLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue) ?? @()
333-
$ServicePrincipals = ($ServicePrincipalsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue) ?? @()
338+
$Users = @(($UsersLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue)) | Select-Object id, displayName, userPrincipalName, accountEnabled
339+
$Groups = @(($GroupsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue)) | Select-Object id, displayName, mailEnabled, securityEnabled
340+
$Devices = @(($DevicesLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue)) | Select-Object id, displayName, deviceId
341+
$ServicePrincipals = @(($ServicePrincipalsLookup.Data | ConvertFrom-Json -ErrorAction SilentlyContinue)) | Select-Object id, appId, displayName, appDisplayName, accountEnabled, servicePrincipalType, tags
334342

335343
# Build hashtables
336344
$UserLookup = @{}

0 commit comments

Comments
 (0)