Skip to content

Commit 2e8921f

Browse files
pull[bot]pull[bot]
authored
Merge pull request #923 from KelvinTegelaar/dev
[pull] dev from KelvinTegelaar:dev
2 parents 1722671 + ae50bee commit 2e8921f

5 files changed

Lines changed: 58 additions & 210 deletions

File tree

‎Modules/CIPPCore/Public/Add-CIPPDbItem.ps1‎

Lines changed: 42 additions & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -2,243 +2,93 @@ function Add-CIPPDbItem {
22
<#
33
.SYNOPSIS
44
Add items to the CIPP Reporting database
5-
6-
.DESCRIPTION
7-
Adds items to the CippReportingDB table with support for bulk inserts, count mode, and pipeline streaming
8-
9-
.PARAMETER TenantFilter
10-
The tenant domain or GUID (used as partition key)
11-
12-
.PARAMETER Type
13-
The type of data being stored (used in row key)
14-
15-
.PARAMETER InputObject
16-
Items to add to the database. Accepts pipeline input for memory-efficient streaming.
17-
Alias: Data (for backward compatibility)
18-
19-
.PARAMETER Count
20-
If specified, stores a single row with count of items processed
21-
22-
.PARAMETER AddCount
23-
If specified, automatically records the total count after processing all items
24-
25-
.PARAMETER Append
26-
If specified, adds items without clearing existing entries for this type/tenant and automatically
27-
increments the count. Useful for accumulating report data over time. By default, existing entries are replaced.
28-
29-
.EXAMPLE
30-
Add-CIPPDbItem -TenantFilter 'contoso.onmicrosoft.com' -Type 'Groups' -Data $GroupsData
31-
32-
.EXAMPLE
33-
New-GraphGetRequest -uri '...' | Add-CIPPDbItem -TenantFilter 'contoso.onmicrosoft.com' -Type 'Users' -AddCount
34-
35-
.EXAMPLE
36-
Add-CIPPDbItem -TenantFilter 'contoso.onmicrosoft.com' -Type 'Groups' -Data $GroupsData -Count
37-
38-
.EXAMPLE
39-
Add-CIPPDbItem -TenantFilter 'contoso.onmicrosoft.com' -Type 'AlertHistory' -Data $AlertData -Append -AddCount
5+
.FUNCTIONALITY
6+
Internal
407
#>
418
[CmdletBinding()]
429
param(
43-
[Parameter(Mandatory = $true)]
10+
[Parameter(Mandatory)]
4411
[string]$TenantFilter,
4512

46-
[Parameter(Mandatory = $true)]
13+
[Parameter(Mandatory)]
4714
[string]$Type,
4815

49-
[Parameter(Mandatory = $true, ValueFromPipeline = $true)]
16+
[Parameter(Mandatory, ValueFromPipeline)]
5017
[Alias('Data')]
5118
[AllowNull()]
5219
[AllowEmptyCollection()]
5320
$InputObject,
5421

55-
[Parameter(Mandatory = $false)]
5622
[switch]$Count,
57-
58-
[Parameter(Mandatory = $false)]
5923
[switch]$AddCount,
60-
61-
[Parameter(Mandatory = $false)]
6224
[switch]$Append
6325
)
6426

6527
begin {
66-
# Initialize pipeline processing with state hashtable for nested function access
6728
$Table = Get-CippTable -tablename 'CippReportingDB'
68-
$BatchAccumulator = [System.Collections.Generic.List[hashtable]]::new(500)
69-
$State = @{
70-
TotalProcessed = 0
71-
BatchNumber = 0
72-
}
29+
$Batch = [System.Collections.Generic.List[hashtable]]::new()
30+
$TotalProcessed = 0
7331

74-
if ($TenantFilter -match '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') {
32+
if ($TenantFilter -match '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$') {
7533
try {
7634
$TenantFilter = (Get-Tenants -TenantFilter $TenantFilter -IncludeErrors | Select-Object -First 1).defaultDomainName
77-
} catch {
78-
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Failed to resolve tenant GUID to default domain: $($_.Exception.Message)" -sev Warning
79-
}
80-
}
81-
82-
# Helper function to format RowKey values by removing disallowed characters
83-
function Format-RowKey {
84-
param([string]$RowKey)
85-
$sanitized = $RowKey -replace '[/\\#?]', '_' -replace '[\u0000-\u001F\u007F-\u009F]', ''
86-
return $sanitized
87-
}
88-
89-
# Function to flush current batch
90-
function Invoke-FlushBatch {
91-
param($State)
92-
if ($BatchAccumulator.Count -eq 0) { return }
93-
94-
$State.BatchNumber++
95-
$batchSize = $BatchAccumulator.Count
96-
$MemoryBeforeGC = [System.GC]::GetTotalMemory($false)
97-
$flushStart = Get-Date
98-
99-
try {
100-
# Entities are already in the accumulator, just write them
101-
$writeStart = Get-Date
102-
Add-CIPPAzDataTableEntity @Table -Entity $BatchAccumulator.ToArray() -Force | Out-Null
103-
$writeEnd = Get-Date
104-
$writeDuration = [math]::Round(($writeEnd - $writeStart).TotalSeconds, 2)
105-
$State.TotalProcessed += $batchSize
106-
107-
} finally {
108-
# Clear and GC
109-
$gcStart = Get-Date
110-
$BatchAccumulator.Clear()
111-
112-
# Single GC pass is sufficient - aggressive GC was causing slowdown
113-
[System.GC]::Collect()
114-
115-
$flushEnd = Get-Date
116-
$gcDuration = [math]::Round(($flushEnd - $gcStart).TotalSeconds, 2)
117-
$flushDuration = [math]::Round(($flushEnd - $flushStart).TotalSeconds, 2)
118-
$MemoryAfterGC = [System.GC]::GetTotalMemory($false)
119-
$FreedMB = [math]::Round(($MemoryBeforeGC - $MemoryAfterGC) / 1MB, 2)
120-
$CurrentMemoryMB = [math]::Round($MemoryAfterGC / 1MB, 2)
121-
#Write-Debug "Batch $($State.BatchNumber): ${flushDuration}s total (write: ${writeDuration}s, gc: ${gcDuration}s) | Processed: $($State.TotalProcessed) | Memory: ${CurrentMemoryMB}MB | Freed: ${FreedMB}MB"
122-
}
35+
} catch {}
12336
}
12437

12538
if (-not $Count.IsPresent -and -not $Append.IsPresent) {
126-
# Delete existing entries for this type
12739
$Filter = "PartitionKey eq '{0}' and RowKey ge '{1}-' and RowKey lt '{1}0'" -f $TenantFilter, $Type
128-
$ExistingEntities = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag
129-
if ($ExistingEntities) {
130-
Remove-AzDataTableEntity @Table -Entity $ExistingEntities -Force | Out-Null
40+
$Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag
41+
if ($Existing) {
42+
$null = Remove-AzDataTableEntity @Table -Entity $Existing -Force
13143
}
132-
$AllocatedMemoryMB = [math]::Round([System.GC]::GetTotalMemory($false) / 1MB, 2)
133-
#Write-Debug "Starting $Type import for $TenantFilter | Allocated Memory: ${AllocatedMemoryMB}MB | Batch Size: 500"
13444
}
13545
}
13646

13747
process {
138-
# Process each item from pipeline
13948
if ($null -eq $InputObject) { return }
14049

141-
# If Count mode and InputObject is an integer, use it directly as count
142-
if ($Count.IsPresent -and $InputObject -is [int]) {
143-
$State.TotalProcessed = $InputObject
144-
return
145-
}
146-
147-
# Handle both single items and arrays (for backward compatibility)
148-
$ItemsToProcess = if ($InputObject -is [array]) {
149-
$InputObject
150-
} else {
151-
@($InputObject)
152-
}
153-
154-
# If Count mode, just count items without processing
15550
if ($Count.IsPresent) {
156-
$itemCount = if ($ItemsToProcess -is [array]) { $ItemsToProcess.Count } else { 1 }
157-
$State.TotalProcessed += $itemCount
51+
if ($InputObject -is [int]) { $TotalProcessed = $InputObject } else { $TotalProcessed += @($InputObject).Count }
15852
return
15953
}
16054

161-
foreach ($Item in $ItemsToProcess) {
55+
foreach ($Item in @($InputObject)) {
16256
if ($null -eq $Item) { continue }
163-
164-
# Convert to entity
165-
$ItemId = $Item.ExternalDirectoryObjectId ?? $Item.id ?? $Item.Identity ?? $Item.skuId ?? $Item.userPrincipalName ?? [Guid]::NewGuid().ToString()
166-
$Entity = @{
167-
PartitionKey = $TenantFilter
168-
RowKey = Format-RowKey "$Type-$ItemId"
169-
Data = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
170-
Type = $Type
171-
}
172-
173-
$BatchAccumulator.Add($Entity)
174-
175-
# Flush when batch reaches 500 items
176-
if ($BatchAccumulator.Count -ge 500) {
177-
Invoke-FlushBatch -State $State
57+
$ItemId = $Item.ExternalDirectoryObjectId ?? $Item.id ?? $Item.Identity ?? $Item.skuId ?? $Item.userPrincipalName ?? [guid]::NewGuid().ToString()
58+
$Batch.Add(@{
59+
PartitionKey = $TenantFilter
60+
RowKey = ("$Type-$ItemId" -replace '[/\\#?]', '_' -replace '[\u0000-\u001F\u007F-\u009F]', '')
61+
Data = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
62+
Type = $Type
63+
})
64+
if ($Batch.Count -ge 500) {
65+
$null = Add-CIPPAzDataTableEntity @Table -Entity $Batch.ToArray() -Force
66+
$TotalProcessed += $Batch.Count
67+
$Batch.Clear()
17868
}
17969
}
18070
}
18171

18272
end {
183-
try {
184-
# Flush any remaining items in final partial batch
185-
if ($BatchAccumulator.Count -gt 0) {
186-
Invoke-FlushBatch -State $State
187-
}
188-
189-
if ($Count.IsPresent -or $Append.IsPresent) {
190-
# Store count record
191-
if ($Append.IsPresent) {
192-
# When appending, always increment the existing count
193-
$Filter = "PartitionKey eq '{0}' and RowKey eq '{1}-Count'" -f $TenantFilter, $Type
194-
$ExistingCount = Get-CIPPAzDataTableEntity @Table -Filter $Filter
195-
$PreviousCount = if ($ExistingCount -and $ExistingCount.DataCount) { [int]$ExistingCount.DataCount } else { 0 }
196-
$NewCount = $PreviousCount + $State.TotalProcessed
197-
} else {
198-
# Normal mode - replace count
199-
$NewCount = $State.TotalProcessed
200-
}
201-
202-
$Entity = @{
203-
PartitionKey = $TenantFilter
204-
RowKey = Format-RowKey "$Type-Count"
205-
DataCount = [int]$NewCount
206-
}
207-
Add-CIPPAzDataTableEntity @Table -Entity $Entity -Force | Out-Null
208-
}
209-
210-
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter `
211-
-message "Added $($State.TotalProcessed) items of type $Type$(if ($Count.IsPresent) { ' (count mode)' })$(if ($Append.IsPresent) { ' (append mode)' })" -sev Debug
73+
if ($Batch.Count -gt 0) {
74+
$null = Add-CIPPAzDataTableEntity @Table -Entity $Batch.ToArray() -Force
75+
$TotalProcessed += $Batch.Count
76+
}
21277

213-
} catch {
214-
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter `
215-
-message "Failed to add items of type $Type : $($_.Exception.Message)" -sev Error `
216-
-LogData (Get-CippException -Exception $_)
217-
#Write-Debug "[Add-CIPPDbItem] $TenantFilter - $(Get-CippException -Exception $_ | ConvertTo-Json -Depth 5 -Compress)"
218-
throw
219-
} finally {
220-
# Record count if AddCount was specified
221-
if ($AddCount.IsPresent -and $State.TotalProcessed -gt 0) {
222-
try {
223-
$countParams = @{
224-
TenantFilter = $TenantFilter
225-
Type = $Type
226-
InputObject = $State.TotalProcessed
227-
Count = $true
228-
}
229-
if ($Append.IsPresent) {
230-
$countParams.Append = $true
231-
}
232-
Add-CIPPDbItem @countParams
233-
} catch {
234-
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter `
235-
-message "Failed to record count for $Type : $($_.Exception.Message)" -sev Warning
236-
}
78+
if ($Count.IsPresent -or $AddCount.IsPresent) {
79+
$NewCount = $TotalProcessed
80+
if ($Append.IsPresent) {
81+
$Filter = "PartitionKey eq '{0}' and RowKey eq '{1}-Count'" -f $TenantFilter, $Type
82+
$ExistingCount = Get-CIPPAzDataTableEntity @Table -Filter $Filter
83+
if ($ExistingCount.DataCount) { $NewCount += [int]$ExistingCount.DataCount }
23784
}
238-
239-
# Final cleanup
240-
$BatchAccumulator = $null
241-
[System.GC]::Collect()
85+
$null = Add-CIPPAzDataTableEntity @Table -Entity @{
86+
PartitionKey = $TenantFilter
87+
RowKey = "$Type-Count"
88+
DataCount = [int]$NewCount
89+
} -Force
24290
}
91+
92+
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Added $TotalProcessed items of type $Type" -sev Debug
24393
}
24494
}

‎Modules/CIPPCore/Public/New-CIPPGroup.ps1‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ function New-CIPPGroup {
4747
$TenMinutesAgo = (Get-Date).AddMinutes(-10).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
4848
$CachedGroup = Get-CIPPAzDataTableEntity @GroupCacheTable -Filter "PartitionKey eq 'GroupCreation' and RowKey eq '$CacheRowKey' and Timestamp ge datetime'$TenMinutesAgo'"
4949
if ($CachedGroup -and $CachedGroup.GroupId) {
50-
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Group '$($GroupObject.displayName)' was recently created (cached id: $($CachedGroup.GroupId)), skipping duplicate creation" -Sev Info
50+
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Group '$($GroupObject.displayName)' was recently created (cached id: $($CachedGroup.GroupId)), skipping duplicate creation" -Sev Info -User $ExecutingUser
5151
return [PSCustomObject]@{
5252
Success = $true
5353
Message = "Group $($GroupObject.displayName) already exists (from cache)"
@@ -104,7 +104,7 @@ function New-CIPPGroup {
104104
}
105105
}
106106

107-
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Creating group $($GroupObject.displayName) of type $NormalizedGroupType$(if ($NeedsEmail) { " with email $Email" })" -Sev Info
107+
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Creating group $($GroupObject.displayName) of type $NormalizedGroupType$(if ($NeedsEmail) { " with email $Email" })" -Sev Info -User $ExecutingUser
108108

109109
# Handle Graph API groups (Security, Generic, AzureRole, Dynamic, M365)
110110
if ($NormalizedGroupType -in @('Generic', 'AzureRole', 'Dynamic', 'M365')) {
@@ -287,12 +287,12 @@ function New-CIPPGroup {
287287
}
288288
}
289289

290-
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Created group $($GroupObject.displayName) with id $($Result.GroupId)" -Sev Info
290+
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Created group $($GroupObject.displayName) with id $($Result.GroupId)" -Sev Info -User $ExecutingUser
291291
return $Result
292292

293293
} catch {
294294
$ErrorMessage = Get-CippException -Exception $_
295-
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Group creation failed for $($GroupObject.displayName): $($ErrorMessage.NormalizedError)" -Sev Error -LogData $ErrorMessage
295+
Write-LogMessage -API $APIName -tenant $TenantFilter -message "Group creation failed for $($GroupObject.displayName): $($ErrorMessage.NormalizedError)" -Sev Error -LogData $ErrorMessage -User $ExecutingUser
296296

297297
return [PSCustomObject]@{
298298
Success = $false

‎Modules/CIPPCore/lib/Cronos.dll‎

52.3 KB
Binary file not shown.

‎Modules/CIPPDB/Public/DBCache/Set-CIPPDBCacheSecureScore.ps1‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,11 @@ function Set-CIPPDBCacheSecureScore {
2121

2222
# Cache secure score history (last 14 days)
2323
$SecureScore = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/secureScores?$top=14' -tenantid $TenantFilter -noPagination $true
24-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScore' -Data $SecureScore
25-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScore' -Data $SecureScore -Count
26-
$SecureScore = $null
24+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScore' -Data $SecureScore -AddCount
2725

2826
# Cache secure score control profiles
29-
$SecureScoreControlProfiles = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/secureScoreControlProfiles' -tenantid $TenantFilter
30-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScoreControlProfiles' -Data $SecureScoreControlProfiles
31-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScoreControlProfiles' -Data $SecureScoreControlProfiles -Count
32-
$SecureScoreControlProfiles = $null
27+
$Profiles = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/secureScoreControlProfiles' -tenantid $TenantFilter
28+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'SecureScoreControlProfiles' -Data $Profiles -AddCount
3329

3430
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Cached secure score and control profiles successfully' -sev Debug
3531

‎Modules/CIPPHTTP/Public/Entrypoints/HTTP Functions/Tenant/Administration/Alerts/Invoke-ListAlertsQueue.ps1‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,16 @@ function Invoke-ListAlertsQueue {
3333
PartitionKey = $Task.PartitionKey
3434
RepeatsEvery = 'When received'
3535
AlertComment = $Task.AlertComment
36+
CustomSubject = $Task.CustomSubject
3637
RawAlert = @{
37-
Conditions = @($Conditions)
38-
Actions = @($($Task.Actions | ConvertFrom-Json -Depth 10 -ErrorAction SilentlyContinue))
39-
Tenants = @($Tenants)
40-
type = $Task.type
41-
RowKey = $Task.RowKey
42-
PartitionKey = $Task.PartitionKey
43-
AlertComment = $Task.AlertComment
38+
Conditions = @($Conditions)
39+
Actions = @($($Task.Actions | ConvertFrom-Json -Depth 10 -ErrorAction SilentlyContinue))
40+
Tenants = @($Tenants)
41+
type = $Task.type
42+
RowKey = $Task.RowKey
43+
PartitionKey = $Task.PartitionKey
44+
AlertComment = $Task.AlertComment
45+
CustomSubject = $Task.CustomSubject
4446
}
4547
}
4648

0 commit comments

Comments
 (0)