Skip to content

Commit a19f8d1

Browse files
pull[bot]pull[bot]
authored
Merge pull request #936 from KelvinTegelaar/dev
[pull] dev from KelvinTegelaar:dev
2 parents aaa2d09 + 2907acc commit a19f8d1

24 files changed

Lines changed: 201 additions & 224 deletions

‎CIPPDBCacheTypes.json‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -204,26 +204,6 @@
204204
"friendlyName": "Exchange Hosted Outbound Spam Filter Policy",
205205
"description": "Exchange Online hosted outbound spam filter policy"
206206
},
207-
{
208-
"type": "ExoAntiPhishPolicy",
209-
"friendlyName": "Exchange Anti-Phish Policy",
210-
"description": "Exchange Online anti-phishing policy"
211-
},
212-
{
213-
"type": "ExoSafeLinksPolicy",
214-
"friendlyName": "Exchange Safe Links Policy",
215-
"description": "Exchange Online Safe Links policy"
216-
},
217-
{
218-
"type": "ExoSafeAttachmentPolicy",
219-
"friendlyName": "Exchange Safe Attachment Policy",
220-
"description": "Exchange Online Safe Attachment policy"
221-
},
222-
{
223-
"type": "ExoMalwareFilterPolicy",
224-
"friendlyName": "Exchange Malware Filter Policy",
225-
"description": "Exchange Online malware filter policy"
226-
},
227207
{
228208
"type": "ExoAtpPolicyForO365",
229209
"friendlyName": "Exchange ATP Policy for O365",

‎Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Mailbox Permissions/Push-GetCalendarPermissionsBatch.ps1‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,9 @@ function Push-GetCalendarPermissionsBatch {
127127
Write-Information "Failed to get calendar permissions for $($Perm.OperationGuid): $($Perm.error)"
128128
continue
129129
}
130+
$AccessStr = if ($Perm.AccessRights -is [array]) { $Perm.AccessRights -join ',' } else { $Perm.AccessRights }
130131
$AllCalendarPermissions.Add([PSCustomObject]@{
131-
id = [guid]::NewGuid().ToString()
132+
id = "CAL-$($Perm.Identity)-$($Perm.User)-$AccessStr"
132133
Identity = $Perm.Identity
133134
User = $Perm.User
134135
AccessRights = $Perm.AccessRights

‎Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/Mailbox Permissions/Push-GetMailboxPermissionsBatch.ps1‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ function Push-GetMailboxPermissionsBatch {
5757
# Normalize MailboxPermission results
5858
if ($MailboxPermissions['Get-MailboxPermission']) {
5959
$NormalizedMailboxPerms = foreach ($Perm in $MailboxPermissions['Get-MailboxPermission']) {
60-
# Create normalized object with consistent property names and unique ID
60+
$AccessStr = if ($Perm.AccessRights -is [array]) { $Perm.AccessRights -join ',' } else { $Perm.AccessRights }
6161
[PSCustomObject]@{
62-
id = [guid]::NewGuid().ToString()
62+
id = "MBP-$($Perm.Identity)-$($Perm.User)-$AccessStr"
6363
Identity = $Perm.Identity
6464
User = $Perm.User
6565
AccessRights = $Perm.AccessRights
@@ -73,11 +73,12 @@ function Push-GetMailboxPermissionsBatch {
7373
# Normalize the results - RecipientPermission uses 'Trustee' instead of 'User'
7474
if ($MailboxPermissions['Get-RecipientPermission']) {
7575
$NormalizedRecipientPerms = foreach ($Perm in $MailboxPermissions['Get-RecipientPermission']) {
76-
# Create normalized object with consistent property names and unique ID
76+
$UserVal = if ($Perm.Trustee) { $Perm.Trustee } else { $Perm.User }
77+
$AccessStr = if ($Perm.AccessRights -is [array]) { $Perm.AccessRights -join ',' } else { $Perm.AccessRights }
7778
[PSCustomObject]@{
78-
id = [guid]::NewGuid().ToString()
79+
id = "RCP-$($Perm.Identity)-$UserVal-$AccessStr"
7980
Identity = $Perm.Identity
80-
User = if ($Perm.Trustee) { $Perm.Trustee } else { $Perm.User }
81+
User = $UserVal
8182
AccessRights = $Perm.AccessRights
8283
IsInherited = $Perm.IsInherited
8384
Deny = $Perm.Deny
@@ -94,10 +95,11 @@ function Push-GetMailboxPermissionsBatch {
9495
# Normalize SendOnBehalf permissions from passed mailbox metadata
9596
$NormalizedSendOnBehalfPerms = foreach ($Mailbox in ($MailboxData | Where-Object { $_.GrantSendOnBehalfTo -and ($Mailboxes -contains $_.UPN) })) {
9697
foreach ($Delegate in (@($Mailbox.GrantSendOnBehalfTo) | Where-Object { $_ -and $MailboxIdentityLookup.ContainsKey([string]$_) })) {
98+
$DelegateUPN = $MailboxIdentityLookup[[string]$Delegate]
9799
[PSCustomObject]@{
98-
id = [guid]::NewGuid().ToString()
100+
id = "SOB-$($Mailbox.UPN)-$DelegateUPN"
99101
Identity = $Mailbox.UPN
100-
User = $MailboxIdentityLookup[[string]$Delegate]
102+
User = $DelegateUPN
101103
AccessRights = @('SendOnBehalf')
102104
IsInherited = $false
103105
Deny = $false

‎Modules/CIPPCore/Public/Add-CIPPAzDataTableEntity.ps1‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ function Add-CIPPAzDataTableEntity {
3939

4040
$MaxRowSize = 500000 - 100
4141
$MaxSize = 30kb
42+
$BatchQueue = [System.Collections.Generic.List[object]]::new()
4243

4344
foreach ($SingleEnt in @($Entity)) {
4445
try {
@@ -85,6 +86,40 @@ function Add-CIPPAzDataTableEntity {
8586
Write-Warning "Error during entity validation: $($_.Exception.Message)"
8687
}
8788

89+
# Check entity size - if under MaxSize, batch it for bulk write
90+
$entityBytes = [System.Text.Encoding]::UTF8.GetByteCount($($SingleEnt | ConvertTo-Json -Compress))
91+
92+
if ($entityBytes -lt $MaxSize) {
93+
# Small entity - add to batch queue
94+
$BatchQueue.Add($SingleEnt)
95+
if ($BatchQueue.Count -ge 100) {
96+
try {
97+
Add-AzDataTableEntity @Parameters -Entity $BatchQueue.ToArray() -ErrorAction Stop
98+
} catch {
99+
# Batch failed - fall back to individual writes
100+
Write-Warning "Batch write failed, falling back to individual writes: $($_.Exception.Message)"
101+
foreach ($batchItem in $BatchQueue) {
102+
Add-AzDataTableEntity @Parameters -Entity $batchItem -ErrorAction Stop
103+
}
104+
}
105+
$BatchQueue.Clear()
106+
}
107+
continue
108+
}
109+
110+
# Large entity - flush any pending batch first, then write individually
111+
if ($BatchQueue.Count -gt 0) {
112+
try {
113+
Add-AzDataTableEntity @Parameters -Entity $BatchQueue.ToArray() -ErrorAction Stop
114+
} catch {
115+
Write-Warning "Batch write failed, falling back to individual writes: $($_.Exception.Message)"
116+
foreach ($batchItem in $BatchQueue) {
117+
Add-AzDataTableEntity @Parameters -Entity $batchItem -ErrorAction Stop
118+
}
119+
}
120+
$BatchQueue.Clear()
121+
}
122+
88123
Add-AzDataTableEntity @Parameters -Entity $SingleEnt -ErrorAction Stop
89124

90125
} catch [System.Exception] {
@@ -237,4 +272,17 @@ function Add-CIPPAzDataTableEntity {
237272
}
238273
}
239274
}
275+
276+
# Flush any remaining batched entities
277+
if ($BatchQueue.Count -gt 0) {
278+
try {
279+
Add-AzDataTableEntity @Parameters -Entity $BatchQueue.ToArray() -ErrorAction Stop
280+
} catch {
281+
Write-Warning "Final batch write failed, falling back to individual writes: $($_.Exception.Message)"
282+
foreach ($batchItem in $BatchQueue) {
283+
Add-AzDataTableEntity @Parameters -Entity $batchItem -ErrorAction Stop
284+
}
285+
}
286+
$BatchQueue.Clear()
287+
}
240288
}

‎Modules/CIPPCore/Public/Add-CIPPDbItem.ps1‎

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,14 @@ function Add-CIPPDbItem {
2727
begin {
2828
$Table = Get-CippTable -tablename 'CippReportingDB'
2929
$Batch = [System.Collections.Generic.List[hashtable]]::new()
30+
$NewRowKeys = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
3031
$TotalProcessed = 0
3132

3233
if ($TenantFilter -match '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$') {
3334
try {
3435
$TenantFilter = (Get-Tenants -TenantFilter $TenantFilter -IncludeErrors | Select-Object -First 1).defaultDomainName
3536
} catch {}
3637
}
37-
38-
if (-not $Count.IsPresent -and -not $Append.IsPresent) {
39-
$Filter = "PartitionKey eq '{0}' and RowKey ge '{1}-' and RowKey lt '{1}0'" -f $TenantFilter, $Type
40-
$Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag
41-
if ($Existing) {
42-
$null = Remove-AzDataTableEntity @Table -Entity $Existing -Force
43-
}
44-
}
4538
}
4639

4740
process {
@@ -55,9 +48,11 @@ function Add-CIPPDbItem {
5548
foreach ($Item in @($InputObject)) {
5649
if ($null -eq $Item) { continue }
5750
$ItemId = $Item.ExternalDirectoryObjectId ?? $Item.id ?? $Item.Identity ?? $Item.skuId ?? $Item.userPrincipalName ?? [guid]::NewGuid().ToString()
51+
$RowKey = "$Type-$ItemId" -replace '[/\\#?]', '_' -replace '[\u0000-\u001F\u007F-\u009F]', ''
52+
[void]$NewRowKeys.Add($RowKey)
5853
$Batch.Add(@{
5954
PartitionKey = $TenantFilter
60-
RowKey = ("$Type-$ItemId" -replace '[/\\#?]', '_' -replace '[\u0000-\u001F\u007F-\u009F]', '')
55+
RowKey = $RowKey
6156
Data = [string]($Item | ConvertTo-Json -Depth 10 -Compress)
6257
Type = $Type
6358
})
@@ -75,7 +70,26 @@ function Add-CIPPDbItem {
7570
$TotalProcessed += $Batch.Count
7671
}
7772

73+
# Clean up orphaned rows (entities that no longer exist in the new dataset)
74+
if (-not $Count.IsPresent -and -not $Append.IsPresent -and $TotalProcessed -gt 0) {
75+
$Filter = "PartitionKey eq '{0}' and RowKey ge '{1}-' and RowKey lt '{1}0'" -f $TenantFilter, $Type
76+
$Existing = Get-CIPPAzDataTableEntity @Table -Filter $Filter -Property PartitionKey, RowKey, ETag, OriginalEntityId
77+
if ($Existing) {
78+
$Orphans = foreach ($Row in @($Existing)) {
79+
if ($Row.RowKey -eq "$Type-Count") { continue }
80+
$ParentKey = $Row.OriginalEntityId ?? $Row.RowKey
81+
if (-not $NewRowKeys.Contains($ParentKey)) {
82+
$Row
83+
}
84+
}
85+
if ($Orphans) {
86+
$null = Remove-AzDataTableEntity @Table -Entity @($Orphans) -Force
87+
}
88+
}
89+
}
90+
7891
if ($Count.IsPresent -or $AddCount.IsPresent) {
92+
$CntStart = $Stopwatch.ElapsedMilliseconds
7993
$NewCount = $TotalProcessed
8094
if ($Append.IsPresent) {
8195
$Filter = "PartitionKey eq '{0}' and RowKey eq '{1}-Count'" -f $TenantFilter, $Type
@@ -87,6 +101,7 @@ function Add-CIPPDbItem {
87101
RowKey = "$Type-Count"
88102
DataCount = [int]$NewCount
89103
} -Force
104+
$CountMs = $Stopwatch.ElapsedMilliseconds - $CntStart
90105
}
91106

92107
Write-LogMessage -API 'CIPPDbItem' -tenant $TenantFilter -message "Added $TotalProcessed items of type $Type" -sev Debug

‎Modules/CIPPCore/Public/Add-CippTestResult.ps1‎

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -78,28 +78,18 @@ function Add-CippTestResult {
7878
[string]$Category
7979
)
8080

81-
try {
82-
$Table = Get-CippTable -tablename 'CippTestResults'
83-
84-
$Entity = @{
85-
PartitionKey = $TenantFilter
86-
RowKey = $TestId
87-
Status = $Status
88-
ResultMarkdown = $ResultMarkdown ?? ''
89-
ResultDataJson = $ResultDataJson ?? ''
90-
Risk = $Risk ?? ''
91-
Name = $Name ?? ''
92-
Pillar = $Pillar ?? ''
93-
UserImpact = $UserImpact ?? ''
94-
ImplementationEffort = $ImplementationEffort ?? ''
95-
Category = $Category ?? ''
96-
TestType = $TestType
97-
}
98-
99-
Add-CIPPAzDataTableEntity @Table -Entity $Entity -Force
100-
Write-LogMessage -API 'CIPPTestResults' -tenant $TenantFilter -message "Added test result: $TestId - $Status" -sev Debug
101-
} catch {
102-
Write-LogMessage -API 'CIPPTestResults' -tenant $TenantFilter -message "Failed to add test result: $($_.Exception.Message)" -sev Error
103-
throw
81+
return @{
82+
PartitionKey = $TenantFilter
83+
RowKey = $TestId
84+
Status = $Status
85+
ResultMarkdown = $ResultMarkdown ?? ''
86+
ResultDataJson = $ResultDataJson ?? ''
87+
Risk = $Risk ?? ''
88+
Name = $Name ?? ''
89+
Pillar = $Pillar ?? ''
90+
UserImpact = $UserImpact ?? ''
91+
ImplementationEffort = $ImplementationEffort ?? ''
92+
Category = $Category ?? ''
93+
TestType = $TestType
10494
}
10595
}

‎Modules/CIPPCore/Public/GraphHelper/New-GraphBulkRequest.ps1‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,48 @@ function New-GraphBulkRequest {
6161
}
6262
Write-Host 'Getting more'
6363
Write-Host $MoreData.body.'@odata.nextLink'
64-
$AdditionalValues = New-GraphGetRequest -ComplexFilter -uri $MoreData.body.'@odata.nextLink' -tenantid $tenantid -NoAuthCheck $NoAuthCheck -scope $scope -AsApp $asapp -headers $Headers
65-
$NewValues = [System.Collections.Generic.List[PSCustomObject]]$MoreData.body.value
66-
$AdditionalValues | ForEach-Object { $NewValues.add($_) }
67-
$MoreData.body.value = $NewValues
64+
# Re-batch nextLink pagination instead of sequential calls
65+
$NextLinkQueue = [System.Collections.Generic.Queue[PSCustomObject]]::new()
66+
$InitialNextUrl = $MoreData.body.'@odata.nextLink' -replace 'https://graph.microsoft.com/(v1\.0|beta)', ''
67+
$NextLinkQueue.Enqueue([PSCustomObject]@{
68+
id = $MoreData.id
69+
url = $InitialNextUrl
70+
})
71+
72+
while ($NextLinkQueue.Count -gt 0) {
73+
# Drain up to 20 nextLinks into a batch
74+
$NextBatchRequests = [System.Collections.Generic.List[PSCustomObject]]::new()
75+
while ($NextLinkQueue.Count -gt 0 -and $NextBatchRequests.Count -lt 20) {
76+
$Item = $NextLinkQueue.Dequeue()
77+
$NextBatchRequests.Add([PSCustomObject]@{
78+
id = $Item.id
79+
method = 'GET'
80+
url = $Item.url
81+
})
82+
}
83+
84+
$NextReqBody = ConvertTo-Json -InputObject @{ requests = @($NextBatchRequests) } -Compress -Depth 100
85+
$NextReturn = Invoke-CIPPRestMethod -Uri $URL -Method POST -Headers $headers -ContentType 'application/json; charset=utf-8' -Body $NextReqBody
86+
if ($NextReturn.headers.'retry-after') {
87+
$headers = Get-GraphToken -tenantid $tenantid -scope $scope -AsApp $asapp
88+
$NextReturn = Invoke-CIPPRestMethod -Uri $URL -Method POST -Headers $headers -ContentType 'application/json; charset=utf-8' -Body $NextReqBody
89+
}
90+
91+
foreach ($NextResponse in $NextReturn.responses) {
92+
if ($NextResponse.body.value) {
93+
$NewValues = [System.Collections.Generic.List[PSCustomObject]]$MoreData.body.value
94+
foreach ($val in $NextResponse.body.value) { $NewValues.Add($val) }
95+
$MoreData.body.value = $NewValues
96+
}
97+
if ($NextResponse.body.'@odata.nextLink' -and $NoPaginateIds -notcontains $NextResponse.id) {
98+
$ContinueUrl = $NextResponse.body.'@odata.nextLink' -replace 'https://graph.microsoft.com/(v1\.0|beta)', ''
99+
$NextLinkQueue.Enqueue([PSCustomObject]@{
100+
id = $NextResponse.id
101+
url = $ContinueUrl
102+
})
103+
}
104+
}
105+
}
68106
}
69107

70108
} catch {

‎Modules/CIPPCore/Public/Invoke-CIPPDBCacheCollection.ps1‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,6 @@ function Invoke-CIPPDBCacheCollection {
8080
'ExoAcceptedDomains'
8181
'ExoHostedContentFilterPolicy'
8282
'ExoHostedOutboundSpamFilterPolicy'
83-
'ExoAntiPhishPolicy'
84-
'ExoSafeLinksPolicy'
85-
'ExoSafeAttachmentPolicy'
86-
'ExoMalwareFilterPolicy'
8783
'ExoAtpPolicyForO365'
8884
'ExoQuarantinePolicy'
8985
'ExoRemoteDomain'

0 commit comments

Comments
 (0)