Skip to content

Commit 658acea

Browse files
JohnDupreyJohnDuprey
committed
fix: add caching to sharepoint/onedrive site listings
1 parent 8c0ca04 commit 658acea

9 files changed

Lines changed: 424 additions & 15 deletions

File tree

‎Config/CIPPDBCacheTypes.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,11 +254,26 @@
254254
"friendlyName": "Mailbox Usage",
255255
"description": "Exchange Online mailbox usage statistics"
256256
},
257+
{
258+
"type": "OneDriveSiteListing",
259+
"friendlyName": "OneDrive Site Listing",
260+
"description": "OneDrive personal site listing details used for usage reporting"
261+
},
257262
{
258263
"type": "OneDriveUsage",
259264
"friendlyName": "OneDrive Usage",
260265
"description": "OneDrive usage statistics"
261266
},
267+
{
268+
"type": "SharePointSiteListing",
269+
"friendlyName": "SharePoint Site Listing",
270+
"description": "SharePoint site listing details used for usage reporting"
271+
},
272+
{
273+
"type": "SharePointSiteUsage",
274+
"friendlyName": "SharePoint Site Usage",
275+
"description": "SharePoint site usage statistics"
276+
},
262277
{
263278
"type": "OfficeActivations",
264279
"friendlyName": "Office Activations",

‎Modules/CIPPActivityTriggers/Public/Entrypoints/Activity Triggers/CIPPDBCache/Push-ExecCIPPDBCache.ps1‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ function Push-ExecCIPPDBCache {
3838

3939
# Single-type mode (legacy) — used by HTTP endpoint for on-demand cache refresh
4040
$Name = $Item.Name
41-
$Types = $Item.Types
41+
$Types = @($Item.Types | Where-Object { -not [string]::IsNullOrWhiteSpace($_) -and $_ -ne 'None' })
4242

4343
Write-Information "Collecting $Name for tenant $TenantFilter"
4444

@@ -62,7 +62,8 @@ function Push-ExecCIPPDBCache {
6262
}
6363

6464
# Add Types if provided (for Mailboxes function)
65-
if ($Types) {
65+
$FunctionSupportsTypes = $Function.Parameters.ContainsKey('Types')
66+
if ($Types.Count -gt 0 -and $FunctionSupportsTypes) {
6667
$CacheFunctionParams.Types = $Types
6768
}
6869

‎Modules/CIPPCore/Public/Get-CIPPOneDriveUsageReport.ps1‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
function Get-CIPPOneDriveUsageReport {
2+
<#
3+
.SYNOPSIS
4+
Generates a OneDrive usage report from the CIPP Reporting database
5+
6+
.DESCRIPTION
7+
Retrieves cached OneDrive site listing and usage data and combines them to match
8+
the payload shape of Invoke-ListSites for Type=OneDriveUsageAccount.
9+
10+
.PARAMETER TenantFilter
11+
The tenant to generate the report for
12+
#>
13+
[CmdletBinding()]
14+
param(
15+
[Parameter(Mandatory = $true)]
16+
[string]$TenantFilter
17+
)
18+
19+
try {
20+
if ($TenantFilter -eq 'AllTenants') {
21+
$AllSiteItems = Get-CIPPDbItem -TenantFilter 'allTenants' -Type 'OneDriveSiteListing'
22+
$Tenants = @($AllSiteItems | Where-Object { $_.RowKey -ne 'OneDriveSiteListing-Count' } | Select-Object -ExpandProperty PartitionKey -Unique)
23+
24+
$TenantList = Get-Tenants -IncludeErrors
25+
$Tenants = $Tenants | Where-Object { $TenantList.defaultDomainName -contains $_ }
26+
27+
$AllResults = [System.Collections.Generic.List[PSCustomObject]]::new()
28+
foreach ($Tenant in $Tenants) {
29+
try {
30+
$TenantResults = Get-CIPPOneDriveUsageReport -TenantFilter $Tenant
31+
foreach ($Result in $TenantResults) {
32+
$Result | Add-Member -NotePropertyName 'Tenant' -NotePropertyValue $Tenant -Force
33+
$AllResults.Add($Result)
34+
}
35+
} catch {
36+
Write-LogMessage -API 'OneDriveUsageReport' -tenant $Tenant -message "Failed to get report for tenant: $($_.Exception.Message)" -sev Warning
37+
}
38+
}
39+
return $AllResults
40+
}
41+
42+
$SiteItems = @(Get-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveSiteListing' | Where-Object { $_.RowKey -ne 'OneDriveSiteListing-Count' })
43+
if (-not $SiteItems) {
44+
throw 'No OneDrive site listing data found in reporting database. Sync OneDriveUsage cache first.'
45+
}
46+
47+
$UsageItems = @(Get-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveUsage' | Where-Object { $_.RowKey -ne 'OneDriveUsage-Count' })
48+
if (-not $UsageItems) {
49+
throw 'No OneDrive usage data found in reporting database. Sync OneDriveUsage cache first.'
50+
}
51+
52+
$LatestSiteTimestamp = ($SiteItems | Where-Object { $_.Timestamp } | Sort-Object Timestamp -Descending | Select-Object -First 1).Timestamp
53+
$LatestUsageTimestamp = ($UsageItems | Where-Object { $_.Timestamp } | Sort-Object Timestamp -Descending | Select-Object -First 1).Timestamp
54+
$CacheTimestamp = if ($LatestSiteTimestamp -and $LatestUsageTimestamp) {
55+
if ($LatestSiteTimestamp -gt $LatestUsageTimestamp) { $LatestSiteTimestamp } else { $LatestUsageTimestamp }
56+
} else {
57+
$LatestSiteTimestamp ?? $LatestUsageTimestamp
58+
}
59+
60+
$UsageBySiteId = [System.Collections.Generic.Dictionary[string, object]]::new([System.StringComparer]::OrdinalIgnoreCase)
61+
foreach ($UsageItem in $UsageItems) {
62+
$UsageRow = $UsageItem.Data | ConvertFrom-Json -Depth 10
63+
if (-not [string]::IsNullOrWhiteSpace($UsageRow.siteId)) {
64+
$UsageBySiteId[[string]$UsageRow.siteId] = $UsageRow
65+
}
66+
}
67+
68+
$Report = [System.Collections.Generic.List[PSCustomObject]]::new()
69+
foreach ($SiteItem in $SiteItems) {
70+
$Site = $SiteItem.Data | ConvertFrom-Json -Depth 10
71+
if ($Site.isPersonalSite -ne $true) {
72+
continue
73+
}
74+
75+
$SiteUsage = $null
76+
[void]$UsageBySiteId.TryGetValue([string]$Site.sharepointIds.siteId, [ref]$SiteUsage)
77+
78+
$StorageUsedInBytes = [double]($SiteUsage.storageUsedInBytes ?? 0)
79+
$StorageAllocatedInBytes = [double]($SiteUsage.storageAllocatedInBytes ?? 0)
80+
81+
$ReportItem = [PSCustomObject]@{
82+
siteId = $Site.sharepointIds.siteId
83+
webId = $Site.sharepointIds.webId
84+
createdDateTime = $Site.createdDateTime
85+
displayName = $Site.displayName
86+
webUrl = $Site.webUrl
87+
ownerDisplayName = $SiteUsage.ownerDisplayName
88+
ownerPrincipalName = $SiteUsage.ownerPrincipalName
89+
lastActivityDate = $SiteUsage.lastActivityDate
90+
fileCount = $SiteUsage.fileCount
91+
storageUsedInGigabytes = [math]::round($StorageUsedInBytes / 1GB, 2)
92+
storageAllocatedInGigabytes = [math]::round($StorageAllocatedInBytes / 1GB, 2)
93+
storageUsedInBytes = $SiteUsage.storageUsedInBytes
94+
storageAllocatedInBytes = $SiteUsage.storageAllocatedInBytes
95+
rootWebTemplate = $SiteUsage.rootWebTemplate
96+
reportRefreshDate = $SiteUsage.reportRefreshDate
97+
AutoMapUrl = $Site.AutoMapUrl
98+
}
99+
100+
if ($CacheTimestamp) {
101+
$ReportItem | Add-Member -NotePropertyName 'CacheTimestamp' -NotePropertyValue $CacheTimestamp -Force
102+
}
103+
104+
$Report.Add($ReportItem)
105+
}
106+
107+
return $Report | Sort-Object -Property displayName
108+
109+
} catch {
110+
Write-LogMessage -API 'OneDriveUsageReport' -tenant $TenantFilter -message "Failed to generate OneDrive usage report: $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
111+
throw
112+
}
113+
}

‎Modules/CIPPCore/Public/Get-CIPPSharePointSiteUsageReport.ps1‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
function Get-CIPPSharePointSiteUsageReport {
2+
<#
3+
.SYNOPSIS
4+
Generates a SharePoint site usage report from the CIPP Reporting database
5+
6+
.DESCRIPTION
7+
Retrieves cached SharePoint site listing and usage data and combines them to match
8+
the payload shape of Invoke-ListSites for Type=SharePointSiteUsage.
9+
10+
.PARAMETER TenantFilter
11+
The tenant to generate the report for
12+
#>
13+
[CmdletBinding()]
14+
param(
15+
[Parameter(Mandatory = $true)]
16+
[string]$TenantFilter
17+
)
18+
19+
try {
20+
if ($TenantFilter -eq 'AllTenants') {
21+
$AllSiteItems = Get-CIPPDbItem -TenantFilter 'allTenants' -Type 'SharePointSiteListing'
22+
$Tenants = @($AllSiteItems | Where-Object { $_.RowKey -ne 'SharePointSiteListing-Count' } | Select-Object -ExpandProperty PartitionKey -Unique)
23+
24+
$TenantList = Get-Tenants -IncludeErrors
25+
$Tenants = $Tenants | Where-Object { $TenantList.defaultDomainName -contains $_ }
26+
27+
$AllResults = [System.Collections.Generic.List[PSCustomObject]]::new()
28+
foreach ($Tenant in $Tenants) {
29+
try {
30+
$TenantResults = Get-CIPPSharePointSiteUsageReport -TenantFilter $Tenant
31+
foreach ($Result in $TenantResults) {
32+
$Result | Add-Member -NotePropertyName 'Tenant' -NotePropertyValue $Tenant -Force
33+
$AllResults.Add($Result)
34+
}
35+
} catch {
36+
Write-LogMessage -API 'SharePointSiteUsageReport' -tenant $Tenant -message "Failed to get report for tenant: $($_.Exception.Message)" -sev Warning
37+
}
38+
}
39+
return $AllResults
40+
}
41+
42+
$SiteItems = @(Get-CIPPDbItem -TenantFilter $TenantFilter -Type 'SharePointSiteListing' | Where-Object { $_.RowKey -ne 'SharePointSiteListing-Count' })
43+
if (-not $SiteItems) {
44+
throw 'No SharePoint site listing data found in reporting database. Sync SharePointSiteUsage cache first.'
45+
}
46+
47+
$UsageItems = @(Get-CIPPDbItem -TenantFilter $TenantFilter -Type 'SharePointSiteUsage' | Where-Object { $_.RowKey -ne 'SharePointSiteUsage-Count' })
48+
if (-not $UsageItems) {
49+
throw 'No SharePoint site usage data found in reporting database. Sync SharePointSiteUsage cache first.'
50+
}
51+
52+
$LatestSiteTimestamp = ($SiteItems | Where-Object { $_.Timestamp } | Sort-Object Timestamp -Descending | Select-Object -First 1).Timestamp
53+
$LatestUsageTimestamp = ($UsageItems | Where-Object { $_.Timestamp } | Sort-Object Timestamp -Descending | Select-Object -First 1).Timestamp
54+
$CacheTimestamp = if ($LatestSiteTimestamp -and $LatestUsageTimestamp) {
55+
if ($LatestSiteTimestamp -gt $LatestUsageTimestamp) { $LatestSiteTimestamp } else { $LatestUsageTimestamp }
56+
} else {
57+
$LatestSiteTimestamp ?? $LatestUsageTimestamp
58+
}
59+
60+
$UsageBySiteId = [System.Collections.Generic.Dictionary[string, object]]::new([System.StringComparer]::OrdinalIgnoreCase)
61+
foreach ($UsageItem in $UsageItems) {
62+
$UsageRow = $UsageItem.Data | ConvertFrom-Json -Depth 10
63+
if (-not [string]::IsNullOrWhiteSpace($UsageRow.siteId)) {
64+
$UsageBySiteId[[string]$UsageRow.siteId] = $UsageRow
65+
}
66+
}
67+
68+
$Report = [System.Collections.Generic.List[PSCustomObject]]::new()
69+
foreach ($SiteItem in $SiteItems) {
70+
$Site = $SiteItem.Data | ConvertFrom-Json -Depth 10
71+
if ($Site.isPersonalSite -eq $true) {
72+
continue
73+
}
74+
75+
$SiteUsage = $null
76+
[void]$UsageBySiteId.TryGetValue([string]$Site.sharepointIds.siteId, [ref]$SiteUsage)
77+
78+
$StorageUsedInBytes = [double]($SiteUsage.storageUsedInBytes ?? 0)
79+
$StorageAllocatedInBytes = [double]($SiteUsage.storageAllocatedInBytes ?? 0)
80+
81+
$ReportItem = [PSCustomObject]@{
82+
siteId = $Site.sharepointIds.siteId
83+
webId = $Site.sharepointIds.webId
84+
createdDateTime = $Site.createdDateTime
85+
displayName = $Site.displayName
86+
webUrl = $Site.webUrl
87+
ownerDisplayName = $SiteUsage.ownerDisplayName
88+
ownerPrincipalName = $SiteUsage.ownerPrincipalName
89+
lastActivityDate = $SiteUsage.lastActivityDate
90+
fileCount = $SiteUsage.fileCount
91+
storageUsedInGigabytes = [math]::round($StorageUsedInBytes / 1GB, 2)
92+
storageAllocatedInGigabytes = [math]::round($StorageAllocatedInBytes / 1GB, 2)
93+
storageUsedInBytes = $SiteUsage.storageUsedInBytes
94+
storageAllocatedInBytes = $SiteUsage.storageAllocatedInBytes
95+
rootWebTemplate = $SiteUsage.rootWebTemplate
96+
reportRefreshDate = $SiteUsage.reportRefreshDate
97+
AutoMapUrl = $Site.AutoMapUrl
98+
}
99+
100+
if ($CacheTimestamp) {
101+
$ReportItem | Add-Member -NotePropertyName 'CacheTimestamp' -NotePropertyValue $CacheTimestamp -Force
102+
}
103+
104+
$Report.Add($ReportItem)
105+
}
106+
107+
return $Report | Sort-Object -Property displayName
108+
109+
} catch {
110+
Write-LogMessage -API 'SharePointSiteUsageReport' -tenant $TenantFilter -message "Failed to generate SharePoint site usage report: $($_.Exception.Message)" -sev Error -LogData (Get-CippException -Exception $_)
111+
throw
112+
}
113+
}

‎Modules/CIPPCore/Public/Invoke-CIPPDBCacheCollection.ps1‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,10 @@ function Invoke-CIPPDBCacheCollection {
9191
ExchangeData = @(
9292
'CASMailboxes'
9393
'MailboxUsage'
94+
'OneDriveSiteListing'
9495
'OneDriveUsage'
96+
'SharePointSiteListing'
97+
'SharePointSiteUsage'
9598
'OfficeActivations'
9699
)
97100
ConditionalAccess = @(

‎Modules/CIPPDB/Public/DBCache/Set-CIPPDBCacheOneDriveUsage.ps1‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,52 @@ function Set-CIPPDBCacheOneDriveUsage {
1717
)
1818

1919
try {
20-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Caching OneDrive usage' -sev Debug
21-
22-
$OneDriveUsage = New-GraphGetRequest -uri "https://graph.microsoft.com/beta/reports/getOneDriveUsageAccountDetail(period='D7')?`$format=application%2fjson" -tenantid $TenantFilter
23-
$OneDriveUsage | ForEach-Object { $_ | Add-Member -NotePropertyName 'userPrincipalName' -NotePropertyValue $_.ownerPrincipalName -Force }
24-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveUsage' -Data $OneDriveUsage
25-
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveUsage' -Data $OneDriveUsage -Count
26-
$OneDriveUsage = $null
27-
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Cached OneDrive usage successfully' -sev Debug
20+
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Caching OneDrive site listing and usage' -sev Debug
21+
22+
$BulkRequests = @(
23+
@{
24+
id = 'listAllSites'
25+
method = 'GET'
26+
url = "sites/getAllSites?`$filter=isPersonalSite eq true&`$select=id,createdDateTime,description,name,displayName,isPersonalSite,lastModifiedDateTime,webUrl,siteCollection,sharepointIds&`$top=999"
27+
}
28+
@{
29+
id = 'usage'
30+
method = 'GET'
31+
url = "reports/getOneDriveUsageAccountDetail(period='D7')?`$format=application/json&`$top=999"
32+
}
33+
)
34+
35+
$Result = New-GraphBulkRequest -tenantid $TenantFilter -Requests @($BulkRequests) -asapp $true
36+
$Sites = @(($Result | Where-Object { $_.id -eq 'listAllSites' }).body.value)
37+
$UsageBase64 = ($Result | Where-Object { $_.id -eq 'usage' }).body
38+
$UsageJson = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($UsageBase64))
39+
$OneDriveUsage = @(($UsageJson | ConvertFrom-Json).value)
40+
41+
foreach ($UsageRow in $OneDriveUsage) {
42+
$UsageRow | Add-Member -NotePropertyName 'id' -NotePropertyValue $UsageRow.siteId -Force
43+
$UsageRow | Add-Member -NotePropertyName 'userPrincipalName' -NotePropertyValue $UsageRow.ownerPrincipalName -Force
44+
}
45+
46+
$OneDriveListing = [System.Collections.Generic.List[object]]::new()
47+
foreach ($Site in $Sites) {
48+
$OneDriveListing.Add([PSCustomObject]@{
49+
id = $Site.id
50+
sharepointIds = $Site.sharepointIds
51+
createdDateTime = $Site.createdDateTime
52+
displayName = $Site.displayName
53+
webUrl = $Site.webUrl
54+
isPersonalSite = $Site.isPersonalSite
55+
AutoMapUrl = ''
56+
})
57+
}
58+
59+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveSiteListing' -Data @($OneDriveListing)
60+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveSiteListing' -Data @($OneDriveListing) -Count
61+
62+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveUsage' -Data @($OneDriveUsage)
63+
Add-CIPPDbItem -TenantFilter $TenantFilter -Type 'OneDriveUsage' -Data @($OneDriveUsage) -Count
64+
65+
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message 'Cached OneDrive site listing and usage successfully' -sev Debug
2866

2967
} catch {
3068
Write-LogMessage -API 'CIPPDBCache' -tenant $TenantFilter -message "Failed to cache OneDrive usage: $($_.Exception.Message)" -sev Error

0 commit comments

Comments
 (0)