forked from KelvinTegelaar/CIPP-API
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
massive audit log update: seperate download and processing into diffe…
…rent jobs
- Loading branch information
1 parent
1e9a80c
commit 8f04d56
Showing
6 changed files
with
194 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
70 changes: 70 additions & 0 deletions
70
Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| function New-CIPPAuditLogSearchResultsCache { | ||
| <# | ||
| .SYNOPSIS | ||
| Cache audit log search results for more efficient processing | ||
| .DESCRIPTION | ||
| Retrieves audit log searches for a tenant, processes them, and stores the results in a cache table. | ||
| Also tracks performance metrics for download and processing times. | ||
| .PARAMETER TenantFilter | ||
| The tenant to filter on. | ||
| #> | ||
| param ( | ||
| [Parameter(Mandatory = $true)] | ||
| [string]$TenantFilter, | ||
| [string]$SearchId | ||
| ) | ||
|
|
||
| try { | ||
| Write-Information "Starting audit log cache process for tenant: $TenantFilter" | ||
| $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' | ||
| $CacheWebhookStatsTable = Get-CippTable -TableName 'CacheWebhookStats' | ||
| # Start tracking download time | ||
| $downloadStartTime = Get-Date | ||
| # Process each search and store results in cache | ||
| try { | ||
| Write-Information "Processing search ID: $($SearchId) for tenant: $TenantFilter" | ||
| # Get the search results | ||
| #check if we haven't already downloaded this search by checking the cache table, if there are items with the same search id and tenant, we skip this search | ||
| $searchEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'" | ||
| if ($searchEntity) { | ||
| Write-Information "Search ID: $SearchId already cached for tenant: $TenantFilter" | ||
| exit 0 | ||
| } | ||
| $searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId | ||
| # Store the results in the cache table | ||
| foreach ($searchResult in $searchResults) { | ||
| $cacheEntity = @{ | ||
| RowKey = $searchResult.id | ||
| PartitionKey = $TenantFilter | ||
| SearchId = $SearchId | ||
| JSON = [string]($searchResult | ConvertTo-Json -Depth 10) | ||
| } | ||
| Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force | ||
| } | ||
| Write-Information "Successfully cached search ID: $($item.id) for tenant: $TenantFilter" | ||
| } catch { | ||
| throw $_ | ||
| } | ||
|
|
||
| # Calculate download time | ||
| $downloadEndTime = Get-Date | ||
| $downloadSeconds = ($downloadEndTime - $downloadStartTime).TotalSeconds | ||
|
|
||
| # Store performance metrics | ||
| $statsEntity = @{ | ||
| RowKey = $TenantFilter | ||
| PartitionKey = 'Stats' | ||
| DownloadSecs = [string]$downloadSeconds | ||
| SearchCount = [string]$logSearches.Count | ||
| } | ||
|
|
||
| Add-CIPPAzDataTableEntity @CacheWebhookStatsTable -Entity $statsEntity -Force | ||
|
|
||
| Write-Information "Completed audit log cache process for tenant: $TenantFilter. Download time: $downloadSeconds seconds" | ||
|
|
||
| return $logSearches.Count | ||
| } catch { | ||
| Write-Information "Error in New-CIPPAuditLogSearchResultsCache for tenant: $TenantFilter. Error: $($_.Exception.Message)" | ||
| throw $_ | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
...les/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantProcess.ps1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| function Push-AuditLogTenantProcess { | ||
| Param($Item) | ||
| $TenantFilter = $Item.TenantFilter | ||
| $RowIds = $Item.RowIds | ||
|
|
||
| try { | ||
| Write-Information "Audit Logs: Processing $($TenantFilter) with $($RowIds.Count) row IDs. We're processing id $($RowIds[0]) to $($RowIds[-1])" | ||
|
|
||
| # Get the CacheWebhooks table | ||
| $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' | ||
| # we do it this way because the rows can grow extremely large, if we get them all it might just hang for minutes at a time. | ||
| $Rows = foreach ($RowId in $RowIds) { | ||
| $CacheEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$RowId'" | ||
| if ($CacheEntity) { | ||
| $AuditData = $CacheEntity.JSON | ConvertFrom-Json -ErrorAction SilentlyContinue | ||
| $AuditData | ||
| } | ||
| } | ||
|
|
||
| if ($Rows.Count -gt 0) { | ||
| Write-Information "Retrieved $($Rows.Count) rows from cache for processing" | ||
| Test-CIPPAuditLogRules -TenantFilter $TenantFilter -Rows $Rows | ||
| } else { | ||
| Write-Information 'No rows found in cache for the provided row IDs' | ||
| } | ||
| } catch { | ||
| Write-Information ('Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters