diff --git a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 index 1aac0b36eeab..693011c25aba 100644 --- a/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 +++ b/Modules/CIPPCore/Public/AuditLogs/Get-CippAuditLogSearches.ps1 @@ -31,7 +31,6 @@ function Get-CippAuditLogSearches { return @() } $Queries = New-GraphBulkRequest -Requests @($BulkRequests) -AsApp $true -TenantId $TenantFilter | Select-Object -ExpandProperty body - $Queries = $Queries | Where-Object { $PendingQueries.RowKey -contains $_.id -and $_.status -eq 'succeeded' } } else { $Queries = New-GraphGetRequest -uri 'https://graph.microsoft.com/beta/security/auditLog/queries' -AsApp $true -tenantid $TenantFilter diff --git a/Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1 b/Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1 new file mode 100644 index 000000000000..9a16976f5204 --- /dev/null +++ b/Modules/CIPPCore/Public/AuditLogs/New-CIPPAuditLogSearchResultsCache.ps1 @@ -0,0 +1,70 @@ +function New-CIPPAuditLogSearchResultsCache { + <# + .SYNOPSIS + Cache audit log search results for more efficient processing + .DESCRIPTION + Retrieves audit log searches for a tenant, processes them, and stores the results in a cache table. + Also tracks performance metrics for download and processing times. + .PARAMETER TenantFilter + The tenant to filter on. + #> + param ( + [Parameter(Mandatory = $true)] + [string]$TenantFilter, + [string]$SearchId + ) + + try { + Write-Information "Starting audit log cache process for tenant: $TenantFilter" + $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' + $CacheWebhookStatsTable = Get-CippTable -TableName 'CacheWebhookStats' + # Start tracking download time + $downloadStartTime = Get-Date + # Process each search and store results in cache + try { + Write-Information "Processing search ID: $($SearchId) for tenant: $TenantFilter" + # Get the search results + #check if we haven't already downloaded this search by checking the cache table, if there are items with the same search id and tenant, we skip this search + $searchEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and SearchId eq '$SearchId'" + if ($searchEntity) { + Write-Information "Search ID: $SearchId already cached for tenant: $TenantFilter" + exit 0 + } + $searchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId + # Store the results in the cache table + foreach ($searchResult in $searchResults) { + $cacheEntity = @{ + RowKey = $searchResult.id + PartitionKey = $TenantFilter + SearchId = $SearchId + JSON = [string]($searchResult | ConvertTo-Json -Depth 10) + } + Add-CIPPAzDataTableEntity @CacheWebhooksTable -Entity $cacheEntity -Force + } + Write-Information "Successfully cached search ID: $($item.id) for tenant: $TenantFilter" + } catch { + throw $_ + } + + # Calculate download time + $downloadEndTime = Get-Date + $downloadSeconds = ($downloadEndTime - $downloadStartTime).TotalSeconds + + # Store performance metrics + $statsEntity = @{ + RowKey = $TenantFilter + PartitionKey = 'Stats' + DownloadSecs = [string]$downloadSeconds + SearchCount = [string]$logSearches.Count + } + + Add-CIPPAzDataTableEntity @CacheWebhookStatsTable -Entity $statsEntity -Force + + Write-Information "Completed audit log cache process for tenant: $TenantFilter. Download time: $downloadSeconds seconds" + + return $logSearches.Count + } catch { + Write-Information "Error in New-CIPPAuditLogSearchResultsCache for tenant: $TenantFilter. Error: $($_.Exception.Message)" + throw $_ + } +} diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 similarity index 63% rename from Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 rename to Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 index b55e4bc85dbf..51556092951f 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenant.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantDownload.ps1 @@ -1,10 +1,10 @@ -function Push-AuditLogTenant { +function Push-AuditLogTenantDownload { Param($Item) $ConfigTable = Get-CippTable -TableName 'WebhookRules' $TenantFilter = $Item.TenantFilter try { - Write-Information "Audit Logs: Processing $($TenantFilter)" + Write-Information "Audit Logs: Downloading $($TenantFilter)" # Get CIPP Url, cleanup legacy tasks $SchedulerConfig = Get-CippTable -TableName 'SchedulerConfig' $LegacyWebhookTasks = Get-CIPPAzDataTableEntity @SchedulerConfig -Filter "PartitionKey eq 'webhookcreation'" @@ -45,20 +45,15 @@ function Push-AuditLogTenant { if ($Configuration) { try { $LogSearches = Get-CippAuditLogSearches -TenantFilter $TenantFilter -ReadyToProcess | Select-Object -First 10 - Write-Information ('Audit Logs: Found {0} searches, begin processing' -f $LogSearches.Count) + Write-Information ('Audit Logs: Found {0} searches, begin downloading' -f $LogSearches.Count) foreach ($Search in $LogSearches) { $SearchEntity = Get-CIPPAzDataTableEntity @LogSearchesTable -Filter "Tenant eq '$($TenantFilter)' and RowKey eq '$($Search.id)'" $SearchEntity.CippStatus = 'Processing' Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force try { - # Test the audit log rules against the search results - $AuditLogTest = Test-CIPPAuditLogRules -TenantFilter $TenantFilter -SearchId $Search.id - - $SearchEntity.CippStatus = 'Completed' - $MatchedRules = [string](ConvertTo-Json -Compress -InputObject $AuditLogTest.MatchedRules) - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedRules -Value $MatchedRules -Force - $SearchEntity | Add-Member -MemberType NoteProperty -Name MatchedLogs -Value $AuditLogTest.MatchedLogs -Force - $SearchEntity | Add-Member -MemberType NoteProperty -Name TotalLogs -Value $AuditLogTest.TotalLogs -Force + Write-Information "Audit Log search: Processing search ID: $($Search.id) for tenant: $TenantFilter" + $Downloads = New-CIPPAuditLogSearchResultsCache -TenantFilter $TenantFilter -searchId $Search.id + $SearchEntity.CippStatus = 'Downloaded' } catch { if ($_.Exception.Message -match 'Request rate is large. More Request Units may be needed, so no changes were made. Please retry this request later.') { $SearchEntity.CippStatus = 'Pending' @@ -74,34 +69,17 @@ function Push-AuditLogTenant { $SearchEntity.CippStatus = 'Failed' Write-Information "Error processing audit log rules: $($_.Exception.Message)" } - $AuditLogTest = [PSCustomObject]@{ - DataToProcess = @() - } + } Add-CIPPAzDataTableEntity @LogSearchesTable -Entity $SearchEntity -Force - $DataToProcess = ($AuditLogTest).DataToProcess - Write-Information "Audit Logs: Data to process found: $($DataToProcess.count) items" - if ($DataToProcess) { - foreach ($AuditLog in $DataToProcess) { - Write-Information "Processing $($AuditLog.operation)" - $Webhook = @{ - Data = $AuditLog - CIPPURL = [string]$CIPPURL - TenantFilter = $TenantFilter - } - try { - Invoke-CippWebhookProcessing @Webhook - } catch { - Write-Information "Error processing webhook: $($_.Exception.Message)" - } - } - } } } catch { - Write-Information ( 'Audit Log search: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + Write-Information ('Audit Log search: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + exit 0 } } } catch { - Write-Information ( 'Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + Write-Information ('Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + exit 0 } } diff --git a/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantProcess.ps1 b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantProcess.ps1 new file mode 100644 index 000000000000..8c3cd08297aa --- /dev/null +++ b/Modules/CIPPCore/Public/Entrypoints/Activity Triggers/Webhooks/Push-AuditLogTenantProcess.ps1 @@ -0,0 +1,29 @@ +function Push-AuditLogTenantProcess { + Param($Item) + $TenantFilter = $Item.TenantFilter + $RowIds = $Item.RowIds + + try { + Write-Information "Audit Logs: Processing $($TenantFilter) with $($RowIds.Count) row IDs. We're processing id $($RowIds[0]) to $($RowIds[-1])" + + # Get the CacheWebhooks table + $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' + # we do it this way because the rows can grow extremely large, if we get them all it might just hang for minutes at a time. + $Rows = foreach ($RowId in $RowIds) { + $CacheEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$RowId'" + if ($CacheEntity) { + $AuditData = $CacheEntity.JSON | ConvertFrom-Json -ErrorAction SilentlyContinue + $AuditData + } + } + + if ($Rows.Count -gt 0) { + Write-Information "Retrieved $($Rows.Count) rows from cache for processing" + Test-CIPPAuditLogRules -TenantFilter $TenantFilter -Rows $Rows + } else { + Write-Information 'No rows found in cache for the provided row IDs' + } + } catch { + Write-Information ('Push-AuditLogTenant: Error {0} line {1} - {2}' -f $_.InvocationInfo.ScriptName, $_.InvocationInfo.ScriptLineNumber, $_.Exception.Message) + } +} diff --git a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 index 25deaf238213..9de12f33de69 100644 --- a/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 +++ b/Modules/CIPPCore/Public/Entrypoints/Orchestrator Functions/Start-AuditLogOrchestrator.ps1 @@ -19,17 +19,48 @@ function Start-AuditLogOrchestrator { } elseif (($WebhookRules | Measure-Object).Count -eq 0) { Write-Information 'No webhook rules defined' } else { - Write-Information "Audit Logs: Processing $($AuditLogSearches.Count) searches" + Write-Information "Audit Logs: Downloading $($AuditLogSearches.Count) searches" if ($PSCmdlet.ShouldProcess('Start-AuditLogOrchestrator', 'Starting Audit Log Polling')) { - $Queue = New-CippQueueEntry -Name 'Audit Log Collection' -Reference 'AuditLogCollection' -TotalTasks ($AuditLogSearches).Count - $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenant' } } - + $Queue = New-CippQueueEntry -Name 'Audit Logs Download' -Reference 'AuditLogsDownload' -TotalTasks ($AuditLogSearches).Count + $Batch = $AuditLogSearches | Sort-Object -Property Tenant -Unique | Select-Object @{Name = 'TenantFilter'; Expression = { $_.Tenant } }, @{Name = 'QueueId'; Expression = { $Queue.RowKey } }, @{Name = 'FunctionName'; Expression = { 'AuditLogTenantDownload' } } $InputObject = [PSCustomObject]@{ - OrchestratorName = 'AuditLogs' + OrchestratorName = 'AuditLogsDownload' Batch = @($Batch) SkipLog = $true } Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($InputObject | ConvertTo-Json -Depth 5 -Compress) + Write-Information 'Starting audit log processing in batches of 1000, per tenant' + $WebhookCacheTable = Get-CippTable -TableName 'CacheWebhooks' + $WebhookCache = Get-CIPPAzDataTableEntity @WebhookCacheTable + $TenantGroups = $WebhookCache | Group-Object -Property PartitionKey + + if ($TenantGroups.Count -gt 0) { + Write-Information "Processing webhook cache for $($TenantGroups.Count) tenants" + $ProcessQueue = New-CippQueueEntry -Name 'Audit Logs Process' -Reference 'AuditLogsProcess' -TotalTasks ($TenantGroups | Measure-Object -Property Count -Sum).Sum + $ProcessBatch = foreach ($TenantGroup in $TenantGroups) { + $TenantFilter = $TenantGroup.Name + $RowIds = $TenantGroup.Group.RowKey + for ($i = 0; $i -lt $RowIds.Count; $i += 1000) { + $BatchRowIds = $RowIds[$i..([Math]::Min($i + 999, $RowIds.Count - 1))] + + [PSCustomObject]@{ + TenantFilter = $TenantFilter + RowIds = $BatchRowIds + QueueId = $ProcessQueue.RowKey + FunctionName = 'AuditLogTenantProcess' + } + } + } + if ($ProcessBatch.Count -gt 0) { + $ProcessInputObject = [PSCustomObject]@{ + OrchestratorName = 'AuditLogsProcess' + Batch = @($ProcessBatch) + SkipLog = $true + } + Start-NewOrchestration -FunctionName 'CIPPOrchestrator' -InputObject ($ProcessInputObject | ConvertTo-Json -Depth 5 -Compress) + Write-Information "Started audit log processing orchestration with $($ProcessBatch.Count) batches" + } + } } } } catch { diff --git a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 index e820377bd909..664e9fd084c4 100644 --- a/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 +++ b/Modules/CIPPCore/Public/Webhooks/Test-CIPPAuditLogRules.ps1 @@ -4,7 +4,7 @@ function Test-CIPPAuditLogRules { [Parameter(Mandatory = $true)] $TenantFilter, [Parameter(Mandatory = $true)] - $SearchId + $Rows ) $Results = [PSCustomObject]@{ @@ -14,6 +14,9 @@ function Test-CIPPAuditLogRules { DataToProcess = @() } + # Get the CacheWebhooks table for removing processed rows + $CacheWebhooksTable = Get-CippTable -TableName 'CacheWebhooks' + $ExtendedPropertiesIgnoreList = @( 'OAuth2:Authorize' 'OAuth2:Token' @@ -35,23 +38,24 @@ function Test-CIPPAuditLogRules { LogType = $_.Type } } - #write-warning 'Getting audit records from Graph API' + try { - $LogCount = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId -CountOnly + $LogCount = $Rows.count $RunGuid = (New-Guid).Guid - Write-Warning "Logs to process: $LogCount - SearchId: $SearchId - RunGuid: $($RunGuid) - $($TenantFilter)" + Write-Warning "Logs to process: $LogCount - RunGuid: $($RunGuid) - $($TenantFilter)" $Results.TotalLogs = $LogCount - Write-Information "RunGuid: $RunGud - Collecting logs" - $SearchResults = Get-CippAuditLogSearchResults -TenantFilter $TenantFilter -QueryId $SearchId + Write-Information "RunGuid: $RunGuid - Collecting logs" + $SearchResults = $Rows } catch { Write-Warning "Error getting audit logs: $($_.Exception.Message)" - Write-LogMessage -API 'Webhooks' -message "Error getting audit logs for search $($SearchId)" -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter + Write-LogMessage -API 'Webhooks' -message 'Error Processing Audit logs' -LogData (Get-CippException -Exception $_) -sev Error -tenant $TenantFilter throw $_ } if ($LogCount -gt 0) { $LocationTable = Get-CIPPTable -TableName 'knownlocationdb' $ProcessedData = foreach ($AuditRecord in $SearchResults) { + Write-Host "Auditlogs: The record is $($AuditRecord.operation) - $($TenantFilter)" $RootProperties = $AuditRecord | Select-Object * -ExcludeProperty auditData $Data = $AuditRecord.auditData | Select-Object *, CIPPAction, CIPPClause, CIPPGeoLocation, CIPPBadRepIP, CIPPHostedIP, CIPPIPDetected, CIPPLocationInfo, CIPPExtendedProperties, CIPPDeviceProperties, CIPPParameters, CIPPModifiedProperties, AuditRecord -ErrorAction SilentlyContinue try { @@ -179,10 +183,10 @@ function Test-CIPPAuditLogRules { $MatchedRules = [System.Collections.Generic.List[string]]::new() $DataToProcess = foreach ($clause in $Where) { - #write-warning "Webhook: Processing clause: $($clause.clause)" + Write-Warning "Webhook: Processing clause: $($clause.clause)" $ReturnedData = $ProcessedData | Where-Object { Invoke-Expression $clause.clause } if ($ReturnedData) { - #write-warning "Webhook: There is matching data: $(($ReturnedData.operation | Select-Object -Unique) -join ', ')" + Write-Warning "Webhook: There is matching data: $(($ReturnedData.operation | Select-Object -Unique) -join ', ')" $ReturnedData = foreach ($item in $ReturnedData) { $item.CIPPAction = $clause.expectedAction $item.CIPPClause = $clause.CIPPClause -join ' and ' @@ -196,6 +200,39 @@ function Test-CIPPAuditLogRules { $Results.MatchedLogs = ($DataToProcess | Measure-Object).Count $Results.DataToProcess = $DataToProcess } - Write-Warning "Finished - RunGuid: $($RunGuid) - $($TenantFilter)" - $Results + + if ($DataToProcess) { + $CippConfigTable = Get-CippTable -tablename Config + $CippConfig = Get-CIPPAzDataTableEntity @CippConfigTable -Filter "PartitionKey eq 'InstanceProperties' and RowKey eq 'CIPPURL'" + $CIPPURL = 'https://{0}' -f $CippConfig.Value + foreach ($AuditLog in $DataToProcess) { + Write-Information "Processing $($AuditLog.operation)" + $Webhook = @{ + Data = $AuditLog + CIPPURL = [string]$CIPPURL + TenantFilter = $TenantFilter + } + try { + Invoke-CippWebhookProcessing @Webhook + } catch { + Write-Information "Error sending final step of auditlog processing: $($_.Exception.Message)" + } + } + } + + # Remove processed rows from the cache table + try { + Write-Information 'Removing processed rows from cache' + foreach ($Row in $Rows) { + if ($Row.id) { + $RowEntity = Get-CIPPAzDataTableEntity @CacheWebhooksTable -Filter "PartitionKey eq '$TenantFilter' and RowKey eq '$($Row.id)'" + if ($RowEntity) { + Remove-AzDataTableEntity @CacheWebhooksTable -Entity $RowEntity -Force + Write-Information "Removed row $($Row.id) from cache" + } + } + } + } catch { + Write-Information "Error removing rows from cache: $($_.Exception.Message)" + } }