From 80fc74ef103f1b7f1d6dbd6bcec1d6167a611a26 Mon Sep 17 00:00:00 2001 From: Dmitrii Date: Thu, 13 Feb 2025 17:50:39 +0100 Subject: [PATCH] Reduce the _review rule upgrade endpoint response size --- .../review_rule_upgrade_route.ts | 72 ++++++- .../rule_management/rule_fields.ts | 1 + .../rule_management/rule_filtering.ts | 13 +- .../rule_management/api/api.ts | 4 + ...tch_prebuilt_rules_upgrade_review_query.ts | 10 +- .../use_prebuilt_rules_upgrade_review.ts | 8 +- .../rule_management/logic/types.ts | 12 +- .../upgrade_prebuilt_rules_table.tsx | 43 +++- .../upgrade_prebuilt_rules_table_context.tsx | 140 ++++++++++--- .../upgrade_prebuilt_rules_table_filters.tsx | 36 ++-- ...rade_rule_customization_filter_popover.tsx | 35 ++-- .../use_filter_prebuilt_rules_to_upgrade.ts | 48 ----- .../calculate_rule_upgrade_info.ts | 59 ++++++ .../review_rule_upgrade_handler.ts | 195 ++++++++++-------- .../review_rule_upgrade_route.ts | 12 +- .../prebuilt_rule_objects_client.ts | 11 +- .../search/get_existing_prepackaged_rules.ts | 14 +- 17 files changed, 477 insertions(+), 236 deletions(-) delete mode 100644 x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/use_filter_prebuilt_rules_to_upgrade.ts create mode 100644 x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/calculate_rule_upgrade_info.ts diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/review_rule_upgrade/review_rule_upgrade_route.ts b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/review_rule_upgrade/review_rule_upgrade_route.ts index 2f2d6e3bd1c26..2f4cf8097534c 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/review_rule_upgrade/review_rule_upgrade_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/review_rule_upgrade/review_rule_upgrade_route.ts @@ -5,9 +5,62 @@ * 2.0. */ -import type { RuleObjectId, RuleSignatureId, RuleTagArray } from '../../model'; +import { z } from '@kbn/zod'; +import { SortOrder, type RuleObjectId, type RuleSignatureId, type RuleTagArray } from '../../model'; import type { PartialRuleDiff } from '../model'; -import type { RuleResponse } from '../../model/rule_schema'; +import type { RuleResponse, RuleVersion } from '../../model/rule_schema'; +import { FindRulesSortField } from '../../rule_management'; + +export enum RuleCustomizationStatus { + CUSTOMIZED = 'CUSTOMIZED', + NOT_CUSTOMIZED = 'NOT_CUSTOMIZED', +} + +export type ReviewRuleUpgradeFilter = z.infer; +export const ReviewRuleUpgradeFilter = z.object({ + /** + * Rule IDs to return upgrade info for + */ + rule_ids: z.array(z.string()).optional(), + /** + * Tags to filter by + */ + tags: z.array(z.string()).optional(), + /** + * Rule name to filter by + */ + name: z.string().optional(), + /** + * Rule customization status to filter by + */ + customization_status: z.nativeEnum(RuleCustomizationStatus).optional(), +}); + +export type ReviewRuleUpgradeSort = z.infer; +export const ReviewRuleUpgradeSort = z.object({ + /** + * Field to sort by + */ + field: FindRulesSortField.optional(), + /** + * Sort order + */ + order: SortOrder.optional(), +}); + +export type ReviewRuleUpgradeRequestBody = z.infer; +export const ReviewRuleUpgradeRequestBody = z + .object({ + filter: ReviewRuleUpgradeFilter.optional(), + sort: ReviewRuleUpgradeSort.optional(), + + page: z.coerce.number().int().min(1).optional().default(1), + /** + * Rules per page + */ + per_page: z.coerce.number().int().min(0).optional().default(20), + }) + .nullable(); export interface ReviewRuleUpgradeResponseBody { /** Aggregated info about all rules available for upgrade */ @@ -15,16 +68,26 @@ export interface ReviewRuleUpgradeResponseBody { /** Info about individual rules: one object per each rule available for upgrade */ rules: RuleUpgradeInfoForReview[]; + + /** The requested page number */ + page: number; + + /** The requested number of items per page */ + per_page: number; } export interface RuleUpgradeStatsForReview { /** Number of installed prebuilt rules available for upgrade (stock + customized) */ num_rules_to_upgrade_total: number; - /** Number of installed prebuilt rules with upgrade conflicts (SOLVABLE or NON_SOLVABLE) */ + /** + * @deprecated Always 0 + */ num_rules_with_conflicts: number; - /** Number of installed prebuilt rules with NON_SOLVABLE upgrade conflicts */ + /** + * @deprecated Always 0 + */ num_rules_with_non_solvable_conflicts: number; /** A union of all tags of all rules available for upgrade */ @@ -34,6 +97,7 @@ export interface RuleUpgradeStatsForReview { export interface RuleUpgradeInfoForReview { id: RuleObjectId; rule_id: RuleSignatureId; + version: RuleVersion; current_rule: RuleResponse; target_rule: RuleResponse; diff: PartialRuleDiff; diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_fields.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_fields.ts index 5d72cd15a96ae..610b2231e8ee7 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_fields.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_fields.ts @@ -22,3 +22,4 @@ export const TAGS_FIELD = 'alert.attributes.tags'; export const PARAMS_TYPE_FIELD = 'alert.attributes.params.type'; export const PARAMS_IMMUTABLE_FIELD = 'alert.attributes.params.immutable'; export const LAST_RUN_OUTCOME_FIELD = 'alert.attributes.lastRun.outcome'; +export const IS_CUSTOMIZED_FIELD = 'alert.attributes.params.ruleSource.isCustomized'; diff --git a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts index 52ace0cfac5da..692f2fa55a5e5 100644 --- a/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts +++ b/x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts @@ -7,10 +7,11 @@ import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { RuleExecutionStatus } from '../../api/detection_engine'; -import { RuleExecutionStatusEnum } from '../../api/detection_engine'; +import { RuleCustomizationStatus, RuleExecutionStatusEnum } from '../../api/detection_engine'; import { prepareKQLStringParam } from '../../utils/kql'; import { ENABLED_FIELD, + IS_CUSTOMIZED_FIELD, LAST_RUN_OUTCOME_FIELD, PARAMS_IMMUTABLE_FIELD, PARAMS_TYPE_FIELD, @@ -23,6 +24,8 @@ export const KQL_FILTER_IMMUTABLE_RULES = `${PARAMS_IMMUTABLE_FIELD}: true`; export const KQL_FILTER_MUTABLE_RULES = `${PARAMS_IMMUTABLE_FIELD}: false`; export const KQL_FILTER_ENABLED_RULES = `${ENABLED_FIELD}: true`; export const KQL_FILTER_DISABLED_RULES = `${ENABLED_FIELD}: false`; +export const KQL_FILTER_CUSTOMIZED_RULES = `${IS_CUSTOMIZED_FIELD}: true`; +export const KQL_FILTER_NOT_CUSTOMIZED_RULES = `${IS_CUSTOMIZED_FIELD}: false`; interface RulesFilterOptions { filter: string; @@ -32,6 +35,7 @@ interface RulesFilterOptions { tags: string[]; excludeRuleTypes: Type[]; ruleExecutionStatus: RuleExecutionStatus; + customizationStatus: RuleCustomizationStatus; ruleIds: string[]; } @@ -50,6 +54,7 @@ export function convertRulesFilterToKQL({ tags, excludeRuleTypes = [], ruleExecutionStatus, + customizationStatus, }: Partial): string { const kql: string[] = []; @@ -85,6 +90,12 @@ export function convertRulesFilterToKQL({ kql.push(`${LAST_RUN_OUTCOME_FIELD}: "failed"`); } + if (customizationStatus === RuleCustomizationStatus.CUSTOMIZED) { + kql.push(KQL_FILTER_CUSTOMIZED_RULES); + } else if (customizationStatus === RuleCustomizationStatus.NOT_CUSTOMIZED) { + kql.push(KQL_FILTER_NOT_CUSTOMIZED_RULES); + } + return kql.join(' AND '); } diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/api.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/api.ts index 5f5fded87ac45..4e1bde51c7394 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/api.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/api.ts @@ -23,6 +23,7 @@ import type { GetPrebuiltRulesStatusResponseBody, ReviewRuleUpgradeResponseBody, ReviewRuleInstallationResponseBody, + ReviewRuleUpgradeRequestBody, } from '../../../../common/api/detection_engine/prebuilt_rules'; import type { BulkDuplicateRules, @@ -637,13 +638,16 @@ export const getPrebuiltRulesStatus = async ({ */ export const reviewRuleUpgrade = async ({ signal, + request, }: { signal: AbortSignal | undefined; + request: ReviewRuleUpgradeRequestBody; }): Promise => KibanaServices.get().http.fetch(REVIEW_RULE_UPGRADE_URL, { method: 'POST', version: '1', signal, + body: JSON.stringify(request), }); /** diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_fetch_prebuilt_rules_upgrade_review_query.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_fetch_prebuilt_rules_upgrade_review_query.ts index 532114b1d4b62..4b779918febc3 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_fetch_prebuilt_rules_upgrade_review_query.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_fetch_prebuilt_rules_upgrade_review_query.ts @@ -9,7 +9,10 @@ import type { UseQueryOptions } from '@tanstack/react-query'; import { useQuery, useQueryClient } from '@tanstack/react-query'; import { reviewRuleUpgrade } from '../../api'; import { REVIEW_RULE_UPGRADE_URL } from '../../../../../../common/api/detection_engine/prebuilt_rules/urls'; -import type { ReviewRuleUpgradeResponseBody } from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import type { + ReviewRuleUpgradeRequestBody, + ReviewRuleUpgradeResponseBody, +} from '../../../../../../common/api/detection_engine/prebuilt_rules'; import { DEFAULT_QUERY_OPTIONS } from '../constants'; import { retryOnRateLimitedError } from './retry_on_rate_limited_error'; import { cappedExponentialBackoff } from './capped_exponential_backoff'; @@ -17,12 +20,13 @@ import { cappedExponentialBackoff } from './capped_exponential_backoff'; export const REVIEW_RULE_UPGRADE_QUERY_KEY = ['POST', REVIEW_RULE_UPGRADE_URL]; export const useFetchPrebuiltRulesUpgradeReviewQuery = ( + request: ReviewRuleUpgradeRequestBody, options?: UseQueryOptions ) => { return useQuery( - REVIEW_RULE_UPGRADE_QUERY_KEY, + [...REVIEW_RULE_UPGRADE_QUERY_KEY, request], async ({ signal }) => { - const response = await reviewRuleUpgrade({ signal }); + const response = await reviewRuleUpgrade({ signal, request }); return response; }, { diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/prebuilt_rules/use_prebuilt_rules_upgrade_review.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/prebuilt_rules/use_prebuilt_rules_upgrade_review.ts index 6e8f008c5ede5..bb0b04174d0dd 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/prebuilt_rules/use_prebuilt_rules_upgrade_review.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/prebuilt_rules/use_prebuilt_rules_upgrade_review.ts @@ -5,7 +5,10 @@ * 2.0. */ import type { UseQueryOptions } from '@tanstack/react-query'; -import type { ReviewRuleUpgradeResponseBody } from '../../../../../common/api/detection_engine/prebuilt_rules'; +import type { + ReviewRuleUpgradeRequestBody, + ReviewRuleUpgradeResponseBody, +} from '../../../../../common/api/detection_engine/prebuilt_rules'; import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; import * as i18n from '../translations'; @@ -18,11 +21,12 @@ import { useFetchPrebuiltRulesUpgradeReviewQuery } from '../../api/hooks/prebuil * @returns useQuery result */ export const usePrebuiltRulesUpgradeReview = ( + request: ReviewRuleUpgradeRequestBody, options?: UseQueryOptions ) => { const { addError } = useAppToasts(); - return useFetchPrebuiltRulesUpgradeReviewQuery({ + return useFetchPrebuiltRulesUpgradeReviewQuery(request, { onError: (error) => addError(error, { title: i18n.RULE_AND_TIMELINE_FETCH_FAILURE }), ...options, }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts index e91639dd0454c..98383e93b444c 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/logic/types.ts @@ -11,7 +11,10 @@ import type { RuleSnooze } from '@kbn/alerting-plugin/common'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import type { NamespaceType } from '@kbn/securitysolution-io-ts-list-types'; import type { RuleSnoozeSettings } from '@kbn/triggers-actions-ui-plugin/public/types'; -import type { WarningSchema } from '../../../../common/api/detection_engine'; +import type { + RuleCustomizationStatus, + WarningSchema, +} from '../../../../common/api/detection_engine'; import type { RuleExecutionStatus } from '../../../../common/api/detection_engine/rule_monitoring'; import { SortOrder } from '../../../../common/api/detection_engine'; @@ -103,7 +106,7 @@ export interface FilterOptions { excludeRuleTypes?: Type[]; enabled?: boolean; // undefined is to display all the rules ruleExecutionStatus?: RuleExecutionStatus; // undefined means "all" - ruleSource?: RuleCustomizationEnum[]; // undefined is to display all the rules + ruleSource?: RuleCustomizationStatus[]; // undefined is to display all the rules showRulesWithGaps?: boolean; gapSearchRange?: GapRangeValue; } @@ -209,8 +212,3 @@ export interface FindRulesReferencedByExceptionsProps { lists: FindRulesReferencedByExceptionsListProp[]; signal?: AbortSignal; } - -export enum RuleCustomizationEnum { - customized = 'CUSTOMIZED', - not_customized = 'NOT_CUSTOMIZED', -} diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table.tsx index dfe8c5787417c..959f20ef72c20 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table.tsx @@ -10,7 +10,7 @@ import { EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem, - EuiInMemoryTable, + EuiBasicTable, EuiProgress, EuiSkeletonLoading, EuiSkeletonText, @@ -19,9 +19,10 @@ import { import React, { useCallback, useState } from 'react'; import type { RuleUpgradeState } from '../../../../rule_management/model/prebuilt_rule_upgrade'; import * as i18n from '../../../../../detections/pages/detection_engine/rules/translations'; -import { RULES_TABLE_INITIAL_PAGE_SIZE, RULES_TABLE_PAGE_SIZE_OPTIONS } from '../constants'; +import { RULES_TABLE_PAGE_SIZE_OPTIONS } from '../constants'; import { RulesChangelogLink } from '../rules_changelog_link'; import { UpgradePrebuiltRulesTableButtons } from './upgrade_prebuilt_rules_table_buttons'; +import type { UpgradePrebuiltRulesSortingOptions } from './upgrade_prebuilt_rules_table_context'; import { useUpgradePrebuiltRulesTableContext } from './upgrade_prebuilt_rules_table_context'; import { UpgradePrebuiltRulesTableFilters } from './upgrade_prebuilt_rules_table_filters'; import { useUpgradePrebuiltRulesTableColumns } from './use_upgrade_prebuilt_rules_table_columns'; @@ -44,20 +45,32 @@ export const UpgradePrebuiltRulesTable = React.memo(() => { ruleUpgradeStates, hasRulesToUpgrade, isLoading, + isFetching, isRefetching, isUpgradingSecurityPackages, + pagination, + sortingOptions, }, + actions: { setPagination, setSortingOptions }, } = useUpgradePrebuiltRulesTableContext(); const [selected, setSelected] = useState([]); const rulesColumns = useUpgradePrebuiltRulesTableColumns(); const shouldShowProgress = isUpgradingSecurityPackages || isRefetching; - const [pageIndex, setPageIndex] = useState(0); const handleTableChange = useCallback( - ({ page: { index } }: CriteriaWithPagination) => { - setPageIndex(index); + ({ page: { index, size }, sort }: CriteriaWithPagination) => { + setPagination({ + page: index + 1, + perPage: size, + }); + if (sort) { + setSortingOptions({ + field: sort.field as UpgradePrebuiltRulesSortingOptions['field'], + order: sort.direction, + }); + } }, - [setPageIndex] + [setPagination, setSortingOptions] ); return ( @@ -104,23 +117,31 @@ export const UpgradePrebuiltRulesTable = React.memo(() => { - true, onSelectionChange: setSelected, initialSelected: selected, }} + sorting={{ + sort: { + // EuiBasicTable has incorrect `sort.field` types which accept only `keyof Item` and reject fields in dot notation + field: sortingOptions.field as keyof RuleUpgradeState, + direction: sortingOptions.order, + }, + }} itemId="rule_id" data-test-subj="rules-upgrades-table" columns={rulesColumns} - onTableChange={handleTableChange} + onChange={handleTableChange} /> ) diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_context.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_context.tsx index 3be956c7cbf94..61b5eb0abd634 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_context.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_context.tsx @@ -10,8 +10,11 @@ import type { Dispatch, SetStateAction } from 'react'; import React, { createContext, useCallback, useContext, useMemo, useState } from 'react'; import { PrebuiltRulesCustomizationDisabledReason } from '../../../../../../common/detection_engine/prebuilt_rules/prebuilt_rule_customization_status'; import type { + FindRulesSortField, + ReviewRuleUpgradeFilter, RuleFieldsToUpgrade, RuleUpgradeSpecifier, + SortOrder, } from '../../../../../../common/api/detection_engine'; import { usePrebuiltRulesCustomizationStatus } from '../../../../rule_management/logic/prebuilt_rules/use_prebuilt_rules_customization_status'; import type { RuleUpgradeState } from '../../../../rule_management/model/prebuilt_rule_upgrade'; @@ -29,8 +32,6 @@ import { usePrebuiltRulesUpgradeReview } from '../../../../rule_management/logic import { RuleDiffTab } from '../../../../rule_management/components/rule_details/rule_diff_tab'; import { FieldUpgradeStateEnum } from '../../../../rule_management/model/prebuilt_rule_upgrade/field_upgrade_state_enum'; import { useRulePreviewFlyout } from '../use_rule_preview_flyout'; -import type { UpgradePrebuiltRulesTableFilterOptions } from './use_filter_prebuilt_rules_to_upgrade'; -import { useFilterPrebuiltRulesToUpgrade } from './use_filter_prebuilt_rules_to_upgrade'; import { usePrebuiltRulesUpgradeState } from './use_prebuilt_rules_upgrade_state'; import { useOutdatedMlJobsUpgradeModal } from './use_ml_jobs_upgrade_modal'; import { useUpgradeWithConflictsModal } from './use_upgrade_with_conflicts_modal'; @@ -39,9 +40,21 @@ import { UpgradeFlyoutSubHeader } from './upgrade_flyout_subheader'; import * as ruleDetailsI18n from '../../../../rule_management/components/rule_details/translations'; import * as i18n from './translations'; import { CustomizationDisabledCallout } from './customization_disabled_callout'; +import { RULES_TABLE_INITIAL_PAGE_SIZE } from '../constants'; +import type { PaginationOptions } from '../../../../rule_management/logic'; +import { usePrebuiltRulesStatus } from '../../../../rule_management/logic/prebuilt_rules/use_prebuilt_rules_status'; const REVIEW_PREBUILT_RULES_UPGRADE_REFRESH_INTERVAL = 5 * 60 * 1000; +export interface UpgradePrebuiltRulesSortingOptions { + field: + | 'current_rule.name' + | 'current_rule.risk_score' + | 'current_rule.severity' + | 'current_rule.last_updated'; + order: SortOrder; +} + export interface UpgradePrebuiltRulesTableState { /** * Rule upgrade state after applying `filterOptions` @@ -50,7 +63,7 @@ export interface UpgradePrebuiltRulesTableState { /** * Currently selected table filter */ - filterOptions: UpgradePrebuiltRulesTableFilterOptions; + filterOptions: ReviewRuleUpgradeFilter; /** * All unique tags for all rules */ @@ -63,6 +76,10 @@ export interface UpgradePrebuiltRulesTableState { * Is true then there is no cached data and the query is currently fetching. */ isLoading: boolean; + /** + * Is true whenever a request is in-flight, which includes initial loading as well as background refetches. + */ + isFetching: boolean; /** * Will be true if the query has been fetched. */ @@ -84,6 +101,14 @@ export interface UpgradePrebuiltRulesTableState { * The timestamp for when the rules were successfully fetched */ lastUpdated: number; + /** + * Current pagination state + */ + pagination: PaginationOptions; + /** + * Currently selected table sorting + */ + sortingOptions: UpgradePrebuiltRulesSortingOptions; } export const PREBUILT_RULE_UPDATE_FLYOUT_ANCHOR = 'updatePrebuiltRulePreview'; @@ -92,7 +117,9 @@ export interface UpgradePrebuiltRulesTableActions { reFetchRules: () => void; upgradeRules: (ruleIds: RuleSignatureId[]) => void; upgradeAllRules: () => void; - setFilterOptions: Dispatch>; + setFilterOptions: Dispatch>; + setPagination: Dispatch>; + setSortingOptions: Dispatch>; openRulePreview: (ruleId: string) => void; } @@ -121,35 +148,74 @@ export const UpgradePrebuiltRulesTableContextProvider = ({ }: UpgradePrebuiltRulesTableContextProviderProps) => { const { isRulesCustomizationEnabled, customizationDisabledReason } = usePrebuiltRulesCustomizationStatus(); + + // Use the data from the prebuilt rules status API to determine if there are + // rules to upgrade because it returns information about all rules without filters + const { data: prebuiltRulesStatusResponse } = usePrebuiltRulesStatus(); + const hasRulesToUpgrade = (prebuiltRulesStatusResponse?.num_prebuilt_rules_to_upgrade ?? 0) > 0; + const [loadingRules, setLoadingRules] = useState([]); - const [filterOptions, setFilterOptions] = useState({ - filter: '', - tags: [], - ruleSource: [], - }); + const [filterOptions, setFilterOptions] = useState({}); const isUpgradingSecurityPackages = useIsUpgradingSecurityPackages(); + const [pagination, setPagination] = useState({ + page: 1, + perPage: RULES_TABLE_INITIAL_PAGE_SIZE, + }); + const [sortingOptions, setSortingOptions] = useState({ + field: 'current_rule.last_updated', + order: 'asc', + }); + + const findRulesSortField = useMemo( + () => + (( + { + 'current_rule.name': 'name', + 'current_rule.risk_score': 'risk_score', + 'current_rule.severity': 'severity', + 'current_rule.last_updated': 'updated_at', + } as const + )[sortingOptions.field]), + [sortingOptions.field] + ); const { - data: { rules: ruleUpgradeInfos, stats: { tags } } = { - rules: [], - stats: { tags: [] }, - }, + data: upgradeReviewResponse, refetch, dataUpdatedAt, isFetched, isLoading, + isFetching, isRefetching, - } = usePrebuiltRulesUpgradeReview({ - refetchInterval: REVIEW_PREBUILT_RULES_UPGRADE_REFRESH_INTERVAL, - keepPreviousData: true, // Use this option so that the state doesn't jump between "success" and "loading" on page change - }); + } = usePrebuiltRulesUpgradeReview( + { + page: pagination.page, + per_page: pagination.perPage, + sort: { + field: findRulesSortField, + order: sortingOptions.order, + }, + filter: filterOptions, + }, + { + refetchInterval: REVIEW_PREBUILT_RULES_UPGRADE_REFRESH_INTERVAL, + keepPreviousData: true, // Use this option so that the state doesn't jump between "success" and "loading" on page change + } + ); + + const upgradeableRules = useMemo( + () => upgradeReviewResponse?.rules ?? [], + [upgradeReviewResponse] + ); + // TODO Remove the stats usage + const tags = useMemo(() => upgradeReviewResponse?.stats.tags ?? [], [upgradeReviewResponse]); + + // TODO should be used explicit pagination total from the response + const totalRulesToUpgrade = upgradeReviewResponse?.stats.num_rules_to_upgrade_total ?? 0; + const { rulesUpgradeState, setRuleFieldResolvedValue } = - usePrebuiltRulesUpgradeState(ruleUpgradeInfos); + usePrebuiltRulesUpgradeState(upgradeableRules); const ruleUpgradeStates = useMemo(() => Object.values(rulesUpgradeState), [rulesUpgradeState]); - const filteredRuleUpgradeStates = useFilterPrebuiltRulesToUpgrade({ - filterOptions, - data: ruleUpgradeStates, - }); const { modal: confirmLegacyMlJobsUpgradeModal, @@ -248,8 +314,8 @@ export const UpgradePrebuiltRulesTableContextProvider = ({ const upgradeAllRules = useCallback( // Upgrade all rules, ignoring filter and selection - () => upgradeRules(ruleUpgradeInfos.map((rule) => rule.rule_id)), - [ruleUpgradeInfos, upgradeRules] + () => upgradeRules(upgradeableRules.map((rule) => rule.rule_id)), + [upgradeableRules, upgradeRules] ); const subHeaderFactory = useCallback( @@ -377,12 +443,8 @@ export const UpgradePrebuiltRulesTableContextProvider = ({ setRuleFieldResolvedValue, ] ); - const filteredRules = useMemo( - () => filteredRuleUpgradeStates.map(({ target_rule: targetRule }) => targetRule), - [filteredRuleUpgradeStates] - ); const { rulePreviewFlyout, openRulePreview } = useRulePreviewFlyout({ - rules: filteredRules, + rules: ruleUpgradeStates.map(({ target_rule: targetRule }) => targetRule), subHeaderFactory, ruleActionsFactory, extraTabsFactory, @@ -399,6 +461,8 @@ export const UpgradePrebuiltRulesTableContextProvider = ({ upgradeAllRules, setFilterOptions, openRulePreview, + setPagination, + setSortingOptions, }), [refetch, upgradeRules, upgradeAllRules, openRulePreview] ); @@ -406,31 +470,41 @@ export const UpgradePrebuiltRulesTableContextProvider = ({ const providerValue = useMemo( () => ({ state: { - ruleUpgradeStates: filteredRuleUpgradeStates, - hasRulesToUpgrade: isFetched && ruleUpgradeInfos.length > 0, + ruleUpgradeStates, + hasRulesToUpgrade, filterOptions, tags, isFetched, isLoading: isLoading || areMlJobsLoading, + isFetching, isRefetching, isUpgradingSecurityPackages, loadingRules, lastUpdated: dataUpdatedAt, + pagination: { + ...pagination, + total: totalRulesToUpgrade, + }, + sortingOptions, }, actions, }), [ - ruleUpgradeInfos.length, - filteredRuleUpgradeStates, + ruleUpgradeStates, + hasRulesToUpgrade, filterOptions, tags, isFetched, isLoading, areMlJobsLoading, + isFetching, isRefetching, isUpgradingSecurityPackages, loadingRules, dataUpdatedAt, + pagination, + totalRulesToUpgrade, + sortingOptions, actions, ] ); diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_filters.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_filters.tsx index d8b8f618cf43d..b62f987b8abe2 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_filters.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_prebuilt_rules_table_filters.tsx @@ -9,11 +9,11 @@ import { EuiFilterGroup, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { isEqual } from 'lodash/fp'; import React, { useCallback } from 'react'; import styled from 'styled-components'; +import type { RuleCustomizationStatus } from '../../../../../../common/api/detection_engine'; import { usePrebuiltRulesCustomizationStatus } from '../../../../rule_management/logic/prebuilt_rules/use_prebuilt_rules_customization_status'; -import type { RuleCustomizationEnum } from '../../../../rule_management/logic'; -import * as i18n from './translations'; -import { TagsFilterPopover } from '../rules_table_filters/tags_filter_popover'; import { RuleSearchField } from '../rules_table_filters/rule_search_field'; +import { TagsFilterPopover } from '../rules_table_filters/tags_filter_popover'; +import * as i18n from './translations'; import { useUpgradePrebuiltRulesTableContext } from './upgrade_prebuilt_rules_table_context'; import { RuleCustomizationFilterPopover } from './upgrade_rule_customization_filter_popover'; @@ -33,13 +33,13 @@ const UpgradePrebuiltRulesTableFiltersComponent = () => { const { isRulesCustomizationEnabled } = usePrebuiltRulesCustomizationStatus(); - const { tags: selectedTags, ruleSource: selectedRuleSource = [] } = filterOptions; + const { tags: selectedTags, customization_status: customizationStatus } = filterOptions; const handleOnSearch = useCallback( - (filterString: string) => { + (nameString: string) => { setFilterOptions((filters) => ({ ...filters, - filter: filterString.trim(), + name: nameString.trim(), })); }, [setFilterOptions] @@ -57,22 +57,20 @@ const UpgradePrebuiltRulesTableFiltersComponent = () => { [selectedTags, setFilterOptions] ); - const handleSelectedRuleSource = useCallback( - (newRuleSource: RuleCustomizationEnum[]) => { - if (!isEqual(newRuleSource, selectedRuleSource)) { - setFilterOptions((filters) => ({ - ...filters, - ruleSource: newRuleSource, - })); - } + const handleCustomizationStatusChange = useCallback( + (newCustomizationStatus: RuleCustomizationStatus | undefined) => { + setFilterOptions((filters) => ({ + ...filters, + customization_status: newCustomizationStatus, + })); }, - [selectedRuleSource, setFilterOptions] + [setFilterOptions] ); return ( @@ -81,8 +79,8 @@ const UpgradePrebuiltRulesTableFiltersComponent = () => { {isRulesCustomizationEnabled && ( @@ -90,7 +88,7 @@ const UpgradePrebuiltRulesTableFiltersComponent = () => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_rule_customization_filter_popover.tsx b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_rule_customization_filter_popover.tsx index 234943e333272..05e68f4ebbccb 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_rule_customization_filter_popover.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/upgrade_rule_customization_filter_popover.tsx @@ -5,23 +5,22 @@ * 2.0. */ -import React, { useState, useMemo } from 'react'; import type { EuiSelectableOption } from '@elastic/eui'; import { EuiFilterButton, EuiPopover, EuiSelectable } from '@elastic/eui'; -import { RuleCustomizationEnum } from '../../../../rule_management/logic'; +import React, { useMemo, useState } from 'react'; +import { RuleCustomizationStatus } from '../../../../../../common/api/detection_engine'; import * as i18n from '../../../../../detections/pages/detection_engine/rules/translations'; -import { toggleSelectedGroup } from '../../../../../common/components/ml_popover/jobs_table/filters/toggle_selected_group'; interface RuleCustomizationFilterPopoverProps { - selectedRuleSource: RuleCustomizationEnum[]; - onSelectedRuleSourceChanged: (newRuleSource: RuleCustomizationEnum[]) => void; + customizationStatus: RuleCustomizationStatus | undefined; + onCustomizationStatusChanged: (newRuleSource: RuleCustomizationStatus | undefined) => void; } const RULE_CUSTOMIZATION_POPOVER_WIDTH = 200; const RuleCustomizationFilterPopoverComponent = ({ - selectedRuleSource, - onSelectedRuleSourceChanged, + customizationStatus, + onCustomizationStatusChanged, }: RuleCustomizationFilterPopoverProps) => { const [isRuleCustomizationPopoverOpen, setIsRuleCustomizationPopoverOpen] = useState(false); @@ -29,18 +28,16 @@ const RuleCustomizationFilterPopoverComponent = ({ () => [ { label: i18n.MODIFIED_LABEL, - key: RuleCustomizationEnum.customized, - checked: selectedRuleSource.includes(RuleCustomizationEnum.customized) ? 'on' : undefined, + key: RuleCustomizationStatus.CUSTOMIZED, + checked: customizationStatus === RuleCustomizationStatus.CUSTOMIZED ? 'on' : undefined, }, { label: i18n.UNMODIFIED_LABEL, - key: RuleCustomizationEnum.not_customized, - checked: selectedRuleSource.includes(RuleCustomizationEnum.not_customized) - ? 'on' - : undefined, + key: RuleCustomizationStatus.NOT_CUSTOMIZED, + checked: customizationStatus === RuleCustomizationStatus.NOT_CUSTOMIZED ? 'on' : undefined, }, ], - [selectedRuleSource] + [customizationStatus] ); const handleSelectableOptionsChange = ( @@ -48,10 +45,8 @@ const RuleCustomizationFilterPopoverComponent = ({ _: unknown, changedOption: EuiSelectableOption ) => { - toggleSelectedGroup( - changedOption.key ?? '', - selectedRuleSource, - onSelectedRuleSourceChanged as (args: string[]) => void + onCustomizationStatusChanged( + changedOption.checked === 'on' ? (changedOption.key as RuleCustomizationStatus) : undefined ); }; @@ -62,8 +57,8 @@ const RuleCustomizationFilterPopoverComponent = ({ onClick={() => setIsRuleCustomizationPopoverOpen(!isRuleCustomizationPopoverOpen)} numFilters={selectableOptions.length} isSelected={isRuleCustomizationPopoverOpen} - hasActiveFilters={selectedRuleSource.length > 0} - numActiveFilters={selectedRuleSource.length} + hasActiveFilters={customizationStatus != null} + numActiveFilters={customizationStatus != null ? 1 : 0} data-test-subj="rule-customization-filter-popover-button" > {i18n.RULE_SOURCE} diff --git a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/use_filter_prebuilt_rules_to_upgrade.ts b/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/use_filter_prebuilt_rules_to_upgrade.ts deleted file mode 100644 index f0a818fd2532e..0000000000000 --- a/x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management_ui/components/rules_table/upgrade_prebuilt_rules_table/use_filter_prebuilt_rules_to_upgrade.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useMemo } from 'react'; -import type { RuleUpgradeState } from '../../../../rule_management/model/prebuilt_rule_upgrade'; -import { RuleCustomizationEnum, type FilterOptions } from '../../../../rule_management/logic/types'; - -export type UpgradePrebuiltRulesTableFilterOptions = Pick< - FilterOptions, - 'filter' | 'tags' | 'ruleSource' ->; - -interface UseFilterPrebuiltRulesToUpgradeParams { - data: RuleUpgradeState[]; - filterOptions: UpgradePrebuiltRulesTableFilterOptions; -} - -export const useFilterPrebuiltRulesToUpgrade = ({ - data, - filterOptions, -}: UseFilterPrebuiltRulesToUpgradeParams): RuleUpgradeState[] => { - return useMemo(() => { - const { filter, tags, ruleSource } = filterOptions; - - return data.filter((ruleInfo) => { - if (filter && !ruleInfo.current_rule.name.toLowerCase().includes(filter.toLowerCase())) { - return false; - } - - if (tags?.length && !tags.every((tag) => ruleInfo.current_rule.tags.includes(tag))) { - return false; - } - - if (ruleSource?.length === 1 && ruleInfo.current_rule.rule_source.type === 'external') { - if (ruleSource.includes(RuleCustomizationEnum.customized)) { - return ruleInfo.current_rule.rule_source.is_customized; - } - return ruleInfo.current_rule.rule_source.is_customized === false; - } - - return true; - }); - }, [filterOptions, data]); -}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/calculate_rule_upgrade_info.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/calculate_rule_upgrade_info.ts new file mode 100644 index 0000000000000..1bfd9f09c43f7 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/calculate_rule_upgrade_info.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { pickBy } from 'lodash'; +import type { RuleResponse } from '../../../../../../common/api/detection_engine/model/rule_schema'; +import type { + RuleUpgradeInfoForReview, + ThreeWayDiff, +} from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import { ThreeWayDiffOutcome } from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import { invariant } from '../../../../../../common/utils/invariant'; +import { convertPrebuiltRuleAssetToRuleResponse } from '../../../rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; +import type { CalculateRuleDiffResult } from '../../logic/diff/calculate_rule_diff'; + +export const calculateRuleUpgradeInfo = ( + results: CalculateRuleDiffResult[] +): RuleUpgradeInfoForReview[] => { + return results.map((result) => { + const { ruleDiff, ruleVersions } = result; + const installedCurrentVersion = ruleVersions.input.current; + const targetVersion = ruleVersions.input.target; + invariant(installedCurrentVersion != null, 'installedCurrentVersion not found'); + invariant(targetVersion != null, 'targetVersion not found'); + + const targetRule: RuleResponse = { + ...convertPrebuiltRuleAssetToRuleResponse(targetVersion), + id: installedCurrentVersion.id, + revision: installedCurrentVersion.revision + 1, + created_at: installedCurrentVersion.created_at, + created_by: installedCurrentVersion.created_by, + updated_at: new Date().toISOString(), + updated_by: installedCurrentVersion.updated_by, + }; + + return { + id: installedCurrentVersion.id, + rule_id: installedCurrentVersion.rule_id, + revision: installedCurrentVersion.revision, + version: installedCurrentVersion.version, + current_rule: installedCurrentVersion, + target_rule: targetRule, + diff: { + fields: pickBy>( + ruleDiff.fields, + (fieldDiff) => + fieldDiff.diff_outcome !== ThreeWayDiffOutcome.StockValueNoUpdate && + fieldDiff.diff_outcome !== ThreeWayDiffOutcome.MissingBaseNoUpdate + ), + num_fields_with_updates: ruleDiff.num_fields_with_updates, + num_fields_with_conflicts: ruleDiff.num_fields_with_conflicts, + num_fields_with_non_solvable_conflicts: ruleDiff.num_fields_with_non_solvable_conflicts, + }, + }; + }); +}; diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_handler.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_handler.ts index 9f4fa7ddc766e..01d13850a7fe2 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_handler.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_handler.ts @@ -5,57 +5,66 @@ * 2.0. */ +import type { RulesClient } from '@kbn/alerting-plugin/server'; import type { KibanaRequest, KibanaResponseFactory } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; -import { pickBy } from 'lodash'; import type { RuleResponse } from '../../../../../../common/api/detection_engine/model/rule_schema'; import type { + ReviewRuleUpgradeFilter, + ReviewRuleUpgradeRequestBody, ReviewRuleUpgradeResponseBody, - RuleUpgradeInfoForReview, - RuleUpgradeStatsForReview, - ThreeWayDiff, + ReviewRuleUpgradeSort, } from '../../../../../../common/api/detection_engine/prebuilt_rules'; -import { ThreeWayDiffOutcome } from '../../../../../../common/api/detection_engine/prebuilt_rules'; -import { invariant } from '../../../../../../common/utils/invariant'; +import { convertRulesFilterToKQL } from '../../../../../../common/detection_engine/rule_management/rule_filtering'; import type { SecuritySolutionRequestHandlerContext } from '../../../../../types'; import { buildSiemResponse } from '../../../routes/utils'; -import { convertPrebuiltRuleAssetToRuleResponse } from '../../../rule_management/logic/detection_rules_client/converters/convert_prebuilt_rule_asset_to_rule_response'; -import type { CalculateRuleDiffResult } from '../../logic/diff/calculate_rule_diff'; +import { internalRuleToAPIResponse } from '../../../rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response'; +import { findRules } from '../../../rule_management/logic/search/find_rules'; import { calculateRuleDiff } from '../../logic/diff/calculate_rule_diff'; +import type { IPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; import { createPrebuiltRuleAssetsClient } from '../../logic/rule_assets/prebuilt_rule_assets_client'; -import { createPrebuiltRuleObjectsClient } from '../../logic/rule_objects/prebuilt_rule_objects_client'; -import { fetchRuleVersionsTriad } from '../../logic/rule_versions/fetch_rule_versions_triad'; -import { getRuleGroups } from '../../model/rule_groups/get_rule_groups'; +import type { RuleVersionSpecifier } from '../../logic/rule_versions/rule_version_specifier'; +import { zipRuleVersions } from '../../logic/rule_versions/zip_rule_versions'; +import { calculateRuleUpgradeInfo } from './calculate_rule_upgrade_info'; + +const DEFAULT_SORT: ReviewRuleUpgradeSort = { + field: 'name', + order: 'asc', +}; export const reviewRuleUpgradeHandler = async ( context: SecuritySolutionRequestHandlerContext, - request: KibanaRequest, + request: KibanaRequest, response: KibanaResponseFactory ) => { const siemResponse = buildSiemResponse(response); + const { page = 1, per_page: perPage = 10_000, sort = DEFAULT_SORT, filter } = request.body ?? {}; try { const ctx = await context.resolve(['core', 'alerting']); const soClient = ctx.core.savedObjects.client; const rulesClient = await ctx.alerting.getRulesClient(); const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); - const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); - const ruleVersionsMap = await fetchRuleVersionsTriad({ + const { diffResults, tags, totalUpgradeableRules } = await calculateUpgradeableRulesDiff({ ruleAssetsClient, - ruleObjectsClient, - }); - const { upgradeableRules } = getRuleGroups(ruleVersionsMap); - - const ruleDiffCalculationResults = upgradeableRules.map(({ current }) => { - const ruleVersions = ruleVersionsMap.get(current.rule_id); - invariant(ruleVersions != null, 'ruleVersions not found'); - return calculateRuleDiff(ruleVersions); + rulesClient, + page, + perPage, + sort, + filter, }); const body: ReviewRuleUpgradeResponseBody = { - stats: calculateRuleStats(ruleDiffCalculationResults), - rules: calculateRuleInfos(ruleDiffCalculationResults), + stats: { + num_rules_to_upgrade_total: totalUpgradeableRules, + num_rules_with_conflicts: 0, + num_rules_with_non_solvable_conflicts: 0, + tags, + }, + rules: calculateRuleUpgradeInfo(diffResults), + page, + per_page: perPage, }; return response.ok({ body }); @@ -67,72 +76,96 @@ export const reviewRuleUpgradeHandler = async ( }); } }; -const calculateRuleStats = (results: CalculateRuleDiffResult[]): RuleUpgradeStatsForReview => { + +interface CalculateUpgradeableRulesDiffArgs { + ruleAssetsClient: IPrebuiltRuleAssetsClient; + rulesClient: RulesClient; + page: number; + perPage: number; + sort: ReviewRuleUpgradeSort; + filter: ReviewRuleUpgradeFilter | undefined; +} + +const BATCH_SIZE = 100; + +async function calculateUpgradeableRulesDiff({ + ruleAssetsClient, + rulesClient, + page, + perPage, + sort, + filter, +}: CalculateUpgradeableRulesDiffArgs) { + const allLatestVersions = await ruleAssetsClient.fetchLatestVersions(); + const latestVersionsMap = new Map(allLatestVersions.map((version) => [version.rule_id, version])); + + let upgradeableRules: Array<{ rule: RuleResponse; targetVersion: RuleVersionSpecifier }> = []; const allTags = new Set(); + let totalUpgradeableRules = 0; - const stats = results.reduce( - (acc, result) => { - acc.num_rules_to_upgrade_total += 1; + const filterKQL = convertRulesFilterToKQL({ + showElasticRules: true, + filter: filter?.name, + tags: filter?.tags, + customizationStatus: filter?.customization_status, + }); - if (result.ruleDiff.num_fields_with_conflicts > 0) { - acc.num_rules_with_conflicts += 1; - } + // Fetch all installed rules that have a newer version available in batches to + // avoid loading all rules at once into memory. We need to iterate over all + // rules, even if the perPage limit is reached, to calculate the total number + // of upgradeable rules and all tags. + // TODO: Get rid of stats in this call and don't iterate over all rules. That should cover tha case when rule_ids are passed in the request. + let batchPage = 1; + while (true) { + const findRulesResult = await findRules({ + rulesClient, + ruleIds: filter?.rule_ids, + filter: filterKQL, + perPage: BATCH_SIZE, + page: batchPage, + sortField: sort.field, + sortOrder: sort.order, + fields: undefined, + }); - if (result.ruleDiff.num_fields_with_non_solvable_conflicts > 0) { - acc.num_rules_with_non_solvable_conflicts += 1; + if (findRulesResult.data.length === 0) { + break; + } + const currentRulesBatch = findRulesResult.data.map((rule) => internalRuleToAPIResponse(rule)); + currentRulesBatch.forEach((rule) => { + const targetVersion = latestVersionsMap.get(rule.rule_id); + if (targetVersion != null && rule.version < targetVersion.version) { + upgradeableRules.push({ rule, targetVersion }); + rule.tags.forEach((tag) => allTags.add(tag)); + totalUpgradeableRules += 1; } + }); + batchPage += 1; + } - result.ruleVersions.input.current?.tags.forEach((tag) => allTags.add(tag)); + // Trim the list of upgradeable rules to the requested page + // TODO handle this in the while loop above + upgradeableRules = upgradeableRules.slice((page - 1) * perPage, page * perPage); - return acc; - }, - { - num_rules_to_upgrade_total: 0, - num_rules_with_conflicts: 0, - num_rules_with_non_solvable_conflicts: 0, - } + // Zip current rules with their base and target versions + const currentRules = upgradeableRules.map(({ rule }) => rule); + const latestRules = await ruleAssetsClient.fetchAssetsByVersion( + upgradeableRules.map(({ targetVersion }) => targetVersion) ); + const baseRules = await ruleAssetsClient.fetchAssetsByVersion(currentRules); + const ruleVersionsMap = zipRuleVersions(currentRules, baseRules, latestRules); + + // Calculate the diff between current, base, and target versions + // Iterate through the current rules array to keep the order of the results + const diffResults = currentRules.map((current) => { + const base = ruleVersionsMap.get(current.rule_id)?.base; + const target = ruleVersionsMap.get(current.rule_id)?.target; + return calculateRuleDiff({ current, base, target }); + }); return { - ...stats, + diffResults, tags: Array.from(allTags), + totalUpgradeableRules, }; -}; -const calculateRuleInfos = (results: CalculateRuleDiffResult[]): RuleUpgradeInfoForReview[] => { - return results.map((result) => { - const { ruleDiff, ruleVersions } = result; - const installedCurrentVersion = ruleVersions.input.current; - const targetVersion = ruleVersions.input.target; - invariant(installedCurrentVersion != null, 'installedCurrentVersion not found'); - invariant(targetVersion != null, 'targetVersion not found'); - - const targetRule: RuleResponse = { - ...convertPrebuiltRuleAssetToRuleResponse(targetVersion), - id: installedCurrentVersion.id, - revision: installedCurrentVersion.revision + 1, - created_at: installedCurrentVersion.created_at, - created_by: installedCurrentVersion.created_by, - updated_at: new Date().toISOString(), - updated_by: installedCurrentVersion.updated_by, - }; - - return { - id: installedCurrentVersion.id, - rule_id: installedCurrentVersion.rule_id, - revision: installedCurrentVersion.revision, - current_rule: installedCurrentVersion, - target_rule: targetRule, - diff: { - fields: pickBy>( - ruleDiff.fields, - (fieldDiff) => - fieldDiff.diff_outcome !== ThreeWayDiffOutcome.StockValueNoUpdate && - fieldDiff.diff_outcome !== ThreeWayDiffOutcome.MissingBaseNoUpdate - ), - num_fields_with_updates: ruleDiff.num_fields_with_updates, - num_fields_with_conflicts: ruleDiff.num_fields_with_conflicts, - num_fields_with_non_solvable_conflicts: ruleDiff.num_fields_with_non_solvable_conflicts, - }, - }; - }); -}; +} diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts index 54f51752b7ab8..94ee1282fae98 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/api/review_rule_upgrade/review_rule_upgrade_route.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { REVIEW_RULE_UPGRADE_URL } from '../../../../../../common/api/detection_engine/prebuilt_rules'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { + REVIEW_RULE_UPGRADE_URL, + ReviewRuleUpgradeRequestBody, +} from '../../../../../../common/api/detection_engine/prebuilt_rules'; import type { SecuritySolutionPluginRouter } from '../../../../../types'; import { routeLimitedConcurrencyTag } from '../../../../../utils/route_limited_concurrency_tag'; import { @@ -34,7 +38,11 @@ export const reviewRuleUpgradeRoute = (router: SecuritySolutionPluginRouter) => .addVersion( { version: '1', - validate: {}, + validate: { + request: { + body: buildRouteValidationWithZod(ReviewRuleUpgradeRequestBody), + }, + }, }, reviewRuleUpgradeHandler ); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts index 1138a48cc39d4..df5ce416a53db 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client.ts @@ -15,8 +15,13 @@ import { findRules } from '../../../rule_management/logic/search/find_rules'; import { getExistingPrepackagedRules } from '../../../rule_management/logic/search/get_existing_prepackaged_rules'; import { internalRuleToAPIResponse } from '../../../rule_management/logic/detection_rules_client/converters/internal_rule_to_api_response'; +interface FetchAllInstalledRulesArgs { + page?: number; + perPage?: number; +} + export interface IPrebuiltRuleObjectsClient { - fetchAllInstalledRules(): Promise; + fetchAllInstalledRules(args?: FetchAllInstalledRulesArgs): Promise; fetchInstalledRulesByIds(ruleIds: string[]): Promise; } @@ -24,9 +29,9 @@ export const createPrebuiltRuleObjectsClient = ( rulesClient: RulesClient ): IPrebuiltRuleObjectsClient => { return { - fetchAllInstalledRules: (): Promise => { + fetchAllInstalledRules: ({ page, perPage } = {}): Promise => { return withSecuritySpan('IPrebuiltRuleObjectsClient.fetchInstalledRules', async () => { - const rulesData = await getExistingPrepackagedRules({ rulesClient }); + const rulesData = await getExistingPrepackagedRules({ rulesClient, page, perPage }); const rules = rulesData.map((rule) => internalRuleToAPIResponse(rule)); return rules; }); diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.ts index 24b2954547e40..9a7eb8fdf6f56 100644 --- a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.ts +++ b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_management/logic/search/get_existing_prepackaged_rules.ts @@ -49,16 +49,20 @@ export const getRulesCount = async ({ export const getRules = async ({ rulesClient, filter, + page = 1, + perPage = MAX_PREBUILT_RULES_COUNT, }: { rulesClient: RulesClient; filter: string; + page?: number; + perPage?: number; }): Promise => withSecuritySpan('getRules', async () => { const rules = await findRules({ rulesClient, filter, - perPage: MAX_PREBUILT_RULES_COUNT, - page: 1, + perPage, + page, sortField: 'createdAt', sortOrder: 'desc', fields: undefined, @@ -80,11 +84,17 @@ export const getNonPackagedRules = async ({ export const getExistingPrepackagedRules = async ({ rulesClient, + page, + perPage, }: { rulesClient: RulesClient; + page?: number; + perPage?: number; }): Promise => { return getRules({ rulesClient, + page, + perPage, filter: KQL_FILTER_IMMUTABLE_RULES, }); };