[8.x] [Security Solution] Reduce the _review rule upgrade endpoint response size (#211045)

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/common/prebuilt_rules_filter.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+
+export enum RuleCustomizationStatus {
+  CUSTOMIZED = 'CUSTOMIZED',
+  NOT_CUSTOMIZED = 'NOT_CUSTOMIZED',
+}
+
+export type PrebuiltRulesFilter = z.infer<typeof PrebuiltRulesFilter>;
+export const PrebuiltRulesFilter = z.object({
+  /**
+   * Tags to filter by
+   */
+  tags: z.array(z.string()).optional(),
+  /**
+   * Rule name to filter by
+   */
+  name: z.string().optional(),
+  /**
+   * Rule customization status to filter by
+   */
+  customization_status: z.nativeEnum(RuleCustomizationStatus).optional(),
+});

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/common/review_prebuilt_rules_upgrade_filter.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import { PrebuiltRulesFilter } from './prebuilt_rules_filter';
+
+export enum RuleCustomizationStatus {
+  CUSTOMIZED = 'CUSTOMIZED',
+  NOT_CUSTOMIZED = 'NOT_CUSTOMIZED',
+}
+
+export type ReviewPrebuiltRuleUpgradeFilter = z.infer<typeof ReviewPrebuiltRuleUpgradeFilter>;
+export const ReviewPrebuiltRuleUpgradeFilter = PrebuiltRulesFilter.merge(
+  z.object({
+    /**
+     * Rule IDs to return upgrade info for
+     */
+    rule_ids: z.array(z.string()).optional(),
+  })
+);

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/get_prebuilt_rules_status/get_prebuilt_rules_status_route.ts

@@ -8,6 +8,18 @@
 export interface GetPrebuiltRulesStatusResponseBody {
   /** Aggregated info about all prebuilt rules */
   stats: PrebuiltRulesStatusStats;
+
+  /**
+   * Aggregated info about upgradeable prebuilt rules. This fields is optional
+   * for backward compatibility. After one serverless release cycle, it can be
+   * made required.
+   * */
+  aggregated_fields?: {
+    upgradeable_rules: {
+      /** List of all tags of the current versions of upgradeable rules */
+      tags: string[];
+    };
+  };
 }
 
 export interface PrebuiltRulesStatusStats {

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/index.ts

@@ -22,3 +22,4 @@ export * from './model/diff/three_way_diff/three_way_diff_outcome';
 export * from './model/diff/three_way_diff/three_way_diff';
 export * from './model/diff/three_way_diff/three_way_diff_conflict';
 export * from './model/diff/three_way_diff/three_way_merge_outcome';
+export * from './common/prebuilt_rules_filter';

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/perform_rule_upgrade/perform_rule_upgrade_route.ts

@@ -10,6 +10,7 @@ import { mapValues } from 'lodash';
 import { RuleResponse } from '../../model/rule_schema/rule_schemas.gen';
 import { AggregatedPrebuiltRuleError, DiffableAllFields } from '../model';
 import { RuleSignatureId, RuleVersion } from '../../model';
+import { PrebuiltRulesFilter } from '../common/prebuilt_rules_filter';
 
 export type Mode = z.infer<typeof Mode>;
 export const Mode = z.enum(['ALL_RULES', 'SPECIFIC_RULES']);
@@ -111,21 +112,31 @@ export const RuleUpgradeSpecifier = z.object({
   fields: RuleFieldsToUpgrade.optional(),
 });
 
+export type UpgradeConflictResolution = z.infer<typeof UpgradeConflictResolution>;
+export const UpgradeConflictResolution = z.enum(['SKIP', 'OVERWRITE']);
+export type UpgradeConflictResolutionEnum = typeof UpgradeConflictResolution.enum;
+export const UpgradeConflictResolutionEnum = UpgradeConflictResolution.enum;
+
 export type UpgradeSpecificRulesRequest = z.infer<typeof UpgradeSpecificRulesRequest>;
 export const UpgradeSpecificRulesRequest = z.object({
   mode: z.literal('SPECIFIC_RULES'),
   rules: z.array(RuleUpgradeSpecifier).min(1),
   pick_version: PickVersionValues.optional(),
+  on_conflict: UpgradeConflictResolution.optional(),
+  dry_run: z.boolean().optional(),
 });
 
 export type UpgradeAllRulesRequest = z.infer<typeof UpgradeAllRulesRequest>;
 export const UpgradeAllRulesRequest = z.object({
   mode: z.literal('ALL_RULES'),
   pick_version: PickVersionValues.optional(),
+  filter: PrebuiltRulesFilter.optional(),
+  on_conflict: UpgradeConflictResolution.optional(),
+  dry_run: z.boolean().optional(),
 });
 
 export type SkipRuleUpgradeReason = z.infer<typeof SkipRuleUpgradeReason>;
-export const SkipRuleUpgradeReason = z.enum(['RULE_UP_TO_DATE']);
+export const SkipRuleUpgradeReason = z.enum(['RULE_UP_TO_DATE', 'CONFLICT']);
 export type SkipRuleUpgradeReasonEnum = typeof SkipRuleUpgradeReason.enum;
 export const SkipRuleUpgradeReasonEnum = SkipRuleUpgradeReason.enum;
 

x-pack/solutions/security/plugins/security_solution/common/api/detection_engine/prebuilt_rules/review_rule_upgrade/review_rule_upgrade_route.ts

@@ -5,35 +5,86 @@
  * 2.0.
  */
 
-import type { RuleObjectId, RuleSignatureId, RuleTagArray } from '../../model';
+import { z } from '@kbn/zod';
+import { SortOrder, type RuleObjectId, type RuleSignatureId, type RuleTagArray } from '../../model';
 import type { PartialRuleDiff } from '../model';
-import type { RuleResponse } from '../../model/rule_schema';
+import type { RuleResponse, RuleVersion } from '../../model/rule_schema';
+import { FindRulesSortField } from '../../rule_management';
+import { PrebuiltRulesFilter } from '../common/prebuilt_rules_filter';
+
+export type ReviewRuleUpgradeSort = z.infer<typeof ReviewRuleUpgradeSort>;
+export const ReviewRuleUpgradeSort = z.object({
+  /**
+   * Field to sort by
+   */
+  field: FindRulesSortField.optional(),
+  /**
+   * Sort order
+   */
+  order: SortOrder.optional(),
+});
+
+export type ReviewRuleUpgradeRequestBody = z.infer<typeof ReviewRuleUpgradeRequestBody>;
+export const ReviewRuleUpgradeRequestBody = z
+  .object({
+    filter: PrebuiltRulesFilter.optional(),
+    sort: ReviewRuleUpgradeSort.optional(),
+
+    page: z.coerce.number().int().min(1).optional().default(1),
+    /**
+     * Rules per page
+     */
+    per_page: z.coerce.number().int().min(0).optional().default(20),
+  })
+  .nullable();
 
 export interface ReviewRuleUpgradeResponseBody {
-  /** Aggregated info about all rules available for upgrade */
+  /**
+   * @deprecated Use the prebuilt rule status API instead. The field is kept
+   * here for backward compatibility but can be removed after one Serverless
+   * release.
+   */
   stats: RuleUpgradeStatsForReview;
 
   /** Info about individual rules: one object per each rule available for upgrade */
   rules: RuleUpgradeInfoForReview[];
+
+  /** The requested page number */
+  page: number;
+
+  /** The requested number of items per page */
+  per_page: number;
+
+  /** The total number of rules available for upgrade that match the filter criteria */
+  total: number;
 }
 
 export interface RuleUpgradeStatsForReview {
-  /** Number of installed prebuilt rules available for upgrade (stock + customized) */
+  /**
+   * @deprecated Always 0
+   */
   num_rules_to_upgrade_total: number;
 
-  /** Number of installed prebuilt rules with upgrade conflicts (SOLVABLE or NON_SOLVABLE) */
+  /**
+   * @deprecated Always 0
+   */
   num_rules_with_conflicts: number;
 
-  /** Number of installed prebuilt rules with NON_SOLVABLE upgrade conflicts */
+  /**
+   * @deprecated Always 0
+   */
   num_rules_with_non_solvable_conflicts: number;
 
-  /** A union of all tags of all rules available for upgrade */
+  /**
+   * @deprecated Always an empty array
+   */
   tags: RuleTagArray;
 }
 
 export interface RuleUpgradeInfoForReview {
   id: RuleObjectId;
   rule_id: RuleSignatureId;
+  version: RuleVersion;
   current_rule: RuleResponse;
   target_rule: RuleResponse;
   diff: PartialRuleDiff;

x-pack/solutions/security/plugins/security_solution/common/constants.ts

@@ -388,7 +388,7 @@ export const STARTED_TRANSFORM_STATES = new Set([
 ]);
 
 /**
- * How many rules to update at a time is set to 50 from errors coming from
+ * How many rules to update at a time is set to 20 from errors coming from
  * the slow environments such as cloud when the rule updates are > 100 we were
  * seeing timeout issues.
  *
@@ -403,14 +403,14 @@ export const STARTED_TRANSFORM_STATES = new Set([
  * Lastly, we saw weird issues where Chrome on upstream 408 timeouts will re-call the REST route
  * which in turn could create additional connections we want to avoid.
  *
- * See file import_rules_route.ts for another area where 50 was chosen, therefore I chose
- * 50 here to mimic it as well. If you see this re-opened or what similar to it, consider
- * reducing the 50 above to a lower number.
+ * See file import_rules_route.ts for another area where 20 was chosen, therefore I chose
+ * 20 here to mimic it as well. If you see this re-opened or what similar to it, consider
+ * reducing the 20 above to a lower number.
  *
  * See the original ticket here:
  * https://github.com/elastic/kibana/issues/94418
  */
-export const MAX_RULES_TO_UPDATE_IN_PARALLEL = 50;
+export const MAX_RULES_TO_UPDATE_IN_PARALLEL = 20;
 
 export const LIMITED_CONCURRENCY_ROUTE_TAG_PREFIX = `${APP_ID}:limitedConcurrency`;
 

x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_fields.ts

@@ -22,3 +22,4 @@ export const TAGS_FIELD = 'alert.attributes.tags';
 export const PARAMS_TYPE_FIELD = 'alert.attributes.params.type';
 export const PARAMS_IMMUTABLE_FIELD = 'alert.attributes.params.immutable';
 export const LAST_RUN_OUTCOME_FIELD = 'alert.attributes.lastRun.outcome';
+export const IS_CUSTOMIZED_FIELD = 'alert.attributes.params.ruleSource.isCustomized';

x-pack/solutions/security/plugins/security_solution/common/detection_engine/rule_management/rule_filtering.ts

@@ -7,10 +7,11 @@
 
 import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
 import type { RuleExecutionStatus } from '../../api/detection_engine';
-import { RuleExecutionStatusEnum } from '../../api/detection_engine';
+import { RuleCustomizationStatus, RuleExecutionStatusEnum } from '../../api/detection_engine';
 import { prepareKQLStringParam } from '../../utils/kql';
 import {
   ENABLED_FIELD,
+  IS_CUSTOMIZED_FIELD,
   LAST_RUN_OUTCOME_FIELD,
   PARAMS_IMMUTABLE_FIELD,
   PARAMS_TYPE_FIELD,
@@ -23,6 +24,8 @@ export const KQL_FILTER_IMMUTABLE_RULES = `${PARAMS_IMMUTABLE_FIELD}: true`;
 export const KQL_FILTER_MUTABLE_RULES = `${PARAMS_IMMUTABLE_FIELD}: false`;
 export const KQL_FILTER_ENABLED_RULES = `${ENABLED_FIELD}: true`;
 export const KQL_FILTER_DISABLED_RULES = `${ENABLED_FIELD}: false`;
+export const KQL_FILTER_CUSTOMIZED_RULES = `${IS_CUSTOMIZED_FIELD}: true`;
+export const KQL_FILTER_NOT_CUSTOMIZED_RULES = `${IS_CUSTOMIZED_FIELD}: false`;
 
 interface RulesFilterOptions {
   filter: string;
@@ -32,6 +35,7 @@ interface RulesFilterOptions {
   tags: string[];
   excludeRuleTypes: Type[];
   ruleExecutionStatus: RuleExecutionStatus;
+  customizationStatus: RuleCustomizationStatus;
   ruleIds: string[];
 }
 
@@ -50,6 +54,7 @@ export function convertRulesFilterToKQL({
   tags,
   excludeRuleTypes = [],
   ruleExecutionStatus,
+  customizationStatus,
 }: Partial<RulesFilterOptions>): string {
   const kql: string[] = [];
 
@@ -85,6 +90,12 @@ export function convertRulesFilterToKQL({
     kql.push(`${LAST_RUN_OUTCOME_FIELD}: "failed"`);
   }
 
+  if (customizationStatus === RuleCustomizationStatus.CUSTOMIZED) {
+    kql.push(KQL_FILTER_CUSTOMIZED_RULES);
+  } else if (customizationStatus === RuleCustomizationStatus.NOT_CUSTOMIZED) {
+    kql.push(KQL_FILTER_NOT_CUSTOMIZED_RULES);
+  }
+
   return kql.join(' AND ');
 }
 

x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/api.ts

@@ -15,14 +15,14 @@ import type { ActionType, AsApiContract } from '@kbn/actions-plugin/common';
 import type { ActionResult } from '@kbn/actions-plugin/server';
 import { convertRulesFilterToKQL } from '../../../../common/detection_engine/rule_management/rule_filtering';
 import type {
-  UpgradeSpecificRulesRequest,
-  PickVersionValues,
   PerformRuleUpgradeResponseBody,
   InstallSpecificRulesRequest,
   PerformRuleInstallationResponseBody,
   GetPrebuiltRulesStatusResponseBody,
   ReviewRuleUpgradeResponseBody,
   ReviewRuleInstallationResponseBody,
+  ReviewRuleUpgradeRequestBody,
+  PerformRuleUpgradeRequestBody,
 } from '../../../../common/api/detection_engine/prebuilt_rules';
 import type {
   BulkDuplicateRules,
@@ -637,13 +637,16 @@ export const getPrebuiltRulesStatus = async ({
  */
 export const reviewRuleUpgrade = async ({
   signal,
+  request,
 }: {
   signal: AbortSignal | undefined;
+  request: ReviewRuleUpgradeRequestBody;
 }): Promise<ReviewRuleUpgradeResponseBody> =>
   KibanaServices.get().http.fetch(REVIEW_RULE_UPGRADE_URL, {
     method: 'POST',
     version: '1',
     signal,
+    body: JSON.stringify(request),
   });
 
 /**
@@ -685,23 +688,13 @@ export const performInstallSpecificRules = async (
     }),
   });
 
-export interface PerformUpgradeRequest {
-  rules: UpgradeSpecificRulesRequest['rules'];
-  pickVersion: PickVersionValues;
-}
-
-export const performUpgradeSpecificRules = async ({
-  rules,
-  pickVersion,
-}: PerformUpgradeRequest): Promise<PerformRuleUpgradeResponseBody> =>
+export const performUpgradeRules = async (
+  body: PerformRuleUpgradeRequestBody
+): Promise<PerformRuleUpgradeResponseBody> =>
   KibanaServices.get().http.fetch(PERFORM_RULE_UPGRADE_URL, {
     method: 'POST',
     version: '1',
-    body: JSON.stringify({
-      mode: 'SPECIFIC_RULES',
-      rules,
-      pick_version: pickVersion,
-    }),
+    body: JSON.stringify(body),
   });
 
 export const bootstrapPrebuiltRules = async (): Promise<BootstrapPrebuiltRulesResponse> =>

x-pack/solutions/security/plugins/security_solution/public/detection_engine/rule_management/api/hooks/prebuilt_rules/use_fetch_prebuilt_rules_status_query.ts

@@ -4,24 +4,24 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { useCallback } from 'react';
 import type { UseQueryOptions } from '@tanstack/react-query';
 import { useQuery, useQueryClient } from '@tanstack/react-query';
-import type { PrebuiltRulesStatusStats } from '../../../../../../common/api/detection_engine/prebuilt_rules';
+import { useCallback } from 'react';
+import type { GetPrebuiltRulesStatusResponseBody } from '../../../../../../common/api/detection_engine/prebuilt_rules';
+import { GET_PREBUILT_RULES_STATUS_URL } from '../../../../../../common/api/detection_engine/prebuilt_rules';
 import { getPrebuiltRulesStatus } from '../../api';
 import { DEFAULT_QUERY_OPTIONS } from '../constants';
-import { GET_PREBUILT_RULES_STATUS_URL } from '../../../../../../common/api/detection_engine/prebuilt_rules';
 
 export const PREBUILT_RULES_STATUS_QUERY_KEY = ['GET', GET_PREBUILT_RULES_STATUS_URL];
 
 export const useFetchPrebuiltRulesStatusQuery = (
-  options?: UseQueryOptions<PrebuiltRulesStatusStats>
+  options?: UseQueryOptions<GetPrebuiltRulesStatusResponseBody>
 ) => {
-  return useQuery<PrebuiltRulesStatusStats>(
+  return useQuery<GetPrebuiltRulesStatusResponseBody>(
     PREBUILT_RULES_STATUS_QUERY_KEY,
     async ({ signal }) => {
       const response = await getPrebuiltRulesStatus({ signal });
-      return response.stats;
+      return response;
     },
     {
       ...DEFAULT_QUERY_OPTIONS,