From 547f67184220ea3eb898d3b9bcd18981725615a4 Mon Sep 17 00:00:00 2001 From: Michael Olorunnisola Date: Mon, 3 Mar 2025 20:22:25 -0500 Subject: [PATCH] [Performance][Security Solution][1/4] - Field Browser Performance (#212469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Part 1 of https://github.com/elastic/kibana/pull/212173 ### Testing For setup see testing section here: https://github.com/elastic/kibana/pull/212173#issue-2870522020 **Areas to test:** - Alert Table (Alerts, Rule Detail, Rule Preview pages) - Security solution field browser component - Flyout table tab. ### Background When investigating the performance of the security solution application, one of the issues that was observed was locking of the page and field browser component when the total number of fields returned were significantly high. This led to cell values not rendering in the alert table, and the field browser in all the security solution pages causing the page to crash. The relevant images can be seen at the bottom of this description In short: The `push(...fields)` is non-performant at scale, and at a significant enough scale (Testing was done with 500k mapped fields), fails to run due to excessive arguments provided to the `push` method. In this PR improvements are made in the `browserFields` transformations that are done for the field browser component, expandable flyout table tab, and alert/rule tables via `CellValue` component. This work was done to get immediate improvements in the security solution UI, but a longer term consideration will be whether or not the `browserFields` is even necessary anymore as a concept based on what is available via the `fields` api. We will revisit once our Sourcerer refactoring work is done. Screenshot 2025-02-26 at 10 15 29 AM Screenshot 2025-02-26 at 10 18 36 AM Screenshot 2025-02-26 at 10 19 48 AM ![image](https://github.com/user-attachments/assets/5d746b21-fa9b-425b-826a-cc7abd444f21) ![image](https://github.com/user-attachments/assets/4dff2378-d61b-4770-b46b-41cb37d6ead4) ### After the fix (Done on [this branch](https://github.com/elastic/kibana/pull/212173) that has the other changes as well) https://github.com/user-attachments/assets/da992296-4eb8-49d4-96ca-b0a19a00f1f0 ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios (cherry picked from commit 750e156c26078015025551c4f10d299ba269fa35) --- .../components/field_items/field_items.tsx | 53 +++--- .../common/containers/source/index.test.tsx | 150 ++--------------- .../public/common/containers/source/index.tsx | 13 +- .../public/common/containers/source/mock.ts | 8 + .../containers/source/use_data_view.test.tsx | 153 ++++++++++++++++++ 5 files changed, 210 insertions(+), 167 deletions(-) create mode 100644 x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx diff --git a/src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx b/src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx index 30e9e551da520..b7d131161b32d 100644 --- a/src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx +++ b/src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx @@ -17,7 +17,6 @@ import { EuiBadge, EuiScreenReaderOnly, } from '@elastic/eui'; -import { uniqBy } from 'lodash/fp'; import type { BrowserFields } from '@kbn/rule-registry-plugin/common'; import { EcsFlat } from '@elastic/ecs'; import { EcsMetadata } from '@kbn/alerts-as-data-utils/src/field_maps/types'; @@ -67,29 +66,41 @@ export const getFieldItemsData = ({ selectedCategoryIds.length > 0 ? selectedCategoryIds : Object.keys(browserFields); const selectedFieldIds = new Set(columnIds); - const fieldItems = uniqBy( - 'name', - categoryIds.reduce((fieldItemsAcc, categoryId) => { + const getFieldItems = () => { + const fieldItemsAcc: BrowserFieldItem[] = []; + const fieldsSeen: Set = new Set(); + + /** + * Both categoryIds and the fields in browserFields can be significantly large. Technically speaking, + * there is no upper cap on how many fields a customer can have. We are using a for loop here to avoid + * the performance issues that can arise from using map/filter/reduce. + */ + for (let i = 0; i < categoryIds.length; i += 1) { + const categoryId = categoryIds[i]; const categoryBrowserFields = Object.values(browserFields[categoryId]?.fields ?? {}); if (categoryBrowserFields.length > 0) { - fieldItemsAcc.push( - ...categoryBrowserFields.map(({ name = '', ...field }) => { - return { - name, - type: field.type, - description: getDescription(name, EcsFlat as Record), - example: field.example?.toString(), - category: getCategory(name), - selected: selectedFieldIds.has(name), - isRuntime: !!field.runtimeField, - }; - }) - ); + for (let j = 0; j < categoryBrowserFields.length; j += 1) { + const field = categoryBrowserFields[j]; + const name = field.name !== undefined ? field.name : ''; + if (fieldsSeen.has(name)) continue; + fieldsSeen.add(name); + const categoryFieldItem = { + name, + type: field.type, + description: getDescription(name, EcsFlat as Record), + example: field.example?.toString(), + category: getCategory(name), + selected: selectedFieldIds.has(name), + isRuntime: !!field.runtimeField, + }; + fieldItemsAcc.push(categoryFieldItem); + } } - return fieldItemsAcc; - }, []) - ); - return { fieldItems }; + } + return fieldItemsAcc; + }; + + return { fieldItems: getFieldItems() }; }; const getDefaultFieldTableColumns = ({ highlight }: { highlight: string }): FieldTableColumns => { diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx index 3f59b65a8fc53..87d58317257aa 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx @@ -5,151 +5,19 @@ * 2.0. */ -import type { IndexFieldSearch } from './use_data_view'; -import { useDataView } from './use_data_view'; -import { mocksSource } from './mock'; -import { mockGlobalState, TestProviders } from '../../mock'; -import { act, renderHook } from '@testing-library/react'; -import { useKibana } from '../../lib/kibana'; - -const mockDispatch = jest.fn(); -jest.mock('react-redux', () => { - const original = jest.requireActual('react-redux'); - - return { - ...original, - useDispatch: () => mockDispatch, - }; -}); -jest.mock('../../lib/kibana'); -jest.mock('../../lib/apm/use_track_http_request'); +import { mockBrowserFields, mockIndexFields, mockIndexFieldsByName } from './mock'; +import * as indexUtils from '.'; describe('source/index.tsx', () => { - describe('useDataView hook', () => { - const mockSearchResponse = { - ...mocksSource, - indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName], - isRestore: false, - rawResponse: {}, - runtimeMappings: {}, - }; - - beforeEach(() => { - jest.clearAllMocks(); - const mock = { - subscribe: ({ next }: { next: Function }) => next(mockSearchResponse), - unsubscribe: jest.fn(), - }; - - (useKibana as jest.Mock).mockReturnValue({ - services: { - data: { - dataViews: { - ...useKibana().services.data.dataViews, - get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => { - const dataViewMock = { - id: dataViewId, - matchedIndices: refreshFields - ? ['hello', 'world', 'refreshed'] - : ['hello', 'world'], - fields: mocksSource.indexFields, - getIndexPattern: () => - refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*', - getRuntimeMappings: () => ({ - myfield: { - type: 'keyword', - }, - }), - }; - return Promise.resolve({ - toSpec: () => dataViewMock, - ...dataViewMock, - }); - }, - getFieldsForWildcard: async () => Promise.resolve(), - getExistingIndices: async (indices: string[]) => Promise.resolve(indices), - }, - search: { - search: jest.fn().mockReturnValue({ - subscribe: ({ next }: { next: Function }) => { - next(mockSearchResponse); - return mock; - }, - unsubscribe: jest.fn(), - }), - }, - }, - }, - }); - }); - it('sets field data for data view', async () => { - const { result } = renderHook(() => useDataView(), { - wrapper: TestProviders, - }); - - await act(async () => { - await result.current.indexFieldsSearch({ dataViewId: 'neato' }); - }); - expect(mockDispatch.mock.calls[0][0]).toEqual({ - type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING', - payload: { id: 'neato', loading: true }, - }); - const { type: sourceType, payload } = mockDispatch.mock.calls[1][0]; - expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW'); - expect(payload.id).toEqual('neato'); - }); - - it('should reuse the result for dataView info when cleanCache not passed', async () => { - let indexFieldsSearch: IndexFieldSearch; - - const { result } = renderHook(() => useDataView(), { - wrapper: TestProviders, - }); - - await act(async () => { - indexFieldsSearch = result.current.indexFieldsSearch; - }); - - await indexFieldsSearch!({ dataViewId: 'neato' }); - const { - payload: { browserFields, indexFields }, - } = mockDispatch.mock.calls[1][0]; - - mockDispatch.mockClear(); - - await indexFieldsSearch!({ dataViewId: 'neato' }); - const { - payload: { browserFields: newBrowserFields, indexFields: newIndexFields }, - } = mockDispatch.mock.calls[1][0]; - - expect(browserFields).toBe(newBrowserFields); - expect(indexFields).toBe(newIndexFields); + describe('getAllBrowserFields', () => { + it('should return the expected browser fields list', () => { + expect(indexUtils.getAllBrowserFields(mockBrowserFields)).toEqual(mockIndexFields); }); + }); - it('should not reuse the result for dataView info when cleanCache passed', async () => { - let indexFieldsSearch: IndexFieldSearch; - const { result } = renderHook(() => useDataView(), { - wrapper: TestProviders, - }); - - await act(async () => { - indexFieldsSearch = result.current.indexFieldsSearch; - }); - - await indexFieldsSearch!({ dataViewId: 'neato' }); - const { - payload: { patternList }, - } = mockDispatch.mock.calls[1][0]; - - mockDispatch.mockClear(); - - await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true }); - const { - payload: { patternList: newPatternList }, - } = mockDispatch.mock.calls[1][0]; - expect(patternList).not.toBe(newPatternList); - expect(patternList).not.toContain('refreshed*'); - expect(newPatternList).toContain('refreshed*'); + describe('getAllFieldsByName', () => { + it('should return the expected browser fields list', () => { + expect(indexUtils.getAllFieldsByName(mockBrowserFields)).toEqual(mockIndexFieldsByName); }); }); }); diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx index a121919b412e5..384f1e88a0fbe 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx @@ -22,10 +22,11 @@ import type { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../../../../common/endpoin export type { BrowserFields }; export function getAllBrowserFields(browserFields: BrowserFields): Array> { - const result: Array> = []; + let result: Array> = []; for (const namespace of Object.values(browserFields)) { if (namespace.fields) { - result.push(...Object.values(namespace.fields)); + const namespaceFields = Object.values(namespace.fields); + result = result.concat(namespaceFields); } } return result; @@ -36,9 +37,11 @@ export function getAllBrowserFields(browserFields: BrowserFields): Array } => keyBy('name', getAllBrowserFields(browserFields)); +export const getAllFieldsByName = memoizeOne( + (browserFields: BrowserFields): { [fieldName: string]: Partial } => + keyBy('name', getAllBrowserFields(browserFields)), + (newArgs, lastArgs) => newArgs[0] === lastArgs[0] +); export const getIndexFields = memoizeOne( (title: string, fields: IIndexPatternFieldList): DataViewBase => diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts index 3b493cf2566b9..ce49d3b834caa 100644 --- a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts +++ b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts @@ -7,6 +7,7 @@ import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { flatten } from 'lodash'; +import type { FieldSpec } from '@kbn/data-views-plugin/common'; import type { BrowserFields } from '../../../../common/search_strategy/index_fields'; export const mocksSource = { @@ -387,6 +388,13 @@ export const mockIndexFields = flatten( Object.values(mockBrowserFields).map((fieldItem) => Object.values(fieldItem.fields ?? {})) ); +export const mockIndexFieldsByName = mockIndexFields.reduce((acc, indexFieldObj) => { + if (indexFieldObj.name) { + acc[indexFieldObj.name] = indexFieldObj; + } + return acc; +}, {} as { [fieldName: string]: Partial }); + const runTimeType: MappingRuntimeFieldType = 'keyword' as const; export const mockRuntimeMappings = { diff --git a/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx new file mode 100644 index 0000000000000..75cabcbb5f566 --- /dev/null +++ b/x-pack/solutions/security/plugins/security_solution/public/common/containers/source/use_data_view.test.tsx @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IndexFieldSearch } from './use_data_view'; +import { useDataView } from './use_data_view'; +import { mocksSource } from './mock'; +import { mockGlobalState, TestProviders } from '../../mock'; +import { act, renderHook } from '@testing-library/react'; +import { useKibana } from '../../lib/kibana'; + +const mockDispatch = jest.fn(); +jest.mock('react-redux', () => { + const original = jest.requireActual('react-redux'); + + return { + ...original, + useDispatch: () => mockDispatch, + }; +}); +jest.mock('../../lib/kibana'); +jest.mock('../../lib/apm/use_track_http_request'); + +describe('useDataView', () => { + const mockSearchResponse = { + ...mocksSource, + indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName], + isRestore: false, + rawResponse: {}, + runtimeMappings: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + const mock = { + subscribe: ({ next }: { next: Function }) => next(mockSearchResponse), + unsubscribe: jest.fn(), + }; + + (useKibana as jest.Mock).mockReturnValue({ + services: { + data: { + dataViews: { + ...useKibana().services.data.dataViews, + get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => { + const dataViewMock = { + id: dataViewId, + matchedIndices: refreshFields + ? ['hello', 'world', 'refreshed'] + : ['hello', 'world'], + fields: mocksSource.indexFields, + getIndexPattern: () => + refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*', + getRuntimeMappings: () => ({ + myfield: { + type: 'keyword', + }, + }), + }; + return Promise.resolve({ + toSpec: () => dataViewMock, + ...dataViewMock, + }); + }, + getFieldsForWildcard: async () => Promise.resolve(), + getExistingIndices: async (indices: string[]) => Promise.resolve(indices), + }, + search: { + search: jest.fn().mockReturnValue({ + subscribe: ({ next }: { next: Function }) => { + next(mockSearchResponse); + return mock; + }, + unsubscribe: jest.fn(), + }), + }, + }, + }, + }); + }); + it('sets field data for data view', async () => { + const { result } = renderHook(() => useDataView(), { + wrapper: TestProviders, + }); + + await act(async () => { + await result.current.indexFieldsSearch({ dataViewId: 'neato' }); + }); + expect(mockDispatch.mock.calls[0][0]).toEqual({ + type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING', + payload: { id: 'neato', loading: true }, + }); + const { type: sourceType, payload } = mockDispatch.mock.calls[1][0]; + expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW'); + expect(payload.id).toEqual('neato'); + }); + + it('should reuse the result for dataView info when cleanCache not passed', async () => { + let indexFieldsSearch: IndexFieldSearch; + + const { result } = renderHook(() => useDataView(), { + wrapper: TestProviders, + }); + + await act(async () => { + indexFieldsSearch = result.current.indexFieldsSearch; + }); + + await indexFieldsSearch!({ dataViewId: 'neato' }); + const { + payload: { browserFields, indexFields }, + } = mockDispatch.mock.calls[1][0]; + + mockDispatch.mockClear(); + + await indexFieldsSearch!({ dataViewId: 'neato' }); + const { + payload: { browserFields: newBrowserFields, indexFields: newIndexFields }, + } = mockDispatch.mock.calls[1][0]; + + expect(browserFields).toBe(newBrowserFields); + expect(indexFields).toBe(newIndexFields); + }); + + it('should not reuse the result for dataView info when cleanCache passed', async () => { + let indexFieldsSearch: IndexFieldSearch; + const { result } = renderHook(() => useDataView(), { + wrapper: TestProviders, + }); + + await act(async () => { + indexFieldsSearch = result.current.indexFieldsSearch; + }); + + await indexFieldsSearch!({ dataViewId: 'neato' }); + const { + payload: { patternList }, + } = mockDispatch.mock.calls[1][0]; + + mockDispatch.mockClear(); + + await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true }); + const { + payload: { patternList: newPatternList }, + } = mockDispatch.mock.calls[1][0]; + expect(patternList).not.toBe(newPatternList); + expect(patternList).not.toContain('refreshed*'); + expect(newPatternList).toContain('refreshed*'); + }); +});