[Performance][Security Solution][1/4] - Field Browser Performance (el…

…astic#212469)

## Summary
Part 1 of elastic#212173

### Testing
For setup see testing section here:
elastic#212173 (comment)

**Areas to test:**
- Alert Table (Alerts, Rule Detail, Rule Preview pages)
- Security solution field browser component
- Flyout table tab.

### Background

When investigating the performance of the security solution application,
one of the issues that was observed was locking of the page and field
browser component when the total number of fields returned were
significantly high.

This led to cell values not rendering in the alert table, and the field
browser in all the security solution pages causing the page to crash.
The relevant images can be seen at the bottom of this description

In short: The `push(...fields)` is non-performant at scale, and at a
significant enough scale (Testing was done with 500k mapped fields),
fails to run due to excessive arguments provided to the `push` method.
In this PR improvements are made in the `browserFields` transformations
that are done for the field browser component, expandable flyout table
tab, and alert/rule tables via `CellValue` component.

This work was done to get immediate improvements in the security
solution UI, but a longer term consideration will be whether or not the
`browserFields` is even necessary anymore as a concept based on what is
available via the `fields` api. We will revisit once our Sourcerer
refactoring work is done.

<img width="1728" alt="Screenshot 2025-02-26 at 10 15 29 AM"
src="https://github.com/user-attachments/assets/a25f577f-f758-415e-9c93-5452eadb8020"
/>

<img width="1445" alt="Screenshot 2025-02-26 at 10 18 36 AM"
src="https://github.com/user-attachments/assets/d70970d3-991a-47ba-b617-5862d18101b6"
/>

<img width="1469" alt="Screenshot 2025-02-26 at 10 19 48 AM"
src="https://github.com/user-attachments/assets/1767aa9b-66ab-46be-bc1a-5311630c2765"
/>

![image](https://github.com/user-attachments/assets/5d746b21-fa9b-425b-826a-cc7abd444f21)

![image](https://github.com/user-attachments/assets/4dff2378-d61b-4770-b46b-41cb37d6ead4)

### After the fix
(Done on [this branch](elastic#212173)
that has the other changes as well)

https://github.com/user-attachments/assets/da992296-4eb8-49d4-96ca-b0a19a00f1f0

### Checklist

Check the PR satisfies following conditions.

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

(cherry picked from commit 750e156)

src/platform/packages/shared/response-ops/alerts-fields-browser/components/field_items/field_items.tsx

@@ -17,7 +17,6 @@ import {
   EuiBadge,
   EuiScreenReaderOnly,
 } from '@elastic/eui';
-import { uniqBy } from 'lodash/fp';
 import type { BrowserFields } from '@kbn/rule-registry-plugin/common';
 import { EcsFlat } from '@elastic/ecs';
 import { EcsMetadata } from '@kbn/alerts-as-data-utils/src/field_maps/types';
@@ -67,29 +66,41 @@ export const getFieldItemsData = ({
     selectedCategoryIds.length > 0 ? selectedCategoryIds : Object.keys(browserFields);
   const selectedFieldIds = new Set(columnIds);
 
-  const fieldItems = uniqBy(
-    'name',
-    categoryIds.reduce<BrowserFieldItem[]>((fieldItemsAcc, categoryId) => {
+  const getFieldItems = () => {
+    const fieldItemsAcc: BrowserFieldItem[] = [];
+    const fieldsSeen: Set<string> = new Set();
+
+    /**
+     * Both categoryIds and the fields in browserFields can be significantly large. Technically speaking,
+     * there is no upper cap on how many fields a customer can have. We are using a for loop here to avoid
+     * the performance issues that can arise from using map/filter/reduce.
+     */
+    for (let i = 0; i < categoryIds.length; i += 1) {
+      const categoryId = categoryIds[i];
       const categoryBrowserFields = Object.values(browserFields[categoryId]?.fields ?? {});
       if (categoryBrowserFields.length > 0) {
-        fieldItemsAcc.push(
-          ...categoryBrowserFields.map(({ name = '', ...field }) => {
-            return {
-              name,
-              type: field.type,
-              description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
-              example: field.example?.toString(),
-              category: getCategory(name),
-              selected: selectedFieldIds.has(name),
-              isRuntime: !!field.runtimeField,
-            };
-          })
-        );
+        for (let j = 0; j < categoryBrowserFields.length; j += 1) {
+          const field = categoryBrowserFields[j];
+          const name = field.name !== undefined ? field.name : '';
+          if (fieldsSeen.has(name)) continue;
+          fieldsSeen.add(name);
+          const categoryFieldItem = {
+            name,
+            type: field.type,
+            description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
+            example: field.example?.toString(),
+            category: getCategory(name),
+            selected: selectedFieldIds.has(name),
+            isRuntime: !!field.runtimeField,
+          };
+          fieldItemsAcc.push(categoryFieldItem);
+        }
       }
-      return fieldItemsAcc;
-    }, [])
-  );
-  return { fieldItems };
+    }
+    return fieldItemsAcc;
+  };
+
+  return { fieldItems: getFieldItems() };
 };
 
 const getDefaultFieldTableColumns = ({ highlight }: { highlight: string }): FieldTableColumns => {

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx

@@ -5,151 +5,19 @@
  * 2.0.
  */
 
-import type { IndexFieldSearch } from './use_data_view';
-import { useDataView } from './use_data_view';
-import { mocksSource } from './mock';
-import { mockGlobalState, TestProviders } from '../../mock';
-import { act, renderHook } from '@testing-library/react';
-import { useKibana } from '../../lib/kibana';
-
-const mockDispatch = jest.fn();
-jest.mock('react-redux', () => {
-  const original = jest.requireActual('react-redux');
-
-  return {
-    ...original,
-    useDispatch: () => mockDispatch,
-  };
-});
-jest.mock('../../lib/kibana');
-jest.mock('../../lib/apm/use_track_http_request');
+import { mockBrowserFields, mockIndexFields, mockIndexFieldsByName } from './mock';
+import * as indexUtils from '.';
 
 describe('source/index.tsx', () => {
-  describe('useDataView hook', () => {
-    const mockSearchResponse = {
-      ...mocksSource,
-      indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
-      isRestore: false,
-      rawResponse: {},
-      runtimeMappings: {},
-    };
-
-    beforeEach(() => {
-      jest.clearAllMocks();
-      const mock = {
-        subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
-        unsubscribe: jest.fn(),
-      };
-
-      (useKibana as jest.Mock).mockReturnValue({
-        services: {
-          data: {
-            dataViews: {
-              ...useKibana().services.data.dataViews,
-              get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
-                const dataViewMock = {
-                  id: dataViewId,
-                  matchedIndices: refreshFields
-                    ? ['hello', 'world', 'refreshed']
-                    : ['hello', 'world'],
-                  fields: mocksSource.indexFields,
-                  getIndexPattern: () =>
-                    refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
-                  getRuntimeMappings: () => ({
-                    myfield: {
-                      type: 'keyword',
-                    },
-                  }),
-                };
-                return Promise.resolve({
-                  toSpec: () => dataViewMock,
-                  ...dataViewMock,
-                });
-              },
-              getFieldsForWildcard: async () => Promise.resolve(),
-              getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
-            },
-            search: {
-              search: jest.fn().mockReturnValue({
-                subscribe: ({ next }: { next: Function }) => {
-                  next(mockSearchResponse);
-                  return mock;
-                },
-                unsubscribe: jest.fn(),
-              }),
-            },
-          },
-        },
-      });
-    });
-    it('sets field data for data view', async () => {
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        await result.current.indexFieldsSearch({ dataViewId: 'neato' });
-      });
-      expect(mockDispatch.mock.calls[0][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
-        payload: { id: 'neato', loading: true },
-      });
-      const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
-      expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
-      expect(payload.id).toEqual('neato');
-    });
-
-    it('should reuse the result for dataView info when cleanCache not passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields, indexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      expect(browserFields).toBe(newBrowserFields);
-      expect(indexFields).toBe(newIndexFields);
+  describe('getAllBrowserFields', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllBrowserFields(mockBrowserFields)).toEqual(mockIndexFields);
     });
+  });
 
-    it('should not reuse the result for dataView info when cleanCache passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { patternList },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
-      const {
-        payload: { patternList: newPatternList },
-      } = mockDispatch.mock.calls[1][0];
-      expect(patternList).not.toBe(newPatternList);
-      expect(patternList).not.toContain('refreshed*');
-      expect(newPatternList).toContain('refreshed*');
+  describe('getAllFieldsByName', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllFieldsByName(mockBrowserFields)).toEqual(mockIndexFieldsByName);
     });
   });
 });

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx

@@ -22,10 +22,11 @@ import type { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../../../../common/endpoin
 export type { BrowserFields };
 
 export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial<FieldSpec>> {
-  const result: Array<Partial<FieldSpec>> = [];
+  let result: Array<Partial<FieldSpec>> = [];
   for (const namespace of Object.values(browserFields)) {
     if (namespace.fields) {
-      result.push(...Object.values(namespace.fields));
+      const namespaceFields = Object.values(namespace.fields);
+      result = result.concat(namespaceFields);
     }
   }
   return result;
@@ -36,9 +37,11 @@ export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial
  * @param browserFields
  * @returns
  */
-export const getAllFieldsByName = (
-  browserFields: BrowserFields
-): { [fieldName: string]: Partial<FieldSpec> } => keyBy('name', getAllBrowserFields(browserFields));
+export const getAllFieldsByName = memoizeOne(
+  (browserFields: BrowserFields): { [fieldName: string]: Partial<FieldSpec> } =>
+    keyBy('name', getAllBrowserFields(browserFields)),
+  (newArgs, lastArgs) => newArgs[0] === lastArgs[0]
+);
 
 export const getIndexFields = memoizeOne(
   (title: string, fields: IIndexPatternFieldList): DataViewBase =>

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts

@@ -7,6 +7,7 @@
 
 import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { flatten } from 'lodash';
+import type { FieldSpec } from '@kbn/data-views-plugin/common';
 import type { BrowserFields } from '../../../../common/search_strategy/index_fields';
 
 export const mocksSource = {
@@ -387,6 +388,13 @@ export const mockIndexFields = flatten(
   Object.values(mockBrowserFields).map((fieldItem) => Object.values(fieldItem.fields ?? {}))
 );
 
+export const mockIndexFieldsByName = mockIndexFields.reduce((acc, indexFieldObj) => {
+  if (indexFieldObj.name) {
+    acc[indexFieldObj.name] = indexFieldObj;
+  }
+  return acc;
+}, {} as { [fieldName: string]: Partial<FieldSpec> });
+
 const runTimeType: MappingRuntimeFieldType = 'keyword' as const;
 
 export const mockRuntimeMappings = {