[9.0] [Performance][Security Solution][1/4] - Field Browser Performance (#212469)

x-pack/platform/plugins/shared/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx

@@ -14,8 +14,7 @@ import {
   EuiBadge,
   EuiScreenReaderOnly,
 } from '@elastic/eui';
-import { uniqBy } from 'lodash/fp';
-import { BrowserFields } from '@kbn/rule-registry-plugin/common';
+import type { BrowserFields } from '@kbn/rule-registry-plugin/common';
 import { EcsFlat } from '@elastic/ecs';
 import { EcsMetadata } from '@kbn/alerts-as-data-utils/src/field_maps/types';
 
@@ -64,29 +63,41 @@ export const getFieldItemsData = ({
     selectedCategoryIds.length > 0 ? selectedCategoryIds : Object.keys(browserFields);
   const selectedFieldIds = new Set(columnIds);
 
-  const fieldItems = uniqBy(
-    'name',
-    categoryIds.reduce<BrowserFieldItem[]>((fieldItemsAcc, categoryId) => {
+  const getFieldItems = () => {
+    const fieldItemsAcc: BrowserFieldItem[] = [];
+    const fieldsSeen: Set<string> = new Set();
+
+    /**
+     * Both categoryIds and the fields in browserFields can be significantly large. Technically speaking,
+     * there is no upper cap on how many fields a customer can have. We are using a for loop here to avoid
+     * the performance issues that can arise from using map/filter/reduce.
+     */
+    for (let i = 0; i < categoryIds.length; i += 1) {
+      const categoryId = categoryIds[i];
       const categoryBrowserFields = Object.values(browserFields[categoryId]?.fields ?? {});
       if (categoryBrowserFields.length > 0) {
-        fieldItemsAcc.push(
-          ...categoryBrowserFields.map(({ name = '', ...field }) => {
-            return {
-              name,
-              type: field.type,
-              description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
-              example: field.example?.toString(),
-              category: getCategory(name),
-              selected: selectedFieldIds.has(name),
-              isRuntime: !!field.runtimeField,
-            };
-          })
-        );
+        for (let j = 0; j < categoryBrowserFields.length; j += 1) {
+          const field = categoryBrowserFields[j];
+          const name = field.name !== undefined ? field.name : '';
+          if (fieldsSeen.has(name)) continue;
+          fieldsSeen.add(name);
+          const categoryFieldItem = {
+            name,
+            type: field.type,
+            description: getDescription(name, EcsFlat as Record<string, EcsMetadata>),
+            example: field.example?.toString(),
+            category: getCategory(name),
+            selected: selectedFieldIds.has(name),
+            isRuntime: !!field.runtimeField,
+          };
+          fieldItemsAcc.push(categoryFieldItem);
+        }
       }
-      return fieldItemsAcc;
-    }, [])
-  );
-  return { fieldItems };
+    }
+    return fieldItemsAcc;
+  };
+
+  return { fieldItems: getFieldItems() };
 };
 
 const getDefaultFieldTableColumns = ({ highlight }: { highlight: string }): FieldTableColumns => {

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.test.tsx

@@ -5,151 +5,19 @@
  * 2.0.
  */
 
-import type { IndexFieldSearch } from './use_data_view';
-import { useDataView } from './use_data_view';
-import { mocksSource } from './mock';
-import { mockGlobalState, TestProviders } from '../../mock';
-import { act, renderHook } from '@testing-library/react';
-import { useKibana } from '../../lib/kibana';
-
-const mockDispatch = jest.fn();
-jest.mock('react-redux', () => {
-  const original = jest.requireActual('react-redux');
-
-  return {
-    ...original,
-    useDispatch: () => mockDispatch,
-  };
-});
-jest.mock('../../lib/kibana');
-jest.mock('../../lib/apm/use_track_http_request');
+import { mockBrowserFields, mockIndexFields, mockIndexFieldsByName } from './mock';
+import * as indexUtils from '.';
 
 describe('source/index.tsx', () => {
-  describe('useDataView hook', () => {
-    const mockSearchResponse = {
-      ...mocksSource,
-      indicesExist: ['auditbeat-*', mockGlobalState.sourcerer.signalIndexName],
-      isRestore: false,
-      rawResponse: {},
-      runtimeMappings: {},
-    };
-
-    beforeEach(() => {
-      jest.clearAllMocks();
-      const mock = {
-        subscribe: ({ next }: { next: Function }) => next(mockSearchResponse),
-        unsubscribe: jest.fn(),
-      };
-
-      (useKibana as jest.Mock).mockReturnValue({
-        services: {
-          data: {
-            dataViews: {
-              ...useKibana().services.data.dataViews,
-              get: async (dataViewId: string, displayErrors?: boolean, refreshFields = false) => {
-                const dataViewMock = {
-                  id: dataViewId,
-                  matchedIndices: refreshFields
-                    ? ['hello', 'world', 'refreshed']
-                    : ['hello', 'world'],
-                  fields: mocksSource.indexFields,
-                  getIndexPattern: () =>
-                    refreshFields ? 'hello*,world*,refreshed*' : 'hello*,world*',
-                  getRuntimeMappings: () => ({
-                    myfield: {
-                      type: 'keyword',
-                    },
-                  }),
-                };
-                return Promise.resolve({
-                  toSpec: () => dataViewMock,
-                  ...dataViewMock,
-                });
-              },
-              getFieldsForWildcard: async () => Promise.resolve(),
-              getExistingIndices: async (indices: string[]) => Promise.resolve(indices),
-            },
-            search: {
-              search: jest.fn().mockReturnValue({
-                subscribe: ({ next }: { next: Function }) => {
-                  next(mockSearchResponse);
-                  return mock;
-                },
-                unsubscribe: jest.fn(),
-              }),
-            },
-          },
-        },
-      });
-    });
-    it('sets field data for data view', async () => {
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        await result.current.indexFieldsSearch({ dataViewId: 'neato' });
-      });
-      expect(mockDispatch.mock.calls[0][0]).toEqual({
-        type: 'x-pack/security_solution/local/sourcerer/SET_DATA_VIEW_LOADING',
-        payload: { id: 'neato', loading: true },
-      });
-      const { type: sourceType, payload } = mockDispatch.mock.calls[1][0];
-      expect(sourceType).toEqual('x-pack/security_solution/local/sourcerer/SET_DATA_VIEW');
-      expect(payload.id).toEqual('neato');
-    });
-
-    it('should reuse the result for dataView info when cleanCache not passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields, indexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { browserFields: newBrowserFields, indexFields: newIndexFields },
-      } = mockDispatch.mock.calls[1][0];
-
-      expect(browserFields).toBe(newBrowserFields);
-      expect(indexFields).toBe(newIndexFields);
+  describe('getAllBrowserFields', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllBrowserFields(mockBrowserFields)).toEqual(mockIndexFields);
     });
+  });
 
-    it('should not reuse the result for dataView info when cleanCache passed', async () => {
-      let indexFieldsSearch: IndexFieldSearch;
-      const { result } = renderHook(() => useDataView(), {
-        wrapper: TestProviders,
-      });
-
-      await act(async () => {
-        indexFieldsSearch = result.current.indexFieldsSearch;
-      });
-
-      await indexFieldsSearch!({ dataViewId: 'neato' });
-      const {
-        payload: { patternList },
-      } = mockDispatch.mock.calls[1][0];
-
-      mockDispatch.mockClear();
-
-      await indexFieldsSearch!({ dataViewId: 'neato', cleanCache: true });
-      const {
-        payload: { patternList: newPatternList },
-      } = mockDispatch.mock.calls[1][0];
-      expect(patternList).not.toBe(newPatternList);
-      expect(patternList).not.toContain('refreshed*');
-      expect(newPatternList).toContain('refreshed*');
+  describe('getAllFieldsByName', () => {
+    it('should return the expected browser fields list', () => {
+      expect(indexUtils.getAllFieldsByName(mockBrowserFields)).toEqual(mockIndexFieldsByName);
     });
   });
 });

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/index.tsx

@@ -22,10 +22,11 @@ import type { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../../../../common/endpoin
 export type { BrowserFields };
 
 export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial<FieldSpec>> {
-  const result: Array<Partial<FieldSpec>> = [];
+  let result: Array<Partial<FieldSpec>> = [];
   for (const namespace of Object.values(browserFields)) {
     if (namespace.fields) {
-      result.push(...Object.values(namespace.fields));
+      const namespaceFields = Object.values(namespace.fields);
+      result = result.concat(namespaceFields);
     }
   }
   return result;
@@ -36,9 +37,11 @@ export function getAllBrowserFields(browserFields: BrowserFields): Array<Partial
  * @param browserFields
  * @returns
  */
-export const getAllFieldsByName = (
-  browserFields: BrowserFields
-): { [fieldName: string]: Partial<FieldSpec> } => keyBy('name', getAllBrowserFields(browserFields));
+export const getAllFieldsByName = memoizeOne(
+  (browserFields: BrowserFields): { [fieldName: string]: Partial<FieldSpec> } =>
+    keyBy('name', getAllBrowserFields(browserFields)),
+  (newArgs, lastArgs) => newArgs[0] === lastArgs[0]
+);
 
 export const getIndexFields = memoizeOne(
   (title: string, fields: IIndexPatternFieldList): DataViewBase =>

x-pack/solutions/security/plugins/security_solution/public/common/containers/source/mock.ts

@@ -7,6 +7,7 @@
 
 import type { MappingRuntimeFieldType } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { flatten } from 'lodash';
+import type { FieldSpec } from '@kbn/data-views-plugin/common';
 import type { BrowserFields } from '../../../../common/search_strategy/index_fields';
 
 export const mocksSource = {
@@ -387,6 +388,13 @@ export const mockIndexFields = flatten(
   Object.values(mockBrowserFields).map((fieldItem) => Object.values(fieldItem.fields ?? {}))
 );
 
+export const mockIndexFieldsByName = mockIndexFields.reduce((acc, indexFieldObj) => {
+  if (indexFieldObj.name) {
+    acc[indexFieldObj.name] = indexFieldObj;
+  }
+  return acc;
+}, {} as { [fieldName: string]: Partial<FieldSpec> });
+
 const runTimeType: MappingRuntimeFieldType = 'keyword' as const;
 
 export const mockRuntimeMappings = {