feat(explore): Auto add columns when searching (#91476)

When searching on attributes, try to auto add columns to the table. This
happens in the following cases:
- adding a negation filter
- adding an array filter
- adding a wildcard filter
Closes EXP-235

Author: Zylphrex

static/app/views/explore/logs/logsTab.tsx

@@ -16,6 +16,7 @@ import {LogsAnalyticsPageSource} from 'sentry/utils/analytics/logsAnalyticsEvent
 import {DiscoverDatasets} from 'sentry/utils/discover/types';
 import {MutableSearch} from 'sentry/utils/tokenizeSearch';
 import usePageFilters from 'sentry/utils/usePageFilters';
+import usePrevious from 'sentry/utils/usePrevious';
 import SchemaHintsList, {
   SchemaHintsSection,
 } from 'sentry/views/explore/components/schemaHints/schemaHintsList';
@@ -43,6 +44,7 @@ import {usePersistentLogsPageParameters} from 'sentry/views/explore/logs/usePers
 import {ColumnEditorModal} from 'sentry/views/explore/tables/columnEditorModal';
 import {TraceItemDataset} from 'sentry/views/explore/types';
 import type {PickableDays} from 'sentry/views/explore/utils';
+import {findSuggestedColumns} from 'sentry/views/explore/utils';
 import {useSortedTimeSeries} from 'sentry/views/insights/common/queries/useSortedTimeSeries';
 
 type LogsTabProps = PickableDays;
@@ -60,6 +62,8 @@ export function LogsTabContent({
   const pageFilters = usePageFilters();
   usePersistentLogsPageParameters(); // persist the columns you chose last time
 
+  const oldLogsSearch = usePrevious(logsSearch);
+
   const columnEditorButtonRef = useRef<HTMLButtonElement>(null);
   // always use the smallest interval possible (the most bars)
   const interval = getIntervalOptionsForPageFilter(pageFilters.selection.datetime)?.[0]
@@ -88,9 +92,18 @@ export function LogsTabContent({
     initialQuery: logsSearch.formatString(),
     searchSource: 'ourlogs',
     onSearch: (newQuery: string) => {
-      const mutableQuery = new MutableSearch(newQuery);
+      const newSearch = new MutableSearch(newQuery);
+      const suggestedColumns = findSuggestedColumns(newSearch, oldLogsSearch, {
+        numberAttributes,
+        stringAttributes,
+      });
+
+      const existingFields = new Set(fields);
+      const newColumns = suggestedColumns.filter(col => !existingFields.has(col));
+
       setLogsPageParams({
-        search: mutableQuery,
+        search: newSearch,
+        fields: newColumns.length ? [...fields, ...newColumns] : undefined,
       });
     },
     numberAttributes,

static/app/views/explore/spans/spansTab.tsx

@@ -29,14 +29,17 @@ import {
 } from 'sentry/utils/fields';
 import {chonkStyled} from 'sentry/utils/theme/theme.chonk';
 import {withChonk} from 'sentry/utils/theme/withChonk';
+import {MutableSearch} from 'sentry/utils/tokenizeSearch';
 import useOrganization from 'sentry/utils/useOrganization';
 import usePageFilters from 'sentry/utils/usePageFilters';
+import usePrevious from 'sentry/utils/usePrevious';
 import {ExploreCharts} from 'sentry/views/explore/charts';
 import SchemaHintsList, {
   SchemaHintsSection,
 } from 'sentry/views/explore/components/schemaHints/schemaHintsList';
 import {SchemaHintsSources} from 'sentry/views/explore/components/schemaHints/schemaHintsUtils';
 import {
+  useExploreFields,
   useExploreId,
   useExploreMode,
   useExploreQuery,
@@ -57,7 +60,11 @@ import {useVisitQuery} from 'sentry/views/explore/hooks/useVisitQuery';
 import {ExploreSpansTour, ExploreSpansTourContext} from 'sentry/views/explore/spans/tour';
 import {ExploreTables} from 'sentry/views/explore/tables';
 import {ExploreToolbar} from 'sentry/views/explore/toolbar';
-import {combineConfidenceForSeries, type PickableDays} from 'sentry/views/explore/utils';
+import {
+  combineConfidenceForSeries,
+  findSuggestedColumns,
+  type PickableDays,
+} from 'sentry/views/explore/utils';
 import {Onboarding} from 'sentry/views/performance/onboarding';
 
 // eslint-disable-next-line no-restricted-imports
@@ -134,18 +141,32 @@ interface SpanTabSearchSectionProps {
 
 function SpanTabSearchSection({datePageFilterProps}: SpanTabSearchSectionProps) {
   const mode = useExploreMode();
+  const fields = useExploreFields();
   const query = useExploreQuery();
   const setExplorePageParams = useSetExplorePageParams();
 
   const {tags: numberTags, isLoading: numberTagsLoading} = useSpanTags('number');
   const {tags: stringTags, isLoading: stringTagsLoading} = useSpanTags('string');
 
+  const search = useMemo(() => new MutableSearch(query), [query]);
+  const oldSearch = usePrevious(search);
+
   const eapSpanSearchQueryBuilderProps = useMemo(
     () => ({
       initialQuery: query,
       onSearch: (newQuery: string) => {
+        const newSearch = new MutableSearch(newQuery);
+        const suggestedColumns = findSuggestedColumns(newSearch, oldSearch, {
+          numberAttributes: numberTags,
+          stringAttributes: stringTags,
+        });
+
+        const existingFields = new Set(fields);
+        const newColumns = suggestedColumns.filter(col => !existingFields.has(col));
+
         setExplorePageParams({
           query: newQuery,
+          fields: newColumns.length ? [...fields, ...newColumns] : undefined,
         });
       },
       searchSource: 'explore',
@@ -165,7 +186,7 @@ function SpanTabSearchSection({datePageFilterProps}: SpanTabSearchSectionProps)
       numberTags,
       stringTags,
     }),
-    [mode, query, setExplorePageParams, numberTags, stringTags]
+    [fields, mode, query, setExplorePageParams, numberTags, stringTags, oldSearch]
   );
 
   const eapSpanSearchQueryProviderProps = useEAPSpanSearchQueryBuilderProps(

static/app/views/explore/utils.spec.tsx

@@ -1,8 +1,9 @@
 import {LocationFixture} from 'sentry-fixture/locationFixture';
 import {ProjectFixture} from 'sentry-fixture/project';
 
+import {MutableSearch} from 'sentry/utils/tokenizeSearch';
 import {Visualize} from 'sentry/views/explore/contexts/pageParamsContext/visualizes';
-import {viewSamplesTarget} from 'sentry/views/explore/utils';
+import {findSuggestedColumns, viewSamplesTarget} from 'sentry/views/explore/utils';
 
 describe('viewSamplesTarget', function () {
   const project = ProjectFixture();
@@ -156,3 +157,128 @@ describe('viewSamplesTarget', function () {
     });
   });
 });
+
+describe('findSuggestedColumns', function () {
+  it.each([
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: '',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:value',
+    },
+    {
+      cols: ['key'],
+      oldQuery: 'key:value1',
+      newQuery: 'key:[value1,value2]',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:[value1,value2]',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: '!key:value',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:*',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:v*',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:\\*',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:v\\*',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:\\\\*',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:v\\\\*',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:\\\\\\*',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:v\\\\\\*',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'has:key',
+    },
+    {
+      cols: [],
+      oldQuery: 'key:value',
+      newQuery: 'has:key',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'key:value has:key',
+    },
+    {
+      cols: ['key'],
+      oldQuery: '',
+      newQuery: 'key:[value1,value2] has:key',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: '!has:a',
+    },
+    {
+      cols: ['num'],
+      oldQuery: '',
+      newQuery: 'num:>0',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'foo:[a,b]',
+    },
+    {
+      cols: [],
+      oldQuery: '',
+      newQuery: 'count():>0',
+    },
+  ])(
+    'should inject $cols when changing from `$oldQuery` to `$newQuery`',
+    function ({cols, oldQuery, newQuery}) {
+      const oldSearch = new MutableSearch(oldQuery);
+      const newSearch = new MutableSearch(newQuery);
+      const suggestion = findSuggestedColumns(newSearch, oldSearch, {
+        numberAttributes: {
+          num: {key: 'num', name: 'num'},
+        },
+        stringAttributes: {
+          key: {key: 'key', name: 'key'},
+        },
+      });
+      expect(new Set(suggestion)).toEqual(new Set(cols));
+    }
+  );
+});

static/app/views/explore/utils.tsx

@@ -9,6 +9,7 @@ import HookOrDefault from 'sentry/components/hookOrDefault';
 import {IconBusiness} from 'sentry/icons/iconBusiness';
 import {t} from 'sentry/locale';
 import type {PageFilters} from 'sentry/types/core';
+import type {TagCollection} from 'sentry/types/group';
 import type {Confidence, Organization} from 'sentry/types/organization';
 import type {Project} from 'sentry/types/project';
 import {defined} from 'sentry/utils';
@@ -457,3 +458,120 @@ export function confirmDeleteSavedQuery({
     onConfirm: handleDelete,
   });
 }
+
+export function findSuggestedColumns(
+  newSearch: MutableSearch,
+  oldSearch: MutableSearch,
+  attributes: {
+    numberAttributes: TagCollection;
+    stringAttributes: TagCollection;
+  }
+): string[] {
+  const oldFilters = oldSearch.filters;
+  const newFilters = newSearch.filters;
+
+  const keys: Set<string> = new Set();
+
+  for (const [key, value] of Object.entries(newFilters)) {
+    if (key === 'has' || key === '!has') {
+      // special key to be handled last
+      continue;
+    }
+
+    const isStringAttribute = key.startsWith('!')
+      ? attributes.stringAttributes.hasOwnProperty(key.slice(1))
+      : attributes.stringAttributes.hasOwnProperty(key);
+    const isNumberAttribute = key.startsWith('!')
+      ? attributes.numberAttributes.hasOwnProperty(key.slice(1))
+      : attributes.numberAttributes.hasOwnProperty(key);
+
+    // guard against unknown keys and aggregate keys
+    if (!isStringAttribute && !isNumberAttribute) {
+      continue;
+    }
+
+    if (isSimpleFilter(key, value, attributes)) {
+      continue;
+    }
+
+    if (
+      !oldFilters.hasOwnProperty(key) || // new filter key
+      isSimpleFilter(key, oldFilters[key] || [], attributes) // existing filter key turned complex
+    ) {
+      keys.add(normalizeKey(key));
+      break;
+    }
+  }
+
+  const oldHas = new Set(oldFilters.has);
+  for (const key of newFilters.has || []) {
+    if (oldFilters.hasOwnProperty(key) || oldHas.has(key)) {
+      // old condition, don't add column
+      continue;
+    }
+
+    // if there's a simple filter on the key, don't add column
+    if (
+      newFilters.hasOwnProperty(key) &&
+      isSimpleFilter(key, newFilters[key] || [], attributes)
+    ) {
+      continue;
+    }
+
+    keys.add(normalizeKey(key));
+  }
+
+  return [...keys];
+}
+
+const PREFIX_WILDCARD_PATTERN = /^(\\\\)*\*/;
+const INFIX_WILDCARD_PATTERN = /[^\\](\\\\)*\*/;
+
+function isSimpleFilter(
+  key: string,
+  value: string[],
+  attributes: {
+    numberAttributes: TagCollection;
+    stringAttributes: TagCollection;
+  }
+): boolean {
+  // negation filters are always considered non trivial
+  // because it matches on multiple values
+  if (key.startsWith('!')) {
+    return false;
+  }
+
+  // all number attributes are considered non trivial because they
+  // almost always match on a range of values
+  if (attributes.numberAttributes.hasOwnProperty(key)) {
+    return false;
+  }
+
+  if (value.length === 1) {
+    const v = value[0]!;
+    // if the value is wrapped in `[...]`, then it's an array value
+    if (v.startsWith('[') && v.endsWith(']')) {
+      return false;
+    }
+
+    // if is wild card search, return false
+    if (v.startsWith('*')) {
+      return false;
+    }
+
+    if (PREFIX_WILDCARD_PATTERN.test(v) || INFIX_WILDCARD_PATTERN.test(v)) {
+      return false;
+    }
+  }
+
+  // if there is more than 1 possible value
+  if (value.length > 1) {
+    return false;
+  }
+
+  return true;
+}
+
+function normalizeKey(key: string): string {
+  return key.startsWith('!') ? key.slice(1) : key;
+}