feat(ourlogs): Add infinite table and auto refresh

static/app/components/events/ourlogs/ourlogsDrawer.tsx

@@ -18,6 +18,7 @@ import type {Group} from 'sentry/types/group';
 import type {Project} from 'sentry/types/project';
 import {getShortEventId} from 'sentry/utils/events';
 import {MutableSearch} from 'sentry/utils/tokenizeSearch';
+import useOrganization from 'sentry/utils/useOrganization';
 import {
   TraceItemSearchQueryBuilder,
   useSearchQueryBuilderProps,
@@ -27,8 +28,8 @@ import {
   useSetLogsSearch,
 } from 'sentry/views/explore/contexts/logs/logsPageParams';
 import {useTraceItemAttributes} from 'sentry/views/explore/contexts/traceItemAttributeContext';
-import {LogsTable} from 'sentry/views/explore/logs/logsTable';
-import {useExploreLogsTable} from 'sentry/views/explore/logs/useLogsQuery';
+import {LogsTable} from 'sentry/views/explore/logs/tables/logsTable';
+import {useLogsQuery} from 'sentry/views/explore/logs/useLogsQuery';
 import {TraceItemDataset} from 'sentry/views/explore/types';
 
 interface LogIssueDrawerProps {
@@ -38,9 +39,9 @@ interface LogIssueDrawerProps {
 }
 
 export function OurlogsDrawer({event, project, group}: LogIssueDrawerProps) {
+  const organization = useOrganization();
   const setLogsSearch = useSetLogsSearch();
   const logsSearch = useLogsSearch();
-  const tableData = useExploreLogsTable({});
   const {attributes: stringAttributes} = useTraceItemAttributes('string');
   const {attributes: numberAttributes} = useTraceItemAttributes('number');
 
@@ -55,6 +56,9 @@ export function OurlogsDrawer({event, project, group}: LogIssueDrawerProps) {
   const searchQueryBuilderProps = useSearchQueryBuilderProps(
     tracesItemSearchQueryBuilderProps
   );
+  const tableData = useLogsQuery({
+    disabled: !organization.features.includes('ourlogs-enabled'),
+  });
 
   return (
     <SearchQueryBuilderProvider {...searchQueryBuilderProps}>

static/app/components/events/ourlogs/ourlogsSection.tsx

@@ -18,8 +18,8 @@ import {
   useLogsSearch,
 } from 'sentry/views/explore/contexts/logs/logsPageParams';
 import {TraceItemAttributeProvider} from 'sentry/views/explore/contexts/traceItemAttributeContext';
-import {LogsTable} from 'sentry/views/explore/logs/logsTable';
-import {useExploreLogsTable} from 'sentry/views/explore/logs/useLogsQuery';
+import {LogsTable} from 'sentry/views/explore/logs/tables/logsTable';
+import {useLogsQuery} from 'sentry/views/explore/logs/useLogsQuery';
 import {TraceItemDataset} from 'sentry/views/explore/types';
 import {SectionKey} from 'sentry/views/issueDetails/streamline/context';
 import {InterimSection} from 'sentry/views/issueDetails/streamline/interimSection';
@@ -56,7 +56,7 @@ function OurlogsSectionContent({
 }) {
   const organization = useOrganization();
   const feature = organization.features.includes('ourlogs-enabled');
-  const tableData = useExploreLogsTable({enabled: feature, limit: 10});
+  const tableData = useLogsQuery({disabled: feature, limit: 10});
   const logsSearch = useLogsSearch();
   const abbreviatedTableData = {...tableData, data: (tableData.data ?? []).slice(0, 5)};
   const {openDrawer} = useDrawer();
@@ -109,7 +109,7 @@ function OurlogsSectionContent({
           allowPagination={false}
           tableData={abbreviatedTableData}
         />
-        {tableData.data?.length > 5 ? (
+        {tableData.data && tableData.data.length > 5 ? (
           <div>
             <Button
               icon={<IconChevron direction="right" />}

static/app/views/explore/contexts/logs/logsPageData.tsx

@@ -1,10 +1,14 @@
 import {createDefinedContext} from 'sentry/utils/performance/contexts/utils';
 import useOrganization from 'sentry/utils/useOrganization';
-import type {UseExploreLogsTableResult} from 'sentry/views/explore/logs/useLogsQuery';
-import {useExploreLogsTable} from 'sentry/views/explore/logs/useLogsQuery';
+import type {
+  UseInfiniteLogsQueryResult,
+  UseLogsQueryResult,
+} from 'sentry/views/explore/logs/useLogsQuery';
+import {useInfiniteLogsQuery, useLogsQuery} from 'sentry/views/explore/logs/useLogsQuery';
 
 interface LogsPageData {
-  logsData: UseExploreLogsTableResult;
+  infiniteLogsQueryResult: UseInfiniteLogsQueryResult;
+  logsQueryResult: UseLogsQueryResult;
 }
 
 const [_LogsPageDataProvider, _useLogsPageData, _ctx] =
@@ -16,8 +20,14 @@ export const useLogsPageData = _useLogsPageData;
 export function LogsPageDataProvider({children}: {children: React.ReactNode}) {
   const organization = useOrganization();
   const feature = organization.features.includes('ourlogs-enabled');
-  const data = useExploreLogsTable({enabled: feature && undefined}); // left as exercise to reader
+  const liveRefresh = organization.features.includes('ourlogs-live-refresh');
+  const logsQueryResult = useLogsQuery({disabled: liveRefresh || !feature});
+  const infiniteLogsQueryResult = useInfiniteLogsQuery({
+    disabled: !liveRefresh || !feature,
+  });
   return (
-    <_LogsPageDataProvider value={{logsData: data}}>{children}</_LogsPageDataProvider>
+    <_LogsPageDataProvider value={{logsQueryResult, infiniteLogsQueryResult}}>
+      {children}
+    </_LogsPageDataProvider>
   );
 }

static/app/views/explore/contexts/logs/logsPageParams.tsx

@@ -7,7 +7,8 @@ import type {LogsAnalyticsPageSource} from 'sentry/utils/analytics/logsAnalytics
 import type {Sort} from 'sentry/utils/discover/fields';
 import localStorage from 'sentry/utils/localStorage';
 import {createDefinedContext} from 'sentry/utils/performance/contexts/utils';
-import {decodeScalar} from 'sentry/utils/queryString';
+import {useQueryClient} from 'sentry/utils/queryClient';
+import {decodeBoolean, decodeInteger, decodeScalar} from 'sentry/utils/queryString';
 import {MutableSearch} from 'sentry/utils/tokenizeSearch';
 import {useLocalStorageState} from 'sentry/utils/useLocalStorageState';
 import {useLocation} from 'sentry/utils/useLocation';
@@ -23,18 +24,25 @@ import {
   updateLocationWithLogSortBys,
 } from 'sentry/views/explore/contexts/logs/sortBys';
 import {OurLogKnownFieldKey} from 'sentry/views/explore/logs/types';
+import {useLogsQueryKeyWithInfinite} from 'sentry/views/explore/logs/useLogsQuery';
 
 const LOGS_PARAMS_VERSION = 1;
 export const LOGS_QUERY_KEY = 'logsQuery'; // Logs may exist on other pages.
 export const LOGS_CURSOR_KEY = 'logsCursor';
 export const LOGS_FIELDS_KEY = 'logsFields';
 
+const LOGS_AUTO_REFRESH_KEY = 'live';
+const LOGS_REFRESH_INTERVAL_KEY = 'refreshEvery';
+const LOGS_REFRESH_INTERVAL_DEFAULT = 5000;
+
 interface LogsPageParams {
   readonly analyticsPageSource: LogsAnalyticsPageSource;
+  readonly autoRefresh: boolean;
   readonly blockRowExpanding: boolean | undefined;
   readonly cursor: string;
   readonly fields: string[];
   readonly isTableFrozen: boolean | undefined;
+  readonly refreshInterval: number;
   readonly search: MutableSearch;
   /**
    * See setSearchForFrozenPages
@@ -86,6 +94,12 @@ export function LogsPageParamsProvider({
   const location = useLocation();
   const logsQuery = decodeLogsQuery(location);
 
+  const autoRefresh = decodeBoolean(location.query[LOGS_AUTO_REFRESH_KEY]) ?? false;
+  const refreshInterval = decodeInteger(
+    location.query[LOGS_REFRESH_INTERVAL_KEY],
+    LOGS_REFRESH_INTERVAL_DEFAULT
+  );
+
   // on embedded pages with search bars, use a useState instead of a URL parameter
   const [searchForFrozenPages, setSearchForFrozenPages] = useState(new MutableSearch(''));
   const [cursorForFrozenPages, setCursorForFrozenPages] = useState('');
@@ -124,6 +138,8 @@ export function LogsPageParamsProvider({
         cursor,
         setCursorForFrozenPages,
         isTableFrozen,
+        autoRefresh,
+        refreshInterval,
         blockRowExpanding,
         baseSearch,
         projectIds,
@@ -152,8 +168,13 @@ function setLogsPageParams(location: Location, pageParams: LogPageParamsUpdate)
   updateNullableLocation(target, LOGS_QUERY_KEY, pageParams.search?.formatString());
   updateNullableLocation(target, LOGS_CURSOR_KEY, pageParams.cursor);
   updateNullableLocation(target, LOGS_FIELDS_KEY, pageParams.fields);
+  updateNullableLocation(target, LOGS_AUTO_REFRESH_KEY, pageParams.autoRefresh);
   if (!pageParams.isTableFrozen) {
     updateLocationWithLogSortBys(target, pageParams.sortBys);
+    if (pageParams.sortBys || pageParams.search) {
+      delete target.query[LOGS_CURSOR_KEY];
+      delete target.query[LOGS_AUTO_REFRESH_KEY];
+    }
   }
   return target;
 }
@@ -166,10 +187,19 @@ function setLogsPageParams(location: Location, pageParams: LogPageParamsUpdate)
 function updateNullableLocation(
   location: Location,
   key: string,
-  value: string | string[] | null | undefined
+  value: boolean | string | string[] | null | undefined
 ): boolean {
+  if (typeof value === 'boolean') {
+    if (value) {
+      location.query[key] = 'true';
+    } else {
+      // Delete boolean keys to minimize the number of query params.
+      delete location.query[key];
+    }
+    return true;
+  }
   if (defined(value) && location.query[key] !== value) {
-    location.query[key] = value;
+    location.query[key] = String(value);
     return true;
   }
   if (value === null && location.query[key]) {
@@ -349,6 +379,32 @@ function getLogsParamsStorageKey(version: number) {
 function getPastLogsParamsStorageKey(version: number) {
   return `logs-params-v${version - 1}`;
 }
+
+export function useLogsAutoRefresh() {
+  const {autoRefresh} = useLogsPageParams();
+  return autoRefresh;
+}
+
+export function useSetLogsAutoRefresh() {
+  const setPageParams = useSetLogsPageParams();
+  const {queryKey} = useLogsQueryKeyWithInfinite({referrer: 'api.explore.logs-table'});
+  const queryClient = useQueryClient();
+  return useCallback(
+    (autoRefresh: boolean) => {
+      setPageParams({autoRefresh});
+      if (autoRefresh) {
+        queryClient.removeQueries({queryKey});
+      }
+    },
+    [setPageParams, queryClient, queryKey]
+  );
+}
+
+export function useLogsRefreshInterval() {
+  const {refreshInterval} = useLogsPageParams();
+  return refreshInterval;
+}
+
 interface ToggleableSortBy {
   field: string;
   defaultDirection?: 'asc' | 'desc'; // Defaults to descending if not provided.

static/app/views/explore/contexts/logs/sortBys.tsx

@@ -3,7 +3,6 @@ import type {Location} from 'history';
 import {defined} from 'sentry/utils';
 import type {Sort} from 'sentry/utils/discover/fields';
 import {decodeSorts} from 'sentry/utils/queryString';
-import {LOGS_CURSOR_KEY} from 'sentry/views/explore/contexts/logs/logsPageParams';
 import {OurLogKnownFieldKey} from 'sentry/views/explore/logs/types';
 
 export const LOGS_SORT_BYS_KEY = 'logsSortBys';
@@ -45,13 +44,11 @@ export function updateLocationWithLogSortBys(
   sortBys: Sort[] | null | undefined
 ) {
   if (defined(sortBys)) {
-    location.query[LOGS_SORT_BYS_KEY] = sortBys.map(sortBy =>
+    const newSortBys = sortBys.map(sortBy =>
       sortBy.kind === 'desc' ? `-${sortBy.field}` : sortBy.field
     );
-
-    // make sure to clear the cursor every time the query is updated
-    delete location.query[LOGS_CURSOR_KEY];
-  } else if (sortBys === null) {
+    location.query[LOGS_SORT_BYS_KEY] = newSortBys;
+  } else {
     delete location.query[LOGS_SORT_BYS_KEY];
   }
 }

static/app/views/explore/hooks/useAnalytics.tsx

@@ -25,7 +25,7 @@ import type {AggregatesTableResult} from 'sentry/views/explore/hooks/useExploreA
 import type {SpansTableResult} from 'sentry/views/explore/hooks/useExploreSpansTable';
 import type {TracesTableResult} from 'sentry/views/explore/hooks/useExploreTracesTable';
 import {useTopEvents} from 'sentry/views/explore/hooks/useTopEvents';
-import type {UseExploreLogsTableResult} from 'sentry/views/explore/logs/useLogsQuery';
+import type {UseLogsQueryResult} from 'sentry/views/explore/logs/useLogsQuery';
 import type {ReadableExploreQueryParts} from 'sentry/views/explore/multiQueryMode/locationUtils';
 import {
   combineConfidenceForSeries,
@@ -397,7 +397,7 @@ export function useLogAnalytics({
   logsTableResult,
   source,
 }: {
-  logsTableResult: UseExploreLogsTableResult;
+  logsTableResult: UseLogsQueryResult;
   source: LogsAnalyticsPageSource;
 }) {
   const organization = useOrganization();

static/app/views/explore/logs/constants.tsx

@@ -13,6 +13,8 @@ export const LogAttributesHumanLabel: Partial<Record<OurLogFieldKey, string>> =
   [OurLogKnownFieldKey.TRACE_ID]: t('Trace'),
 };
 
+export const LOG_INGEST_DELAY = 10_000;
+
 /**
  * These are required fields are always added to the query when fetching the log table.
  */
@@ -23,12 +25,15 @@ export const AlwaysPresentLogFields: OurLogFieldKey[] = [
   OurLogKnownFieldKey.SEVERITY_NUMBER,
   OurLogKnownFieldKey.SEVERITY,
   OurLogKnownFieldKey.TIMESTAMP,
-];
+  OurLogKnownFieldKey.TIMESTAMP_PRECISE,
+] as const;
 
 const AlwaysHiddenLogFields: OurLogFieldKey[] = [
   OurLogKnownFieldKey.ID,
   OurLogKnownFieldKey.ORGANIZATION_ID,
   OurLogKnownFieldKey.ITEM_TYPE,
+  OurLogKnownFieldKey.PROJECT,
+  OurLogKnownFieldKey.TIMESTAMP_PRECISE,
   'project.id',
   'project_id', // these are both aliases that might show up
 ];
@@ -43,7 +48,6 @@ export const HiddenLogDetailFields: OurLogFieldKey[] = [
   // deprecated/otel fields that clutter the UI
   'sentry.timestamp_nanos',
   'sentry.observed_timestamp_nanos',
-  'tags[sentry.timestamp_precise,number]',
   'tags[sentry.trace_flags,number]',
   'span_id',
 ];
@@ -56,4 +60,9 @@ const LOGS_FILTERS: FilterKeySection = {
   children: [...SENTRY_LOG_STRING_TAGS, ...SENTRY_LOG_NUMBER_TAGS],
 };
 
+export const LOGS_INSTRUCTIONS_URL =
+  'https://docs.sentry.io/product/explore/logs/getting-started/';
+
 export const LOGS_FILTER_KEY_SECTIONS: FilterKeySection[] = [LOGS_FILTERS];
+
+export const VIRTUAL_STREAMED_INTERVAL_MS = 333;