Merge branch 'master' into tkdodo/ref/eslint-plugin-boundaries

Author: TkDodo

.github/CODEOWNERS

@@ -20,8 +20,9 @@
 /src/sentry/tagstore/snuba/                              @getsentry/owners-snuba
 /src/sentry/sentry_metrics/                              @getsentry/owners-snuba
 /tests/sentry/sentry_metrics/                            @getsentry/owners-snuba
-/src/sentry/snuba/metrics/                               @getsentry/owners-snuba @getsentry/telemetry-experience
+/src/sentry/snuba/metrics/                               @getsentry/owners-snuba
 /src/sentry/snuba/metrics/query.py                       @getsentry/owners-snuba @getsentry/telemetry-experience
+/src/sentry/snuba/metrics/extraction.py                  @getsentry/owners-snuba @getsentry/telemetry-experience
 /src/sentry/snuba/metrics_layer/                         @getsentry/owners-snuba
 /src/sentry/search/events/datasets/metrics_layer.py      @getsentry/owners-snuba
 /tests/snuba/test_snql_snuba.py                          @getsentry/owners-snuba
@@ -242,9 +243,6 @@ yarn.lock                                                @getsentry/owners-js-de
 /tests/snuba/api/endpoints/test_organization_events_vitals.py               @getsentry/visibility
 /tests/snuba/api/endpoints/test_organization_tagkey_values.py               @getsentry/visibility
 
-/src/sentry/spans/                                                          @getsentry/visibility
-/tests/sentry/spans/                                                        @getsentry/visibility
-
 /src/sentry/api/serializers/snuba.py                                        @getsentry/visibility
 
 /src/sentry/snuba/discover.py                                               @getsentry/visibility
@@ -510,8 +508,6 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/utils/platform_categories.py                                         @getsentry/telemetry-experience
 /src/sentry/sentry_metrics/querying/                                             @getsentry/telemetry-experience
 /tests/sentry/sentry_metrics/querying/                                           @getsentry/telemetry-experience
-/src/sentry/snuba/metrics/                                                       @getsentry/telemetry-experience
-/tests/sentry/snuba/metrics/                                                     @getsentry/telemetry-experience
 
 /static/app/actionCreators/metrics.tsx                                           @getsentry/telemetry-experience
 /static/app/data/platformCategories.tsx                                          @getsentry/telemetry-experience
@@ -673,6 +669,10 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/tempest/                                       @getsentry/gdx
 /tests/sentry/tempest/                                     @getsentry/gdx
 
+# Span buffer + process-segments are co-owned by streaming platform and vis for now.
+/src/sentry/spans/                                @getsentry/visibility @getsentry/streaming-platform
+/tests/sentry/spans/                              @getsentry/visibility @getsentry/streaming-platform
+
 # Workflow engine
 /src/sentry/workflow_engine/                               @getsentry/alerts-create-issues
 /tests/sentry/workflow_engine/                             @getsentry/alerts-create-issues

requirements-base.txt

@@ -65,7 +65,7 @@ rfc3339-validator>=0.1.2
 rfc3986-validator>=0.1.1
 # [end] jsonschema format validators
 sentry-arroyo>=2.21.0
-sentry-kafka-schemas>=1.3.2
+sentry-kafka-schemas>=1.3.6
 sentry-ophio>=1.1.3
 sentry-protos==0.2.0
 sentry-redis-tools>=0.5.0

requirements-dev-frozen.txt

@@ -185,7 +185,7 @@ sentry-devenv==1.21.0
 sentry-forked-django-stubs==5.2.0.post3
 sentry-forked-djangorestframework-stubs==3.16.0.post1
 sentry-forked-email-reply-parser==0.5.12.post1
-sentry-kafka-schemas==1.3.2
+sentry-kafka-schemas==1.3.6
 sentry-ophio==1.1.3
 sentry-protos==0.2.0
 sentry-redis-tools==0.5.0

requirements-dev.txt

@@ -16,7 +16,7 @@ pytest-django>=4.9.0
 pytest-fail-slow>=0.3.0
 pytest-json-report>=1.5.0
 pytest-rerunfailures>=15
-pytest-sentry>=0.3.0
+pytest-sentry>=0.3.0,<0.4.0
 pytest-workaround-12888
 pytest-xdist>=3
 responses>=0.23.1

requirements-frozen.txt

@@ -123,7 +123,7 @@ rsa==4.8
 s3transfer==0.10.0
 sentry-arroyo==2.21.0
 sentry-forked-email-reply-parser==0.5.12.post1
-sentry-kafka-schemas==1.3.2
+sentry-kafka-schemas==1.3.6
 sentry-ophio==1.1.3
 sentry-protos==0.2.0
 sentry-redis-tools==0.5.0

src/sentry/spans/consumers/process_segments/enrichment.py

@@ -1,6 +1,8 @@
+from collections import defaultdict
 from typing import Any, cast
 
-from sentry.spans.consumers.process_segments.types import Span
+from sentry.models.project import Project
+from sentry.spans.consumers.process_segments.types import MeasurementValue, Span
 
 # Keys in `sentry_tags` that are shared across all spans in a segment. This list
 # is taken from `extract_shared_tags` in Relay.
@@ -127,7 +129,7 @@ def set_exclusive_time(spans: list[Span]) -> None:
     span_map: dict[str, list[tuple[int, int]]] = {}
     for span in spans:
         if parent_span_id := span.get("parent_span_id"):
-            interval = (_us(span["start_timestamp_precise"]), _us(span["end_timestamp_precise"]))
+            interval = _span_interval(span)
             span_map.setdefault(parent_span_id, []).append(interval)
 
     for span in spans:
@@ -136,7 +138,7 @@ def set_exclusive_time(spans: list[Span]) -> None:
         intervals.sort(key=lambda x: (x[0], -x[1]))
 
         exclusive_time_us: int = 0  # microseconds to prevent rounding issues
-        start, end = _us(span["start_timestamp_precise"]), _us(span["end_timestamp_precise"])
+        start, end = _span_interval(span)
 
         # Progressively add time gaps before the next span and then skip to its end.
         for child_start, child_end in intervals:
@@ -155,7 +157,75 @@ def set_exclusive_time(spans: list[Span]) -> None:
         span["exclusive_time_ms"] = exclusive_time_us / 1_000
 
 
+def _span_interval(span: Span) -> tuple[int, int]:
+    """Get the start and end timestamps of a span in microseconds."""
+    return _us(span["start_timestamp_precise"]), _us(span["end_timestamp_precise"])
+
+
 def _us(timestamp: float) -> int:
     """Convert the floating point duration or timestamp to integer microsecond
     precision."""
     return int(timestamp * 1_000_000)
+
+
+def compute_breakdowns(segment: Span, spans: list[Span], project: Project) -> None:
+    """
+    Computes breakdowns from all spans and writes them to the segment span.
+
+    Breakdowns are measurements that are derived from the spans in the segment.
+    By convention, their unit is in milliseconds. In the end, these measurements
+    are converted into attributes on the span trace item.
+    """
+
+    config = project.get_option("sentry:breakdowns")
+
+    for breakdown_name, breakdown_config in config.items():
+        ty = breakdown_config.get("type")
+
+        if ty == "spanOperations":
+            breakdowns = _compute_span_ops(spans, breakdown_config)
+        else:
+            continue
+
+        measurements = segment.setdefault("measurements", {})
+        for key, value in breakdowns.items():
+            measurements[f"{breakdown_name}.{key}"] = value
+
+
+def _compute_span_ops(spans: list[Span], config: Any) -> dict[str, MeasurementValue]:
+    matches = config.get("matches")
+    if not matches:
+        return {}
+
+    intervals_by_op = defaultdict(list)
+    for span in spans:
+        op = span.get("sentry_tags", {}).get("op", "")
+        if operation_name := next(filter(lambda m: op.startswith(m), matches), None):
+            intervals_by_op[operation_name].append(_span_interval(span))
+
+    measurements: dict[str, MeasurementValue] = {}
+    for operation_name, intervals in intervals_by_op.items():
+        duration = _get_duration_us(intervals)
+        measurements[f"ops.{operation_name}"] = {"value": duration / 1000, "unit": "millisecond"}
+    return measurements
+
+
+def _get_duration_us(intervals: list[tuple[int, int]]) -> int:
+    """
+    Get the wall clock time duration covered by the intervals in microseconds.
+
+    Overlapping intervals are merged so that they are not counted twice. For
+    example, the intervals [(1, 3), (2, 4)] would yield a duration of 3, not 4.
+    """
+
+    duration = 0
+    last_end = 0
+
+    intervals.sort(key=lambda x: (x[0], -x[1]))
+    for start, end in intervals:
+        # Ensure the current interval doesn't overlap with the last one
+        start = max(start, last_end)
+        duration += max(end - start, 0)
+        last_end = end
+
+    return duration

src/sentry/spans/consumers/process_segments/message.py

@@ -24,6 +24,7 @@
     record_release_received,
 )
 from sentry.spans.consumers.process_segments.enrichment import (
+    compute_breakdowns,
     match_schemas,
     set_exclusive_time,
     set_shared_tags,
@@ -50,6 +51,7 @@ def process_segment(unprocessed_spans: list[UnprocessedSpan]) -> list[Span]:
         # If the project does not exist then it might have been deleted during ingestion.
         return []
 
+    compute_breakdowns(segment_span, spans, project)
     _create_models(segment_span, project)
     _detect_performance_problems(segment_span, spans, project)
     _record_signals(segment_span, spans, project)

src/sentry/spans/consumers/process_segments/types.py

@@ -1,8 +1,10 @@
-from typing import Any, NotRequired
+from typing import NotRequired
 
+from sentry_kafka_schemas.schema_types.buffered_segments_v1 import MeasurementValue
 from sentry_kafka_schemas.schema_types.buffered_segments_v1 import SegmentSpan as UnprocessedSpan
 
 __all__ = (
+    "MeasurementValue",
     "Span",
     "UnprocessedSpan",
 )
@@ -14,11 +16,6 @@ class Span(UnprocessedSpan, total=True):
     extracted.
     """
 
-    # Missing in schema
-    start_timestamp_precise: float
-    end_timestamp_precise: float
-    data: NotRequired[dict[str, Any]]  # currently unused
-
     # Added in enrichment
     exclusive_time: float
     exclusive_time_ms: float

static/app/components/charts/chartWidgetLoader-unmocked-imports.spec.tsx

@@ -152,6 +152,29 @@ jest.mock(
     })),
   })
 );
+jest.mock('sentry/views/insights/common/queries/useDiscover', () => ({
+  useEAPSpans: jest.fn(() => ({
+    data: [
+      {
+        'avg(span.duration)': 123,
+        'sum(span.duration)': 456,
+        'span.group': 'abc123',
+        'span.description': 'span1',
+        'sentry.normalized_description': 'span1',
+        transaction: 'transaction_a',
+      },
+    ],
+    isPending: false,
+    error: null,
+  })),
+}));
+jest.mock('sentry/views/insights/common/queries/useTopNDiscoverSeries', () => ({
+  useTopNSpanEAPSeries: jest.fn(() => ({
+    data: [mockDiscoverSeries('transaction_a,abc123')],
+    isPending: false,
+    error: null,
+  })),
+}));
 jest.mock('sentry/views/insights/common/queries/useDiscoverSeries', () => ({
   useEAPSeries: jest.fn(() => ({
     data: {

static/app/components/charts/chartWidgetLoader.tsx

@@ -155,16 +155,34 @@ const CHART_MAP = {
     import(
       'sentry/views/insights/common/components/widgets/httpDomainSummaryDurationChartWidget'
     ),
+  overviewAgentsRunsChartWidget: () =>
+    import(
+      'sentry/views/insights/common/components/widgets/overviewAgentsRunsChartWidget'
+    ),
+  overviewAgentsDurationChartWidget: () =>
+    import(
+      'sentry/views/insights/common/components/widgets/overviewAgentsDurationChartWidget'
+    ),
   overviewApiLatencyChartWidget: () =>
     import(
       'sentry/views/insights/common/components/widgets/overviewApiLatencyChartWidget'
     ),
+  overviewCacheMissChartWidget: () =>
+    import(
+      'sentry/views/insights/common/components/widgets/overviewCacheMissChartWidget'
+    ),
+  overviewJobsChartWidget: () =>
+    import('sentry/views/insights/common/components/widgets/overviewJobsChartWidget'),
   overviewPageloadsChartWidget: () =>
     import(
       'sentry/views/insights/common/components/widgets/overviewPageloadsChartWidget'
     ),
   overviewRequestsChartWidget: () =>
     import('sentry/views/insights/common/components/widgets/overviewRequestsChartWidget'),
+  overviewSlowQueriesChartWidget: () =>
+    import(
+      'sentry/views/insights/common/components/widgets/overviewSlowQueriesChartWidget'
+    ),
 } satisfies Record<string, () => Promise<{default: React.FC<LoadableChartWidgetProps>}>>;
 
 /**

static/app/views/insights/agentMonitoring/utils/query.tsx

@@ -0,0 +1,7 @@
+// These are the span op we are currently ingesting.
+// They will probably change.
+const AI_PIPELINE_OPS = ['ai.pipeline.generateText', 'ai.pipeline.generateObject'];
+
+export const getAgentRunsFilter = () => {
+  return `span.op:[${AI_PIPELINE_OPS.join(',')}]`;
+};

static/app/views/insights/agentMonitoring/views/agentsOverviewPage.tsx

@@ -26,8 +26,11 @@ import * as ModuleLayout from 'sentry/views/insights/common/components/moduleLay
 import {ModulePageProviders} from 'sentry/views/insights/common/components/modulePageProviders';
 import {ModuleBodyUpsellHook} from 'sentry/views/insights/common/components/moduleUpsellHookWrapper';
 import {ToolRibbon} from 'sentry/views/insights/common/components/ribbon';
+import OverviewAgentsDurationChartWidget from 'sentry/views/insights/common/components/widgets/overviewAgentsDurationChartWidget';
+import OverviewAgentsRunsChartWidget from 'sentry/views/insights/common/components/widgets/overviewAgentsRunsChartWidget';
 import {useOnboardingProject} from 'sentry/views/insights/common/queries/useOnboardingProject';
 import {AgentsPageHeader} from 'sentry/views/insights/pages/agents/agentsPageHeader';
+import {IssuesWidget} from 'sentry/views/insights/pages/platform/shared/issuesWidget';
 import {WidgetGrid} from 'sentry/views/insights/pages/platform/shared/styles';
 import {useTransactionNameQuery} from 'sentry/views/insights/pages/platform/shared/useTransactionNameQuery';
 import {ModuleName} from 'sentry/views/insights/types';
@@ -76,13 +79,13 @@ function AgentsMonitoringPage() {
               <ModuleLayout.Full>
                 <WidgetGrid>
                   <WidgetGrid.Position1>
-                    <PlaceholderWidget title="Traffic" />
+                    <OverviewAgentsRunsChartWidget />
                   </WidgetGrid.Position1>
                   <WidgetGrid.Position2>
-                    <PlaceholderWidget title="Duration" />
+                    <OverviewAgentsDurationChartWidget />
                   </WidgetGrid.Position2>
                   <WidgetGrid.Position3>
-                    <PlaceholderWidget title="Issues" />
+                    <IssuesWidget />
                   </WidgetGrid.Position3>
                   <WidgetGrid.Position4>
                     <PlaceholderWidget title="LLM Generations" />

static/app/views/insights/common/components/widgets/overviewAgentsDurationChartWidget.tsx

@@ -0,0 +1,17 @@
+import {t} from 'sentry/locale';
+import {getAgentRunsFilter} from 'sentry/views/insights/agentMonitoring/utils/query';
+import type {LoadableChartWidgetProps} from 'sentry/views/insights/common/components/widgets/types';
+import BaseLatencyWidget from 'sentry/views/insights/pages/platform/shared/baseLatencyWidget';
+
+export default function OverviewAgentsDurationChartWidget(
+  props: LoadableChartWidgetProps
+) {
+  return (
+    <BaseLatencyWidget
+      id="overviewAgentsDurationChartWidget"
+      title={t('Duration')}
+      baseQuery={getAgentRunsFilter()}
+      {...props}
+    />
+  );
+}

static/app/views/insights/common/components/widgets/overviewAgentsRunsChartWidget.tsx

@@ -0,0 +1,16 @@
+import {t} from 'sentry/locale';
+import {getAgentRunsFilter} from 'sentry/views/insights/agentMonitoring/utils/query';
+import type {LoadableChartWidgetProps} from 'sentry/views/insights/common/components/widgets/types';
+import {BaseTrafficWidget} from 'sentry/views/insights/pages/platform/shared/baseTrafficWidget';
+
+export default function OverviewAgentsRunsChartWidget(props: LoadableChartWidgetProps) {
+  return (
+    <BaseTrafficWidget
+      id="overviewAgentsRunsChartWidget"
+      title={t('Traffic')}
+      trafficSeriesName={t('Runs')}
+      baseQuery={getAgentRunsFilter()}
+      {...props}
+    />
+  );
+}