Skip to content

Commit 9992799

Browse files
obostjancicobostjancic
authored
feat(agents-insights): LLM Generations and Tool Usage widgets (#92202)
1 parent 6e87a42 commit 9992799

File tree

4 files changed

+360
-3
lines changed

4 files changed

+360
-3
lines changed

‎static/app/views/insights/agentMonitoring/components/llmGenerationsWidget.tsx

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import {Fragment} from 'react';
2+
import {useTheme} from '@emotion/react';
3+
import styled from '@emotion/styled';
4+
5+
import {openInsightChartModal} from 'sentry/actionCreators/modal';
6+
import {t} from 'sentry/locale';
7+
import useOrganization from 'sentry/utils/useOrganization';
8+
import {Bars} from 'sentry/views/dashboards/widgets/timeSeriesWidget/plottables/bars';
9+
import {TimeSeriesWidgetVisualization} from 'sentry/views/dashboards/widgets/timeSeriesWidget/timeSeriesWidgetVisualization';
10+
import {Widget} from 'sentry/views/dashboards/widgets/widget/widget';
11+
import {Mode} from 'sentry/views/explore/contexts/pageParamsContext/mode';
12+
import {
13+
AI_MODEL_ID_ATTRIBUTE,
14+
getLLMGenerationsFilter,
15+
} from 'sentry/views/insights/agentMonitoring/utils/query';
16+
import {ChartType} from 'sentry/views/insights/common/components/chart';
17+
import {useEAPSpans} from 'sentry/views/insights/common/queries/useDiscover';
18+
import {useTopNSpanEAPSeries} from 'sentry/views/insights/common/queries/useTopNDiscoverSeries';
19+
import {convertSeriesToTimeseries} from 'sentry/views/insights/common/utils/convertSeriesToTimeseries';
20+
import {Referrer} from 'sentry/views/insights/pages/platform/laravel/referrers';
21+
import {usePageFilterChartParams} from 'sentry/views/insights/pages/platform/laravel/utils';
22+
import {WidgetVisualizationStates} from 'sentry/views/insights/pages/platform/laravel/widgetVisualizationStates';
23+
import {
24+
ModalChartContainer,
25+
ModalTableWrapper,
26+
SeriesColorIndicator,
27+
WidgetFooterTable,
28+
} from 'sentry/views/insights/pages/platform/shared/styles';
29+
import {Toolbar} from 'sentry/views/insights/pages/platform/shared/toolbar';
30+
import {useTransactionNameQuery} from 'sentry/views/insights/pages/platform/shared/useTransactionNameQuery';
31+
import {TimeSpentInDatabaseWidgetEmptyStateWarning} from 'sentry/views/performance/landing/widgets/components/selectableList';
32+
33+
export default function LLMGenerationsWidget() {
34+
const organization = useOrganization();
35+
const {query} = useTransactionNameQuery();
36+
const pageFilterChartParams = usePageFilterChartParams({
37+
granularity: 'spans-low',
38+
});
39+
40+
const theme = useTheme();
41+
42+
const fullQuery = `${getLLMGenerationsFilter()} ${query}`.trim();
43+
44+
const generationsRequest = useEAPSpans(
45+
{
46+
// @ts-expect-error TODO(telex): Add model id attribute to Fields
47+
fields: [AI_MODEL_ID_ATTRIBUTE, 'count(span.duration)'],
48+
sorts: [{field: 'count(span.duration)', kind: 'desc'}],
49+
search: fullQuery,
50+
limit: 3,
51+
},
52+
Referrer.QUERIES_CHART // TODO
53+
);
54+
55+
const timeSeriesRequest = useTopNSpanEAPSeries(
56+
{
57+
...pageFilterChartParams,
58+
search: fullQuery,
59+
fields: [AI_MODEL_ID_ATTRIBUTE, 'count(span.duration)'],
60+
yAxis: ['count(span.duration)'],
61+
sort: {field: 'count(span.duration)', kind: 'desc'},
62+
topN: 3,
63+
enabled: !!generationsRequest.data,
64+
},
65+
Referrer.QUERIES_CHART // TODO
66+
);
67+
68+
const timeSeries = timeSeriesRequest.data.filter(ts => ts.seriesName !== 'Other');
69+
70+
const isLoading = timeSeriesRequest.isLoading || generationsRequest.isLoading;
71+
const error = timeSeriesRequest.error || generationsRequest.error;
72+
73+
// TODO(telex): Add model id attribute to Fields and get rid of this cast
74+
const models = generationsRequest.data as unknown as
75+
| Array<{
76+
[AI_MODEL_ID_ATTRIBUTE]: string;
77+
'count(span.duration)': number;
78+
}>
79+
| undefined;
80+
81+
const hasData = models && models.length > 0 && timeSeries.length > 0;
82+
83+
const colorPalette = theme.chart.getColorPalette(timeSeries.length - 2);
84+
85+
const visualization = (
86+
<WidgetVisualizationStates
87+
isEmpty={!hasData}
88+
isLoading={isLoading}
89+
error={error}
90+
emptyMessage={<TimeSpentInDatabaseWidgetEmptyStateWarning />}
91+
VisualizationType={TimeSeriesWidgetVisualization}
92+
visualizationProps={{
93+
showLegend: 'never',
94+
plottables: timeSeries.map(
95+
(ts, index) =>
96+
new Bars(convertSeriesToTimeseries(ts), {
97+
color: colorPalette[index],
98+
alias: ts.seriesName,
99+
stack: 'stack',
100+
})
101+
),
102+
}}
103+
/>
104+
);
105+
106+
const footer = hasData && (
107+
<WidgetFooterTable>
108+
{models?.map((item, index) => (
109+
<Fragment key={item[AI_MODEL_ID_ATTRIBUTE]}>
110+
<div>
111+
<SeriesColorIndicator
112+
style={{
113+
backgroundColor: colorPalette[index],
114+
}}
115+
/>
116+
</div>
117+
<div>
118+
<ModelText>{item[AI_MODEL_ID_ATTRIBUTE]}</ModelText>
119+
</div>
120+
<span>{item['count(span.duration)']}</span>
121+
</Fragment>
122+
))}
123+
</WidgetFooterTable>
124+
);
125+
126+
return (
127+
<Widget
128+
Title={<Widget.WidgetTitle title={t('LLM Generations')} />}
129+
Visualization={visualization}
130+
Actions={
131+
organization.features.includes('visibility-explore-view') &&
132+
hasData && (
133+
<Toolbar
134+
exploreParams={{
135+
mode: Mode.SAMPLES,
136+
visualize: [
137+
{
138+
chartType: ChartType.BAR,
139+
yAxes: ['count(span.duration)'],
140+
},
141+
],
142+
query: fullQuery,
143+
interval: pageFilterChartParams.interval,
144+
}}
145+
onOpenFullScreen={() => {
146+
openInsightChartModal({
147+
title: t('LLM Generations'),
148+
children: (
149+
<Fragment>
150+
<ModalChartContainer>{visualization}</ModalChartContainer>
151+
<ModalTableWrapper>{footer}</ModalTableWrapper>
152+
</Fragment>
153+
),
154+
});
155+
}}
156+
/>
157+
)
158+
}
159+
noFooterPadding
160+
Footer={footer}
161+
/>
162+
);
163+
}
164+
165+
const ModelText = styled('div')`
166+
${p => p.theme.overflowEllipsis};
167+
color: ${p => p.theme.subText};
168+
font-size: ${p => p.theme.fontSizeSmall};
169+
line-height: 1.2;
170+
min-width: 0px;
171+
`;

‎static/app/views/insights/agentMonitoring/components/toolUsageWidget.tsx

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import {Fragment} from 'react';
2+
import {useTheme} from '@emotion/react';
3+
import styled from '@emotion/styled';
4+
5+
import {openInsightChartModal} from 'sentry/actionCreators/modal';
6+
import {t} from 'sentry/locale';
7+
import useOrganization from 'sentry/utils/useOrganization';
8+
import {Bars} from 'sentry/views/dashboards/widgets/timeSeriesWidget/plottables/bars';
9+
import {TimeSeriesWidgetVisualization} from 'sentry/views/dashboards/widgets/timeSeriesWidget/timeSeriesWidgetVisualization';
10+
import {Widget} from 'sentry/views/dashboards/widgets/widget/widget';
11+
import {Mode} from 'sentry/views/explore/contexts/pageParamsContext/mode';
12+
import {
13+
AI_TOOL_NAME_ATTRIBUTE,
14+
getAIToolCallsFilter,
15+
} from 'sentry/views/insights/agentMonitoring/utils/query';
16+
import {ChartType} from 'sentry/views/insights/common/components/chart';
17+
import {useEAPSpans} from 'sentry/views/insights/common/queries/useDiscover';
18+
import {useTopNSpanEAPSeries} from 'sentry/views/insights/common/queries/useTopNDiscoverSeries';
19+
import {convertSeriesToTimeseries} from 'sentry/views/insights/common/utils/convertSeriesToTimeseries';
20+
import {Referrer} from 'sentry/views/insights/pages/platform/laravel/referrers';
21+
import {usePageFilterChartParams} from 'sentry/views/insights/pages/platform/laravel/utils';
22+
import {WidgetVisualizationStates} from 'sentry/views/insights/pages/platform/laravel/widgetVisualizationStates';
23+
import {
24+
ModalChartContainer,
25+
ModalTableWrapper,
26+
SeriesColorIndicator,
27+
WidgetFooterTable,
28+
} from 'sentry/views/insights/pages/platform/shared/styles';
29+
import {Toolbar} from 'sentry/views/insights/pages/platform/shared/toolbar';
30+
import {useTransactionNameQuery} from 'sentry/views/insights/pages/platform/shared/useTransactionNameQuery';
31+
import {TimeSpentInDatabaseWidgetEmptyStateWarning} from 'sentry/views/performance/landing/widgets/components/selectableList';
32+
33+
export default function ToolUsageWidget() {
34+
const organization = useOrganization();
35+
const {query} = useTransactionNameQuery();
36+
const pageFilterChartParams = usePageFilterChartParams({
37+
granularity: 'spans-low',
38+
});
39+
40+
const theme = useTheme();
41+
42+
const fullQuery = `${getAIToolCallsFilter()} ${query}`.trim();
43+
44+
const toolsRequest = useEAPSpans(
45+
{
46+
// @ts-expect-error TODO(telex): Add tool name attribute to Fields
47+
fields: [AI_TOOL_NAME_ATTRIBUTE, 'count(span.duration)'],
48+
sorts: [{field: 'count(span.duration)', kind: 'desc'}],
49+
search: fullQuery,
50+
limit: 3,
51+
},
52+
Referrer.QUERIES_CHART // TODO
53+
);
54+
55+
const timeSeriesRequest = useTopNSpanEAPSeries(
56+
{
57+
...pageFilterChartParams,
58+
search: fullQuery,
59+
fields: [AI_TOOL_NAME_ATTRIBUTE, 'count(span.duration)'],
60+
yAxis: ['count(span.duration)'],
61+
sort: {field: 'count(span.duration)', kind: 'desc'},
62+
topN: 3,
63+
enabled: !!toolsRequest.data,
64+
},
65+
Referrer.QUERIES_CHART // TODO
66+
);
67+
68+
const timeSeries = timeSeriesRequest.data.filter(ts => ts.seriesName !== 'Other');
69+
70+
const isLoading = timeSeriesRequest.isLoading || toolsRequest.isLoading;
71+
const error = timeSeriesRequest.error || toolsRequest.error;
72+
73+
// TODO(telex): Add tool name attribute to Fields and get rid of this cast
74+
const tools = toolsRequest.data as unknown as
75+
| Array<{
76+
[AI_TOOL_NAME_ATTRIBUTE]: string;
77+
'count(span.duration)': number;
78+
}>
79+
| undefined;
80+
81+
const hasData = tools && tools.length > 0 && timeSeries.length > 0;
82+
83+
const colorPalette = theme.chart.getColorPalette(timeSeries.length - 2);
84+
85+
const visualization = (
86+
<WidgetVisualizationStates
87+
isEmpty={!hasData}
88+
isLoading={isLoading}
89+
error={error}
90+
emptyMessage={<TimeSpentInDatabaseWidgetEmptyStateWarning />}
91+
VisualizationType={TimeSeriesWidgetVisualization}
92+
visualizationProps={{
93+
showLegend: 'never',
94+
plottables: timeSeries.map(
95+
(ts, index) =>
96+
new Bars(convertSeriesToTimeseries(ts), {
97+
color: colorPalette[index],
98+
alias: ts.seriesName,
99+
stack: 'stack',
100+
})
101+
),
102+
}}
103+
/>
104+
);
105+
106+
const footer = hasData && (
107+
<WidgetFooterTable>
108+
{tools.map((item, index) => (
109+
<Fragment key={item[AI_TOOL_NAME_ATTRIBUTE]}>
110+
<div>
111+
<SeriesColorIndicator
112+
style={{
113+
backgroundColor: colorPalette[index],
114+
}}
115+
/>
116+
</div>
117+
<div>
118+
<ModelText>{item[AI_TOOL_NAME_ATTRIBUTE]}</ModelText>
119+
</div>
120+
<span>{item['count(span.duration)']}</span>
121+
</Fragment>
122+
))}
123+
</WidgetFooterTable>
124+
);
125+
126+
return (
127+
<Widget
128+
Title={<Widget.WidgetTitle title={t('Tool Usage')} />}
129+
Visualization={visualization}
130+
Actions={
131+
organization.features.includes('visibility-explore-view') &&
132+
hasData && (
133+
<Toolbar
134+
exploreParams={{
135+
mode: Mode.SAMPLES,
136+
visualize: [
137+
{
138+
chartType: ChartType.BAR,
139+
yAxes: ['count(span.duration)'],
140+
},
141+
],
142+
query: fullQuery,
143+
interval: pageFilterChartParams.interval,
144+
}}
145+
onOpenFullScreen={() => {
146+
openInsightChartModal({
147+
title: t('Tool Usage'),
148+
children: (
149+
<Fragment>
150+
<ModalChartContainer>{visualization}</ModalChartContainer>
151+
<ModalTableWrapper>{footer}</ModalTableWrapper>
152+
</Fragment>
153+
),
154+
});
155+
}}
156+
/>
157+
)
158+
}
159+
noFooterPadding
160+
Footer={footer}
161+
/>
162+
);
163+
}
164+
165+
const ModelText = styled('div')`
166+
${p => p.theme.overflowEllipsis};
167+
color: ${p => p.theme.subText};
168+
font-size: ${p => p.theme.fontSizeSmall};
169+
line-height: 1.2;
170+
min-width: 0px;
171+
`;

‎static/app/views/insights/agentMonitoring/utils/query.tsx

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,20 @@
11
// These are the span op we are currently ingesting.
2-
// They will probably change.
2+
// They will probably change and maybe it will be enough to inline them in the widgets.
33
const AI_PIPELINE_OPS = ['ai.pipeline.generateText', 'ai.pipeline.generateObject'];
4+
const AI_GENERATION_OPS = ['ai.run.doGenerate'];
5+
const AI_TOOL_CALL_OPS = ['ai.toolCall'];
6+
7+
export const AI_MODEL_ID_ATTRIBUTE = 'ai.model_id';
8+
export const AI_TOOL_NAME_ATTRIBUTE = 'ai.toolCall.name';
49

510
export const getAgentRunsFilter = () => {
611
return `span.op:[${AI_PIPELINE_OPS.join(',')}]`;
712
};
13+
14+
export const getLLMGenerationsFilter = () => {
15+
return `span.op:[${AI_GENERATION_OPS.join(',')}]`;
16+
};
17+
18+
export const getAIToolCallsFilter = () => {
19+
return `span.description:[${AI_TOOL_CALL_OPS.join(',')}]`;
20+
};

‎static/app/views/insights/agentMonitoring/views/agentsOverviewPage.tsx

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@ import {useLocation} from 'sentry/utils/useLocation';
1414
import useOrganization from 'sentry/utils/useOrganization';
1515
import {Widget} from 'sentry/views/dashboards/widgets/widget/widget';
1616
import {limitMaxPickableDays} from 'sentry/views/explore/utils';
17+
import LLMGenerationsWidget from 'sentry/views/insights/agentMonitoring/components/llmGenerationsWidget';
1718
import {ModelsTable} from 'sentry/views/insights/agentMonitoring/components/modelsTable';
1819
import {ToolsTable} from 'sentry/views/insights/agentMonitoring/components/toolsTable';
20+
import ToolUsageWidget from 'sentry/views/insights/agentMonitoring/components/toolUsageWidget';
1921
import {TracesTable} from 'sentry/views/insights/agentMonitoring/components/tracesTable';
2022
import {
2123
TableType,
@@ -88,10 +90,10 @@ function AgentsMonitoringPage() {
8890
<IssuesWidget />
8991
</WidgetGrid.Position3>
9092
<WidgetGrid.Position4>
91-
<PlaceholderWidget title="LLM Generations" />
93+
<LLMGenerationsWidget />
9294
</WidgetGrid.Position4>
9395
<WidgetGrid.Position5>
94-
<PlaceholderWidget title="Tool Usage" />
96+
<ToolUsageWidget />
9597
</WidgetGrid.Position5>
9698
<WidgetGrid.Position6>
9799
<PlaceholderWidget title="Token Usage" />

0 commit comments

Comments
 (0)