Skip to content

Commit c3f20b2

Browse files
hubertdeng123hubertdeng123
authored
feat(devservices): Add ingest full mode (#93079)
Now ingestion should work with the devserver without bringing up the devserver with --ingest flag if `devservices up --mode ingest-all` is run
1 parent cbda1db commit c3f20b2

File tree

2 files changed

+203
-7
lines changed

2 files changed

+203
-7
lines changed

‎devservices/config.yml

Lines changed: 104 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ x-sentry-service-config:
33
version: 0.1
44
service_name: sentry
55
dependencies:
6+
#########################################################
7+
# Docker Compose Services
8+
#########################################################
69
snuba:
710
description: Service that provides fast aggregation and query capabilities on top of Clickhouse
811
remote:
@@ -62,10 +65,60 @@ x-sentry-service-config:
6265
description: Memcached used for caching
6366
spotlight:
6467
description: Spotlight server for local debugging
68+
#########################################################
69+
# Supervisor Programs
70+
#########################################################
71+
# Taskworker services
6572
taskworker:
6673
description: Workers that process tasks from the taskbroker
6774
taskworker-scheduler:
6875
description: Task scheduler that can spawn tasks based on their schedules
76+
# Kafka consumer services for event ingestion
77+
ingest-events:
78+
description: Kafka consumer for processing ingested events
79+
ingest-attachments:
80+
description: Kafka consumer for processing ingested attachments
81+
ingest-transactions:
82+
description: Kafka consumer for processing ingested transactions
83+
ingest-monitors:
84+
description: Kafka consumer for processing monitor check-ins
85+
ingest-profiles:
86+
description: Kafka consumer for processing profiling data
87+
ingest-occurrences:
88+
description: Kafka consumer for processing issue occurrences
89+
ingest-feedback-events:
90+
description: Kafka consumer for processing user feedback events
91+
# Monitor-related services
92+
monitors-clock-tick:
93+
description: Kafka consumer for monitor clock ticks
94+
monitors-clock-tasks:
95+
description: Kafka consumer for monitor clock tasks
96+
monitors-incident-occurrences:
97+
description: Kafka consumer for monitor incident occurrences
98+
# Spans and performance monitoring
99+
process-spans:
100+
description: Kafka consumer for processing spans data
101+
process-segments:
102+
description: Kafka consumer for processing performance segments
103+
# Metrics-related consumers
104+
ingest-metrics:
105+
description: Kafka consumer for processing metrics data
106+
ingest-generic-metrics:
107+
description: Kafka consumer for processing generic metrics
108+
billing-metrics-consumer:
109+
description: Kafka consumer for billing metrics processing
110+
# Post-processing forwarders
111+
post-process-forwarder-errors:
112+
description: Post-process forwarder for error events
113+
post-process-forwarder-transactions:
114+
description: Post-process forwarder for transaction events
115+
post-process-forwarder-issue-platform:
116+
description: Post-process forwarder for issue platform events
117+
# Uptime monitoring
118+
uptime-results:
119+
description: Kafka consumer for uptime monitoring results
120+
worker:
121+
description: Worker that processes tasks from celery
69122

70123
modes:
71124
default: [snuba, postgres, relay, spotlight]
@@ -80,18 +133,62 @@ x-sentry-service-config:
80133
memcached: [postgres, snuba, memcached, spotlight]
81134
profiling: [postgres, snuba, vroom, spotlight]
82135
minimal: [postgres, snuba]
83-
full:
136+
ingest-all:
84137
[
138+
billing-metrics-consumer,
139+
ingest-attachments,
140+
ingest-events,
141+
ingest-feedback-events,
142+
ingest-generic-metrics,
143+
ingest-metrics,
144+
ingest-monitors,
145+
ingest-occurrences,
146+
ingest-profiles,
147+
ingest-transactions,
148+
monitors-clock-tasks,
149+
monitors-clock-tick,
150+
monitors-incident-occurrences,
151+
post-process-forwarder-errors,
152+
post-process-forwarder-issue-platform,
153+
post-process-forwarder-transactions,
85154
postgres,
86-
snuba,
155+
process-segments,
156+
process-spans,
87157
relay,
88-
redis,
89-
redis-cluster,
90-
symbolicator,
91-
taskbroker,
92-
rabbitmq,
158+
snuba,
159+
spotlight,
160+
worker,
93161
vroom,
162+
]
163+
full:
164+
[
165+
snuba,
166+
postgres,
167+
relay,
94168
spotlight,
169+
vroom,
170+
ingest-events,
171+
ingest-attachments,
172+
ingest-transactions,
173+
ingest-monitors,
174+
ingest-feedback-events,
175+
monitors-clock-tick,
176+
monitors-clock-tasks,
177+
monitors-incident-occurrences,
178+
ingest-profiles,
179+
ingest-occurrences,
180+
process-spans,
181+
process-segments,
182+
ingest-metrics,
183+
ingest-generic-metrics,
184+
billing-metrics-consumer,
185+
uptime-results,
186+
post-process-forwarder-errors,
187+
post-process-forwarder-transactions,
188+
post-process-forwarder-issue-platform,
189+
taskworker,
190+
taskworker-scheduler,
191+
worker,
95192
]
96193

97194
services:

‎devservices/programs.conf

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,102 @@ autorestart=true
1111
command=sentry run taskworker-scheduler
1212
autostart=false
1313
autorestart=true
14+
15+
[program:ingest-events]
16+
command=sentry run consumer ingest-events --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
17+
autostart=false
18+
autorestart=true
19+
20+
[program:ingest-attachments]
21+
command=sentry run consumer ingest-attachments --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
22+
autostart=false
23+
autorestart=true
24+
25+
[program:ingest-transactions]
26+
command=sentry run consumer ingest-transactions --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
27+
autostart=false
28+
autorestart=true
29+
30+
[program:ingest-monitors]
31+
command=sentry run consumer ingest-monitors --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
32+
autostart=false
33+
autorestart=true
34+
35+
[program:monitors-clock-tick]
36+
command=sentry run consumer monitors-clock-tick --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
37+
autostart=false
38+
autorestart=true
39+
40+
[program:monitors-clock-tasks]
41+
command=sentry run consumer monitors-clock-tasks --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
42+
autostart=false
43+
autorestart=true
44+
45+
[program:monitors-incident-occurrences]
46+
command=sentry run consumer monitors-incident-occurrences --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
47+
autostart=false
48+
autorestart=true
49+
50+
[program:ingest-profiles]
51+
command=sentry run consumer ingest-profiles --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
52+
autostart=false
53+
autorestart=true
54+
55+
[program:ingest-occurrences]
56+
command=sentry run consumer ingest-occurrences --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
57+
autostart=false
58+
autorestart=true
59+
60+
[program:process-spans]
61+
command=sentry run consumer process-spans --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
62+
autostart=false
63+
64+
[program:process-segments]
65+
command=sentry run consumer process-segments --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
66+
autostart=false
67+
autorestart=true
68+
69+
[program:uptime-results]
70+
command=sentry run consumer uptime-results --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
71+
autostart=false
72+
autorestart=true
73+
74+
[program:ingest-metrics]
75+
command=sentry run consumer ingest-metrics --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
76+
autostart=false
77+
autorestart=true
78+
79+
[program:ingest-generic-metrics]
80+
command=sentry run consumer ingest-generic-metrics --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
81+
autostart=false
82+
autorestart=true
83+
84+
[program:billing-metrics-consumer]
85+
command=sentry run consumer billing-metrics-consumer --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
86+
autostart=false
87+
autorestart=true
88+
89+
[program:post-process-forwarder-errors]
90+
command=sentry run consumer post-process-forwarder-errors --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
91+
autostart=false
92+
autorestart=true
93+
94+
[program:post-process-forwarder-transactions]
95+
command=sentry run consumer post-process-forwarder-transactions --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
96+
autostart=false
97+
autorestart=true
98+
99+
[program:post-process-forwarder-issue-platform]
100+
command=sentry run consumer post-process-forwarder-issue-platform --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
101+
autostart=false
102+
autorestart=true
103+
104+
[program:ingest-feedback-events]
105+
command=sentry run consumer ingest-feedback-events --consumer-group=sentry-consumer --auto-offset-reset=latest --no-strict-offset-reset
106+
autostart=false
107+
autorestart=true
108+
109+
[program:worker]
110+
command=sentry run worker -c 1 --autoreload
111+
autostart=false
112+
autorestart=true

0 commit comments

Comments
 (0)