[wip] feat: forward webhooks to codecov

TODO: make this work for monolith mode, i didn't consider monolith
mode when I made these changes. I think the path forward for that is
to define another middleware that takes care of that and only operates
in monolith mode.

depends on: #91830

We want to forward webhooks Sentry receives to Codecov since Codecov
depends on receiving webhooks from the code forges it supports. We're
starting with only GitHub webhooks for now, but plan to support more
in the future.

We're using the existing WebhookPayload system to forward webhooks to
Codecov by creating an additional WebhookPayload object in the
GithubRequestParser. Codecov will do additional filtering on its end
to filter out irrelevant webhooks so we don't have to do any filtering
on Sentry's end, this is to avoid keeping track of what's relevant to
Codecov in Sentry.

I'm adding a destination_type column to the WebhookPayload so we can
distinguish between payloads meant for Codecov and other Sentry
regions because when we make the request in the drain_mailbox we want
to run different logic, using the CodecovApiClient.

I also made the choice to use a different mailbox_name so that the
forwarding doesn't interfere with the webhooks being sent to the
region silos.

Finally, because we want to forward ALL webhooks to Codecov, that
means we're forwarding installation events, a webhook that in Sentry
gets processed in the control silo, so that gets its own mailbox
identifier "installation".

Author: joseph-sentry

migrations_lockfile.txt

@@ -9,7 +9,7 @@ explore: 0004_add_explore_last_visited_table
 
 feedback: 0004_index_together
 
-hybridcloud: 0021_django_arrayfield_scope_list
+hybridcloud: 0022_update_webhook_payload
 
 insights: 0001_add_starred_transactions_model
 

src/sentry/codecov/client.py

@@ -58,7 +58,7 @@ def _create_jwt(self):
 
     def __init__(
         self,
-        git_provider_user: GitProviderId,
+        git_provider_user: GitProviderId | None,
         git_provider: GitProvider = GitProvider.GitHub,
     ):
         """

src/sentry/conf/server.py

@@ -3383,6 +3383,8 @@ def custom_parameter_sort(parameter: dict) -> tuple[str, int]:
 
 SENTRY_REPLAYS_SERVICE_URL = "http://localhost:8090"
 
+# Codecov is disabled by default
+SENTRY_CODECOV_URL = None
 
 SENTRY_ISSUE_ALERT_HISTORY = "sentry.rules.history.backends.postgres.PostgresRuleHistoryBackend"
 SENTRY_ISSUE_ALERT_HISTORY_OPTIONS: dict[str, Any] = {}

src/sentry/hybridcloud/migrations/0022_update_webhook_payload.py

@@ -0,0 +1,38 @@
+# Generated by Django 5.2.1 on 2025-05-21 20:33
+
+from django.db import migrations, models
+
+from sentry.new_migrations.migrations import CheckedMigration
+
+
+class Migration(CheckedMigration):
+    # This flag is used to mark that a migration shouldn't be automatically run in production.
+    # This should only be used for operations where it's safe to run the migration after your
+    # code has deployed. So this should not be used for most operations that alter the schema
+    # of a table.
+    # Here are some things that make sense to mark as post deployment:
+    # - Large data migrations. Typically we want these to be run manually so that they can be
+    #   monitored and not block the deploy for a long period of time while they run.
+    # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
+    #   run this outside deployments so that we don't block them. Note that while adding an index
+    #   is a schema change, it's completely safe to run the operation after the code has deployed.
+    # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment
+
+    is_post_deployment = False
+
+    dependencies = [
+        ("hybridcloud", "0021_django_arrayfield_scope_list"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="webhookpayload",
+            name="destination_type",
+            field=models.CharField(db_default="sentry_region"),
+        ),
+        migrations.AlterField(
+            model_name="webhookpayload",
+            name="region_name",
+            field=models.CharField(null=True),
+        ),
+    ]

src/sentry/hybridcloud/models/webhookpayload.py

@@ -4,7 +4,7 @@
 from typing import Any, Self
 
 from django.db import models
-from django.db.models import Case, ExpressionWrapper, F, IntegerField, Value, When
+from django.db.models import Case, ExpressionWrapper, F, IntegerField, TextChoices, Value, When
 from django.http import HttpRequest
 from django.utils import timezone
 
@@ -19,13 +19,25 @@
 BACKOFF_RATE = 1.4
 
 
+class DestinationType(TextChoices):
+    SENTRY_REGION = "sentry_region"
+    CODECOV = "codecov"
+
+
 @control_silo_model
 class WebhookPayload(Model):
     __relocation_scope__ = RelocationScope.Excluded
 
     mailbox_name = models.CharField(null=False, blank=False)
     provider = models.CharField(null=True, blank=True)
-    region_name = models.CharField(null=False)
+
+    # Destination attributes
+    # Table is constantly being deleted from so let's make this non-nullable with a default value, since the table should be small at any given point in time.
+    destination_type = models.CharField(
+        choices=DestinationType.choices, null=False, db_default=DestinationType.SENTRY_REGION
+    )
+    region_name = models.CharField(null=True)
+
     # May need to add organization_id in the future for debugging.
     integration_id = models.BigIntegerField(null=True)
 
@@ -69,6 +81,7 @@ class Meta:
 
     __repr__ = sane_repr(
         "mailbox_name",
+        "destination_type",
         "region_name",
         "schedule_for",
         "attempts",
@@ -93,7 +106,8 @@ def get_attributes_from_request(
     def create_from_request(
         cls,
         *,
-        region: str,
+        destination_type: DestinationType,
+        region: str | None,
         provider: str,
         identifier: int | str,
         request: HttpRequest,
@@ -103,6 +117,7 @@ def create_from_request(
         return cls.objects.create(
             mailbox_name=f"{provider}:{identifier}",
             provider=provider,
+            destination_type=destination_type,
             region_name=region,
             integration_id=integration_id,
             **cls.get_attributes_from_request(request),

src/sentry/hybridcloud/tasks/deliver_webhooks.py

@@ -11,8 +11,14 @@
 from rest_framework import status
 
 from sentry import options
+from sentry.codecov.client import CodecovApiClient, ConfigurationError
 from sentry.exceptions import RestrictedIPAddress
-from sentry.hybridcloud.models.webhookpayload import BACKOFF_INTERVAL, MAX_ATTEMPTS, WebhookPayload
+from sentry.hybridcloud.models.webhookpayload import (
+    BACKOFF_INTERVAL,
+    MAX_ATTEMPTS,
+    DestinationType,
+    WebhookPayload,
+)
 from sentry.shared_integrations.exceptions import (
     ApiConflictError,
     ApiConnectionResetError,
@@ -25,7 +31,7 @@
 from sentry.tasks.base import instrumented_task
 from sentry.taskworker.config import TaskworkerConfig
 from sentry.taskworker.namespaces import hybridcloud_control_tasks
-from sentry.types.region import get_region_by_name
+from sentry.types.region import Region, get_region_by_name
 from sentry.utils import metrics
 
 logger = logging.getLogger(__name__)
@@ -403,12 +409,24 @@ def deliver_message(payload: WebhookPayload) -> None:
 
 
 def perform_request(payload: WebhookPayload) -> None:
+    destination_type = payload.destination_type
+
+    match destination_type:
+        case DestinationType.SENTRY_REGION:
+            region = get_region_by_name(name=payload.region_name)
+            perform_region_request(region, payload)
+        case DestinationType.CODECOV:
+            perform_codecov_request(payload)
+        case _:
+            raise ValueError(f"Unknown destination type: {destination_type!r}")
+
+
+def perform_region_request(region: Region, payload: WebhookPayload) -> None:
     logging_context: dict[str, str | int] = {
         "payload_id": payload.id,
         "mailbox_name": payload.mailbox_name,
         "attempt": payload.attempts,
     }
-    region = get_region_by_name(name=payload.region_name)
 
     try:
         client = RegionSiloClient(region=region)
@@ -521,3 +539,47 @@ def perform_request(payload: WebhookPayload) -> None:
             extra={"error": str(err), "response_code": response_code, **logging_context},
         )
         raise DeliveryFailed() from err
+
+
+def perform_codecov_request(payload: WebhookPayload) -> None:
+    logging_context: dict[str, str | int] = {
+        "payload_id": payload.id,
+        "mailbox_name": payload.mailbox_name,
+        "attempt": payload.attempts,
+        "request_method": payload.request_method,
+        "request_path": payload.request_path,
+    }
+
+    with metrics.timer(
+        "hybridcloud.deliver_webhooks.send_request_to_codecov",
+    ):
+        try:
+            client = CodecovApiClient(None)
+            response = client.post(
+                endpoint=payload.request_path,
+                data=payload.request_body.encode("utf-8"),
+                headers=orjson.loads(payload.request_headers),
+            )
+        except ConfigurationError as err:
+            metrics.incr(
+                "hybridcloud.deliver_webhooks.send_request_to_codecov.codecov_configuration_error",
+            )
+            logger.warning(
+                "deliver_webhooks.send_request_to_codecov.codecov_configuration_error",
+                extra={"error": str(err), **logging_context},
+            )
+            return
+
+    if response is None:
+        metrics.incr(
+            "hybridcloud.deliver_webhooks.send_request_to_codecov.failure",
+        )
+        return
+
+    logger.debug(
+        "deliver_webhooks.send_request_to_codecov.success",
+        extra={
+            "status": response.status_code,
+            **logging_context,
+        },
+    )

src/sentry/integrations/middleware/hybrid_cloud/parser.py

@@ -14,7 +14,7 @@
 
 from sentry.api.base import ONE_DAY
 from sentry.constants import ObjectStatus
-from sentry.hybridcloud.models.webhookpayload import WebhookPayload
+from sentry.hybridcloud.models.webhookpayload import DestinationType, WebhookPayload
 from sentry.hybridcloud.outbox.category import WebhookProviderIdentifier
 from sentry.hybridcloud.services.organization_mapping import organization_mapping_service
 from sentry.integrations.middleware.metrics import (
@@ -178,6 +178,7 @@ def get_response_from_webhookpayload(
         shard_identifier = identifier or self.webhook_identifier.value
         for region in regions:
             WebhookPayload.create_from_request(
+                destination_type=DestinationType.SENTRY_REGION,
                 region=region.name,
                 provider=self.provider,
                 identifier=shard_identifier,
@@ -337,3 +338,24 @@ def get_regions_from_organizations(
 
     def get_default_missing_integration_response(self) -> HttpResponse:
         return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
+
+    # Forwarding Helpers
+
+    def forward_to_codecov(
+        self,
+        identifier: int | str | None = None,
+        integration_id: int | None = None,
+    ):
+        shard_identifier = f"codecov:{identifier or self.webhook_identifier.value}"
+
+        # create webhookpayloads for each service
+        WebhookPayload.create_from_request(
+            destination_type=DestinationType.CODECOV,
+            region=None,
+            provider=self.provider,
+            identifier=shard_identifier,
+            integration_id=integration_id,
+            request=self.request,
+        )
+
+        return HttpResponse(status=status.HTTP_202_ACCEPTED)

src/sentry/middleware/integrations/parsers/github.py

@@ -5,6 +5,7 @@
 from typing import Any
 
 import orjson
+from django.conf import settings
 from django.http import HttpResponse
 
 from sentry.hybridcloud.outbox.category import WebhookProviderIdentifier
@@ -53,6 +54,11 @@ def get_response(self):
             return HttpResponse(status=400)
 
         if event.get("installation") and event.get("action") in {"created", "deleted"}:
+            if settings.SENTRY_CODECOV_URL:  # check if codecov is enabled
+                self.forward_to_codecov(
+                    identifier="installation",
+                    integration_id=None,
+                )
             return self.get_response_from_control_silo()
 
         try:
@@ -67,6 +73,12 @@ def get_response(self):
         if len(regions) == 0:
             return self.get_default_missing_integration_response()
 
+        if settings.SENTRY_CODECOV_URL:  # check if codecov is enabled
+            self.forward_to_codecov(
+                identifier=integration.id,
+                integration_id=integration.id,
+            )
+
         return self.get_response_from_webhookpayload(
             regions=regions, identifier=integration.id, integration_id=integration.id
         )

src/sentry/testutils/factories.py

@@ -1983,7 +1983,9 @@ def create_request_access(
 
     @staticmethod
     @assume_test_silo_mode(SiloMode.CONTROL)
-    def create_webhook_payload(mailbox_name: str, region_name: str, **kwargs) -> WebhookPayload:
+    def create_webhook_payload(
+        mailbox_name: str, region_name: str | None, **kwargs
+    ) -> WebhookPayload:
         payload_kwargs = {
             "request_method": "POST",
             "request_path": "/extensions/github/webhook/",

src/sentry/testutils/outbox.py

@@ -8,7 +8,7 @@
 from django.core.handlers.wsgi import WSGIRequest
 
 from sentry.hybridcloud.models.outbox import OutboxBase
-from sentry.hybridcloud.models.webhookpayload import THE_PAST, WebhookPayload
+from sentry.hybridcloud.models.webhookpayload import THE_PAST, DestinationType, WebhookPayload
 from sentry.hybridcloud.tasks.deliver_from_outbox import (
     enqueue_outbox_jobs,
     enqueue_outbox_jobs_control,
@@ -65,19 +65,21 @@ def assert_webhook_payloads_for_mailbox(
     request: WSGIRequest,
     mailbox_name: str,
     region_names: list[str],
+    destination_types: dict[DestinationType, int] | None = None,
 ):
     """
     A test method for asserting that a webhook payload is properly queued for
      the given request
 
     :param request:
     :param mailbox_name: The mailbox name that messages should be found in.
-    :param region_names: The regions each messages should be queued for
+    :param region_names: Optional list of regions each messages should be queued for
+    :param destination_type: The destination type of the messages
     """
     expected_payload = WebhookPayload.get_attributes_from_request(request=request)
     region_names_set = set(region_names)
     messages = WebhookPayload.objects.filter(mailbox_name=mailbox_name)
-    message_count = messages.count()
+    message_count = messages.filter(region_name__isnull=False).count()
     if message_count != len(region_names_set):
         raise Exception(
             f"Mismatch: Found {message_count} WebhookPayload but {len(region_names_set)} region_names"
@@ -89,11 +91,24 @@ def assert_webhook_payloads_for_mailbox(
         assert message.request_body == expected_payload["request_body"]
         assert message.schedule_for == THE_PAST
         assert message.attempts == 0
-        try:
-            region_names_set.remove(message.region_name)
-        except KeyError:
-            raise Exception(
-                f"Found ControlOutbox for '{message.region_name}', which was not in region_names: {str(region_names_set)}"
-            )
+
+        if destination_types:
+            assert message.destination_type in destination_types
+            destination_types[message.destination_type] -= 1
+            if destination_types[message.destination_type] == 0:
+                del destination_types[message.destination_type]
+
+        if message.region_name:
+            try:
+                region_names_set.remove(message.region_name)
+            except KeyError:
+                raise Exception(
+                    f"Found ControlOutbox for '{message.region_name}', which was not in region_names: {str(region_names_set)}"
+                )
     if len(region_names_set) != 0:
         raise Exception(f"WebhookPayload not found for some region_names: {str(region_names_set)}")
+
+    if destination_types and len(destination_types) != 0:
+        raise Exception(
+            f"WebhookPayload not found for some destination_types: {str(destination_types)}"
+        )