|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import logging |
| 4 | +import os |
| 5 | +from collections import defaultdict |
| 6 | +from collections.abc import Mapping |
| 7 | +from datetime import timedelta |
| 8 | +from typing import Any |
| 9 | +from urllib.parse import urlparse |
| 10 | + |
| 11 | +from sentry.issues.grouptype import PerformanceNPlusOneAPICallsGroupType |
| 12 | +from sentry.issues.issue_occurrence import IssueEvidence |
| 13 | +from sentry.models.organization import Organization |
| 14 | +from sentry.models.project import Project |
| 15 | +from sentry.utils.performance_issues.detectors.utils import get_total_span_duration |
| 16 | +from sentry.utils.performance_issues.performance_problem import PerformanceProblem |
| 17 | + |
| 18 | +from ..base import ( |
| 19 | + DetectorType, |
| 20 | + PerformanceDetector, |
| 21 | + fingerprint_http_spans, |
| 22 | + get_notification_attachment_body, |
| 23 | + get_span_evidence_value, |
| 24 | + get_url_from_span, |
| 25 | + parameterize_url, |
| 26 | + parameterize_url_with_result, |
| 27 | +) |
| 28 | +from ..types import PerformanceProblemsMap, Span |
| 29 | + |
| 30 | +logger = logging.getLogger(__name__) |
| 31 | + |
| 32 | + |
| 33 | +class DummyNPlusOneAPICallsDetector(PerformanceDetector): |
| 34 | + """ |
| 35 | + Detect parallel network calls to the same endpoint. |
| 36 | +
|
| 37 | + [-------- transaction -----------] |
| 38 | + [-------- parent span -----------] |
| 39 | + [n0] https://service.io/resources/?id=12443 |
| 40 | + [n1] https://service.io/resources/?id=13342 |
| 41 | + [n2] https://service.io/resources/?id=13441 |
| 42 | + ... |
| 43 | + """ |
| 44 | + |
| 45 | + __slots__: list[str] = [] |
| 46 | + type = DetectorType.N_PLUS_ONE_API_CALLS |
| 47 | + settings_key = DetectorType.N_PLUS_ONE_API_CALLS |
| 48 | + |
| 49 | + def __init__(self, settings: dict[DetectorType, Any], event: dict[str, Any]) -> None: |
| 50 | + super().__init__(settings, event) |
| 51 | + |
| 52 | + # TODO: Only store the span IDs and timestamps instead of entire span objects |
| 53 | + self.stored_problems: PerformanceProblemsMap = {} |
| 54 | + self.spans: list[Span] = [] |
| 55 | + |
| 56 | + def visit_span(self, span: Span) -> None: |
| 57 | + if not DummyNPlusOneAPICallsDetector.is_span_eligible(span): |
| 58 | + return |
| 59 | + |
| 60 | + op = span.get("op", None) |
| 61 | + if op not in self.settings.get("allowed_span_ops", []): |
| 62 | + return |
| 63 | + |
| 64 | + previous_span = self.spans[-1] if len(self.spans) > 0 else None |
| 65 | + |
| 66 | + if previous_span is None: |
| 67 | + self.spans.append(span) |
| 68 | + elif self._spans_are_concurrent(previous_span, span) and self._spans_are_similar( |
| 69 | + previous_span, span |
| 70 | + ): |
| 71 | + self.spans.append(span) |
| 72 | + else: |
| 73 | + self._maybe_store_problem() |
| 74 | + self.spans = [span] |
| 75 | + |
| 76 | + def is_creation_allowed_for_organization(self, organization: Organization) -> bool: |
| 77 | + return True |
| 78 | + |
| 79 | + def is_creation_allowed_for_project(self, project: Project) -> bool: |
| 80 | + return self.settings["detection_enabled"] |
| 81 | + |
| 82 | + @classmethod |
| 83 | + def is_event_eligible(cls, event: dict[str, Any], project: Project | None = None) -> bool: |
| 84 | + trace_op = event.get("contexts", {}).get("trace", {}).get("op") |
| 85 | + if trace_op and trace_op not in ["navigation", "pageload", "ui.load", "ui.action"]: |
| 86 | + return False |
| 87 | + |
| 88 | + return True |
| 89 | + |
| 90 | + @classmethod |
| 91 | + def is_span_eligible(cls, span: Span) -> bool: |
| 92 | + span_id = span.get("span_id", None) |
| 93 | + op = span.get("op", None) |
| 94 | + hash = span.get("hash", None) |
| 95 | + |
| 96 | + if not span_id or not op or not hash: |
| 97 | + return False |
| 98 | + |
| 99 | + description = span.get("description") |
| 100 | + if not description: |
| 101 | + return False |
| 102 | + |
| 103 | + if description.strip()[:3].upper() != "GET": |
| 104 | + return False |
| 105 | + |
| 106 | + url = get_url_from_span(span) |
| 107 | + |
| 108 | + # GraphQL URLs have complicated queries in them. Until we parse those |
| 109 | + # queries to check for what's duplicated, we can't tell what is being |
| 110 | + # duplicated. Ignore them for now |
| 111 | + if "graphql" in url: |
| 112 | + return False |
| 113 | + |
| 114 | + # Next.js infixes its data URLs with a build ID. (e.g., |
| 115 | + # /_next/data/<uuid>/some-endpoint) This causes a fingerprinting |
| 116 | + # explosion, since every deploy would change this ID and create new |
| 117 | + # fingerprints. Since we're not parameterizing URLs yet, we need to |
| 118 | + # exclude them |
| 119 | + if "_next/data" in url: |
| 120 | + return False |
| 121 | + |
| 122 | + # Next.js error pages cause an N+1 API Call that isn't useful to anyone |
| 123 | + if "__nextjs_original-stack-frame" in url: |
| 124 | + return False |
| 125 | + |
| 126 | + if not url: |
| 127 | + return False |
| 128 | + |
| 129 | + # Once most users update their SDKs to use the latest standard, we |
| 130 | + # won't have to do this, since the URLs will be sent in as `span.data` |
| 131 | + # in a parsed format |
| 132 | + parsed_url = urlparse(str(url)) |
| 133 | + |
| 134 | + # Ignore anything that looks like an asset. Some frameworks (and apps) |
| 135 | + # fetch assets via XHR, which is not our concern |
| 136 | + _pathname, extension = os.path.splitext(parsed_url.path) |
| 137 | + if extension and extension in [".js", ".css", ".svg", ".png", ".mp3", ".jpg", ".jpeg"]: |
| 138 | + return False |
| 139 | + |
| 140 | + return True |
| 141 | + |
| 142 | + def on_complete(self) -> None: |
| 143 | + self._maybe_store_problem() |
| 144 | + self.spans = [] |
| 145 | + |
| 146 | + def _maybe_store_problem(self) -> None: |
| 147 | + if len(self.spans) < 1: |
| 148 | + return |
| 149 | + |
| 150 | + if len(self.spans) < self.settings["count"]: |
| 151 | + return |
| 152 | + |
| 153 | + total_duration = get_total_span_duration(self.spans) |
| 154 | + if total_duration < self.settings["total_duration"]: |
| 155 | + return |
| 156 | + |
| 157 | + last_span = self.spans[-1] |
| 158 | + |
| 159 | + fingerprint = self._fingerprint() |
| 160 | + if not fingerprint: |
| 161 | + return |
| 162 | + |
| 163 | + offender_span_ids = [span["span_id"] for span in self.spans] |
| 164 | + problem_description = self._get_parameterized_url(self.spans[0]) |
| 165 | + if problem_description == "": |
| 166 | + problem_description = os.path.commonprefix( |
| 167 | + [span.get("description", "") or "" for span in self.spans] |
| 168 | + ) |
| 169 | + |
| 170 | + problem = PerformanceProblem( |
| 171 | + fingerprint=fingerprint, |
| 172 | + op=last_span["op"], |
| 173 | + desc=problem_description, |
| 174 | + type=PerformanceNPlusOneAPICallsGroupType, |
| 175 | + cause_span_ids=[], |
| 176 | + parent_span_ids=[last_span.get("parent_span_id", None)], |
| 177 | + offender_span_ids=offender_span_ids, |
| 178 | + evidence_data={ |
| 179 | + "op": last_span["op"], |
| 180 | + "cause_span_ids": [], |
| 181 | + "parent_span_ids": [last_span.get("parent_span_id", None)], |
| 182 | + "offender_span_ids": offender_span_ids, |
| 183 | + "transaction_name": self._event.get("transaction", ""), |
| 184 | + "num_repeating_spans": str(len(offender_span_ids)) if offender_span_ids else "", |
| 185 | + "repeating_spans": self._get_path_prefix(self.spans[0]), |
| 186 | + "repeating_spans_compact": get_span_evidence_value(self.spans[0], include_op=False), |
| 187 | + "parameters": self._get_parameters()["query_params"], |
| 188 | + "path_parameters": self._get_parameters()["path_params"], |
| 189 | + }, |
| 190 | + evidence_display=[ |
| 191 | + IssueEvidence( |
| 192 | + name="Offending Spans", |
| 193 | + value=get_notification_attachment_body( |
| 194 | + op=last_span["op"], desc=problem_description |
| 195 | + ), |
| 196 | + # Has to be marked important to be displayed in the notifications |
| 197 | + important=True, |
| 198 | + ) |
| 199 | + ], |
| 200 | + ) |
| 201 | + logger.info("new_issue", extra={"problem": problem.to_dict()}) |
| 202 | + |
| 203 | + def _get_parameters(self) -> dict[str, list[str]]: |
| 204 | + if not self.spans or len(self.spans) == 0: |
| 205 | + return {"query_params": [], "path_params": []} |
| 206 | + |
| 207 | + parameterized_urls = [ |
| 208 | + parameterize_url_with_result(get_url_from_span(span)) for span in self.spans |
| 209 | + ] |
| 210 | + path_params = [param["path_params"] for param in parameterized_urls] |
| 211 | + query_dict: Mapping[str, list[str]] = defaultdict(list) |
| 212 | + |
| 213 | + for parameterized_url in parameterized_urls: |
| 214 | + query_params = parameterized_url["query_params"] |
| 215 | + |
| 216 | + for key, value in query_params.items(): |
| 217 | + query_dict[key] += value |
| 218 | + return { |
| 219 | + "path_params": [f"{', '.join(param_group)}" for param_group in path_params], |
| 220 | + "query_params": [f"{key}: {', '.join(values)}" for key, values in query_dict.items()], |
| 221 | + } |
| 222 | + |
| 223 | + def _get_parameterized_url(self, span: Span) -> str: |
| 224 | + return parameterize_url(get_url_from_span(span)) |
| 225 | + |
| 226 | + def _get_path_prefix(self, repeating_span: Span) -> str: |
| 227 | + if not repeating_span: |
| 228 | + return "" |
| 229 | + |
| 230 | + url = get_url_from_span(repeating_span) |
| 231 | + parsed_url = urlparse(url) |
| 232 | + return parsed_url.path or "" |
| 233 | + |
| 234 | + def _fingerprint(self) -> str | None: |
| 235 | + first_url = get_url_from_span(self.spans[0]) |
| 236 | + parameterized_first_url = parameterize_url(first_url) |
| 237 | + |
| 238 | + # Check if we parameterized the URL at all. If not, do not attempt |
| 239 | + # fingerprinting. Unparameterized URLs run too high a risk of |
| 240 | + # fingerprinting explosions. |
| 241 | + if parameterized_first_url == first_url: |
| 242 | + return None |
| 243 | + |
| 244 | + fingerprint = fingerprint_http_spans([self.spans[0]]) |
| 245 | + |
| 246 | + return f"1-{PerformanceNPlusOneAPICallsGroupType.type_id}-{fingerprint}" |
| 247 | + |
| 248 | + def _spans_are_concurrent(self, span_a: Span, span_b: Span) -> bool: |
| 249 | + span_a_start: int = span_a.get("start_timestamp", 0) or 0 |
| 250 | + span_b_start: int = span_b.get("start_timestamp", 0) or 0 |
| 251 | + |
| 252 | + return timedelta(seconds=abs(span_a_start - span_b_start)) < timedelta( |
| 253 | + milliseconds=self.settings["concurrency_threshold"] |
| 254 | + ) |
| 255 | + |
| 256 | + def _spans_are_similar(self, span_a: Span, span_b: Span) -> bool: |
| 257 | + return ( |
| 258 | + self._get_parameterized_url(span_a) == self._get_parameterized_url(span_b) |
| 259 | + and span_a["parent_span_id"] == span_b["parent_span_id"] |
| 260 | + ) |
0 commit comments