Skip to content

Commit 7323263

Browse files
authored
feat(relocation): Allow relocation artifacts to be downloaded (#69444)
If the relocation artifact is a `.tar` file, it is first decrypted. This ability is only available to Superusers/staff who have the `relocation.admin` permission.
1 parent edadfa8 commit 7323263

File tree

3 files changed

+341
-0
lines changed

3 files changed

+341
-0
lines changed
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import logging
2+
3+
from cryptography.fernet import Fernet
4+
from rest_framework.exceptions import PermissionDenied
5+
from rest_framework.request import Request
6+
from rest_framework.response import Response
7+
8+
from sentry.api.api_owners import ApiOwner
9+
from sentry.api.api_publish_status import ApiPublishStatus
10+
from sentry.api.base import Endpoint, region_silo_endpoint
11+
from sentry.api.exceptions import ResourceDoesNotExist, StaffRequired, SuperuserRequired
12+
from sentry.api.permissions import SuperuserOrStaffFeatureFlaggedPermission
13+
from sentry.auth.elevated_mode import has_elevated_mode
14+
from sentry.auth.staff import has_staff_option
15+
from sentry.backup.crypto import (
16+
GCPKMSDecryptor,
17+
get_default_crypto_key_version,
18+
unwrap_encrypted_export_tarball,
19+
)
20+
from sentry.models.files.utils import get_relocation_storage
21+
from sentry.models.relocation import Relocation
22+
from sentry.utils import json
23+
24+
ERR_NEED_RELOCATION_ADMIN = (
25+
"Cannot view relocation artifacts, as you do not have the appropriate permissions."
26+
)
27+
28+
logger = logging.getLogger(__name__)
29+
30+
31+
@region_silo_endpoint
32+
class RelocationArtifactDetailsEndpoint(Endpoint):
33+
owner = ApiOwner.OPEN_SOURCE
34+
publish_status = {
35+
# TODO(getsentry/team-ospo#214): Stabilize before GA.
36+
"GET": ApiPublishStatus.EXPERIMENTAL,
37+
}
38+
permission_classes = (SuperuserOrStaffFeatureFlaggedPermission,)
39+
40+
def get(
41+
self, request: Request, relocation_uuid: str, artifact_kind: str, file_name: str
42+
) -> Response:
43+
"""
44+
Get a single relocation artifact.
45+
``````````````````````````````````````````````````
46+
47+
:pparam string relocation_uuid: a UUID identifying the relocation.
48+
:pparam string artifact_kind: one of `conf` | `in` | `out` | `findings`.
49+
:pparam string file_name: The name of the file itself.
50+
51+
:auth: required
52+
"""
53+
54+
logger.info("relocations.artifact.details.get.start", extra={"caller": request.user.id})
55+
56+
# TODO(schew2381): Remove the superuser reference below after feature flag is removed.
57+
# Must be superuser/staff AND have a `UserPermission` of `relocation.admin` to see access!
58+
if not has_elevated_mode(request):
59+
if has_staff_option(request.user):
60+
raise StaffRequired
61+
raise SuperuserRequired
62+
63+
if not request.access.has_permission("relocation.admin"):
64+
raise PermissionDenied(
65+
"Cannot view relocation artifacts, as you do not have the appropriate permissions."
66+
)
67+
68+
try:
69+
relocation: Relocation = Relocation.objects.get(uuid=relocation_uuid)
70+
except Relocation.DoesNotExist:
71+
raise ResourceDoesNotExist
72+
73+
file_path = f"runs/{relocation.uuid}/{artifact_kind}/{file_name}"
74+
relocation_storage = get_relocation_storage()
75+
if not relocation_storage.exists(file_path):
76+
raise ResourceDoesNotExist
77+
78+
# TODO(azaslavsky): We can probably get all clever and stream these files, but it's not
79+
# necessary for now.
80+
with relocation_storage.open(file_path) as fp:
81+
if not file_name.endswith(".tar"):
82+
return self.respond({"contents": fp.read()})
83+
84+
unwrapped = unwrap_encrypted_export_tarball(fp)
85+
decryptor = GCPKMSDecryptor.from_bytes(
86+
json.dumps(get_default_crypto_key_version()).encode("utf-8")
87+
)
88+
plaintext_data_encryption_key = decryptor.decrypt_data_encryption_key(unwrapped)
89+
fernet = Fernet(plaintext_data_encryption_key)
90+
return self.respond(
91+
{"contents": fernet.decrypt(unwrapped.encrypted_json_blob).decode("utf-8")}
92+
)

src/sentry/api/urls.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
ReleaseThresholdStatusIndexEndpoint,
4242
)
4343
from sentry.api.endpoints.relocations.abort import RelocationAbortEndpoint
44+
from sentry.api.endpoints.relocations.artifacts.details import RelocationArtifactDetailsEndpoint
4445
from sentry.api.endpoints.relocations.artifacts.index import RelocationArtifactIndexEndpoint
4546
from sentry.api.endpoints.relocations.cancel import RelocationCancelEndpoint
4647
from sentry.api.endpoints.relocations.details import RelocationDetailsEndpoint
@@ -870,6 +871,11 @@ def create_group_urls(name_prefix: str) -> list[URLPattern | URLResolver]:
870871
RelocationArtifactIndexEndpoint.as_view(),
871872
name="sentry-api-0-relocations-artifacts-index",
872873
),
874+
re_path(
875+
r"^(?P<relocation_uuid>[^\/]+)/artifacts/(?P<artifact_kind>[^\/]+)/(?P<file_name>[^\/]+)$",
876+
RelocationArtifactDetailsEndpoint.as_view(),
877+
name="sentry-api-0-relocations-artifacts-details",
878+
),
873879
]
874880

875881
RELAY_URLS = [
Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
from datetime import datetime, timezone
2+
from io import BytesIO, StringIO
3+
from pathlib import Path
4+
from tempfile import TemporaryDirectory
5+
from types import SimpleNamespace
6+
from unittest.mock import patch
7+
from uuid import uuid4
8+
9+
from google_crc32c import value as crc32c
10+
11+
from sentry.api.endpoints.relocations.artifacts.index import ERR_NEED_RELOCATION_ADMIN
12+
from sentry.backup.crypto import (
13+
LocalFileDecryptor,
14+
LocalFileEncryptor,
15+
create_encrypted_export_tarball,
16+
unwrap_encrypted_export_tarball,
17+
)
18+
from sentry.models.files.utils import get_relocation_storage
19+
from sentry.models.relocation import Relocation
20+
from sentry.testutils.cases import APITestCase
21+
from sentry.testutils.helpers.backups import FakeKeyManagementServiceClient, generate_rsa_key_pair
22+
from sentry.testutils.helpers.options import override_options
23+
from sentry.utils.relocation import OrderedTask
24+
25+
TEST_DATE_ADDED = datetime(2023, 1, 23, 1, 23, 45, tzinfo=timezone.utc)
26+
RELOCATION_ADMIN_PERMISSION = "relocation.admin"
27+
28+
29+
class GetRelocationArtifactDetailsTest(APITestCase):
30+
endpoint = "sentry-api-0-relocations-artifacts-details"
31+
method = "GET"
32+
33+
def setUp(self):
34+
super().setUp()
35+
self.owner = self.create_user(email="owner@example.com", is_superuser=False, is_staff=False)
36+
self.superuser = self.create_user(is_superuser=True)
37+
self.staff_user = self.create_user(is_staff=True)
38+
self.relocation: Relocation = Relocation.objects.create(
39+
date_added=TEST_DATE_ADDED,
40+
creator_id=self.owner.id,
41+
owner_id=self.owner.id,
42+
status=Relocation.Status.PAUSE.value,
43+
step=Relocation.Step.PREPROCESSING.value,
44+
want_org_slugs=["foo"],
45+
want_usernames=["alice", "bob"],
46+
latest_notified=Relocation.EmailKind.STARTED.value,
47+
latest_task=OrderedTask.PREPROCESSING_SCAN.name,
48+
latest_task_attempts=1,
49+
)
50+
51+
52+
class GetRelocationArtifactDetailsGoodTest(GetRelocationArtifactDetailsTest):
53+
def setUp(self):
54+
super().setUp()
55+
dir = f"runs/{self.relocation.uuid}"
56+
self.relocation_storage = get_relocation_storage()
57+
58+
# These files are unencrypted, so just save the file name as the content for testing
59+
# purposes.
60+
self.relocation_storage.save(
61+
f"{dir}/somedir/file.json", StringIO(f'"{dir}/somedir/file.json"')
62+
)
63+
64+
# `.tar` files should be encrypted.
65+
with TemporaryDirectory() as tmp_dir:
66+
(priv_key_pem, pub_key_pem) = generate_rsa_key_pair()
67+
tmp_priv_key_path = Path(tmp_dir).joinpath("key")
68+
self.priv_key_pem = priv_key_pem
69+
with open(tmp_priv_key_path, "wb") as f:
70+
f.write(priv_key_pem)
71+
72+
tmp_pub_key_path = Path(tmp_dir).joinpath("key.pub")
73+
self.pub_key_pem = pub_key_pem
74+
with open(tmp_pub_key_path, "wb") as f:
75+
f.write(pub_key_pem)
76+
77+
with open(tmp_pub_key_path, "rb") as p:
78+
self.tarball = create_encrypted_export_tarball(
79+
f"{dir}/encrypted/file.tar", LocalFileEncryptor(p)
80+
).getvalue()
81+
self.relocation_storage.save(f"{dir}/encrypted/file.tar", BytesIO(self.tarball))
82+
83+
def mock_kms_client(self, fake_kms_client: FakeKeyManagementServiceClient):
84+
fake_kms_client.asymmetric_decrypt.call_count = 0
85+
fake_kms_client.get_public_key.call_count = 0
86+
87+
unwrapped = unwrap_encrypted_export_tarball(BytesIO(self.tarball))
88+
plaintext_dek = LocalFileDecryptor.from_bytes(
89+
self.priv_key_pem
90+
).decrypt_data_encryption_key(unwrapped)
91+
92+
fake_kms_client.asymmetric_decrypt.return_value = SimpleNamespace(
93+
plaintext=plaintext_dek,
94+
plaintext_crc32c=crc32c(plaintext_dek),
95+
)
96+
fake_kms_client.asymmetric_decrypt.side_effect = None
97+
98+
fake_kms_client.get_public_key.return_value = SimpleNamespace(
99+
pem=self.pub_key_pem.decode("utf-8")
100+
)
101+
fake_kms_client.get_public_key.side_effect = None
102+
103+
@patch(
104+
"sentry.backup.crypto.KeyManagementServiceClient",
105+
new_callable=lambda: FakeKeyManagementServiceClient,
106+
)
107+
def test_good_unencrypted_with_superuser(
108+
self, fake_kms_client: FakeKeyManagementServiceClient
109+
) -> None:
110+
self.mock_kms_client(fake_kms_client)
111+
self.add_user_permission(self.superuser, RELOCATION_ADMIN_PERMISSION)
112+
self.login_as(user=self.superuser, superuser=True)
113+
response = self.get_success_response(str(self.relocation.uuid), "somedir", "file.json")
114+
115+
assert fake_kms_client.asymmetric_decrypt.call_count == 0
116+
assert (
117+
response.data["contents"] == f'"runs/{self.relocation.uuid}/somedir/file.json"'.encode()
118+
)
119+
120+
@patch(
121+
"sentry.backup.crypto.KeyManagementServiceClient",
122+
new_callable=lambda: FakeKeyManagementServiceClient,
123+
)
124+
def test_good_encrypted_with_superuser(
125+
self, fake_kms_client: FakeKeyManagementServiceClient
126+
) -> None:
127+
self.mock_kms_client(fake_kms_client)
128+
self.add_user_permission(self.superuser, RELOCATION_ADMIN_PERMISSION)
129+
self.login_as(user=self.superuser, superuser=True)
130+
response = self.get_success_response(str(self.relocation.uuid), "encrypted", "file.tar")
131+
132+
assert fake_kms_client.asymmetric_decrypt.call_count == 1
133+
assert str(response.data["contents"]) == f'"runs/{self.relocation.uuid}/encrypted/file.tar"'
134+
135+
@override_options({"staff.ga-rollout": True})
136+
@patch(
137+
"sentry.backup.crypto.KeyManagementServiceClient",
138+
new_callable=lambda: FakeKeyManagementServiceClient,
139+
)
140+
def test_good_unencrypted_with_staff(
141+
self, fake_kms_client: FakeKeyManagementServiceClient
142+
) -> None:
143+
self.mock_kms_client(fake_kms_client)
144+
self.add_user_permission(self.staff_user, RELOCATION_ADMIN_PERMISSION)
145+
self.login_as(user=self.staff_user, staff=True)
146+
response = self.get_success_response(str(self.relocation.uuid), "somedir", "file.json")
147+
148+
assert fake_kms_client.asymmetric_decrypt.call_count == 0
149+
assert (
150+
response.data["contents"] == f'"runs/{self.relocation.uuid}/somedir/file.json"'.encode()
151+
)
152+
153+
@override_options({"staff.ga-rollout": True})
154+
@patch(
155+
"sentry.backup.crypto.KeyManagementServiceClient",
156+
new_callable=lambda: FakeKeyManagementServiceClient,
157+
)
158+
def test_good_encrypted_with_staff(
159+
self, fake_kms_client: FakeKeyManagementServiceClient
160+
) -> None:
161+
self.mock_kms_client(fake_kms_client)
162+
self.add_user_permission(self.staff_user, RELOCATION_ADMIN_PERMISSION)
163+
self.login_as(user=self.staff_user, staff=True)
164+
response = self.get_success_response(str(self.relocation.uuid), "encrypted", "file.tar")
165+
166+
assert fake_kms_client.asymmetric_decrypt.call_count == 1
167+
assert str(response.data["contents"]) == f'"runs/{self.relocation.uuid}/encrypted/file.tar"'
168+
169+
170+
class GetRelocationArtifactDetailsBadTest(GetRelocationArtifactDetailsTest):
171+
def setUp(self):
172+
super().setUp()
173+
dir = f"runs/{self.relocation.uuid}"
174+
self.relocation_storage = get_relocation_storage()
175+
176+
# These files are unencrypted, so just save the file name as the content for testing
177+
# purposes.
178+
self.relocation_storage.save(
179+
f"{dir}/somedir/file.json", StringIO(f'"{dir}/somedir/file.json"')
180+
)
181+
182+
@override_options({"staff.ga-rollout": True})
183+
def test_bad_unprivileged_user(self):
184+
self.login_as(user=self.owner, superuser=False, staff=False)
185+
186+
# Ensures we don't reveal existence info to improperly authenticated users.
187+
does_not_exist_uuid = uuid4().hex
188+
self.get_error_response(str(does_not_exist_uuid), "somedir", "file.json", status_code=403)
189+
190+
def test_bad_superuser_disabled(self):
191+
self.add_user_permission(self.superuser, RELOCATION_ADMIN_PERMISSION)
192+
self.login_as(user=self.superuser, superuser=False)
193+
194+
# Ensures we don't reveal existence info to improperly authenticated users.
195+
does_not_exist_uuid = uuid4().hex
196+
self.get_error_response(str(does_not_exist_uuid), "somedir", "file.json", status_code=403)
197+
198+
@override_options({"staff.ga-rollout": True})
199+
def test_bad_staff_disabled(self):
200+
self.add_user_permission(self.staff_user, RELOCATION_ADMIN_PERMISSION)
201+
self.login_as(user=self.staff_user, staff=False)
202+
203+
# Ensures we don't reveal existence info to improperly authenticated users.
204+
does_not_exist_uuid = uuid4().hex
205+
self.get_error_response(str(does_not_exist_uuid), "somedir", "file.json", status_code=403)
206+
207+
def test_bad_has_superuser_but_no_relocation_admin_permission(self):
208+
self.login_as(user=self.superuser, superuser=True)
209+
210+
# Ensures we don't reveal existence info to improperly authenticated users.
211+
does_not_exist_uuid = uuid4().hex
212+
response = self.get_error_response(
213+
str(does_not_exist_uuid), "somedir", "file.json", status_code=403
214+
)
215+
216+
assert response.data.get("detail") == ERR_NEED_RELOCATION_ADMIN
217+
218+
@override_options({"staff.ga-rollout": True})
219+
def test_bad_has_staff_but_no_relocation_admin_permission(self):
220+
self.login_as(user=self.staff_user, staff=True)
221+
222+
# Ensures we don't reveal existence info to improperly authenticated users.
223+
does_not_exist_uuid = uuid4().hex
224+
response = self.get_error_response(
225+
str(does_not_exist_uuid), "somedir", "file.json", status_code=403
226+
)
227+
228+
assert response.data.get("detail") == ERR_NEED_RELOCATION_ADMIN
229+
230+
@override_options({"staff.ga-rollout": True})
231+
def test_bad_relocation_not_found(self):
232+
self.add_user_permission(self.staff_user, RELOCATION_ADMIN_PERMISSION)
233+
self.login_as(user=self.staff_user, staff=True)
234+
does_not_exist_uuid = uuid4().hex
235+
self.get_error_response(str(does_not_exist_uuid), "somedir", "file.json", status_code=404)
236+
237+
@override_options({"staff.ga-rollout": True})
238+
def test_bad_file_not_found(self):
239+
self.add_user_permission(self.staff_user, RELOCATION_ADMIN_PERMISSION)
240+
self.login_as(user=self.staff_user, staff=True)
241+
self.get_error_response(
242+
str(self.relocation.uuid), "nonexistent", "file.json", status_code=404
243+
)

0 commit comments

Comments
 (0)