feat(nodestore) FilestoreNodeStorage, all credits to #76250

src/sentry/nodestore/filestore/__init__.py

@@ -0,0 +1 @@
+from .backend import FilestoreNodeStorage  # NOQA

src/sentry/nodestore/filestore/backend.py

@@ -0,0 +1,63 @@
+import datetime
+import logging
+from io import BytesIO
+
+from sentry import options as options_store
+from sentry.models.file import get_storage
+from sentry.nodestore.base import NodeStorage
+
+logger = logging.getLogger("sentry.nodestore")
+
+
+class FilestoreNodeStorage(NodeStorage):
+    """
+    A backend that persist nodes to configured File Store.
+    Intended for "s3" or "gcs", which performnace may not be ideal.
+    config.yml Configuration reference:
+    >>> filestore.backend: "s3"
+    ... filestore.options:
+    ...   access_key: "xxx"
+    ...   secret_key: "xxx"
+    ...   endpoint_url: "https://s3.us-east-1.amazonaws.com"
+    ...   bucket_name: "sentry"
+    ...   location: "/sentry"
+    """
+
+    def __init__(self, prefix_path=None):
+        self.prefix_path: str = "nodestore/"
+        backend = options_store.get("filestore.backend")
+        if backend not in ["s3", "gcs"]:
+            logger.warning(
+                "FilestoreNodeStorage was intended for s3 or gcs, currently using: %s", backend
+            )
+        if prefix_path:
+            self.prefix_path = prefix_path
+
+    def _get_bytes(self, id: str):
+        storage = get_storage()
+        path = self.node_path(id)
+        return storage.open(path).read()
+
+    def _set_bytes(self, id: str, data: bytes, ttl=0):
+        storage = get_storage()
+        path = self.node_path(id)
+        storage.save(path, BytesIO(data))
+
+    def delete(self, id):
+        storage = get_storage()
+        path = self.node_path(id)
+        storage.delete(path)
+
+    def cleanup(self, cutoff: datetime.datetime):
+        """
+        This driver does not have managed TTLs.  To enable TTLs you will need to enable it on your
+        bucket.
+        """
+        raise NotImplementedError
+
+    def bootstrap(self):
+        # Nothing for this backend to do during bootstrap
+        pass
+
+    def node_path(self, id: str):
+        return f"{self.prefix_path}{id}.json"

tests/sentry/nodestore/filestore/test_backend.py

@@ -0,0 +1,51 @@
+from datetime import timedelta
+
+import pytest
+from django.utils import timezone
+
+from sentry.nodestore.filestore.backend import FilestoreNodeStorage
+from sentry.testutils.pytest.fixtures import django_db_all
+
+
+@django_db_all
+class TestFilestoreNodeStorage:
+
+    def setup_method(self):
+        self.ns = FilestoreNodeStorage()
+
+    def test_set_and_get(self):
+        id = "d2502ebbd7df41ceba8d3275595cac33"
+        data = {"foo": "bar"}
+        self.ns.set(id, data)
+        result = self.ns.get(id=id)
+        assert result == data
+
+    def test_get_not_exist(self):
+        with pytest.raises(FileNotFoundError):
+            self.ns.get("not_exist_123")
+
+    def test_set_and_get_multi(self):
+        self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
+        self.ns.set("5394aa025b8e401ca6bc3ddee3130edc", {"foo": "baz"})
+
+        result = self.ns.get_multi(
+            ["d2502ebbd7df41ceba8d3275595cac33", "5394aa025b8e401ca6bc3ddee3130edc"]
+        )
+        assert result == {
+            "d2502ebbd7df41ceba8d3275595cac33": {"foo": "bar"},
+            "5394aa025b8e401ca6bc3ddee3130edc": {"foo": "baz"},
+        }
+
+    def test_set_and_delete(self):
+        self.ns.set("d2502ebbd7df41ceba8d3275595cac33", {"foo": "bar"})
+        self.ns.delete("d2502ebbd7df41ceba8d3275595cac33")
+        result = self.ns.get(id="d2502ebbd7df41ceba8d3275595cac33")
+
+        # It seems the default Filestore does not delete file immediately, is this expected?
+        assert result == {"foo": "bar"}
+
+    def test_cleanup(self):
+        now = timezone.now()
+        cutoff = now - timedelta(days=1)
+        with pytest.raises(NotImplementedError):
+            self.ns.cleanup(cutoff)