Merge pull request #209 from ironArray/fixCaching

Improve security of caching

Author: lshaw8317

caterva2/services/sub.py

@@ -817,7 +817,8 @@ async def fetch_data(
         if field:
             raise ArgumentError("Cannot handle both field and filter parameters at the same time")
         filter = filter.strip()
-        container, _ = get_filtered_array(abspath, path, filter, sortby=None)
+        mtime = abspath.stat().st_mtime
+        container, _ = get_filtered_array(abspath, path, filter, sortby=None, mtime=mtime)
     else:
         container = open_b2(abspath, path)
 
@@ -1900,9 +1901,9 @@ async def htmx_path_info(
     return response
 
 
-# Global dictionary to store objects
+# Added mtime to implicitly check when underlying files are changed, and so can't use cache (see issue #207)
 @functools.lru_cache(maxsize=16)
-def get_filtered_array(abspath, path, filter, sortby):
+def get_filtered_array(abspath, path, filter, sortby, mtime):
     arr = open_b2(abspath, path)
     has_ndfields = hasattr(arr, "fields") and arr.fields != {}
     assert has_ndfields
@@ -1954,7 +1955,8 @@ async def htmx_path_view(
     filter = filter.strip()
     if filter or sortby:
         try:
-            arr, idx = get_filtered_array(abspath, path, filter, sortby)
+            mtime = abspath.stat().st_mtime
+            arr, idx = get_filtered_array(abspath, path, filter, sortby, mtime)
         except TypeError as exc:
             return htmx_error(request, f"Error in filter: {exc}")
         except NameError as exc:

caterva2/tests/test_api.py

@@ -758,6 +758,47 @@ def test_lazyexpr_fields(auth_client):
     [np.testing.assert_array_equal(servered[f], downloaded[f]) for f in downloaded.dtype.fields]
 
 
+def test_lazyexpr_cache(auth_client):
+    if not auth_client:
+        pytest.skip("authentication support needed")
+
+    root = auth_client.get("@personal")
+    oppt = f"{root.name}/sa-1M.b2nd"
+
+    N = 1000
+    rng = np.random.default_rng(seed=1)
+    it = ((-x + 1, x - 2, x) for x in range(N))
+    sa = blosc2.fromiter(
+        it, dtype=[("A", "i4"), ("B", "f4"), ("C", "f8")], shape=(N,), urlpath="sa-1M.b2nd", mode="w"
+    )
+    path = auth_client.upload("sa-1M.b2nd", oppt)
+    arr = auth_client.get(path)
+
+    # Test a lazyexpr
+    arr = auth_client.get(oppt)
+    servered = arr["(A < 500) & (B >= .1)"][:]
+    downloaded = arr.slice(None)["(A < 500) & (B >= .1)"][:]
+    [np.testing.assert_array_equal(servered[f], downloaded[f]) for f in downloaded.dtype.fields]
+
+    # Overwrite the file and check that cache isn't used
+    N = 10000
+    rng = np.random.default_rng(seed=1)
+    it = ((-x + 1, x - 2, x) for x in range(N))
+    sa = blosc2.fromiter(
+        it, dtype=[("A", "i4"), ("B", "f4"), ("C", "f8")], shape=(N,), urlpath="sa-1M.b2nd", mode="w"
+    )
+    path = auth_client.upload("sa-1M.b2nd", oppt)
+    arr = auth_client.get(path)
+
+    # Test lazyexpr again
+    servered = arr["(A < - 500) & (B >= .1)"][:]
+    downloaded = arr.slice(None)["(A < - 500) & (B >= .1)"][:]
+    [np.testing.assert_allclose(servered[f], downloaded[f]) for f in downloaded.dtype.fields]
+
+    # remove file
+    arr.remove()
+
+
 def test_expr_from_expr(auth_client):
     if not auth_client:
         pytest.skip("authentication support needed")