Gzip zenodo export

internetofwater · Mar 4, 2025 · aa814b5 · aa814b5
1 parent 662659e
commit aa814b5
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/userCode/exports.py b/userCode/exports.py
@@ -163,7 +163,7 @@ def nquads_to_zenodo(
 
     # Use the deposit ID to upload the file
     response = requests.put(
-        deposit["links"]["bucket"] + "/geoconnex-graph.nq",
+        deposit["links"]["bucket"] + "/geoconnex-graph.nq.gz",
         data=stream,
         headers={"Authorization": f"Bearer {TOKEN}"},
     )

diff --git a/userCode/lib/classes.py b/userCode/lib/classes.py
@@ -114,13 +114,15 @@ def read(self, remote_path: str) -> bytes:
         response.release_conn()
         return data
 
-    def read_stream(self, remote_path: str):
+    def read_stream(self, remote_path: str, decode_content: bool = False):
         """Read an object from S3 as a stream"""
         response: BaseHTTPResponse = self.client.get_object(
             GLEANER_MINIO_BUCKET, remote_path
         )
         try:
-            for chunk in response.stream(8 * 1024 * 1024, decode_content=True):
+            for chunk in response.stream(
+                8 * 1024 * 1024, decode_content=decode_content
+            ):
                 yield chunk
         finally:
             response.close()