Fix Zenodo export

internetofwater · Mar 3, 2025 · cafa06a · cafa06a
1 parent 58db0a0
commit cafa06a
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 5 deletions.
diff --git a/tests/lib_test/classes_test.py b/tests/lib_test/classes_test.py
@@ -21,6 +21,6 @@ def test_s3_read_stream():
     stream = S3().read_stream("streamKey")
     assert stream is not None
     data = b""
-    for chunk in stream.stream(amt=1024 * 1024):
+    for chunk in stream:
         data += chunk
     assert data == longData
diff --git a/userCode/exports.py b/userCode/exports.py
@@ -164,9 +164,9 @@ def nquads_to_zenodo(
     get_dagster_logger().info(f"Deposit created with ID: {deposit_id}")
 
     # Use the deposit ID to upload the file
-    response = requests.post(
-        f"{ZENODO_API_URL}/{deposit_id}/files",
-        files={"file": ("nquads.nt", stream)},
+    response = requests.put(
+        deposit["links"]["bucket"] + "/" + "nquads.nt",
+        data=stream,
         headers={"Authorization": f"Bearer {TOKEN}"},
     )
 

diff --git a/userCode/lib/classes.py b/userCode/lib/classes.py
@@ -119,7 +119,12 @@ def read_stream(self, remote_path: str):
         response: BaseHTTPResponse = self.client.get_object(
             GLEANER_MINIO_BUCKET, remote_path
         )
-        return response  # Return the stream directly
+        try:
+            for chunk in response.stream(8 * 1024 * 1024):
+                yield chunk
+        finally:
+            response.close()
+            response.release_conn()
 
 
 class RcloneClient: