Fix frink upload

internetofwater · Mar 3, 2025 · 7e5ab89 · 7e5ab89
1 parent cafa06a
commit 7e5ab89
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 9 deletions.
diff --git a/userCode/lib/classes.py b/userCode/lib/classes.py
@@ -204,7 +204,7 @@ def copy_to_lakefs(
 
         new_branch = lakefs_client.create_branch_if_not_exists(destination_branch)
 
-        cmd_to_run = f"{self.rclone_client()} copyto s3:{GLEANER_MINIO_BUCKET}/{path_to_file} lakefs:geoconnex/{destination_branch}/{destination_filename} -v"
+        cmd_to_run = f"{self.rclone_client()} copyto storage:{GLEANER_MINIO_BUCKET}/{path_to_file} lakefs:geoconnex/{destination_branch}/{destination_filename} -v --s3-upload-concurrency 8"
         get_dagster_logger().info(f"Running bash command: {cmd_to_run}")
         self._run_subprocess(cmd_to_run)
 

diff --git a/userCode/templates/rclone.conf.j2 b/userCode/templates/rclone.conf.j2
@@ -22,11 +22,9 @@ secret_access_key = {{ MINIO_SECRET_KEY }}
 endpoint = http://{{ GLEANERIO_MINIO_ADDRESS }}:{{ GLEANERIO_MINIO_PORT }}
 {%- else -%}
 [s3]
-type = s3
-provider = GCS
-env_auth = false
-region   = {{ GLEANERIO_MINIO_REGION }}
-access_key_id = {{ MINIO_ACCESS_KEY }}
-secret_access_key = {{ MINIO_SECRET_KEY }}
-endpoint = https://{{ GLEANERIO_MINIO_ADDRESS }}
-{%- endif %}
+type = google cloud storage
+object_acl = projectPrivate
+bucket_acl = publicReadWrite
+location = {{ GLEANERIO_MINIO_REGION }}
+env_auth = true
+{%- endif %}