Skip to content

Commit

Permalink
Patch rclone path
Browse files Browse the repository at this point in the history
  • Loading branch information
webb-ben committed Mar 4, 2025
1 parent 6080da6 commit 66457dc
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions userCode/lib/classes.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,10 @@ def read_stream(self, remote_path: str, decode_content: bool = False):
class RcloneClient:
"""Helper class to transfer files from minio to lakefs using rclone"""

@staticmethod
def get_rclone():
return "rclone" if RUNNING_AS_TEST_OR_DEV() else "/root/.local/bin/rclone"

@classmethod
def get_config_path(cls) -> Path:
"""
Expand All @@ -140,7 +144,7 @@ def get_config_path(cls) -> Path:
"""
# Run the command and capture its output
result = subprocess.run(
["rclone", "config", "file"],
[cls.get_rclone(), "config", "file"],
text=True, # Ensure output is returned as a string
stdout=subprocess.PIPE, # Capture standard output
stderr=subprocess.PIPE, # Capture standard error
Expand Down Expand Up @@ -208,7 +212,7 @@ def copy_to_lakefs(
else f"gs:{GLEANER_MINIO_BUCKET}/{path_to_file} --gcs-decompress --s3-use-accept-encoding-gzip=true --s3-might-gzip=true"
)
dst_ = f"lakefs:geoconnex/{destination_branch}/{destination_filename} --s3-upload-concurrency 8"
cmd_to_run = f"rclone copyto {src_} {dst_} -v"
cmd_to_run = f"{self.get_rclone()} copyto {src_} {dst_} -v"
get_dagster_logger().info(f"Running bash command: {cmd_to_run}")
self._run_subprocess(cmd_to_run)

Expand Down

0 comments on commit 66457dc

Please sign in to comment.