From 82650ef39ed8a1fb77cefe203276a247b22fd8e2 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sat, 21 Dec 2024 13:06:58 -0500 Subject: [PATCH 1/4] Add cat to CLI --- python/hdfs_native/cli.py | 20 ++++++++++++++++++++ python/tests/test_cli.py | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/python/hdfs_native/cli.py b/python/hdfs_native/cli.py index b931162..b60ce1b 100644 --- a/python/hdfs_native/cli.py +++ b/python/hdfs_native/cli.py @@ -1,5 +1,6 @@ import functools import os +import sys from argparse import ArgumentParser, Namespace from typing import List, Optional, Sequence from urllib.parse import urlparse @@ -48,6 +49,17 @@ def _glob_path(client: Client, glob: str) -> List[str]: return [glob] +def cat(args: Namespace): + for src in args.src: + client = _client_for_url(src) + for path in _glob_path(client, src): + with client.read(path) as file: + while chunk := file.read(1024 * 1024): + sys.stdout.buffer.write(chunk) + + sys.stdout.buffer.flush() + + def mkdir(args: Namespace): create_parent = args.parent @@ -95,6 +107,14 @@ def main(in_args: Optional[Sequence[str]] = None): subparsers = parser.add_subparsers(title="Subcommands", required=True) + cat_parser = subparsers.add_parser( + "cat", + help="Print the contents of a file", + description="Print the contents of a file to stdout", + ) + cat_parser.add_argument("src", nargs="+", help="File pattern to print") + cat_parser.set_defaults(func=cat) + mkdir_parser = subparsers.add_parser( "mkdir", help="Create a directory", diff --git a/python/tests/test_cli.py b/python/tests/test_cli.py index 0047629..bbbe506 100644 --- a/python/tests/test_cli.py +++ b/python/tests/test_cli.py @@ -1,3 +1,6 @@ +import contextlib +import io + import pytest from hdfs_native import Client @@ -7,6 +10,29 @@ def test_cli(minidfs: str): client = Client(minidfs) + # cat + with client.create("/testfile") as file: + file.write(b"1234") + + buf = io.BytesIO() + with contextlib.redirect_stdout(io.TextIOWrapper(buf)): + cli_main(["cat", "/testfile"]) + assert buf.getvalue() == b"1234" + + with client.create("/testfile2") as file: + file.write(b"5678") + + buf = io.BytesIO() + with contextlib.redirect_stdout(io.TextIOWrapper(buf)): + cli_main(["cat", "/testfile", "/testfile2"]) + assert buf.getvalue() == b"12345678" + + with pytest.raises(FileNotFoundError): + cli_main(["cat", "/nonexistent"]) + + client.delete("/testfile") + client.delete("/testfile2") + # mkdir cli_main(["mkdir", "/testdir"]) assert client.get_file_info("/testdir").isdir From b495d8e47e747a2d4da49937b25faa195aa40363 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Wed, 1 Jan 2025 13:34:07 -0500 Subject: [PATCH 2/4] Extract path from URL --- python/hdfs_native/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/hdfs_native/cli.py b/python/hdfs_native/cli.py index b60ce1b..b3c7a65 100644 --- a/python/hdfs_native/cli.py +++ b/python/hdfs_native/cli.py @@ -52,7 +52,7 @@ def _glob_path(client: Client, glob: str) -> List[str]: def cat(args: Namespace): for src in args.src: client = _client_for_url(src) - for path in _glob_path(client, src): + for path in _glob_path(client, _path_for_url(src)): with client.read(path) as file: while chunk := file.read(1024 * 1024): sys.stdout.buffer.write(chunk) From 57fd9fe310835482f52b65305100481d0a493acc Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Wed, 1 Jan 2025 13:36:22 -0500 Subject: [PATCH 3/4] Add a few missing path extractions --- python/hdfs_native/cli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/python/hdfs_native/cli.py b/python/hdfs_native/cli.py index b3c7a65..49e265c 100644 --- a/python/hdfs_native/cli.py +++ b/python/hdfs_native/cli.py @@ -63,9 +63,9 @@ def cat(args: Namespace): def mkdir(args: Namespace): create_parent = args.parent - for path in args.path: - client = _client_for_url(path) - client.mkdirs(path, create_parent=create_parent) + for url in args.path: + client = _client_for_url(url) + client.mkdirs(_path_for_url(url), create_parent=create_parent) def mv(args: Namespace): @@ -81,7 +81,9 @@ def mv(args: Namespace): pass resolved_src = [ - path for pattern in args.src for path in _glob_path(client, pattern) + path + for pattern in args.src + for path in _glob_path(client, _path_for_url(pattern)) ] if len(resolved_src) > 1 and not dst_isdir: From 6fcfe4254438ca147687026bd719eb36fe9cab21 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Wed, 1 Jan 2025 13:40:26 -0500 Subject: [PATCH 4/4] Add dynamic version --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index a99cca4..068c8b4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -7,6 +7,7 @@ name = "hdfs-native" description = "Python bindings for hdfs-native Rust library" readme = "README.md" requires-python = ">=3.9" +dynamic = ["version"] classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython",