From 7fe4cfbc9012bb13c7cba4a96222f6ca282abc23 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:02:15 -0500 Subject: [PATCH 1/6] Add basic setup for python docs --- .readthedocs.yml | 28 ++++++++++++++++++++++++++++ python/docs/Makefile | 20 ++++++++++++++++++++ python/docs/make.bat | 35 +++++++++++++++++++++++++++++++++++ python/docs/source/conf.py | 27 +++++++++++++++++++++++++++ python/docs/source/index.rst | 17 +++++++++++++++++ python/pyproject.toml | 5 +++-- 6 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 python/docs/Makefile create mode 100644 python/docs/make.bat create mode 100644 python/docs/source/conf.py create mode 100644 python/docs/source/index.rst diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..076f905 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,28 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: python/docs/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt \ No newline at end of file diff --git a/python/docs/Makefile b/python/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/python/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/python/docs/make.bat b/python/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/python/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py new file mode 100644 index 0000000..9bda321 --- /dev/null +++ b/python/docs/source/conf.py @@ -0,0 +1,27 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "hdfs-native" +copyright = "2024, Adam Binford" +author = "Adam Binford" +release = "0.11.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ["_templates"] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/python/docs/source/index.rst b/python/docs/source/index.rst new file mode 100644 index 0000000..e179a91 --- /dev/null +++ b/python/docs/source/index.rst @@ -0,0 +1,17 @@ +.. hdfs-native documentation master file, created by + sphinx-quickstart on Mon Dec 16 14:57:12 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +hdfs-native documentation +========================= + +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + diff --git a/python/pyproject.toml b/python/pyproject.toml index b00e4c4..7144a29 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,9 +21,10 @@ dependencies = [ [project.optional-dependencies] devel = [ "mypy~=1.13.0", - "ruff~=0.7.2", "pytest~=8.3", - "pytest-benchmark~=5.1" + "pytest-benchmark~=5.1", + "ruff~=0.7.2", + "sphinx~=8.1" ] [project.urls] From 2a96895abd8ef608e4b03eb46155433b0058c9ae Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:43:23 -0500 Subject: [PATCH 2/6] Add initial python doc generation --- python/docs/source/api.rst | 5 ++ .../docs/source/api/hdfs_native.AclEntry.rst | 23 ++++++++ .../docs/source/api/hdfs_native.AclStatus.rst | 25 +++++++++ python/docs/source/api/hdfs_native.Client.rst | 53 +++++++++++++++++++ .../source/api/hdfs_native.ContentSummary.rst | 27 ++++++++++ .../source/api/hdfs_native.FileReader.rst | 41 ++++++++++++++ .../source/api/hdfs_native.FileStatus.rst | 35 ++++++++++++ .../source/api/hdfs_native.FileWriter.rst | 21 ++++++++ .../source/api/hdfs_native.WriteOptions.rst | 25 +++++++++ python/docs/source/conf.py | 12 ++++- python/docs/source/index.rst | 6 +-- python/docs/source/usage.rst | 19 +++++++ python/hdfs_native/__init__.py | 11 ++++ python/pyproject.toml | 3 +- 14 files changed, 300 insertions(+), 6 deletions(-) create mode 100644 python/docs/source/api.rst create mode 100644 python/docs/source/api/hdfs_native.AclEntry.rst create mode 100644 python/docs/source/api/hdfs_native.AclStatus.rst create mode 100644 python/docs/source/api/hdfs_native.Client.rst create mode 100644 python/docs/source/api/hdfs_native.ContentSummary.rst create mode 100644 python/docs/source/api/hdfs_native.FileReader.rst create mode 100644 python/docs/source/api/hdfs_native.FileStatus.rst create mode 100644 python/docs/source/api/hdfs_native.FileWriter.rst create mode 100644 python/docs/source/api/hdfs_native.WriteOptions.rst create mode 100644 python/docs/source/usage.rst diff --git a/python/docs/source/api.rst b/python/docs/source/api.rst new file mode 100644 index 0000000..191b3de --- /dev/null +++ b/python/docs/source/api.rst @@ -0,0 +1,5 @@ +API +=== + +.. automodapi:: hdfs_native + :no-inheritance-diagram: \ No newline at end of file diff --git a/python/docs/source/api/hdfs_native.AclEntry.rst b/python/docs/source/api/hdfs_native.AclEntry.rst new file mode 100644 index 0000000..bf3b2ca --- /dev/null +++ b/python/docs/source/api/hdfs_native.AclEntry.rst @@ -0,0 +1,23 @@ +AclEntry +======== + +.. currentmodule:: hdfs_native + +.. autoclass:: AclEntry + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~AclEntry.name + ~AclEntry.permissions + ~AclEntry.scope + ~AclEntry.type + + .. rubric:: Attributes Documentation + + .. autoattribute:: name + .. autoattribute:: permissions + .. autoattribute:: scope + .. autoattribute:: type diff --git a/python/docs/source/api/hdfs_native.AclStatus.rst b/python/docs/source/api/hdfs_native.AclStatus.rst new file mode 100644 index 0000000..eb510f1 --- /dev/null +++ b/python/docs/source/api/hdfs_native.AclStatus.rst @@ -0,0 +1,25 @@ +AclStatus +========= + +.. currentmodule:: hdfs_native + +.. autoclass:: AclStatus + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~AclStatus.entries + ~AclStatus.group + ~AclStatus.owner + ~AclStatus.permission + ~AclStatus.sticky + + .. rubric:: Attributes Documentation + + .. autoattribute:: entries + .. autoattribute:: group + .. autoattribute:: owner + .. autoattribute:: permission + .. autoattribute:: sticky diff --git a/python/docs/source/api/hdfs_native.Client.rst b/python/docs/source/api/hdfs_native.Client.rst new file mode 100644 index 0000000..7ac4286 --- /dev/null +++ b/python/docs/source/api/hdfs_native.Client.rst @@ -0,0 +1,53 @@ +Client +====== + +.. currentmodule:: hdfs_native + +.. autoclass:: Client + :show-inheritance: + + .. rubric:: Methods Summary + + .. autosummary:: + + ~Client.append + ~Client.create + ~Client.delete + ~Client.get_acl_status + ~Client.get_content_summary + ~Client.get_file_info + ~Client.list_status + ~Client.mkdirs + ~Client.modify_acl_entries + ~Client.read + ~Client.remove_acl + ~Client.remove_acl_entries + ~Client.remove_default_acl + ~Client.rename + ~Client.set_acl + ~Client.set_owner + ~Client.set_permission + ~Client.set_replication + ~Client.set_times + + .. rubric:: Methods Documentation + + .. automethod:: append + .. automethod:: create + .. automethod:: delete + .. automethod:: get_acl_status + .. automethod:: get_content_summary + .. automethod:: get_file_info + .. automethod:: list_status + .. automethod:: mkdirs + .. automethod:: modify_acl_entries + .. automethod:: read + .. automethod:: remove_acl + .. automethod:: remove_acl_entries + .. automethod:: remove_default_acl + .. automethod:: rename + .. automethod:: set_acl + .. automethod:: set_owner + .. automethod:: set_permission + .. automethod:: set_replication + .. automethod:: set_times diff --git a/python/docs/source/api/hdfs_native.ContentSummary.rst b/python/docs/source/api/hdfs_native.ContentSummary.rst new file mode 100644 index 0000000..3f3f922 --- /dev/null +++ b/python/docs/source/api/hdfs_native.ContentSummary.rst @@ -0,0 +1,27 @@ +ContentSummary +============== + +.. currentmodule:: hdfs_native + +.. autoclass:: ContentSummary + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~ContentSummary.directory_count + ~ContentSummary.file_count + ~ContentSummary.length + ~ContentSummary.quota + ~ContentSummary.space_consumed + ~ContentSummary.space_quota + + .. rubric:: Attributes Documentation + + .. autoattribute:: directory_count + .. autoattribute:: file_count + .. autoattribute:: length + .. autoattribute:: quota + .. autoattribute:: space_consumed + .. autoattribute:: space_quota diff --git a/python/docs/source/api/hdfs_native.FileReader.rst b/python/docs/source/api/hdfs_native.FileReader.rst new file mode 100644 index 0000000..c1cb8f6 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileReader.rst @@ -0,0 +1,41 @@ +FileReader +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileReader + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~FileReader.size + + .. rubric:: Methods Summary + + .. autosummary:: + + ~FileReader.close + ~FileReader.read + ~FileReader.read_range + ~FileReader.readable + ~FileReader.readall + ~FileReader.seek + ~FileReader.seekable + ~FileReader.tell + + .. rubric:: Attributes Documentation + + .. autoattribute:: size + + .. rubric:: Methods Documentation + + .. automethod:: close + .. automethod:: read + .. automethod:: read_range + .. automethod:: readable + .. automethod:: readall + .. automethod:: seek + .. automethod:: seekable + .. automethod:: tell diff --git a/python/docs/source/api/hdfs_native.FileStatus.rst b/python/docs/source/api/hdfs_native.FileStatus.rst new file mode 100644 index 0000000..4584d55 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileStatus.rst @@ -0,0 +1,35 @@ +FileStatus +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileStatus + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~FileStatus.access_time + ~FileStatus.blocksize + ~FileStatus.group + ~FileStatus.isdir + ~FileStatus.length + ~FileStatus.modification_time + ~FileStatus.owner + ~FileStatus.path + ~FileStatus.permission + ~FileStatus.replication + + .. rubric:: Attributes Documentation + + .. autoattribute:: access_time + .. autoattribute:: blocksize + .. autoattribute:: group + .. autoattribute:: isdir + .. autoattribute:: length + .. autoattribute:: modification_time + .. autoattribute:: owner + .. autoattribute:: path + .. autoattribute:: permission + .. autoattribute:: replication diff --git a/python/docs/source/api/hdfs_native.FileWriter.rst b/python/docs/source/api/hdfs_native.FileWriter.rst new file mode 100644 index 0000000..ec27221 --- /dev/null +++ b/python/docs/source/api/hdfs_native.FileWriter.rst @@ -0,0 +1,21 @@ +FileWriter +========== + +.. currentmodule:: hdfs_native + +.. autoclass:: FileWriter + :show-inheritance: + + .. rubric:: Methods Summary + + .. autosummary:: + + ~FileWriter.close + ~FileWriter.writable + ~FileWriter.write + + .. rubric:: Methods Documentation + + .. automethod:: close + .. automethod:: writable + .. automethod:: write diff --git a/python/docs/source/api/hdfs_native.WriteOptions.rst b/python/docs/source/api/hdfs_native.WriteOptions.rst new file mode 100644 index 0000000..de210e6 --- /dev/null +++ b/python/docs/source/api/hdfs_native.WriteOptions.rst @@ -0,0 +1,25 @@ +WriteOptions +============ + +.. currentmodule:: hdfs_native + +.. autoclass:: WriteOptions + :show-inheritance: + + .. rubric:: Attributes Summary + + .. autosummary:: + + ~WriteOptions.block_size + ~WriteOptions.create_parent + ~WriteOptions.overwrite + ~WriteOptions.permission + ~WriteOptions.replication + + .. rubric:: Attributes Documentation + + .. autoattribute:: block_size + .. autoattribute:: create_parent + .. autoattribute:: overwrite + .. autoattribute:: permission + .. autoattribute:: replication diff --git a/python/docs/source/conf.py b/python/docs/source/conf.py index 9bda321..6a39b8e 100644 --- a/python/docs/source/conf.py +++ b/python/docs/source/conf.py @@ -14,14 +14,22 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx_rtd_theme", + "sphinx_automodapi.automodapi", +] + templates_path = ["_templates"] exclude_patterns = [] +autosummary_imported_members = False + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] diff --git a/python/docs/source/index.rst b/python/docs/source/index.rst index e179a91..a4327a5 100644 --- a/python/docs/source/index.rst +++ b/python/docs/source/index.rst @@ -6,12 +6,12 @@ hdfs-native documentation ========================= -Add your content using ``reStructuredText`` syntax. See the -`reStructuredText `_ -documentation for details. +Python bindings for Rust-based `hdfs-native `_ library. .. toctree:: :maxdepth: 2 :caption: Contents: + usage + api diff --git a/python/docs/source/usage.rst b/python/docs/source/usage.rst new file mode 100644 index 0000000..cf2df92 --- /dev/null +++ b/python/docs/source/usage.rst @@ -0,0 +1,19 @@ +Usage +===== + +Simply create a :py:class:`Client `. + +.. code-block:: + + from hdfs_native import Client + + # Load the fs.defaultFS from your core-site.xml config file + client = Client() + + # Connect to a specific NameNode + client = Client("hdfs://nn-1:9000") + + # Connect to a Name Service + client = Client("hdfs://ns") + +See :py:class:`Client ` for supported methods on a client. \ No newline at end of file diff --git a/python/hdfs_native/__init__.py b/python/hdfs_native/__init__.py index 44fa40a..b9d16ba 100644 --- a/python/hdfs_native/__init__.py +++ b/python/hdfs_native/__init__.py @@ -20,6 +20,17 @@ RawFileWriter, ) +__all__ = [ + "Client", + "FileReader", + "FileWriter", + "FileStatus", + "ContentSummary", + "WriteOptions", + "AclEntry", + "AclStatus", +] + class FileReader(io.RawIOBase): def __init__(self, inner: "RawFileReader"): diff --git a/python/pyproject.toml b/python/pyproject.toml index 7144a29..d37a2b6 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -24,7 +24,8 @@ devel = [ "pytest~=8.3", "pytest-benchmark~=5.1", "ruff~=0.7.2", - "sphinx~=8.1" + "sphinx~=8.1", + "sphinx-rtd-theme=~3.0" ] [project.urls] From e4cdbbe3137bfdc359a2eb1e1e34cc72be937702 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:47:52 -0500 Subject: [PATCH 3/6] Cleanup dep management and add doc building test --- .github/workflows/python-test.yml | 7 ++++++- .readthedocs.yml | 9 ++++++--- python/pyproject.toml | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 48fbf7c..d5a87fd 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -50,7 +50,7 @@ jobs: sccache: 'true' container: 'off' working-directory: ./python - args: --extras devel + args: --extras devel,docs - name: Run lints run: | @@ -64,6 +64,11 @@ jobs: source .venv/bin/activate pytest + - name: Build docs + run: | + source .venv/bin/activate + sphinx-build -M html docs/source/ docs/build/ + build: runs-on: ubuntu-latest steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index 076f905..3a7cf4d 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -23,6 +23,9 @@ sphinx: # Optional but recommended, declare the Python requirements required # to build your documentation # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -# python: -# install: -# - requirements: docs/requirements.txt \ No newline at end of file +python: + install: + - method: pip + path: python + extra_requirements: + - docs \ No newline at end of file diff --git a/python/pyproject.toml b/python/pyproject.toml index d37a2b6..489f661 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -23,9 +23,13 @@ devel = [ "mypy~=1.13.0", "pytest~=8.3", "pytest-benchmark~=5.1", - "ruff~=0.7.2", + "ruff~=0.7.2" +] + +docs = [ "sphinx~=8.1", - "sphinx-rtd-theme=~3.0" + "sphinx-automodapi~=0.18", + "sphinx-rtd-theme~=3.0" ] [project.urls] From 34bfc33fdcf7f390053a14a7f349669d6882e633 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:51:12 -0500 Subject: [PATCH 4/6] Add documentation link --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 489f661..a069d79 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -34,6 +34,7 @@ docs = [ [project.urls] repository = "https://github.com/Kimahriman/hdfs-native" +documentation = "https://hdfs-native.readthedocs.io/" [project.entry-points."fsspec.specs"] hdfs = "hdfs_native.fsspec.HdfsFileSystem" From 1dd7aa6819f7f50bbf178a3ed4d2de0db4308e9d Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:56:01 -0500 Subject: [PATCH 5/6] Only run docs on 3.13 --- .github/workflows/python-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index d5a87fd..bce38aa 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -51,7 +51,7 @@ jobs: container: 'off' working-directory: ./python args: --extras devel,docs - + - name: Run lints run: | source .venv/bin/activate @@ -65,6 +65,7 @@ jobs: pytest - name: Build docs + if: ${{ matrix.python-version == '3.13' }} run: | source .venv/bin/activate sphinx-build -M html docs/source/ docs/build/ From d36708eea688ea1cf26698bda2c0e474e64d9334 Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Mon, 16 Dec 2024 15:59:35 -0500 Subject: [PATCH 6/6] Create separate job for docs --- .github/workflows/python-test.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index bce38aa..1c5397d 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -50,7 +50,7 @@ jobs: sccache: 'true' container: 'off' working-directory: ./python - args: --extras devel,docs + args: --extras devel - name: Run lints run: | @@ -64,8 +64,29 @@ jobs: source .venv/bin/activate pytest + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.13 + + - name: Create virtualenv + run: | + python3 -m venv .venv + + - uses: PyO3/maturin-action@v1 + with: + command: develop + sccache: 'true' + container: 'off' + working-directory: ./python + args: --extras docs + - name: Build docs - if: ${{ matrix.python-version == '3.13' }} run: | source .venv/bin/activate sphinx-build -M html docs/source/ docs/build/