feat: configure python path based on command line and package options

Implements: #77

Author: silvester747

CHANGELOG.adoc

@@ -18,6 +18,15 @@ The format is based on https://keepachangelog.com/en/1.0.0/[Keep a Changelog],
 and this project adheres to https://semver.org/spec/v2.0.0.html[Semantic Versioning].
 
 
+== Unreleased
+
+=== Added
+
+  * Python path management to easily include external python scripts. By default the directory
+    containing the document being generated is added to the python path when the file is generated. 
+    The python path can be overridden on the command line. For packages python scripts can be 
+    included in a separate directory and specified in the contents file.
+
 == 0.8.6 (4 Sep 2022)
 
 === Added

asciidoxy/config.py

@@ -63,6 +63,7 @@ class Configuration(argparse.Namespace):
     image_dir: Optional[Path] = None
     spec_file: Optional[Path] = None
     version_file: Optional[Path] = None
+    python_dir: Optional[Path] = None
 
     build_dir: Path
     destination_dir: Path
@@ -106,6 +107,13 @@ def parse_args(argv):
         help="Directory containing images to include. If no image directory is"
         " specified, only images in the `images` directory next to the input file"
         " can be included.")
+    input_group.add_argument("--python-dir",
+                             metavar="PYTHON_DIR",
+                             type=PathArgument(existing_dir=True),
+                             help="Directory containg Python code to include in the documentation."
+                             " If no explicit Python directory is specified, either `BASE_DIR`"
+                             " or the directory containing the input file is used to find Python"
+                             " code")
 
     external_data_group = parser.add_argument_group(title="Loading external data")
     external_data_group.add_argument("-s",

asciidoxy/document.py

@@ -42,6 +42,9 @@ class Package:
         adoc_image_dir:    Directory containing images to include in the documentation.
         adoc_root_doc:     Entry point document for the package. To be linked to if no specific file
                                in the package is mentioned.
+        python_dir:        Directory containing Python code to include in the documentation. This
+                               directory will be added to the Python path for documents in this
+                               package
         scoped:            True if this is a new-style, scoped package.
         copy_adoc_src_dir: True if the content of `adoc_src_dir` should be copied to the working
                                directory.
@@ -54,6 +57,7 @@ class Package:
     adoc_src_dir: Optional[Path] = None
     adoc_image_dir: Optional[Path] = None
     adoc_root_doc: Optional[Path] = None
+    python_dir: Optional[Path] = None
     scoped: bool = False
     copy_adoc_src_dir: bool = True
 
@@ -82,6 +86,10 @@ def load_from_toml(self, pkg_root: Path, data: Mapping[str, Any]) -> None:
             if self.adoc_src_dir is not None:
                 self.adoc_root_doc = path_from_toml(adoc, "root_doc", self.adoc_src_dir)
 
+        python = data.get("python", None)
+        if python is not None:
+            self.python_dir = path_from_toml(python, "dir", pkg_root)
+
         self.scoped = True
 
 

asciidoxy/generator/cache.py

@@ -14,7 +14,7 @@
 """Cache implementation for Mako templates supporting package resources."""
 
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from mako.exceptions import TopLevelLookupException
 from mako.lookup import TemplateLookup
@@ -41,10 +41,28 @@ def __init__(self, cache_name: str, cache_dir: Optional[Path] = None, *args, **k
         super().__init__(*args, **kwargs)
 
 
+def _generate_python_path_code(python_paths: Optional[List[Path]]) -> List[str]:
+    if python_paths:
+        imports = ["import sys"]
+        imports.extend(f"sys.path.insert(1, \"{p}\")" for p in python_paths)
+        imports.append("del sys")
+        return imports
+
+    return []
+
+
 class DocumentCache(BaseCache):
     """Cache for input documents."""
-    def __init__(self, cache_dir: Optional[Path] = None, *args, **kwargs):
-        super().__init__("documents", cache_dir, *args, **kwargs)
+    def __init__(self,
+                 cache_dir: Optional[Path] = None,
+                 python_paths: Optional[List[Path]] = None,
+                 *args,
+                 **kwargs):
+        super().__init__("documents",
+                         cache_dir,
+                         imports=_generate_python_path_code(python_paths),
+                         *args,
+                         **kwargs)
 
     def get_document(self, document: Document) -> Template:
         return self.get_template(str(document.original_file))

asciidoxy/generator/context.py

@@ -181,7 +181,7 @@ def __init__(self, reference: ApiReference, package_manager: PackageManager, doc
         self.document_stack = [document]
 
         self.templates = TemplateCache(config.template_dir, config.cache_dir)
-        self.document_cache = DocumentCache(config.cache_dir)
+        self.document_cache = DocumentCache(config.cache_dir, package_manager.python_paths())
 
         self.config = config
 

asciidoxy/packaging/collect.py

@@ -447,9 +447,8 @@ def _combine_dict(a: Dict[Any, Any], b: Dict[Any, Any]):
     return a
 
 
-def specs_from_file(
-        spec_file: Union[PurePath, str],
-        version_file: Optional[Union[PurePath, str]] = None) -> Sequence[PackageSpec]:
+def specs_from_file(spec_file: Union[PurePath, str],
+                    version_file: Optional[Union[PurePath, str]] = None) -> Sequence[PackageSpec]:
     """Load package specifications from a file.
 
     Optionally a version CSV file is used to provide package versions.

asciidoxy/packaging/manager.py

@@ -17,7 +17,7 @@
 import logging
 import shutil
 from pathlib import Path
-from typing import Dict, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
 
@@ -81,7 +81,8 @@ def image_work_dir(self) -> Path:
     def set_input_files(self,
                         in_file: Path,
                         include_dir: Optional[Path] = None,
-                        image_dir: Optional[Path] = None) -> None:
+                        image_dir: Optional[Path] = None,
+                        python_dir: Optional[Path] = None) -> None:
         """Set the input files to collect.
 
         Args:
@@ -91,6 +92,9 @@ def set_input_files(self,
             image_dir:   Directory containing iamges to include from AsciiDoc files. If `None` and
                             directory named `images` is present next to the `in_file`, that
                             directory is used for images. Otherwise, no images are copied.
+            python_dir:  Directory containing Python code to include in the documentation. This
+                            directory will be added to the Python path for the input file and any
+                            document in the include_dir.
         """
         pkg = Package(Package.INPUT_PACKAGE_NAME)
         pkg.adoc_src_dir = include_dir
@@ -106,6 +110,11 @@ def set_input_files(self,
             pkg.copy_adoc_src_dir = False
             pkg.adoc_src_dir = in_file.parent
 
+        if python_dir:
+            pkg.python_dir = python_dir
+        else:
+            pkg.python_dir = pkg.adoc_src_dir
+
         self.packages[Package.INPUT_PACKAGE_NAME] = pkg
 
     def collect(self,
@@ -298,6 +307,14 @@ def make_document(self,
 
         return doc
 
+    def python_paths(self) -> List[Path]:
+        """Get the paths containing python code to be included in the generated documents.
+
+        Returns:
+            A list of python paths. Can be empty.
+        """
+        return [pkg.python_dir for pkg in self.packages.values() if pkg.python_dir]
+
     def _warning_or_error(self, error: Exception):
         if self.warnings_are_errors:
             raise error

documentation/getting-started/using-python.adoc

@@ -105,6 +105,20 @@ This results in:
 The factorial of 5 is ${math.factorial(5)}.
 ====
 
+=== Python path handling
+
+For `import` statements to work, the imported modules need to be on the python path. For libraries 
+included with python or installed with `pip` this is automatic. To include your own python modules, 
+AsciiDoxy needs to know where to look.
+
+By default, the directory containing the input AsciiDoc file is used to look for additional python 
+modules. If needed, you can specify a different path to be included using the `--python-dir` 
+command line option.
+
+When using ${cross_document_ref("packages.adoc", link_text="packages")}, you can specify a 
+${cross_document_ref("../reference/packages.adoc", anchor="_python_section",
+link_text="subdirectory in the package to load python code from")}.
+
 == Custom functions
 
 Module-level blocks can also be used to define custom functions to reuse in your AsciiDoc files.

documentation/reference/packages.adoc

@@ -195,6 +195,7 @@ following sections:
 `package`:: Metadata for the package.
 `reference`:: The API reference information contained in the package.
 `asciidoc`:: AsciiDoc and other files to be included in AsciiDoc generation.
+`python`:: Python scripts to include in AsciiDoc generation.
 
 === Package section
 
@@ -226,3 +227,11 @@ AsciiDoc files. These files are copied to the image directory used for all Ascii
 `root_doc`:: (Optional) Document to include by default for the package. Used if not specific file
 in the package is mentioned.
 
+=== Python section
+
+The `python` section describes files in the package that are to be included when generating the 
+AsciiDoc files. The subdirectory containing the python files is automatically added to the python 
+path for all files.
+
+`dir`:: (Required) Subdirectory inside the package containing python code to be included in 
+generation.

pyproject.toml

@@ -24,6 +24,7 @@ markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
 ]
 junit_family = "legacy"
+asyncio_mode = "auto"
 
 [tool.isort]
 profile = "hug"

tests/data/adoc/include_python.input.adoc

@@ -0,0 +1,6 @@
+= Include some python code from another file
+
+<%
+import include_python
+%>
+${include_python.test()}

tests/data/adoc/include_python.py

@@ -0,0 +1,2 @@
+def test():
+    return "This is included python code"

tests/unit/generator/test_generator.py

@@ -1486,10 +1486,8 @@ def test_process_adoc_custom_templates(warnings_are_errors, single_and_multipage
 
 
 def test_process_adoc_access_config(warnings_are_errors, single_and_multipage, adoc_data,
-                                    api_reference, package_manager, update_expected_results,
-                                    doxygen_version, tmp_path, default_config):
+                                    api_reference, package_manager, tmp_path, default_config):
     input_file = adoc_data / "access_config.input.adoc"
-    adoc_data_expected_result_file(input_file, single_and_multipage, doxygen_version)
 
     package_manager.set_input_files(input_file)
     doc = package_manager.prepare_work_directory(input_file)
@@ -1507,6 +1505,26 @@ def test_process_adoc_access_config(warnings_are_errors, single_and_multipage, a
     assert f"Build dir: {default_config.build_dir}" in content
 
 
+def test_process_adoc_include_python(warnings_are_errors, single_and_multipage, adoc_data,
+                                     api_reference, package_manager, tmp_path, default_config):
+    input_file = adoc_data / "include_python.input.adoc"
+
+    package_manager.set_input_files(input_file)
+    doc = package_manager.prepare_work_directory(input_file)
+
+    progress_mock = ProgressMock()
+    default_config.warnings_are_errors = warnings_are_errors
+    output_doc = process_adoc(doc,
+                              api_reference,
+                              package_manager,
+                              config=default_config,
+                              progress=progress_mock)[0]
+    assert output_doc.work_file.is_file()
+
+    content = output_doc.work_file.read_text()
+    assert "This is included python code" in content
+
+
 @pytest.mark.parametrize("api_reference_set", [("cpp/default", "cpp/consumer")])
 @pytest.mark.parametrize(
     "test_file_name",