merge in main

Author: eveleighoj

.github/workflows/continuous-integration.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: [3.8, 3.9]

.github/workflows/deploy-documentation.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main  # Adjust to your main branch
   workflow_dispatch: null
-    
+
 # Set permissions of GITHUB_TOKEN
 permissions:
   contents: read
@@ -19,11 +19,13 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -32,25 +34,24 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install sphinx sphinx-autobuild sphinx_rtd_theme
+        make init
 
     - name: Generate Documentation
       run: |
-        sphinx-build -b html docs/ docs/_build
+        make docs
 
     - name: Upload artifact
-      uses: actions/upload-pages-artifact@v1
+      uses: actions/upload-pages-artifact@v3
       with:
         path: docs/_build
-        
+
   deploy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: build
     permissions:
       pages: write
       id-token: write
 
     steps:
     - name: Deploy to GitHub Pages
-      uses: actions/deploy-pages@v1
+      uses: actions/deploy-pages@v4

.gitignore

@@ -28,3 +28,5 @@ pyrightconfig.json
 
 # don't store html build file for docs
 /docs/_build
+docs/digital_land*
+docs/modules.rst

Makefile

@@ -40,17 +40,15 @@ endif
 # install pre-commits
 	pre-commit install
 
-# build docs from doc strings
-.PHONY: api-docs
-api-docs:
-	sphinx-apidoc -o docs/ api/digital_land
-# Build documentation with Sphinx
+# Build documentation with Sphinx use for deploying documentation
 .PHONY: docs
-docs: api-docs
+docs:
+	sphinx-apidoc -o docs/ digital_land
 	sphinx-build -b html docs/ docs/_build
 
-# Serve the documentation locally with live-reloading
-serve-docs: api-docs
+# Serve the documentation locally with live-reloading. Use this for local building
+serve-docs:
+	sphinx-apidoc -o docs/ digital_land
 	sphinx-autobuild docs/ docs/_build --host 0.0.0.0 --port 8000
 
 # Clean up the generated documentation

digital_land/cli.py

@@ -30,6 +30,7 @@
     organisation_check,
     save_state,
     compare_state,
+    add_data,
 )
 
 from digital_land.command_arguments import (
@@ -366,6 +367,52 @@ def retire_endpoints_cmd(config_collections_dir, csv_path):
     return collection_retire_endpoints_and_sources(config_collections_dir, csv_path)
 
 
+@cli.command("add-data")
+@click.argument("csv-path", nargs=1, type=click.Path())
+@click.argument("collection-name", nargs=1, type=click.STRING)
+@click.option("--collection-dir", "-c", nargs=1, type=click.Path(exists=True))
+@click.option("--pipeline-dir", "-p", nargs=1, type=click.Path(exists=True))
+@click.option(
+    "--specification-dir", "-s", type=click.Path(exists=True), default="specification/"
+)
+@click.option(
+    "--organisation-path",
+    "-o",
+    type=click.Path(exists=True),
+    default="var/cache/organisation.csv",
+)
+@click.option(
+    "--cache-dir",
+    type=click.Path(exists=True),
+)
+def add_data_cmd(
+    csv_path,
+    collection_name,
+    collection_dir,
+    pipeline_dir,
+    specification_dir,
+    organisation_path,
+    cache_dir,
+):
+    csv_file_path = Path(csv_path)
+    if not csv_file_path.is_file():
+        logging.error(f"CSV file not found at path: {csv_path}")
+        sys.exit(2)
+    collection_dir = Path(collection_dir)
+    pipeline_dir = Path(pipeline_dir)
+    specification_dir = Path(specification_dir)
+
+    return add_data(
+        csv_file_path,
+        collection_name,
+        collection_dir,
+        pipeline_dir,
+        specification_dir,
+        organisation_path,
+        cache_dir=cache_dir,
+    )
+
+
 # edit to add collection_name in
 @cli.command("add-endpoints-and-lookups")
 @click.argument("csv-path", nargs=1, type=click.Path())

digital_land/collection.py

@@ -168,6 +168,11 @@ def load(
             organisations = set()
             datasets = set()
             for endpoint in resource["endpoints"]:
+                if endpoint not in source.records:
+                    raise RuntimeError(
+                        f"Endpoint '{endpoint}' not found in source. Check the endpoint.csv and source.csv files."
+                    )
+
                 for entry in source.records[endpoint]:
                     organisations.add(entry["organisation"])
                     datasets = set(
@@ -354,7 +359,7 @@ def load(self, directory=None):
         self.source.load(directory=directory)
         self.endpoint.load(directory=directory)
 
-        regenerate_resouces = False
+        regenerate_resources = False
 
         # Try to load log store from csv first
         try:
@@ -363,22 +368,22 @@ def load(self, directory=None):
         except FileNotFoundError:
             logging.info("No log.csv - building from log items")
             self.load_log_items(directory=directory)
-            regenerate_resouces = True
+            regenerate_resources = True
 
-        # Now try to load resoucres, unless we need to rebuild them anyway
-        if not regenerate_resouces:
+        # Now try to load resources, unless we need to rebuild them anyway
+        if not regenerate_resources:
             try:
                 self.resource.load_csv(directory=directory)
                 logging.info(
                     f"Resource loaded from CSV - {len(self.resource.entries)} entries"
                 )
             except FileNotFoundError:
-                logging.info("No resources.csv - genereating from log.csv")
-                regenerate_resouces = True
+                logging.info("No resources.csv - generating from log.csv")
+                regenerate_resources = True
 
         # Do we need to regenerate resources?
-        if regenerate_resouces:
-            logging.info("Generating resouces from log.csv")
+        if regenerate_resources:
+            logging.info("Generating resources from log.csv")
             self.resource.load(log=self.log, source=self.source, directory=directory)
 
         # attempts to load in old-resources if the file exists, many use cases won't have any