Feat fast 2

Makefile

@@ -26,14 +26,15 @@ endif
 DATASTORE_URL = https://files.planning.data.gov.uk/
 
 first-pass::
+	export PYTHONPATH=$(pwd)
 	mkdir -p dataset/
 	bin/download-collection.sh
 	bin/download-pipeline.sh
 	bin/concat.sh
-	bin/download-issues.sh
+	python3 bin/download-issues.py
 	bin/download-operational-issues.sh
-	bin/download-column-field.sh
-	bin/download-converted-resources.sh
+	python3 bin/download-column-field.py
+	python3 bin/download-converted-resources.py
 	#bin/download-resources.sh
 	./bin/concat-issues.py
 	./bin/concat-column-field.py

bin/download-column-field.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import logging
+import click
+from datetime import datetime
+
+from bin.resources import get_resources
+from file_downloader import download_urls
+
+logger =  logging.getLogger("__name__")
+
+@click.command()
+@click.option("--timestamp",default=None)
+def download_column_field(timestamp=None):
+    resources = get_resources("collection/")
+    url_map = {}
+    now = datetime.now()
+    timestamp = int(now.replace(minute=0, second=0, microsecond=0).timestamp())
+
+    for resource in resources:
+        collection = resources[resource]["collection"]
+        for pipeline in resources[resource]["pipelines"]:
+            if not pipeline:
+                logger.error(f"no pipeline for {resource} in {collection} so cannot download")
+            else:
+                url = f"https://files.planning.data.gov.uk/{collection}-collection/var/column-field/{pipeline}/{resource}.csv?version={timestamp}"
+                output_path = f"var/column-field/{pipeline}/{resource}.csv"
+                url_map[url]=output_path
+
+    download_urls(url_map)
+
+
+
+if __name__ == "__main__":
+    download_column_field()

bin/download-converted-resources.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import logging
+import click
+from datetime import datetime
+
+from bin.resources import get_resources
+from file_downloader import download_urls
+
+logger =  logging.getLogger("__name__")
+
+@click.command()
+@click.option("--timestamp",default=None)
+def download_converted_resource(timestamp=None):
+    resources = get_resources("collection/")
+    url_map = {}
+    now = datetime.now()
+    timestamp = int(now.replace(minute=0, second=0, microsecond=0).timestamp())
+
+    for resource in resources:
+        collection = resources[resource]["collection"]
+        for pipeline in resources[resource]["pipelines"]:
+            if not pipeline:
+                logger.error(f"no pipeline for {resource} in {collection} so cannot download")
+            else:
+                url = f"https://files.planning.data.gov.uk/{collection}-collection/var/converted-resource/{pipeline}/{resource}.csv?version={timestamp}"
+                output_path = f"var/converted-resource/{pipeline}/{resource}.csv"
+                url_map[url]=output_path
+
+    download_urls(url_map)
+
+
+
+if __name__ == "__main__":
+    download_converted_resource()

bin/download-issues.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import logging
+import click
+from datetime import datetime
+
+from bin.resources import get_resources
+from file_downloader import download_urls
+
+
+logger =  logging.getLogger("__name__")
+
+@click.command()
+@click.option("--timestamp",default=None)
+def download_issues(timestamp=None):
+    resources = get_resources("collection/")
+    url_map = {}
+    now = datetime.now()
+    timestamp = int(now.replace(minute=0, second=0, microsecond=0).timestamp())
+
+    for resource in resources:
+        collection = resources[resource]["collection"]
+        for pipeline in resources[resource]["pipelines"]:
+            if not pipeline:
+                logger.error(f"no pipeline for {resource} in {collection} so cannot download")
+            else:
+                url = f"https://files.planning.data.gov.uk/{collection}-collection/issue/{pipeline}/{resource}.csv?version={timestamp}"
+                output_path = f"var/issue/{pipeline}/{resource}.csv"
+                url_map[url]=output_path
+    download_urls(url_map)
+
+
+
+if __name__ == "__main__":
+    download_issues()

bin/file_downloader.py

@@ -10,17 +10,21 @@
 
 logger = logging.getLogger("__name__")
 
-def download_file(url, output_path,raise_error=False):
+def download_file(url, output_path,raise_error=False, max_retries=5):
     """Downloads a file using urllib and saves it to the output directory."""
     output_path = Path(output_path)
     output_path.parent.mkdir(parents=True,exist_ok=True)
-    try:
-        urlretrieve(url, output_path)
-    except Exception as e:
-        if raise_error:
-            raise e
-        else:
-            logger.error(f"error downloading file {e}")
+    retries = 0
+    while retries < max_retries:
+        try:
+            urlretrieve(url, output_path)
+            break
+        except Exception as e:
+            if raise_error:
+                raise e
+            else:
+                logger.error(f"error downloading file from url {url}: {e}")
+        retries += 1
 
 def download_urls(url_map, max_threads=4):
     """Downloads multiple files concurrently using threads."""

bin/resources.py

@@ -4,15 +4,7 @@
 import csv
 import click
 
-@click.command()
-@click.option(
-    "--input-dir", 
-    default="collection/", 
-    help="Directory containing the CSV files"
-)
-def process_data(input_dir):
-    """Process CSV files to map resources, endpoints, and pipelines."""
-
+def get_resources(input_dir):
     endpoints = {}
     resources = {}
 
@@ -74,12 +66,22 @@ def process_data(input_dir):
     except FileNotFoundError:
         print("Error: resource.csv not found", file=sys.stderr)
         sys.exit(1)
+    return resources
 
+@click.command()
+@click.option(
+    "--input-dir", 
+    default="collection/", 
+    help="Directory containing the CSV files"
+)
+def process_data(input_dir):
+    """Process CSV files to map resources, endpoints, and pipelines."""
+    resources = get_resources(input_dir)
     # Print results
     for resource in resources:
         collection = resources[resource]["collection"]
         for pipeline in resources[resource]["pipelines"]:
             print(collection, pipeline, resource)
 
 if __name__ == "__main__":
-    process_data()
+    process_data()

requirements.txt

@@ -7,6 +7,7 @@ chardet
 PyPDF2
 pandas
 tqdm
+click
 pyarrow
 click==8.1.8
 -e git+https://github.com/digital-land/digital-land-python.git#egg=digital-land