adds get_processing_option to makerules.py

CarlosCoelhoSL · CarlosCoelhoSL · commit 9fad971e5c9b · 2025-03-11T16:55:17.000Z
diff --git a/digital_land/cli.py b/digital_land/cli.py
@@ -111,8 +111,47 @@ def collection_list_resources_cmd(collection_dir):
     short_help="generate makerules for processing a collection",
 )
 @collection_dir
-def collection_pipeline_makerules_cmd(collection_dir):
-    return collection_pipeline_makerules(collection_dir)
+@click.option(
+    "--specification-dir",
+    type=click.Path(),
+    default="specification",
+    help="directory containing the specification",
+)
+@click.option(
+    "--pipeline-dir",
+    type=click.Path(),
+    default="pipeline",
+    help="directory containing the pipeline",
+)
+@click.option(
+    "--resource-dir",
+    type=click.Path(),
+    default="collection/resource",
+    help="directory containing resources",
+)
+@click.option("--incremental-loading-override", type=click.BOOL, default=False)
+@click.option(
+    "--state-path",
+    type=click.Path(),
+    default=None,
+    help="path of the output state file",
+)
+def collection_pipeline_makerules_cmd(
+    collection_dir,
+    specification_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_loading_override,
+    state_path,
+):
+    return collection_pipeline_makerules(
+        collection_dir,
+        specification_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_loading_override,
+        state_path,
+    )
 
 
 @cli.command("collection-save-csv", short_help="save collection as CSV package")
@@ -661,15 +700,15 @@ def save_state_cmd(
     collection_dir,
     pipeline_dir,
     resource_dir,
-    incremental_override,
+    incremental_loading_override,
     output_path,
 ):
     save_state(
         specification_dir,
         collection_dir,
         pipeline_dir,
         resource_dir,
-        incremental_override,
+        incremental_loading_override,
         output_path,
     )
 
@@ -714,14 +753,14 @@ def check_state_cmd(
     collection_dir,
     pipeline_dir,
     resource_dir,
-    incremental_override,
+    incremental_loading_override,
     state_path,
 ):
     # If the state isn't the same, use a non-zero return code so scripts can
     # detect this, and print a message. If it is the same, exit silenty wirh a
     # 0 retun code.
 
-    if incremental_override:
+    if incremental_loading_override:
         print("State comparison skipped as incremental override enabled")
         sys.exit(1)
 
@@ -730,7 +769,7 @@ def check_state_cmd(
         collection_dir,
         pipeline_dir,
         resource_dir,
-        incremental_override,
+        incremental_loading_override,
         state_path,
     )
     if diffs:
diff --git a/digital_land/collection.py b/digital_land/collection.py
@@ -412,8 +412,22 @@ def resource_organisations(self, resource):
     def resource_path(self, resource):
         return resource_path(resource, self.dir)
 
-    def pipeline_makerules(self):
-        pipeline_makerules(self)
+    def pipeline_makerules(
+        self,
+        specification_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_loading_override,
+        state_path,
+    ):
+        pipeline_makerules(
+            self,
+            specification_dir,
+            pipeline_dir,
+            resource_dir,
+            incremental_loading_override,
+            state_path,
+        )
 
     def dataset_resource_map(self):
         "a map of resources needed by each dataset in a collection"
diff --git a/digital_land/commands.py b/digital_land/commands.py
@@ -103,10 +103,23 @@ def collection_list_resources(collection_dir):
         print(resource_path(resource, directory=collection_dir))
 
 
-def collection_pipeline_makerules(collection_dir):
+def collection_pipeline_makerules(
+    collection_dir,
+    specification_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_loading_override,
+    state_path,
+):
     collection = Collection(name=None, directory=collection_dir)
     collection.load()
-    collection.pipeline_makerules()
+    collection.pipeline_makerules(
+        specification_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_loading_override,
+        state_path,
+    )
 
 
 def collection_save_csv(collection_dir, refill_todays_logs=False):
@@ -1476,15 +1489,15 @@ def save_state(
     collection_dir,
     pipeline_dir,
     resource_dir,
-    incremental_override,
+    incremental_loading_override,
     output_path,
 ):
     state = State.build(
         specification_dir=specification_dir,
         collection_dir=collection_dir,
         pipeline_dir=pipeline_dir,
         resource_dir=resource_dir,
-        incremental_override=incremental_override,
+        incremental_loading_override=incremental_loading_override,
     )
     state.save(
         output_path=output_path,
@@ -1496,7 +1509,7 @@ def compare_state(
     collection_dir,
     pipeline_dir,
     resource_dir,
-    incremental_override,
+    incremental_loading_override,
     state_path,
 ):
     """Compares the current state against the one in state_path.
@@ -1506,14 +1519,14 @@ def compare_state(
         collection_dir=collection_dir,
         pipeline_dir=pipeline_dir,
         resource_dir=resource_dir,
-        incremental_override=incremental_override,
+        incremental_loading_override=incremental_loading_override,
     )
     # in here current incremental override must be false
 
     compare = State.load(state_path)
     # we don't want to include whether the previous state was an incremental override in comparison
-    current.pop("incremental_override", None)
-    compare.pop("incremental_override", None)
+    current.pop("incremental_loading_override", None)
+    compare.pop("incremental_loading_override", None)
 
     if current == compare:
         return None
diff --git a/digital_land/makerules.py b/digital_land/makerules.py
@@ -6,6 +6,16 @@
 #
 
 
+from digital_land.commands import compare_state
+from enum import Enum
+
+
+class ProcessingOption(Enum):
+    PROCESS_ALL = "all"
+    PROCESS_PARTIAL = "partial"
+    PROCESS_NONE = "none"
+
+
 def transformed_path(resource, dataset):
     return "$(TRANSFORMED_DIR)" + dataset + "/" + resource + ".csv"
 
@@ -14,8 +24,62 @@ def dataset_path(dataset):
     return "$(DATASET_DIR)" + dataset + ".csv"
 
 
-def pipeline_makerules(collection):
+def get_processing_option(
+    collection,
+    specification_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_loading_override,
+    state_path,
+):
+    # If there's no previous state, process everything
+    if not state_path:
+        return ProcessingOption.PROCESS_ALL
+
+    # Compare current state with the previous state
+    diffs = compare_state(
+        specification_dir,
+        collection.dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_loading_override,
+        state_path,
+    )
+
+    # If incremental loading is overridden or critical configs changed, process everything
+    critical_changes = {"code", "pipeline", "collection", "specification"}
+    if incremental_loading_override or critical_changes & set(diffs):
+        return ProcessingOption.PROCESS_ALL
+
+    # New resources downloaded
+    if "resource" in diffs:
+        return ProcessingOption.PROCESS_ALL  # To be changed to partial in the future
+
+    if not diffs:
+        return ProcessingOption.PROCESS_NONE
+
+    # If there are diffs we don't recognise then play safe and reprocess everything
+    return ProcessingOption.PROCESS_ALL
+
+
+def pipeline_makerules(
+    collection,
+    specification_dir,
+    pipeline_dir,
+    resource_dir,
+    incremental_loading_override,
+    state_path,
+):
     dataset_resource = collection.dataset_resource_map()
+    process = get_processing_option(
+        collection,
+        specification_dir,
+        pipeline_dir,
+        resource_dir,
+        incremental_loading_override,
+        state_path,
+    )
+
     redirect = {}
     for entry in collection.old_resource.entries:
         redirect[entry["old-resource"]] = entry["resource"]
@@ -35,6 +99,15 @@ def pipeline_makerules(collection):
                 print("\\\n    %s" % (transformed_path(resource, dataset)), end="")
         print()
 
+        if process == ProcessingOption.PROCESS_NONE:
+            print("\n$(%s)::" % dataset_var)
+            print("\techo skipping dataset")
+            print("\ntransformed::")
+            print("\techo skipping transformed")
+            print("\ndataset::")
+            print("\techo skipping dataset")
+            continue  # Skip the rest of the loop for this dataset
+
         for resource in sorted(dataset_resource[dataset]):
             old_resource = resource
             resource = redirect.get(resource, resource)
@@ -55,20 +128,22 @@ def pipeline_makerules(collection):
                         resource_path,
                     )
                 )
-                call_pipeline = (
-                    "\t$(call run-pipeline,"
-                    + f" --endpoints '{endpoints}'"
-                    + f" --organisations '{organisations}'"
-                    + f" --entry-date '{entry_date}'"
-                )
-                # we will include the resource arguement if the old resource
-                # is  different so it's processed as the old_resource
-                if resource != old_resource:
-                    call_pipeline = call_pipeline + f" --resource '{old_resource}'"
 
-                call_pipeline = call_pipeline + " )"
+                if process == ProcessingOption.PROCESS_ALL:
+                    call_pipeline = (
+                        "\t$(call run-pipeline,"
+                        + " --endpoints '%s'" % endpoints
+                        + " --organisations '%s'" % organisations
+                        + " --entry-date '%s'" % entry_date
+                    )
+                    # we will include the resource arguement if the old resource
+                    # is  different so it's processed as the old_resource
+                    if resource != old_resource:
+                        call_pipeline = call_pipeline + f" --resource '{old_resource}'"
+
+                    call_pipeline = call_pipeline + " )"
 
-                print(call_pipeline)
+                    print(call_pipeline)
 
         print("\n$(%s): $(%s)" % (dataset_var, dataset_files_var))
         print("\t$(build-dataset)")
diff --git a/digital_land/state.py b/digital_land/state.py
@@ -17,7 +17,7 @@ def build(
         collection_dir,
         pipeline_dir,
         resource_dir,
-        incremental_override,
+        incremental_loading_override,
     ):
         """Build a state object from the current configuration and code"""
         return State(
@@ -29,7 +29,7 @@ def build(
                 ),
                 "resource": State.get_dir_hash(resource_dir),
                 "pipeline": State.get_dir_hash(pipeline_dir),
-                "incremental_override": incremental_override,
+                "incremental_loading_override": incremental_loading_override,
             }
         )
 
diff --git a/tests/e2e/test_state.py b/tests/e2e/test_state.py
