fixes #341

stimela/commands/run.py

@@ -169,8 +169,12 @@ def load_recipe_files(filenames: List[str]):
                 help="""explicitly skips steps wth the given tags.
                 Use commas, or give multiple times for multiple tags.""")
 @click.option("-e", "--enable-step", "enable_steps", metavar="STEP(s)", multiple=True,
-                help="""Force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times
+                help="""force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times
                 for multiple steps.""")
+@click.option("-f", "--disable-fresh-skips", "disable_fresh_skips",  is_flag=True,
+                help="""forces execution of steps with a skip_if_outputs: fresh property.""")
+@click.option("-F", "--disable-exist-skips", "disable_exist_skips",  is_flag=True,
+                help="""forces execution of steps with a skip_if_outputs: exist property.""")
 @click.option("-c", "--config", "config_equals", metavar="X.Y.Z=VALUE", nargs=1, multiple=True,
                 help="""tweak configuration options.""")
 @click.option("-a", "--assign", metavar="PARAM VALUE", nargs=2, multiple=True,
@@ -202,6 +206,7 @@ def run(parameters: List[str] = [], dump_config: bool = False, dry_run: bool = F
     config_assign: List[Tuple[str, str]] = [],
     step_ranges: List[str] = [], tags: List[str] = [], skip_tags: List[str] = [], enable_steps: List[str] = [],
     skip_ranges: List[str] = [],
+    disable_fresh_skips=False, disable_exist_skips=False,
     build=False, rebuild=False, build_skips=False,
     enable_native=False,
     enable_singularity=False,
@@ -214,6 +219,11 @@ def run(parameters: List[str] = [], dump_config: bool = False, dry_run: bool = F
     recipe_or_cab = None
     files_to_load = []
 
+    if disable_fresh_skips:
+        stimela.CONFIG.opts.disable_skips.fresh = True
+    if disable_exist_skips:
+        stimela.CONFIG.opts.disable_skips.exist = True
+
     def convert_value(value):
         if value == "=UNSET":
             return UNSET

stimela/config.py

@@ -41,7 +41,12 @@ class StimelaLogConfig(object):
 class StimelaProfilingOptions(object):
     print_depth: int = 9999
     unroll_loops: bool = False
-
+
+@dataclass
+class StimelaDisableSkipOptions(object):
+    fresh: bool = False
+    exist: bool = False
+
 @dataclass
 class StimelaOptions(object):
     backend: StimelaBackendOptions = EmptyClassDefault(StimelaBackendOptions)
@@ -52,8 +57,8 @@ class StimelaOptions(object):
     runtime: Dict[str, Any] = EmptyDictDefault()    
     ## Profiling options
     profile: StimelaProfilingOptions = EmptyClassDefault(StimelaProfilingOptions)
-
-
+    ## Disables skip_if_outputs checks
+    disable_skips: StimelaDisableSkipOptions = EmptyClassDefault(StimelaDisableSkipOptions)
 
 def DefaultDirs():
     return field(default_factory=lambda:dict(indir='.', outdir='.'))

stimela/kitchen/step.py

@@ -487,11 +487,25 @@ def run(self, backend: Optional[Dict] = None, subst: Optional[Dict[str, Any]] =
                     raise StepValidationError(f"step '{self.name}': invalid inputs: {join_quote(invalid)}", log=self.log)
 
             ## check if we need to skip based on existing/fresh file outputs
+            skip_if_outputs = self.skip_if_outputs
+            # don't check if skipping anyway
+            if skip:
+                skip_if_outputs = None
+            # don't check if remote filesystem
+            elif backend_runner.is_remote_fs:
+                parent_log_info(f"ignoring skip_if_outputs: {skip_if_outputs} because backend has remote filesystem")
+                skip_if_outputs = None
+            # don't check if force-disabled
+            elif (skip_if_outputs == OUTPUTS_EXISTS and stimela.CONFIG.opts.disable_skips.exist) or \
+                    (skip_if_outputs == OUTPUTS_FRESH and stimela.CONFIG.opts.disable_skips.fresh):
+                parent_log_info(f"ignoring skip_if_outputs: {skip_if_outputs} because it has been force-disabled")
+                skip_if_outputs = None
+
             ## if skip on fresh outputs is in effect, find mtime of most recent input 
-            if not backend_runner.is_remote_fs and not skip and self.skip_if_outputs:
+            if skip_if_outputs:
                 # max_mtime will remain 0 if we're not echecking for freshness, or if there are no file-type inputs
                 max_mtime, max_mtime_path = 0, None
-                if self.skip_if_outputs == OUTPUTS_FRESH:
+                if skip_if_outputs == OUTPUTS_FRESH:
                     parent_log_info("checking if file-type outputs of step are fresh")
                     for name, value in params.items():
                         schema = self.inputs_outputs[name]

tests/stimela_tests/test_conditional_skips.py

@@ -0,0 +1,56 @@
+import os, re, subprocess, pytest
+from .test_recipe import change_test_dir, run, verify_output
+
+def test_conditional_skips():
+    os.system("rm -fr test_conditional_skips[1234].tmp")
+    print("===== all three files touched =====")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml")
+    assert retcode == 0
+    print(output)
+    assert verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== 2 and 3 touched =====")
+    os.system("touch test_conditional_skips4.tmp")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml")
+    assert retcode == 0
+    print(output)
+    assert not verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== only 3 touched =====")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml")
+    assert retcode == 0
+    print(output)
+    assert not verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert not verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== 2 and 3 touched =====")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml -f")
+    assert retcode == 0
+    print(output)
+    assert not verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== 1 and 3 touched =====")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml -F")
+    assert retcode == 0
+    print(output)
+    assert verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert not verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== all three files touched =====")
+    retcode, output = run("stimela -b native run test_conditional_skips.yml -f -F")
+    assert retcode == 0
+    print(output)
+    assert verify_output(output, "running touch test_conditional_skips1.tmp")
+    assert verify_output(output, "running touch test_conditional_skips2.tmp")
+    assert verify_output(output, "running touch test_conditional_skips3.tmp")
+
+    print("===== cleaning up =====")
+    os.system("rm -fr test_conditional_skips[1234].tmp")

tests/stimela_tests/test_conditional_skips.yml

@@ -0,0 +1,34 @@
+cabs:
+  touch:
+    command: touch
+    inputs:
+      conditional-file:
+        dtype: File
+        must_exist: false
+        policies:
+          skip: true  # not passed to command
+    outputs:
+      file: 
+        dtype: File
+        policies:
+          positional: true
+
+recipe:
+  steps:
+    touch1:
+      cab: touch
+      params:
+        file: test_conditional_skips1.tmp
+      skip_if_outputs: exist
+
+    touch2:
+      cab: touch
+      params:
+        conditional-file: test_conditional_skips4.tmp
+        file: test_conditional_skips2.tmp
+      skip_if_outputs: fresh
+
+    touch3:
+      cab: touch
+      params:
+        file: test_conditional_skips3.tmp