From b80156620ad74792873514fb0bfe0054a3511c69 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Sat, 1 Mar 2025 20:40:14 +0200 Subject: [PATCH] fixes #341 --- stimela/commands/run.py | 12 +++- stimela/config.py | 11 +++- stimela/kitchen/step.py | 18 +++++- tests/stimela_tests/test_conditional_skips.py | 56 +++++++++++++++++++ .../stimela_tests/test_conditional_skips.yml | 34 +++++++++++ 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 tests/stimela_tests/test_conditional_skips.py create mode 100644 tests/stimela_tests/test_conditional_skips.yml diff --git a/stimela/commands/run.py b/stimela/commands/run.py index 55221e2..d4ac099 100644 --- a/stimela/commands/run.py +++ b/stimela/commands/run.py @@ -169,8 +169,12 @@ def load_recipe_files(filenames: List[str]): help="""explicitly skips steps wth the given tags. Use commas, or give multiple times for multiple tags.""") @click.option("-e", "--enable-step", "enable_steps", metavar="STEP(s)", multiple=True, - help="""Force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times + help="""force-enable steps even if the recipe marks them as skipped. Use commas, or give multiple times for multiple steps.""") +@click.option("-f", "--disable-fresh-skips", "disable_fresh_skips", is_flag=True, + help="""forces execution of steps with a skip_if_outputs: fresh property.""") +@click.option("-F", "--disable-exist-skips", "disable_exist_skips", is_flag=True, + help="""forces execution of steps with a skip_if_outputs: exist property.""") @click.option("-c", "--config", "config_equals", metavar="X.Y.Z=VALUE", nargs=1, multiple=True, help="""tweak configuration options.""") @click.option("-a", "--assign", metavar="PARAM VALUE", nargs=2, multiple=True, @@ -202,6 +206,7 @@ def run(parameters: List[str] = [], dump_config: bool = False, dry_run: bool = F config_assign: List[Tuple[str, str]] = [], step_ranges: List[str] = [], tags: List[str] = [], skip_tags: List[str] = [], enable_steps: List[str] = [], skip_ranges: List[str] = [], + disable_fresh_skips=False, disable_exist_skips=False, build=False, rebuild=False, build_skips=False, enable_native=False, enable_singularity=False, @@ -214,6 +219,11 @@ def run(parameters: List[str] = [], dump_config: bool = False, dry_run: bool = F recipe_or_cab = None files_to_load = [] + if disable_fresh_skips: + stimela.CONFIG.opts.disable_skips.fresh = True + if disable_exist_skips: + stimela.CONFIG.opts.disable_skips.exist = True + def convert_value(value): if value == "=UNSET": return UNSET diff --git a/stimela/config.py b/stimela/config.py index a535c44..74635a0 100644 --- a/stimela/config.py +++ b/stimela/config.py @@ -41,7 +41,12 @@ class StimelaLogConfig(object): class StimelaProfilingOptions(object): print_depth: int = 9999 unroll_loops: bool = False - + +@dataclass +class StimelaDisableSkipOptions(object): + fresh: bool = False + exist: bool = False + @dataclass class StimelaOptions(object): backend: StimelaBackendOptions = EmptyClassDefault(StimelaBackendOptions) @@ -52,8 +57,8 @@ class StimelaOptions(object): runtime: Dict[str, Any] = EmptyDictDefault() ## Profiling options profile: StimelaProfilingOptions = EmptyClassDefault(StimelaProfilingOptions) - - + ## Disables skip_if_outputs checks + disable_skips: StimelaDisableSkipOptions = EmptyClassDefault(StimelaDisableSkipOptions) def DefaultDirs(): return field(default_factory=lambda:dict(indir='.', outdir='.')) diff --git a/stimela/kitchen/step.py b/stimela/kitchen/step.py index ca98b38..b979ad7 100644 --- a/stimela/kitchen/step.py +++ b/stimela/kitchen/step.py @@ -487,11 +487,25 @@ def run(self, backend: Optional[Dict] = None, subst: Optional[Dict[str, Any]] = raise StepValidationError(f"step '{self.name}': invalid inputs: {join_quote(invalid)}", log=self.log) ## check if we need to skip based on existing/fresh file outputs + skip_if_outputs = self.skip_if_outputs + # don't check if skipping anyway + if skip: + skip_if_outputs = None + # don't check if remote filesystem + elif backend_runner.is_remote_fs: + parent_log_info(f"ignoring skip_if_outputs: {skip_if_outputs} because backend has remote filesystem") + skip_if_outputs = None + # don't check if force-disabled + elif (skip_if_outputs == OUTPUTS_EXISTS and stimela.CONFIG.opts.disable_skips.exist) or \ + (skip_if_outputs == OUTPUTS_FRESH and stimela.CONFIG.opts.disable_skips.fresh): + parent_log_info(f"ignoring skip_if_outputs: {skip_if_outputs} because it has been force-disabled") + skip_if_outputs = None + ## if skip on fresh outputs is in effect, find mtime of most recent input - if not backend_runner.is_remote_fs and not skip and self.skip_if_outputs: + if skip_if_outputs: # max_mtime will remain 0 if we're not echecking for freshness, or if there are no file-type inputs max_mtime, max_mtime_path = 0, None - if self.skip_if_outputs == OUTPUTS_FRESH: + if skip_if_outputs == OUTPUTS_FRESH: parent_log_info("checking if file-type outputs of step are fresh") for name, value in params.items(): schema = self.inputs_outputs[name] diff --git a/tests/stimela_tests/test_conditional_skips.py b/tests/stimela_tests/test_conditional_skips.py new file mode 100644 index 0000000..ff5e5e6 --- /dev/null +++ b/tests/stimela_tests/test_conditional_skips.py @@ -0,0 +1,56 @@ +import os, re, subprocess, pytest +from .test_recipe import change_test_dir, run, verify_output + +def test_conditional_skips(): + os.system("rm -fr test_conditional_skips[1234].tmp") + print("===== all three files touched =====") + retcode, output = run("stimela -b native run test_conditional_skips.yml") + assert retcode == 0 + print(output) + assert verify_output(output, "running touch test_conditional_skips1.tmp") + assert verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== 2 and 3 touched =====") + os.system("touch test_conditional_skips4.tmp") + retcode, output = run("stimela -b native run test_conditional_skips.yml") + assert retcode == 0 + print(output) + assert not verify_output(output, "running touch test_conditional_skips1.tmp") + assert verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== only 3 touched =====") + retcode, output = run("stimela -b native run test_conditional_skips.yml") + assert retcode == 0 + print(output) + assert not verify_output(output, "running touch test_conditional_skips1.tmp") + assert not verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== 2 and 3 touched =====") + retcode, output = run("stimela -b native run test_conditional_skips.yml -f") + assert retcode == 0 + print(output) + assert not verify_output(output, "running touch test_conditional_skips1.tmp") + assert verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== 1 and 3 touched =====") + retcode, output = run("stimela -b native run test_conditional_skips.yml -F") + assert retcode == 0 + print(output) + assert verify_output(output, "running touch test_conditional_skips1.tmp") + assert not verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== all three files touched =====") + retcode, output = run("stimela -b native run test_conditional_skips.yml -f -F") + assert retcode == 0 + print(output) + assert verify_output(output, "running touch test_conditional_skips1.tmp") + assert verify_output(output, "running touch test_conditional_skips2.tmp") + assert verify_output(output, "running touch test_conditional_skips3.tmp") + + print("===== cleaning up =====") + os.system("rm -fr test_conditional_skips[1234].tmp") diff --git a/tests/stimela_tests/test_conditional_skips.yml b/tests/stimela_tests/test_conditional_skips.yml new file mode 100644 index 0000000..d6dd27c --- /dev/null +++ b/tests/stimela_tests/test_conditional_skips.yml @@ -0,0 +1,34 @@ +cabs: + touch: + command: touch + inputs: + conditional-file: + dtype: File + must_exist: false + policies: + skip: true # not passed to command + outputs: + file: + dtype: File + policies: + positional: true + +recipe: + steps: + touch1: + cab: touch + params: + file: test_conditional_skips1.tmp + skip_if_outputs: exist + + touch2: + cab: touch + params: + conditional-file: test_conditional_skips4.tmp + file: test_conditional_skips2.tmp + skip_if_outputs: fresh + + touch3: + cab: touch + params: + file: test_conditional_skips3.tmp