diff --git a/flow/test/test_autotuner.sh b/flow/test/test_autotuner.sh index b1591c9349..e1248f83ec 100755 --- a/flow/test/test_autotuner.sh +++ b/flow/test/test_autotuner.sh @@ -22,6 +22,9 @@ python3 -m unittest tools.AutoTuner.test.smoke_test_sweep.${PLATFORM_WITHOUT_DAS echo "Running Autotuner smoke tests for --sample and --iteration." python3 -m unittest tools.AutoTuner.test.smoke_test_sample_iteration.${PLATFORM_WITHOUT_DASHES}SampleIterationSmokeTest.test_sample_iteration +echo "Running Autotuner smoke algorithm eval test" +python3 -m unittest tools.AutoTuner.test.smoke_test_algo_eval.${PLATFORM_WITHOUT_DASHES}AlgoEvalSmokeTest.test_algo_eval + if [ "$PLATFORM_WITHOUT_DASHES" == "asap7" ] && [ "$DESIGN_NAME" == "gcd" ]; then echo "Running Autotuner ref file test (only once)" python3 -m unittest tools.AutoTuner.test.ref_file_check.RefFileCheck.test_files diff --git a/tools/AutoTuner/test/autotuner_test_utils.py b/tools/AutoTuner/test/autotuner_test_utils.py new file mode 100644 index 0000000000..6786288dc2 --- /dev/null +++ b/tools/AutoTuner/test/autotuner_test_utils.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import os + + +class AutoTunerTestUtils: + @staticmethod + def get_exec_cmd(): + """ + Returns the execution command based on whether this is a coverage run or + not. + + Note that you need to run coverage combine after the runs complete to + get the coverage of the parent plus the child invocations + """ + + if "COVERAGE_RUN" in os.environ: + exec = "coverage run --parallel-mode --omit=*/site-packages/*,*/dist-packages/*" + else: # pragma: no cover + exec = "python3" + return exec + " -m autotuner.distributed" + + +if __name__ == "__main__": # pragma: no cover + print(AutoTunerTestUtils.get_exec_cmd()) diff --git a/tools/AutoTuner/test/ref_file_check.py b/tools/AutoTuner/test/ref_file_check.py index 4de1fb1c74..3301162f91 100644 --- a/tools/AutoTuner/test/ref_file_check.py +++ b/tools/AutoTuner/test/ref_file_check.py @@ -1,6 +1,7 @@ import unittest import subprocess import os +from .autotuner_test_utils import AutoTunerTestUtils cur_dir = os.path.dirname(os.path.abspath(__file__)) src_dir = os.path.join(cur_dir, "../src") @@ -18,8 +19,9 @@ def setUp(self): "../../test/files/no_sdc_ref.json", "../../test/files/no_fr_ref.json", ] - self.commands = [ - f"python3 -m autotuner.distributed" + self.exec = AutoTunerTestUtils.get_exec_cmd() + self.command = ( + f"{self.exec}" f" --design {self.design}" f" --platform {self.platform}" f" --config {c}" diff --git a/tools/AutoTuner/test/resume_check.py b/tools/AutoTuner/test/resume_check.py index 90c42079ed..e56496deb9 100644 --- a/tools/AutoTuner/test/resume_check.py +++ b/tools/AutoTuner/test/resume_check.py @@ -2,6 +2,7 @@ import subprocess import os import time +from .autotuner_test_utils import AutoTunerTestUtils from contextlib import contextmanager @@ -45,8 +46,9 @@ def setUp(self): # Cast to 1 decimal place res_per_trial = float("{:.1f}".format(self.num_cpus / self.samples)) options = ["", "--resume"] + self.exec = AutoTunerTestUtils.get_exec_cmd() self.commands = [ - f"python3 -m autotuner.distributed" + f"{self.exec}" f" --design {self.design}" f" --platform {self.platform}" f" --config {self.config}" diff --git a/tools/AutoTuner/test/smoke_test_algo_eval.py b/tools/AutoTuner/test/smoke_test_algo_eval.py index 6a026224f5..5f716cbc9a 100644 --- a/tools/AutoTuner/test/smoke_test_algo_eval.py +++ b/tools/AutoTuner/test/smoke_test_algo_eval.py @@ -1,6 +1,7 @@ import unittest import subprocess import os +from .autotuner_test_utils import AutoTunerTestUtils cur_dir = os.path.dirname(os.path.abspath(__file__)) orfs_dir = os.path.join(cur_dir, "../../../flow") @@ -19,6 +20,7 @@ def setUp(self): _algo = ["hyperopt", "ax", "optuna", "pbt", "random"] _eval = ["default", "ppa-improv"] self.matrix = [(a, e) for a in _algo for e in _eval] + self.exec = AutoTunerTestUtils.get_exec_cmd() self.commands = [ f"python3 -m autotuner.distributed" f" --design {self.design}" diff --git a/tools/AutoTuner/test/smoke_test_sample_iteration.py b/tools/AutoTuner/test/smoke_test_sample_iteration.py index 6208554d0c..beaee2e964 100644 --- a/tools/AutoTuner/test/smoke_test_sample_iteration.py +++ b/tools/AutoTuner/test/smoke_test_sample_iteration.py @@ -1,6 +1,7 @@ import unittest import subprocess import os +from .autotuner_test_utils import AutoTunerTestUtils cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -16,7 +17,9 @@ def setUp(self): ) self.experiment = f"smoke-test-sample-iteration-{self.platform}" self.matrix = [(5, 1), (1, 5), (2, 2), (1, 1)] + self.exec = AutoTunerTestUtils.get_exec_cmd() self.commands = [ + f"{self.exec}" f"python3 -m autotuner.distributed" f" --design {self.design}" f" --platform {self.platform}" diff --git a/tools/AutoTuner/test/smoke_test_sweep.py b/tools/AutoTuner/test/smoke_test_sweep.py index 72e62e39a7..6cc9e0af43 100644 --- a/tools/AutoTuner/test/smoke_test_sweep.py +++ b/tools/AutoTuner/test/smoke_test_sweep.py @@ -2,6 +2,7 @@ import subprocess import os import json +from .autotuner_test_utils import AutoTunerTestUtils cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -30,8 +31,9 @@ def setUp(self): core = os.cpu_count() self.jobs = 4 if core >= 4 else core self.experiment = f"smoke-test-sweep-{self.platform}" + self.exec = AutoTunerTestUtils.get_exec_cmd() self.command = ( - "python3 -m autotuner.distributed" + f"{self.exec}" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}" diff --git a/tools/AutoTuner/test/smoke_test_tune.py b/tools/AutoTuner/test/smoke_test_tune.py index 8d75d5ad5c..59e927ac95 100644 --- a/tools/AutoTuner/test/smoke_test_tune.py +++ b/tools/AutoTuner/test/smoke_test_tune.py @@ -1,6 +1,7 @@ import unittest import subprocess import os +from .autotuner_test_utils import AutoTunerTestUtils cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -15,8 +16,9 @@ def setUp(self): f"../../../flow/designs/{self.platform}/{self.design}/autotuner.json", ) self.experiment = f"smoke-test-tune-{self.platform}" + self.exec = AutoTunerTestUtils.get_exec_cmd() self.command = ( - "python3 -m autotuner.distributed" + f"{self.exec}" f" --design {self.design}" f" --platform {self.platform}" f" --experiment {self.experiment}"