diff --git a/payu/git_utils.py b/payu/git_utils.py index 1565c20a..6880e578 100644 --- a/payu/git_utils.py +++ b/payu/git_utils.py @@ -102,7 +102,7 @@ def commit(self, untracked_files = [Path(self.repo_path) / path for path in self.repo.untracked_files] for path in paths_to_commit: - if self.repo.git.diff(None, path) or path in untracked_files: + if self.repo.git.diff(None, path) or Path(path) in untracked_files: self.repo.index.add([path]) changes = True diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index 6e4e2a4c..e4298401 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -19,7 +19,7 @@ from payu.fsops import mkdir_p, make_symlink from payu.models.model import Model from payu.models.fms import fms_collate -from payu.models.mom6 import mom6_add_parameter_files +from payu.models.mom6 import mom6_add_parameter_files, mom6_save_docs_files NUOPC_CONFIG = "nuopc.runconfig" NUOPC_RUNSEQ = "nuopc.runseq" @@ -272,6 +272,11 @@ def _setup_checks(self): return True def archive(self): + + # Move any the MOM_parameter_docs output back into the control repo and commit it for documentation + if 'mom' in self.components.values() : + mom6_save_docs_files(self) + super().archive() mkdir_p(self.restart_path) @@ -387,14 +392,16 @@ class AccessOm3(CesmCmeps): def get_components(self): super().get_components() - assert self.components["atm"] == "datm", ( - "Access-OM3 comprises a data atmosphere model, but the atmospheric model in nuopc.runconfig is set " - f"to {self.components['atm']}." - ) - assert self.components["rof"] == "drof", ( - "Access-OM3 comprises a data runoff model, but the runoff model in nuopc.runconfig is set " - f"to {self.components['rof']}." - ) + if (self.components["atm"] != "datm") : + raise RuntimeError( + "Access-OM3 comprises a data atmosphere model, but the atmospheric model in nuopc.runconfig is set " + f"to {self.components['atm']}." + ) + if (self.components["rof"] != "drof") : + raise RuntimeError( + "Access-OM3 comprises a data runoff model, but the runoff model in nuopc.runconfig is set " + f"to {self.components['rof']}." + ) class Runconfig: diff --git a/payu/models/mom6.py b/payu/models/mom6.py index d4860356..73f7efb6 100644 --- a/payu/models/mom6.py +++ b/payu/models/mom6.py @@ -13,11 +13,17 @@ # Extensions import f90nml +import shutil +from warnings import warn +from glob import glob # Local +from payu.fsops import mkdir_p from payu.models.fms import Fms from payu.models.mom_mixin import MomMixin +from payu.git_utils import GitRepository +MOM6_DOCS = ["MOM_parameter_doc.*","available_diags.*"] def mom6_add_parameter_files(model): """Add parameter files defined in input.nml to model configuration files. @@ -41,6 +47,32 @@ def mom6_add_parameter_files(model): else: model.config_files.extend(filenames) +def mom6_save_docs_files(model): + """Add docs files created as MOM output back to the control directory""" + docs_folder = os.path.join(model.control_path, 'docs') + mkdir_p(docs_folder) + + # copy everything that matches MOM_parameter_doc.* to the control dir + for pattern in MOM6_DOCS: + for f in glob(os.path.join(model.work_path, pattern)): + try: + shutil.copy(f, docs_folder) + except Exception as e: + warn(e) + + if model.expt.runlog.enabled: #if runlog true, default to true + # commit new files to the control dir + repo = GitRepository(repo_path = model.control_path) + + paths_to_commit = [] + for pattern in MOM6_DOCS: + for i in glob(os.path.join(docs_folder, pattern)): + paths_to_commit.append(i) + + repo.commit( + commit_message = "payu archive: documentation of MOM6 run-time configuration" , + paths_to_commit = paths_to_commit + ) class Mom6(MomMixin, Fms): """Interface to GFDL's MOM6 ocean model.""" @@ -93,3 +125,10 @@ def init_config(self): input_nml['SIS_input_nml']['input_filename'] = input_type f90nml.write(input_nml, input_fpath, force=True) + + def archive(self): + # Move any the MOM_parameter_docs output back into the control repo + # and commit it for documentation + mom6_save_docs_files(self) + + super().archive() diff --git a/test/common.py b/test/common.py index 2ef76b3b..05b53e77 100644 --- a/test/common.py +++ b/test/common.py @@ -7,8 +7,6 @@ import yaml -import payu - # Namespace clash if import setup_cmd.runcmd as setup. For # consistency use payu_ prefix for all commands from payu.subcommands.init_cmd import runcmd as payu_init diff --git a/test/models/access-om3/test_access_om3.py b/test/models/access-om3/test_access_om3.py index e1995cf4..61597a2a 100644 --- a/test/models/access-om3/test_access_om3.py +++ b/test/models/access-om3/test_access_om3.py @@ -1,7 +1,6 @@ import copy import os import shutil -from pathlib import Path import pytest import payu @@ -421,4 +420,3 @@ def test_get_restart_datetime_badcal(start_dt, calendar, cmeps_calendar, expecte teardown_cmeps_config() remove_expt_archive_dirs(type='restart') - diff --git a/test/models/test_mom6.py b/test/models/test_mom6.py index 941fe3f1..050527da 100644 --- a/test/models/test_mom6.py +++ b/test/models/test_mom6.py @@ -1,6 +1,6 @@ -import copy import os import shutil +from pathlib import Path import pytest import f90nml @@ -11,6 +11,7 @@ from test.common import tmpdir, ctrldir, labdir, expt_workdir, ctrldir_basename from test.common import write_config, write_metadata from test.common import make_random_file, make_inputs, make_exe +from test.test_git_utils import create_new_repo verbose = True @@ -126,6 +127,146 @@ def test_mom6_add_parameter_files(input_nml, # Tidy up input.nml os.remove(input_nml_fp) +@pytest.fixture +def mom_parameter_doc(request): + + with cd(ctrldir): + lab = payu.laboratory.Laboratory(lab_path=str(labdir)) + expt = payu.experiment.Experiment(lab, reproduce=False) + model = expt.models[0] + + # Create docs + if (request.param != None) : + for file in request.param: + filename = os.path.join(model.work_path, file) + make_random_file(filename, 8) + + yield model + + # and Tidy up + if (request.param != None) : + for file in request.param: + filename = os.path.join(model.work_path, file) + os.remove(filename) + + +@pytest.mark.parametrize( + "mom_parameter_doc", + [["MOM_parameter_doc.all","MOM_parameter_doc.debug","MOM_parameter_docs.debug", "available_diags.000000"]], + indirect=True +) +@pytest.mark.filterwarnings("error") +def test_mom6_save_doc_files(mom_parameter_doc): + # Confirm that mom6_save_doc_filse moves files names MOM_parameter_doc.* into the docs folder of a config + # and doesn't move files that don't match that name + + # don't try and commit during tests + mom_parameter_doc.expt.runlog.enabled = False + + # Function to test + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + # Check MOM_parameter_doc.* are added to control_path + for file in ["MOM_parameter_doc.all","MOM_parameter_doc.debug", "available_diags.000000"]: + filename = os.path.join(mom_parameter_doc.control_path, "docs", file) + assert os.path.isfile(filename)==True , "Payu did not move MOM_parameter_doc.* files into docs folder" + os.remove(filename) + + # Check fake files are not added to control_path + for file in ["MOM_parameter_docs.debug"]: + filename = os.path.join(mom_parameter_doc.control_path, "docs", file) + assert os.path.isfile(filename)==False, "Payu incorrectly moved MOM_parameter_docs.* files into docs folder" + + +@pytest.mark.parametrize( + "mom_parameter_doc", + [["MOM_parameter_doc.layout"]], + indirect=True +) +@pytest.mark.filterwarnings("error") +def test_mom6_commit_doc_files(mom_parameter_doc): + # Confirm that mom6_save_doc_files commits files named MOM_parameter_doc.* into the docs folder of a config + mom_parameter_doc.expt.runlog.enabled = True + + #init a git repo + repo = create_new_repo(Path(mom_parameter_doc.control_path)) + initial_commit = repo.head.commit + + # Function to test + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + # Check files are added to control_path + for file in ["MOM_parameter_doc.layout"]: + filename = os.path.join(mom_parameter_doc.control_path, "docs", file) + assert os.path.isfile(filename)==True , "docs/MOM_parameter_doc.* do not exist" + os.remove(filename) + + assert repo.head.commit != initial_commit, "Payu did not commit MOM_parameter_doc.layout" + + # Confirm it doesn't commit twice if unchanged + initial_commit = repo.head.commit + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + assert repo.head.commit == initial_commit, "Payu commit MOM_parameter_doc incorrectly" + + # Confirm it does commit twice correctly + file = "MOM_parameter_doc.all" + filename = os.path.join(mom_parameter_doc.work_path, file) + make_random_file(filename, 8) + + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + assert repo.head.commit != initial_commit, "Payu did not commit MOM_parameter_doc.all" + + # and Tidy up + filename = os.path.join(mom_parameter_doc.work_path, file) + os.remove(filename) + + +@pytest.mark.parametrize( + "mom_parameter_doc", + [["MOM_parameter_doc.layout"]], + indirect=True +) +@pytest.mark.filterwarnings("error") +def test_mom6_not_commit_doc_files(mom_parameter_doc): + # Confirm that mom6_save_doc_files doesn't commits files if runlog is False + + mom_parameter_doc.expt.runlog.enabled = False + + #init a git repo + repo = create_new_repo(Path(mom_parameter_doc.control_path)) + initial_commit = repo.head.commit + + # Function to test + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + # Check files are added to control_path + for file in ["MOM_parameter_doc.layout"]: + filename = os.path.join(mom_parameter_doc.control_path, "docs", file) + assert os.path.isfile(filename)==True , "docs/MOM_parameter_doc.* do not exist" + os.remove(filename) + + assert repo.head.commit == initial_commit, "Payu incorrectly committed MOM_parameter_docs.layout" + +@pytest.mark.parametrize( + "mom_parameter_doc", + [None], + indirect=True +) +@pytest.mark.filterwarnings("error") +def test_mom6_not_commit_doc_files(mom_parameter_doc): + # Confirm that mom6_save_doc_files doesn't commits files if runlog is False + + #init a git repo + repo = create_new_repo(Path(mom_parameter_doc.control_path)) + initial_commit = repo.head.commit + + # Function to test + payu.models.mom6.mom6_save_docs_files(mom_parameter_doc) + + assert repo.head.commit == initial_commit, "Payu incorrectly committed with no docs to add" + def test_setup(): input_nml = { diff --git a/test/test_setup.py b/test/test_setup.py index 857ada83..2f9eb5d8 100644 --- a/test/test_setup.py +++ b/test/test_setup.py @@ -257,3 +257,36 @@ def test_check_payu_version_configured_invalid_version(minimum_version): with pytest.raises(ValueError): expt.check_payu_version() + + + +# model.expt.runlog.enabled is used in code for writing runlog +# it can be set as runlog:true or runlog:enable:true in config.yaml +@pytest.mark.parametrize( + "runlog, enabled", + [ + (None, True), #default is True + (True, True), + (False, False), + ({"enable":True}, True), + ({"enable":False}, False) + ] +) +@pytest.mark.filterwarnings("error") +def test_runlog_enable(runlog, enabled): + config = copy.deepcopy(config_orig) + if runlog == None: + config.pop('runlog') #remove from config for default case + else: + config['runlog'] = runlog + + write_config(config) + + make_inputs() + + with cd(ctrldir): + lab = payu.laboratory.Laboratory(lab_path=str(labdir)) + expt = payu.experiment.Experiment(lab, reproduce=False) + model = expt.models[0] + + assert model.expt.runlog.enabled == enabled \ No newline at end of file