From 6051f55634973cbfb5ac000306e9f2cf33ce6897 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 7 Feb 2025 12:00:11 +1100 Subject: [PATCH 1/9] draft of restart pruning in om3 --- payu/models/cesm_cmeps.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index d9d9d41c..06c3f48b 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -13,6 +13,7 @@ import os import re import shutil +import cftime from warnings import warn from payu.fsops import mkdir_p, make_symlink @@ -336,6 +337,38 @@ def collate(self): else: super().collate() + def get_restart_datetime(self, restart_path): + """Given a restart path, parse the restart files and + return a cftime datetime (for date-based restart pruning)""" + + # Check for rpointer.cpl file + rpointer_path = os.path.join(restart_path, 'rpointer.cpl') + if not os.path.exists(rpointer_path): + raise FileNotFoundError( + 'Cannot find rpointer.cpl file, which is required for ' + 'date-based restart pruning') + + with open(rpointer_path, 'r') as ocean_solo: + lines = ocean_solo.readlines() + # example lines would be access-om3.cpl.r.1900-01-02-00000.nc + + date_str = lines[0].split('.')[3] + year, month, day, hms = date_str.split('-') + + self.get_runconfig(self.expt.work_path) + run_calendar = self.runconfig.get("CLOCK_attributes", "calendar") + + cftime_calendars = { + "NO_LEAP" : "noleap" , + "GREGORIAN" : "proleptic_gregorian" + } + + return cftime.datetime( + int(year), int(month), int(day), + int(hms[0:2]), int(hms[2:4]), int(hms[4:6]), + calendar = cftime_calendars[run_calendar] + ) + class AccessOm3(CesmCmeps): From 84457a82cd11001307baf5366f589009db8b1da7 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Fri, 7 Feb 2025 13:59:48 +1100 Subject: [PATCH 2/9] correct path --- payu/models/cesm_cmeps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index 06c3f48b..34026bd1 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -355,7 +355,7 @@ def get_restart_datetime(self, restart_path): date_str = lines[0].split('.')[3] year, month, day, hms = date_str.split('-') - self.get_runconfig(self.expt.work_path) + self.get_runconfig(self.expt.control_path) run_calendar = self.runconfig.get("CLOCK_attributes", "calendar") cftime_calendars = { From ebf05c0d849806381b169486fe217e8547c4684b Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Mon, 10 Feb 2025 15:59:02 +1100 Subject: [PATCH 3/9] tests and fixes --- payu/models/cesm_cmeps.py | 30 ++++-- test/models/access-om3/test_access_om3.py | 110 ++++++++++++++++++++++ 2 files changed, 132 insertions(+), 8 deletions(-) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index 34026bd1..6178d945 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -24,6 +24,12 @@ NUOPC_CONFIG = "nuopc.runconfig" NUOPC_RUNSEQ = "nuopc.runseq" +#mapping of runconfig to cftime calendars: +CFTIME_CALENDARS = { + "NO_LEAP" : "noleap" , + "GREGORIAN" : "proleptic_gregorian" +} + # Add as needed component_info = { "mom": { @@ -339,7 +345,8 @@ def collate(self): def get_restart_datetime(self, restart_path): """Given a restart path, parse the restart files and - return a cftime datetime (for date-based restart pruning)""" + return a cftime datetime (for date-based restart pruning) + Supports noleap and proleptic gregorian calendars""" # Check for rpointer.cpl file rpointer_path = os.path.join(restart_path, 'rpointer.cpl') @@ -353,20 +360,27 @@ def get_restart_datetime(self, restart_path): # example lines would be access-om3.cpl.r.1900-01-02-00000.nc date_str = lines[0].split('.')[3] - year, month, day, hms = date_str.split('-') + year, month, day, seconds = date_str.split('-') + + seconds = int(seconds) + hour = seconds // 3600 ; min = (seconds % 3600) // 60 ; sec = seconds % 60 self.get_runconfig(self.expt.control_path) run_calendar = self.runconfig.get("CLOCK_attributes", "calendar") - cftime_calendars = { - "NO_LEAP" : "noleap" , - "GREGORIAN" : "proleptic_gregorian" - } + try: + cf_cal = CFTIME_CALENDARS[run_calendar] + except KeyError: + warn( + f"Unsupported calendar for restart pruning: {run_calendar}," + f" try {' or '.join(CFTIME_CALENDARS.keys())}" + ) + return False return cftime.datetime( int(year), int(month), int(day), - int(hms[0:2]), int(hms[2:4]), int(hms[4:6]), - calendar = cftime_calendars[run_calendar] + hour, min, sec, + calendar = cf_cal ) diff --git a/test/models/access-om3/test_access_om3.py b/test/models/access-om3/test_access_om3.py index 67a3b179..6ae5b121 100644 --- a/test/models/access-om3/test_access_om3.py +++ b/test/models/access-om3/test_access_om3.py @@ -5,10 +5,12 @@ import pytest import payu +import cftime from test.common import cd, tmpdir, ctrldir, labdir, workdir, write_config, config_path from test.common import config as config_orig from test.common import make_inputs, make_exe +from test.common import list_expt_archive_dirs, make_expt_archive_dir, remove_expt_archive_dirs MODEL = 'access-om3' @@ -312,3 +314,111 @@ def test__setup_checks_bad_io(pio_numiotasks, pio_stride): model._setup_checks() teardown_cmeps_config() + + +# test restart datetime pruning + +def make_restart_dir(start_dt): + """Create restart directory with rpointer.cpl file""" + # Create restart directory + restart_path = make_expt_archive_dir(type='restart') + + rpath = os.path.join(restart_path, "rpointer.cpl") + with open(rpath, "w") as rpointer_file: + rpointer_file.write( + f"access-om3.cpl.r.{start_dt}.nc" + ) + +@pytest.mark.parametrize( + "start_dt, calendar, cmeps_calendar, expected_cftime", + [ + ( + "0001-01-01-00000", + "proleptic_gregorian", + "GREGORIAN", + cftime.datetime(1, 1, 1, calendar="proleptic_gregorian") + ), + ( + "9999-12-31-86399", + "proleptic_gregorian", + "GREGORIAN", + cftime.datetime(9999, 12, 31,23,59,59, calendar="proleptic_gregorian") + ), + ( + "1900-02-01-00000", + "noleap", + "NO_LEAP", + cftime.datetime(1900, 2, 1, calendar="noleap") + ), + ]) +@pytest.mark.filterwarnings("error") +def test_get_restart_datetime(start_dt, calendar, cmeps_calendar, expected_cftime): + + cmeps_config(1) + + make_restart_dir(start_dt) + + test_runconf = { + "CLOCK_attributes": { + "calendar":cmeps_calendar + } + } + + with cd(ctrldir): + lab = payu.laboratory.Laboratory(lab_path=str(labdir)) + expt = payu.experiment.Experiment(lab, reproduce=False) + + model = expt.model + model.get_runconfig = lambda a : (True) # mock reading runconf from file + model.runconfig = MockRunConfig(test_runconf) + + print(model.runconfig.get("CLOCK_attributes","calendar")) + + restart_path = list_expt_archive_dirs()[0] + parsed_run_dt = expt.model.get_restart_datetime(restart_path) + assert parsed_run_dt == expected_cftime + + teardown_cmeps_config() + remove_expt_archive_dirs(type='restart') + +@pytest.mark.parametrize( + "start_dt, calendar, cmeps_calendar, expected_cftime", + [ + ( + "1900-02-01-00000", + "julian", + "JULIAN", + cftime.datetime(1900, 2, 1, calendar="julian") + ), + ]) +def test_get_restart_datetime_badcal(start_dt, calendar, cmeps_calendar, expected_cftime): + + cmeps_config(1) + + make_restart_dir(start_dt) + + test_runconf = { + "CLOCK_attributes": { + "calendar":cmeps_calendar + } + } + + with cd(ctrldir): + lab = payu.laboratory.Laboratory(lab_path=str(labdir)) + expt = payu.experiment.Experiment(lab, reproduce=False) + + model = expt.model + model.get_runconfig = lambda a : (True) # mock reading runconf from file + model.runconfig = MockRunConfig(test_runconf) + + print(model.runconfig.get("CLOCK_attributes","calendar")) + + restart_path = list_expt_archive_dirs()[0] + with pytest.warns( + Warning, match="Unsupported calendar" + ): + expt.model.get_restart_datetime(restart_path) + + teardown_cmeps_config() + remove_expt_archive_dirs(type='restart') + From a5fa0d7a089b4548d452fdbdbfa3626d42ff27f0 Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Mon, 10 Feb 2025 16:04:41 +1100 Subject: [PATCH 4/9] docs --- docs/source/config.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/config.rst b/docs/source/config.rst index 4ed48113..3a093fc8 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -217,9 +217,9 @@ configuration. For example, ``restart_freq: 10YS`` would save earliest restart of the year, 10 years from the last permanently archived restart's datetime. - Please note that currently, only ACCESS-OM2, MOM5 and MOM6 models support - date-based restart frequency, as it depends on the payu model driver being - able to parse restarts files for a datetime. + Please note that currently, only ACCESS-ESM1.5, ACCESS-OM2, ACCESS-OM3, MOM5 + and MOM6 models support date-based restart frequency, as it depends on the payu + model driver being able to parse restarts files for a datetime. ``restart_history`` Specifies how many of the most recent restart files to retain regardless of From 6af75db7ebe20fb5b4171b7db9a866c9b64b1a14 Mon Sep 17 00:00:00 2001 From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:55:32 +1100 Subject: [PATCH 5/9] Update payu/models/cesm_cmeps.py --- payu/models/cesm_cmeps.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index c229cbc4..72f6a4a5 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -24,7 +24,9 @@ NUOPC_CONFIG = "nuopc.runconfig" NUOPC_RUNSEQ = "nuopc.runseq" -#mapping of runconfig to cftime calendars: +# mapping of runconfig to cftime calendars: +# these match the supported calendars in CMEPS +# https://github.com/ESCOMP/CMEPS/blob/bc29792d76c16911046dbbfcfc7f4c3ae89a6f00/cesm/driver/ensemble_driver.F90#L196 CFTIME_CALENDARS = { "NO_LEAP" : "noleap" , "GREGORIAN" : "proleptic_gregorian" From caa094b8bb63b640cb0a8f95fd1e62eb8f9803af Mon Sep 17 00:00:00 2001 From: anton-seaice Date: Tue, 11 Feb 2025 15:29:51 +1100 Subject: [PATCH 6/9] use error not warning for unsupported cal --- payu/models/cesm_cmeps.py | 11 ++++++----- test/models/access-om3/test_access_om3.py | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index 72f6a4a5..adc646f0 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -358,6 +358,7 @@ def get_restart_datetime(self, restart_path): date_str = lines[0].split('.')[3] year, month, day, seconds = date_str.split('-') + # convert seconds into hours, mins, seconds seconds = int(seconds) hour = seconds // 3600 ; min = (seconds % 3600) // 60 ; sec = seconds % 60 @@ -366,11 +367,11 @@ def get_restart_datetime(self, restart_path): try: cf_cal = CFTIME_CALENDARS[run_calendar] - except KeyError: - warn( - f"Unsupported calendar for restart pruning: {run_calendar}," - f" try {' or '.join(CFTIME_CALENDARS.keys())}" - ) + except KeyError as e: + raise RuntimeError( + f"Unsupported calendar for restart pruning: {run_calendar}, in {NUOPC_CONFIG}." + f" Try {' or '.join(CFTIME_CALENDARS.keys())}" + ) from e return False return cftime.datetime( diff --git a/test/models/access-om3/test_access_om3.py b/test/models/access-om3/test_access_om3.py index 6ae5b121..e1995cf4 100644 --- a/test/models/access-om3/test_access_om3.py +++ b/test/models/access-om3/test_access_om3.py @@ -414,8 +414,8 @@ def test_get_restart_datetime_badcal(start_dt, calendar, cmeps_calendar, expecte print(model.runconfig.get("CLOCK_attributes","calendar")) restart_path = list_expt_archive_dirs()[0] - with pytest.warns( - Warning, match="Unsupported calendar" + with pytest.raises( + RuntimeError, match="Unsupported calendar" ): expt.model.get_restart_datetime(restart_path) From ccfcac44835e4668f43ed14e5ea008690f5d72b0 Mon Sep 17 00:00:00 2001 From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:44:13 +1100 Subject: [PATCH 7/9] Update docs/source/config.rst --- docs/source/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config.rst b/docs/source/config.rst index 3a093fc8..9d0a4780 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -218,7 +218,7 @@ configuration. 10 years from the last permanently archived restart's datetime. Please note that currently, only ACCESS-ESM1.5, ACCESS-OM2, ACCESS-OM3, MOM5 - and MOM6 models support date-based restart frequency, as it depends on the payu + , MOM6 and UM7 models support date-based restart frequency, as it depends on the payu model driver being able to parse restarts files for a datetime. ``restart_history`` From 8f21de7ece1ac8085a897bb19260ada1be8f79cc Mon Sep 17 00:00:00 2001 From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> Date: Tue, 11 Feb 2025 16:33:57 +1100 Subject: [PATCH 8/9] Update payu/models/cesm_cmeps.py --- payu/models/cesm_cmeps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/payu/models/cesm_cmeps.py b/payu/models/cesm_cmeps.py index adc646f0..6e4e2a4c 100644 --- a/payu/models/cesm_cmeps.py +++ b/payu/models/cesm_cmeps.py @@ -362,6 +362,7 @@ def get_restart_datetime(self, restart_path): seconds = int(seconds) hour = seconds // 3600 ; min = (seconds % 3600) // 60 ; sec = seconds % 60 + # TODO: change to self.control_path if https://github.com/payu-org/payu/issues/509 is implemented self.get_runconfig(self.expt.control_path) run_calendar = self.runconfig.get("CLOCK_attributes", "calendar") From 2c56dc9e8e64ccd9ef6394f2cf2f8748818f3980 Mon Sep 17 00:00:00 2001 From: Anton Steketee <79179784+anton-seaice@users.noreply.github.com> Date: Fri, 14 Feb 2025 08:06:49 +1100 Subject: [PATCH 9/9] Update docs/source/config.rst Co-authored-by: Dougie Squire <42455466+dougiesquire@users.noreply.github.com> --- docs/source/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/config.rst b/docs/source/config.rst index 9d0a4780..10b71ec9 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -218,7 +218,7 @@ configuration. 10 years from the last permanently archived restart's datetime. Please note that currently, only ACCESS-ESM1.5, ACCESS-OM2, ACCESS-OM3, MOM5 - , MOM6 and UM7 models support date-based restart frequency, as it depends on the payu + , MOM6 and UM7 models support date-based restart frequency, as it depends on the payu model driver being able to parse restarts files for a datetime. ``restart_history``