Model log compression

payu/experiment.py

@@ -503,7 +503,7 @@ def setup(self, force_archive=False):
 
         # Check restart pruning for valid configuration values and
         # warns user if more restarts than expected would be pruned
-        if self.config.get('archive', True):
+        if self.archiving():
             self.get_restarts_to_prune()
 
     def run(self, *user_flags):
@@ -769,8 +769,25 @@ def run(self, *user_flags):
         if run_script:
             self.run_userscript(run_script)
 
+    def archiving(self):
+        """
+        Determine whether to run archive step based on config.yaml settings.
+        Default to True when archive settings are absent.
+        """
+        archive_config = self.config.get('archive', {})
+        if isinstance(archive_config, dict):
+            return archive_config.get('enable', True)
+
+        # Backwards compatibility for configs with boolean archive setting
+        elif isinstance(archive_config, bool):
+            return archive_config
+
+        else:
+            msg = "Incorrect format for archive settings in config.yaml"
+            raise RuntimeError(msg)
+
     def archive(self, force_prune_restarts=False):
-        if not self.config.get('archive', True):
+        if not self.archiving():
             print('payu: not archiving due to config.yaml setting.')
             return
 

payu/models/cice.py

@@ -342,10 +342,29 @@ def archive(self, **kwargs):
         else:
             shutil.rmtree(self.work_input_path)
 
-        if self.expt.config.get('compress_logs', False):
+        if self.compression_enabled():
             self.compress_log_files()
 
+    def compression_enabled(self):
+        """
+        Determine whether to run log compression based on config.yaml settings.
+        Default to True when 'compress_logs' setting is absent.
+        """
+        archive_config = self.expt.config.get('archive', {})
+        if isinstance(archive_config, dict):
+            return archive_config.get('compress_logs', True)
+        else:
+            return True
+
     def get_log_files(self):
+        """
+        Find model log files in the work directory based on regex patterns
+        in self.logs_to_compress.
+
+        Returns
+        -------
+        log_files: list of paths to model log files.
+        """
         log_files = []
         for filename in os.listdir(self.work_path):
             if any((re.match(pattern, filename)
@@ -354,6 +373,9 @@ def get_log_files(self):
         return log_files
 
     def compress_log_files(self):
+        """
+        Compress model log files into tarball.
+        """
         log_files = self.get_log_files()
         with tarfile.open(name=os.path.join(self.work_path, self.log_tar_name),
                           mode="w:gz") as tar:

test/models/test_cice.py

@@ -4,6 +4,7 @@
 import pytest
 import f90nml
 import tarfile
+from pathlib import Path
 
 import payu
 
@@ -142,6 +143,7 @@ def cice_nml():
     # Cleanup
     os.remove(nml_path)
 
+
 # Important to test None case without separate ice history file
 @pytest.fixture(params=[None,
                         {"icefields_nml": {"f_icy": "m"}},
@@ -325,22 +327,19 @@ def cice4_log_files():
     """
     Create cice log files matching those produced during ESM1.5 simulations.
     """
-    log_names = ["ice_diag_out", "ice_diag.d", "debug.root.03",
-                 "iceout086", "iceout088", "iceout090", "iceout092",
-                 "iceout094", "iceout096", "iceout085", "iceout087",
-                 "iceout089", "iceout091", "iceout093", "iceout095"]
-    log_paths = [os.path.join(expt_workdir, name) for name in log_names]
+    log_names = (["ice_diag_out", "ice_diag.d", "debug.root.03"]
+                 + [f'iceout{x:03d}' for x in range(85, 96)])
+    log_paths = [Path(expt_workdir)/name for name in log_names]
 
     for log_file in log_paths:
-        with open(log_file, "w") as f:
-            f.close()
+        log_file.touch()
 
     yield log_paths
 
     # Cleanup
     for log_file in log_paths:
         try:
-            os.remove(log_file)
+            log_file.unlink()
         except FileNotFoundError:
             pass
 
@@ -351,14 +350,13 @@ def non_log_file():
     Create a cice4 output file to be ignored by log compression.
     Use cice_in.nml which is copied to the work directory in ESM1.5.
     """
-    non_log_path = os.path.join(expt_workdir, CICE_NML_NAME)
-    with open(non_log_path, "w") as f:
-        f.close()
+    non_log_path = Path(expt_workdir)/CICE_NML_NAME
+    non_log_path.touch()
 
     yield non_log_path
 
     # Cleanup
-    os.remove(non_log_path)
+    non_log_path.unlink()
 
 
 @pytest.mark.parametrize("config", [CONFIG_WITH_COMPRESSION],
@@ -381,11 +379,12 @@ def test_log_compression(config, cice4_log_files, non_log_file,
 
     # Check that log tarball created and no original logs remain
     assert set(os.listdir(expt_workdir)) == {model.log_tar_name,
-                                             os.path.basename(non_log_file)}
+                                             non_log_file.name}
 
     # Check all logs present in tarball
-    log_file_names = {os.path.basename(log_path) for
+    log_file_names = {log_path.name for
                       log_path in cice4_log_files}
+
     with tarfile.open(os.path.join(expt_workdir, model.log_tar_name),
                       "r") as tar:
         assert set(tar.getnames()) == log_file_names