From 522d86a2b39cb4ccdc2134f9ccae39aa435941b8 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Mon, 14 Oct 2024 13:30:42 +1100 Subject: [PATCH] Initialise lab in payu checkout and Experiment() to check directory is writable Checking base labdirectory for just write access fails for laboratory directories that don't exist yet. --- payu/branch.py | 35 +++++++++++------------------------ payu/experiment.py | 2 ++ payu/laboratory.py | 25 +++++++++++++++++++++---- test/test_branch.py | 36 ++++++++++++++++++++++++++---------- 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/payu/branch.py b/payu/branch.py index 9f0d6f77..8d944138 100644 --- a/payu/branch.py +++ b/payu/branch.py @@ -22,20 +22,12 @@ from payu.git_utils import GitRepository, git_clone, PayuBranchError LAB_WRITE_ACCESS_ERROR = """ -Do not have write access to the configured laboratory directory. -Skipping creating metadata, setting up restart configuration, and archive/work -symlinks. - -To fix the errors, edit one (or more) of the following config.yaml options that -determine the laboratory path: - - 'laboratory': (Top-level directory for the model laboratory - Default: /scratch/${PROJECT}/${USER}/${MODEL} - - 'shortpath' (Top-level directory for laboratory. - Default: /scratch/${PROJECT} - - 'project': (The project to use for payu PBS jobs. Default: ${PROJECT}) - -Then either run 'payu setup' or rerun checkout command with the current git -branch, e.g. `payu checkout -r ` +Failed to initialise laboratory directories. Skipping creating metadata, +setting up restart configuration, and archive/work symlinks. + +To fix, first modify/remove the config.yaml options that determine laboratory +path. Then either run 'payu setup' or rerun checkout command with the current +git branch, e.g. `payu checkout -r ` """ NO_CONFIG_FOUND_MESSAGE = """No configuration file found on this branch. @@ -138,15 +130,6 @@ def check_config_path(config_path: Optional[Path] = None) -> Optional[Path]: return config_path -def check_laboratory_path(lab_basepath: str) -> None: - """Check if a laboratory path has write access""" - if not os.access(lab_basepath, os.W_OK | os.X_OK): - print(LAB_WRITE_ACCESS_ERROR) - raise PermissionError( - f"Laboratory directory does not have write access: {lab_basepath}" - ) - - def checkout_branch(branch_name: str, is_new_branch: bool = False, is_new_experiment: bool = False, @@ -198,7 +181,11 @@ def checkout_branch(branch_name: str, # Initialise Lab lab = Laboratory(model_type, config_path, lab_path) - check_laboratory_path(lab.basepath) + try: + lab.initialize() + except PermissionError: + print(LAB_WRITE_ACCESS_ERROR) + raise # Initialise metadata metadata = Metadata(Path(lab.archive_path), diff --git a/payu/experiment.py b/payu/experiment.py index 62b531ee..40832736 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -52,6 +52,8 @@ class Experiment(object): def __init__(self, lab, reproduce=False, force=False, metadata_off=False): self.lab = lab + # Check laboratory directories are writable + self.lab.initialize() if not force: # check environment for force flag under PBS diff --git a/payu/laboratory.py b/payu/laboratory.py index d571f661..cf74ea08 100644 --- a/payu/laboratory.py +++ b/payu/laboratory.py @@ -10,6 +10,17 @@ from payu.fsops import mkdir_p, read_config +LAB_INITIALIZE_ERROR = """ +The configured laboratory directory may not have write access. Edit/remove one +(or more) of the following config.yaml options that determine the laboratory +path: + - 'laboratory': Top-level directory for the model laboratory + Default: /scratch/${PROJECT}/${USER}/${MODEL} + - 'shortpath' Top-level directory for laboratory. + Default: /scratch/${PROJECT} + - 'project': The project to use for payu PBS jobs. Default: ${PROJECT} +""" + class Laboratory(object): """Interface to the numerical model's laboratory.""" @@ -84,7 +95,13 @@ def get_default_lab_path(self, config): def initialize(self): """Create the laboratory directories.""" - mkdir_p(self.archive_path) - mkdir_p(self.bin_path) - mkdir_p(self.codebase_path) - mkdir_p(self.input_basepath) + try: + mkdir_p(self.archive_path) + mkdir_p(self.bin_path) + mkdir_p(self.codebase_path) + mkdir_p(self.input_basepath) + except PermissionError as e: + print(LAB_INITIALIZE_ERROR) + raise PermissionError( + f"Failed to initialise laboratory directories. Error: {e}" + ) diff --git a/test/test_branch.py b/test/test_branch.py index 41f10559..352dcf1d 100644 --- a/test/test_branch.py +++ b/test/test_branch.py @@ -9,7 +9,6 @@ from payu.branch import add_restart_to_config, check_restart, switch_symlink from payu.branch import checkout_branch, clone, list_branches -from payu.branch import check_laboratory_path from payu.metadata import MetadataWarning from payu.fsops import read_config @@ -515,20 +514,37 @@ def test_checkout_branch_with_restart_path(mock_uuid): expected_parent_uuid=uuid1) -@patch("os.access") -def test_check_laboratory_path(mock_os_access): - # Test seems a bit useless as was unsure on how to create a directory - # without write permissions without locking myself out the directory - mock_os_access.return_value = False +@patch("payu.laboratory.Laboratory.initialize") +def test_checkout_laboratory_path_error(mock_lab_initialise): + mock_lab_initialise.side_effect = PermissionError + + repo = setup_control_repository() + current_commit = repo.active_branch.object.hexsha + with cd(ctrldir): # Test raises a permission error with pytest.raises(PermissionError): - check_laboratory_path("some/path") + checkout_branch(branch_name="Branch1", + is_new_branch=True, + lab_path=labdir) + + # Assert new commit has not been added + assert repo.active_branch.object.hexsha == current_commit - mock_os_access.return_value = True + assert str(repo.active_branch) == "Branch1" + assert not (ctrldir / "metadata.yaml").exists() + + # Test removing lab directory + shutil.rmtree(labdir) + mock_lab_initialise.side_effect = None with cd(ctrldir): - # Test check runs without errors - check_laboratory_path("some/path") + # Test runs without an error - directory is initialised + checkout_branch(branch_name="Branch2", + is_new_branch=True, + lab_path=labdir) + + # Assert new commit has been added + assert repo.active_branch.object.hexsha != current_commit @patch("uuid.uuid4")