diff --git a/test/test_telemetry.py b/test/test_telemetry.py index 090aaa9c..f7746206 100644 --- a/test/test_telemetry.py +++ b/test/test_telemetry.py @@ -1,8 +1,6 @@ -import datetime -import os +from datetime import datetime import json -import warnings -from pathlib import Path +import os import pytest from unittest.mock import patch, Mock @@ -11,6 +9,7 @@ from payu.telemetry import Telemetry from payu.telemetry import TELEMETRY_CONFIG, SERVER_URL_FIELD, HOSTNAME_FIELD + @pytest.fixture def setup_env(tmp_path): config_path = tmp_path / "telemetry_config.json" @@ -82,26 +81,17 @@ def test_post_telemetry_data_missing_access_py_telemetry(mock_api_handler): mock_api = Mock() mock_api_handler.return_value = mock_api + expected_warning = ( + "access_py_telemetry module not found. Skipping posting telemetry." + ) with patch.dict('sys.modules', {'access_py_telemetry.api': None}): - with pytest.warns(UserWarning, match="access_py_telemetry module not found. Skipping posting telemetry."): - post_telemetry_data("http://example.com", {}, "service_name", "function_name") + with pytest.warns(UserWarning, match=expected_warning): + post_telemetry_data("http://example.com", {}, "service_name", + "function_name") mock_api_handler.assert_not_called() -# def test_post_telemetry_data(): -# """Test posting to ApiHandler().send_api_request - -# Just confirming that an non-existent server_url, service name and function -# name doesn't run into errors""" -# #TODO: Should access_py_telemetry be imported in payu source -# # code or just imported as test dependency? -# post_telemetry_data("http://example.com", {}, "service_name", "function_name", -# request_timeout=1) - - # It does run into errors - service name needs to be defined. - - @patch('payu.telemetry.get_scheduler_run_info') @patch('payu.telemetry.get_external_telemetry_config') @patch('payu.telemetry.post_telemetry_data') @@ -112,17 +102,17 @@ def test_telemetry_record_run_no_telemetry_config( tmp_path ): mock_telemetry_scheduler_run_info.return_value = { - "test_field": "test_value" + "test_field": "test_value" } mock_telemetry_get_external_config.return_value = None mock_post_telemetry_data.return_value = None # Setup Telemetry class - telemetry = Telemetry(config = {}, scheduler=None) + telemetry = Telemetry(config={}, scheduler=None) job_info_filepath = tmp_path / "job.json" telemetry.set_run_info_filepath(job_info_filepath) telemetry.telemetry_enabled = False - + # Run method telemetry.record_run() @@ -134,7 +124,7 @@ def test_telemetry_record_run_no_telemetry_config( assert job_info_filepath.exists() with open(job_info_filepath, 'r') as f: assert json.load(f) == { - "test_field": "test_value" + "test_field": "test_value" } @@ -142,7 +132,7 @@ def test_telemetry_not_enabled_no_environment_config(monkeypatch): # Ensure telemetry config is not in os environment if TELEMETRY_CONFIG in os.environ: monkeypatch.delenv(TELEMETRY_CONFIG, raising=False) - + telemetry = Telemetry(config={}, scheduler=None) assert not telemetry.telemetry_enabled @@ -151,7 +141,7 @@ def test_telemetry_not_enabled_config(monkeypatch, tmp_path): # Set up the environment variable config_path = tmp_path / "telemetry_config.json" monkeypatch.setenv(TELEMETRY_CONFIG, str(config_path)) - + config = { "telemetry": { "enable": False @@ -170,58 +160,87 @@ def test_telemetry_enabled(monkeypatch, tmp_path): assert telemetry.telemetry_enabled -# TODO: Finish larger test -# @patch('payu.__version__', new='2.0.0') -# def test_telemetry_record_run(monkeypatch, tmp_path): - -# # Mock out experiment values -# experiment = Mock() -# experiment.run_id = "test-commit-hash" -# experiment.counter = 0 -# experiment.n_runs = 0 -# experiment.start_time = datetime(2025, 1, 1, 0, 0, 0) -# experiment.finish_time = datetime(2025, 1, 1, 0, 0, 30) -# experiment.payu_path = "path/to/testenv/payu" -# experiment.control_path = "path/to/control/dir" -# experiment.archive_path = "path/to/archive/dir" - -# # Mock metadata -# metadata = Mock() -# metadata.read_file.return_value = { -# "experiment_uuid": "test-uuid", -# "created": "2025-01-01", -# "name": "test-expt-name", -# "model": "test-model" -# } -# experiment.metadata = metadata - -# # Mock scheduler -# scheduler = Mock() -# scheduler.get_job_id.return_value = "test-job-id" -# scheduler.get_job_info.return_value = { -# "job_id": "test-job-id", -# "project": "test-project" -# } -# scheduler.name = "test-scheduler" - -# # Create telemetry config and environment variable -# telemetry_config = { -# "server_url": "test-persistent-session-hostname", -# "hostname": "test-host" -# } - -# telemetry_config_path = tmp_path / "telemetry_config.json" -# with open(telemetry_config_path, 'w') as f: -# json.dump(telemetry_config, f) -# monkeypatch.setenv(TELEMETRY_CONFIG, str(telemetry_config_path)) - -# # How to access API handler? - -# # Setup Telemetry class -# telemetry = Telemetry(config={}, scheduler=scheduler) -# telemetry.set_run_info(experiment=experiment) -# telemetry.record_run() +@patch('payu.__version__', new='2.0.0') +def test_telemetry_payu_run(monkeypatch, tmp_path): + """Test whole telemetry build run info and record run""" + + # Mock out experiment values + experiment = Mock() + experiment.run_id = "test-commit-hash" + experiment.counter = 0 + experiment.n_runs = 0 + experiment.run_job_status = 0 + experiment.start_time = datetime(2025, 1, 1, 0, 0, 0) + experiment.finish_time = datetime(2025, 1, 1, 0, 0, 30) + experiment.payu_path = "path/to/testenv/payu" + experiment.control_path = "path/to/control/dir" + experiment.archive_path = "path/to/archive/dir" + + # Mock metadata + metadata = Mock() + metadata.read_file.return_value = { + "experiment_uuid": "test-uuid", + "created": "2025-01-01", + "name": "test-expt-name", + "model": "test-model" + } + experiment.metadata = metadata + + # Mock scheduler + scheduler = Mock() + scheduler.get_job_id.return_value = "test-job-id" + scheduler.get_job_info.return_value = { + "job_id": "test-job-id", + "project": "test-project" + } + scheduler.name = "test-scheduler" + # Create telemetry config and environment variable + telemetry_config = { + "server_url": "test-persistent-session-hostname", + "hostname": "test-host" + } + telemetry_config_path = tmp_path / "telemetry_config.json" + with open(telemetry_config_path, 'w') as f: + json.dump(telemetry_config, f) + monkeypatch.setenv(TELEMETRY_CONFIG, str(telemetry_config_path)) + # Setup Telemetry class + telemetry = Telemetry(config={}, scheduler=scheduler) + # Save run state information during experiment run + telemetry.set_run_info(experiment=experiment) + # Configure job info path during experiment run + job_info_filepath = tmp_path / "job.json" + telemetry.set_run_info_filepath(job_info_filepath) + # Store & post run job information + telemetry.record_run() + # Should this be tested here - how likely is this to change + # - would there be a fixed access_py_telemetry version shipped with payu? + from access_py_telemetry.api import ApiHandler + record = ApiHandler()._last_record + assert record['function'] == 'payu.subcommands.run_cmd.runscript' + assert record['args'] == {} + assert record['kwargs'] == {} + assert record['experiment_uuid'] == 'test-uuid' + assert record['experiment_created'] == '2025-01-01' + assert record['experiment_name'] == 'test-expt-name' + assert record['model'] == 'test-model' + assert record['payu_run_id'] == 'test-commit-hash' + assert record['payu_current_run'] == 0 + assert record['payu_n_runs'] == 0 + assert record['payu_job_status'] == 0 + assert record['payu_start_time'] == '2025-01-01T00:00:00' + assert record['payu_finish_time'] == '2025-01-01T00:00:30' + assert record['payu_walltime_seconds'] == 30.0 + assert record['payu_version'] == '2.0.0' + assert record['payu_path'] == 'path/to/testenv' + assert record['payu_control_dir'] == 'path/to/control/dir' + assert record['payu_archive_dir'] == 'path/to/archive/dir' + assert record['scheduler_type'] == 'test-scheduler' + assert record['scheduler_job_id'] == 'test-job-id' + assert record['hostname'] == 'test-host' + + telemetry.clear_run_info() + assert telemetry.run_info == {}