Merge pull request #930 from CliMA/gb/walltime

Increase AMIP walltime

.buildkite/amip/pipeline.yml

@@ -1,6 +1,6 @@
 agents:
   queue: clima
-  slurm_time: 76:00:00
+  slurm_time: 96:00:00
   modules: common
 
 env:
@@ -9,7 +9,7 @@ env:
   SLURM_KILL_BAD_EXIT: 1
   CONFIG_PATH: "config/amip_configs"
 
-timeout_in_minutes: 4560
+timeout_in_minutes: 5760
 
 steps:
 
@@ -46,7 +46,7 @@ steps:
           - echo "--- Run simulation"
           - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip.yml --job_id amip"
         artifact_paths: "experiments/ClimaEarth/output/amip/amip_artifacts/*"
-        timeout_in_minutes: 4320
+        timeout_in_minutes: 5760
         env:
           CLIMACOMMS_DEVICE: "CUDA"
         agents:
@@ -70,4 +70,3 @@ steps:
             find experiments/ClimaEarth/output/amip/amip_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
               slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)"
             done
-          - SYPD="$(cat experiments/ClimaEarth/output/amip/amip_artifacts/sypd.txt)"; WALL="$(cat experiments/ClimaEarth/output/amip/amip_artifacts/walltime_per_atmos_step.txt)"; slack-message -c "#coupler-report" -x "SYPD: $SYPD Walltime per Atmos step: $WALL"

experiments/ClimaEarth/run_amip.jl

@@ -808,15 +808,18 @@ walltime_per_atmos_step = es.walltime / n_atmos_steps
 
 ## Save the SYPD and allocation information
 if ClimaComms.iamroot(comms_ctx)
-    sypd_filename = joinpath(dir_paths.artifacts, "sypd.txt")
-    write(sypd_filename, "$sypd")
+    open(joinpath(dir_paths.artifacts, "sypd.txt"), "w") do sypd_filename
+        println(sypd_filename, "$sypd")
+    end
 
-    walltime_per_atmos_step_filename = joinpath(dir_paths.artifacts, "walltime_per_atmos_step.txt")
-    write(walltime_per_atmos_step_filename, "$(walltime_per_atmos_step)")
+    open(joinpath(dir_paths.artifacts, "walltime_per_atmos_step.txt"), "w") do walltime_per_atmos_step_filename
+        println(walltime_per_atmos_step_filename, "$(walltime_per_atmos_step)")
+    end
 
-    cpu_max_rss_GB = Utilities.show_memory_usage(comms_ctx)
-    cpu_max_rss_filename = joinpath(dir_paths.artifacts, "max_rss_cpu.txt")
-    write(cpu_max_rss_filename, cpu_max_rss_GB)
+    open(joinpath(dir_paths.artifacts, "max_rss_cpu.txt"), "w") do cpu_max_rss_filename
+        cpu_max_rss_GB = Utilities.show_memory_usage(comms_ctx)
+        println(cpu_max_rss_filename, cpu_max_rss_GB)
+    end
 end
 
 #=

test/component_model_tests/climaatmos_standalone/atmos_driver.jl

@@ -71,11 +71,11 @@ sypd = CA.simulated_years_per_day(es)
 comms_ctx = atmos_config.comms_ctx
 if ClimaComms.iamroot(comms_ctx)
     open(joinpath(output_dir, "sypd.txt"), "w") do sypd_filename
-        write(sypd_filename, "$sypd")
+        println(sypd_filename, "$sypd")
     end
 
     open(joinpath(output_dir, "max_rss_cpu.txt"), "w") do cpu_max_rss_filename
         cpu_max_rss_GB = Utilities.show_memory_usage(comms_ctx)
-        write(cpu_max_rss_filename, cpu_max_rss_GB)
+        println(cpu_max_rss_filename, cpu_max_rss_GB)
     end
 end