Skip to content

Commit cd0ac1b

Browse files
juliasloan25juliasloan25
committed
add nightly coarse amip pipeline
1 parent f5b8c7b commit cd0ac1b

File tree

5 files changed

+160
-1
lines changed

5 files changed

+160
-1
lines changed

‎.buildkite/nightly/pipeline.yml

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
agents:
2+
queue: clima
3+
slurm_time: 14:00:00
4+
modules: common
5+
6+
env:
7+
JULIA_MAX_NUM_PRECOMPILE_FILES: 100
8+
GKSwstype: 100
9+
SLURM_KILL_BAD_EXIT: 1
10+
CONFIG_PATH: "config/nightly_configs"
11+
12+
timeout_in_minutes: 840
13+
14+
15+
steps:
16+
- label: "init :GPU:"
17+
key: "init_gpu_env"
18+
command:
19+
- "echo $$JULIA_DEPOT_PATH"
20+
21+
- echo "--- Instantiate AMIP env"
22+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
23+
24+
# For this pipeline, use the main branches of certain upstream packages
25+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"ClimaAtmos\", rev=\"main\"))'"
26+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"ClimaLand\", rev=\"main\"))'"
27+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"ClimaCore\", rev=\"main\"))'"
28+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"Thermodynamics\", rev=\"main\"))'"
29+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"SurfaceFluxes\", rev=\"main\"))'"
30+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(Pkg.PackageSpec(;name=\"RRTMGP\", rev=\"main\"))'"
31+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.resolve()'"
32+
33+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.add(\"MPI\"); Pkg.add(\"CUDA\")'"
34+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
35+
- "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
36+
37+
- echo "--- Download artifacts"
38+
- "julia --project=experiments/ClimaEarth/ artifacts/download_artifacts.jl"
39+
40+
agents:
41+
slurm_gpus: 1
42+
slurm_cpus_per_task: 8
43+
env:
44+
JULIA_NUM_PRECOMPILE_TASKS: 8
45+
JULIA_MAX_NUM_PRECOMPILE_FILES: 50
46+
47+
- wait
48+
49+
- group: "AMIP coarse ensemble"
50+
51+
steps:
52+
53+
- label: "1 Coarse current AMIP: diagedmf + 0M + bucket land"
54+
key: "amip1"
55+
command:
56+
- echo "--- Run simulation"
57+
- "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse.yml --job_id amip_coarse1"
58+
artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse1_artifacts/*"
59+
timeout_in_minutes: 1080
60+
env:
61+
CLIMACOMMS_DEVICE: "CUDA"
62+
agents:
63+
slurm_gpus_per_task: 1
64+
slurm_cpus_per_task: 4
65+
slurm_ntasks: 1
66+
slurm_mem: 30GB
67+
soft_fail: true
68+
69+
- label: "2 Coarse current AMIP: diagedmf + 0M + bucket land"
70+
key: "amip2"
71+
command:
72+
- echo "--- Run simulation"
73+
- "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse.yml --job_id amip_coarse2"
74+
artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse2_artifacts/*"
75+
timeout_in_minutes: 1080
76+
env:
77+
CLIMACOMMS_DEVICE: "CUDA"
78+
agents:
79+
slurm_gpus_per_task: 1
80+
slurm_cpus_per_task: 4
81+
slurm_ntasks: 1
82+
slurm_mem: 30GB
83+
soft_fail: true
84+
85+
- label: "3 Coarse current AMIP: diagedmf + 0M + bucket land"
86+
key: "amip3"
87+
command:
88+
- echo "--- Run simulation"
89+
- "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse.yml --job_id amip_coarse3"
90+
artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse3_artifacts/*"
91+
timeout_in_minutes: 1080
92+
env:
93+
CLIMACOMMS_DEVICE: "CUDA"
94+
agents:
95+
slurm_gpus_per_task: 1
96+
slurm_cpus_per_task: 4
97+
slurm_ntasks: 1
98+
slurm_mem: 30GB
99+
soft_fail: true
100+
101+
- label: "4 Coarse current AMIP: diagedmf + 0M + bucket land"
102+
key: "amip4"
103+
command:
104+
- echo "--- Run simulation"
105+
- "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse.yml --job_id amip_coarse4"
106+
artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse4_artifacts/*"
107+
timeout_in_minutes: 1080
108+
env:
109+
CLIMACOMMS_DEVICE: "CUDA"
110+
agents:
111+
slurm_gpus_per_task: 1
112+
slurm_cpus_per_task: 4
113+
slurm_ntasks: 1
114+
slurm_mem: 30GB
115+
soft_fail: true

‎Project.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,6 @@ Logging = "1"
2828
NCDatasets = "0.11, 0.12, 0.13, 0.14.2"
2929
SciMLBase = "1, 2"
3030
StaticArrays = "1"
31-
SurfaceFluxes = "0.11"
31+
SurfaceFluxes = "0.11, 0.12"
3232
Thermodynamics = "0.12.4"
3333
julia = "1.8"

‎config/nightly_configs/amip_coarse.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
FLOAT_TYPE: "Float32"
2+
albedo_model: "CouplerAlbedo"
3+
anim: false
4+
atmos_config_file: "config/longrun_configs/amip_target_diagedmf.yml"
5+
coupler_toml_file: "toml/amip.toml"
6+
dt: "120secs"
7+
dt_cpl: 120
8+
dt_save_state_to_disk: "30days"
9+
dt_save_to_sol: "30days"
10+
dz_bottom: 30.0
11+
energy_check: false
12+
h_elem: 8
13+
hourly_checkpoint: true
14+
hourly_checkpoint_dt: 720
15+
land_albedo_type: "map_temporal"
16+
mode_name: "amip"
17+
mono_surface: false
18+
netcdf_output_at_levels: true
19+
output_default_diagnostics: true
20+
rayleigh_sponge: true
21+
start_date: "20100101"
22+
surface_setup: "PrescribedSurface"
23+
t_end: "366days"
24+
topo_smoothing: true
25+
topography: "Earth"
26+
turb_flux_partition: "CombinedStateFluxesMOST"
27+
unique_seed: true
28+
viscous_sponge: false
29+
z_elem: 31
30+
z_max: 55000.0

‎experiments/ClimaEarth/cli_options.jl

+4
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ function argparse_settings()
8484
help = "Boolean flag indicating whether to compute and output coupler diagnostics [`true` (default), `false`]"
8585
arg_type = Bool
8686
default = true
87+
"--unique_seed"
88+
help = "Boolean flag indicating whether to set the random number seed to a unique value [`false` (default), `true`]"
89+
arg_type = Bool
90+
default = false
8791
# ClimaAtmos specific
8892
"--surface_setup"
8993
help = "Triggers ClimaAtmos into the coupled mode [`PrescribedSurface` (default)]" # retained here for standalone Atmos benchmarks

‎experiments/ClimaEarth/run_amip.jl

+10
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,16 @@ We can additionally pass the configuration dictionary to the component model ini
100100
include("cli_options.jl")
101101
parsed_args = parse_commandline(argparse_settings())
102102

103+
## set unique random seed if desired, otherwise use default
104+
if parsed_args["unique_seed"]
105+
time_ns = time_ns()
106+
Random.seed!(time_ns)
107+
@info("Random seed set to $time_ns")
108+
else
109+
Random.seed!(1234)
110+
@info("Random seed set to 1234")
111+
end
112+
103113
## modify parsed args for fast testing from REPL #hide
104114
if isinteractive()
105115
parsed_args["config_file"] =

0 commit comments

Comments
 (0)