Skip to content

Commit 7d4027f

Browse files
juliasloan25juliasloan25
authored
Merge pull request #804 from CliMA/js/jobid
pass in job_id as cmd line argument
2 parents da361b4 + cc32cb2 commit 7d4027f

File tree

78 files changed

+146
-592
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+146
-592
lines changed

‎.buildkite/benchmarks/pipeline.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ steps:
5757

5858
- label: "CPU AMIP with diagnostic EDMF"
5959
key: "amip_diagedmf"
60-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml"
60+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id amip_diagedmf"
6161
artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_artifacts/*"
6262
env:
6363
BUILD_HISTORY_HANDLE: ""
@@ -71,7 +71,7 @@ steps:
7171
steps:
7272
- label: "GPU ClimaAtmos with diagnostic EDMF"
7373
key: "gpu_climaatmos_diagedmf"
74-
command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/gpu_climaatmos_diagedmf.yml --job_id gpu_climaatmos_diagedmf"
74+
command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml --job_id gpu_climaatmos_diagedmf"
7575
artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_diagedmf_artifacts/*"
7676
agents:
7777
slurm_gpus_per_task: 1
@@ -81,7 +81,7 @@ steps:
8181

8282
- label: "GPU AMIP with diagnostic EDMF"
8383
key: "gpu_amip_diagedmf"
84-
command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/gpu_amip_diagedmf.yml"
84+
command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id gpu_amip_diagedmf"
8585
artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_artifacts/*"
8686
agents:
8787
slurm_gpus_per_task: 1
@@ -93,7 +93,7 @@ steps:
9393
steps:
9494
- label: "Compare AMIP/Atmos-only with diagnostic EDMF"
9595
key: "compare_amip_climaatmos_amip_diagedmf"
96-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/user_io/benchmarks.jl --cpu_run_name_coupled amip_diagedmf --cpu_run_name_atmos climaatmos_diagedmf --gpu_run_name_coupled gpu_amip_diagedmf --gpu_run_name_atmos gpu_climaatmos_diagedmf --mode_name amip --build_id $BUILDKITE_BUILD_NUMBER"
96+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/user_io/benchmarks.jl --cpu_job_id_coupled amip_diagedmf --cpu_job_id_atmos climaatmos_diagedmf --gpu_job_id_coupled gpu_amip_diagedmf --gpu_job_id_atmos gpu_climaatmos_diagedmf --mode_name amip --build_id $BUILDKITE_BUILD_NUMBER"
9797
artifact_paths: "experiments/ClimaEarth/output/compare_amip_climaatmos_amip_diagedmf/*"
9898
depends_on:
9999
- "climaatmos_diagedmf"

‎.buildkite/longruns/pipeline.yml

+17-17
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ steps:
7070
# Integration tests - the expected results were also confirmed locally
7171
- label: "Slabplanet_aqua: nocouple"
7272
key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
73-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml"
73+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml --job_id slabplanet_aqua_atmos_sf_nocouple"
7474
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
7575
env:
7676
BUILD_HISTORY_HANDLE: ""
@@ -82,7 +82,7 @@ steps:
8282

8383
- label: "Slabplanet_aqua: couple"
8484
key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
85-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml"
85+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml --job_id slabplanet_aqua_atmos_sf_couple"
8686
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
8787
env:
8888
BUILD_HISTORY_HANDLE: ""
@@ -94,7 +94,7 @@ steps:
9494

9595
- label: "Slabplanet_aqua: coupler fluxes"
9696
key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
97-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml"
97+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml --job_id slabplanet_aqua_coupler_sf"
9898
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
9999
env:
100100
BUILD_HISTORY_HANDLE: ""
@@ -106,7 +106,7 @@ steps:
106106

107107
- label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
108108
key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
109-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml"
109+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml --job_id slabplanet_aqua_coupler_sf_evolve_ocn"
110110
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
111111
env:
112112
BUILD_HISTORY_HANDLE: ""
@@ -118,7 +118,7 @@ steps:
118118

119119
- label: "Slabplanet_terra: coupler fluxes, evolving bucket"
120120
key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
121-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml"
121+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml --job_id slabplanet_terra"
122122
artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/slabplanet_terra_artifacts/*"
123123
env:
124124
BUILD_HISTORY_HANDLE: ""
@@ -130,7 +130,7 @@ steps:
130130

131131
- label: "Slabplanet: coupler fluxes, evolving ocean and land"
132132
key: "slabplanet_coupler_sf_evolve_ocn"
133-
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml"
133+
command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml --job_id slabplanet_coupler_sf_evolve_ocn"
134134
artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
135135
env:
136136
BUILD_HISTORY_HANDLE: ""
@@ -159,7 +159,7 @@ steps:
159159

160160
- label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
161161
key: "slabplanet_aqua_target_nocouple"
162-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml"
162+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml --job_id slabplanet_aqua_target_nocouple"
163163
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*"
164164
env:
165165
CLIMACORE_DISTRIBUTED: "MPI"
@@ -172,7 +172,7 @@ steps:
172172

173173
- label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
174174
key: "slabplanet_aqua_target"
175-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml"
175+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml --job_id slabplanet_aqua_target"
176176
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*"
177177
env:
178178
CLIMACORE_DISTRIBUTED: "MPI"
@@ -185,7 +185,7 @@ steps:
185185

186186
- label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T"
187187
key: "slabplanet_aqua_target_evolve_ocn"
188-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml"
188+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml --job_id slabplanet_aqua_target_evolve_ocn"
189189
artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*"
190190
env:
191191
CLIMACORE_DISTRIBUTED: "MPI"
@@ -198,7 +198,7 @@ steps:
198198

199199
- label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
200200
key: "slabplanet_target"
201-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target.yml"
201+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target.yml --job_id slabplanet_target"
202202
artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_artifacts/*"
203203
env:
204204
CLIMACORE_DISTRIBUTED: "MPI"
@@ -211,7 +211,7 @@ steps:
211211

212212
- label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
213213
key: "slabplanet_target_evolve_ocn"
214-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml"
214+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml --job_id slabplanet_target_evolve_ocn"
215215
artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*"
216216
env:
217217
CLIMACORE_DISTRIBUTED: "MPI"
@@ -229,7 +229,7 @@ steps:
229229

230230
- label: "MPI AMIP FINE: new target amip"
231231
key: "amip_target"
232-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target.yml"
232+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target.yml --job_id amip_target"
233233
artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_artifacts/*"
234234
env:
235235
CLIMACORE_DISTRIBUTED: "MPI"
@@ -242,7 +242,7 @@ steps:
242242

243243
- label: "MPI AMIP FINE: new target amip: topo"
244244
key: "amip_target_topo"
245-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml"
245+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id amip_target_topo"
246246
artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_artifacts/*"
247247
env:
248248
CLIMACORE_DISTRIBUTED: "MPI"
@@ -255,7 +255,7 @@ steps:
255255

256256
- label: "MPI AMIP FINE: new target amip: topo + diagedmf"
257257
key: "amip_target_topo_diagedmf"
258-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml"
258+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml --job_id amip_target_topo_diagedmf"
259259
artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_artifacts/*"
260260
env:
261261
CLIMACORE_DISTRIBUTED: "MPI"
@@ -272,7 +272,7 @@ steps:
272272

273273
- label: "GPU AMIP FINE: new target amip: topo"
274274
key: "gpu_amip_target_topo"
275-
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml"
275+
command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id gpu_amip_target_topo"
276276
artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_target_topo_artifacts/*"
277277
agents:
278278
slurm_gpus: 1
@@ -283,7 +283,7 @@ steps:
283283
key: "gpu_amip_target_topo_diagedmf"
284284
command:
285285
- echo "--- Run simulation"
286-
- "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml"
286+
- "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml --job_id gpu_amip_target_topo_diagedmf"
287287
artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*"
288288
timeout_in_minutes: 1440
289289
env:
@@ -302,7 +302,7 @@ steps:
302302
key: "gpu_longrun_amip_dyamond"
303303
command:
304304
- echo "--- Run simulation"
305-
- "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_longrun_amip_dyamond.yml"
305+
- "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/longrun_amip_dyamond.yml --job_id gpu_longrun_amip_dyamond"
306306
artifact_paths: "experiments/ClimaEarth/output/amip/gpu_longrun_amip_dyamond_artifacts/*"
307307
agents:
308308
queue: clima

0 commit comments

Comments
 (0)