diff --git a/projects/benchmarking/batch.sh b/projects/benchmarking/batch.sh index c58697d80..ab6bf402b 100755 --- a/projects/benchmarking/batch.sh +++ b/projects/benchmarking/batch.sh @@ -27,7 +27,7 @@ echo -e "\nuname: $(uname -a)" # Compile libraries... echo -e "\nCompile libraries..." -if [ $libs = "compute" ] ; then +if [ "$libs" = "compute" ] ; then cd $trac/libs && ./build.sh -a ; cd - fi du -h $trac/libs/build/lib/ || exit @@ -35,10 +35,10 @@ du -h $trac/libs/build/lib/ || exit # Load modules... echo -e "\nLoad modules..." ml purge -if [ $compiler = "gcc" ] ; then +if [ "$compiler" = "gcc" ] ; then ml GCC ParaStationMPI echo -e "\ngcc version:" ; gcc --version -elif [ $compiler = "nvc" ] ; then +elif [ "$compiler" = "nvc" ] ; then ml NVHPC ParaStationMPI echo -e "\nnvc version:" ; nvc --version else @@ -80,14 +80,14 @@ for np in $(echo $npmin $npmax $npfac | awk '{for(np=$1; np<=$2; np*=$3) print n # Set compile flags... echo -e "\nCompile MPTRAC..." np_comp=$(echo $np | awk '{if($1<100) print 100; else print $1}') - [ $meteo = "erai" ] && defs="-DNP=$np_comp -DNQ=8 -DEX=482 -DEY=242 -DEP=62" - [ $meteo = "era5" ] && defs="-DNP=$np_comp -DNQ=8 -DEX=1202 -DEY=602 -DEP=140" + [ "$meteo" = "erai" ] && defs="-DNP=$np_comp -DNQ=8 -DEX=482 -DEY=242 -DEP=62" + [ "$meteo" = "era5" ] && defs="-DNP=$np_comp -DNQ=8 -DEX=1202 -DEY=602 -DEP=140" flags="COMPILER=$compiler MPI=1 STATIC=0 " - [ $gpu = "1" ] && flags+=" GPU=1" - [ $gpu = "2" ] && flags+=" GPU=1 GPU_PIN=1" - [ $rng = "2" ] && flags+=" CURAND=1" - [ $sort != "0" ] && flags+=" THRUST=1" - [ $cache = "2" ] && flags+=" ASYNCIO=1" + [ "$gpu" = "1" ] && flags+=" GPU=1" + [ "$gpu" = "2" ] && flags+=" GPU=1 GPU_PIN=1" + [ "$rng" = "2" ] && flags+=" CURAND=1" + [ "$sort" != "0" ] && flags+=" THRUST=1" + [ "$cache" = "2" ] && flags+=" ASYNCIO=1" # Compile... cd $trac/src && make clean && make -j DEFINES="$defs" $flags || exit @@ -95,13 +95,14 @@ for np in $(echo $npmin $npmax $npfac | awk '{for(np=$1; np<=$2; np*=$3) print n # MPTRAC setup... qnt="NQ 8 QNT_NAME[0] theta QNT_NAME[1] pv QNT_NAME[2] h2o QNT_NAME[3] o3 QNT_NAME[4] Cn2o QNT_NAME[5] Cccl4 QNT_NAME[6] Cccl3f QNT_NAME[7] Cccl2f2" - #clim="CLIM_PHOTO ./mptrac/data/clams_photolysis_rates.nc CLIM_HNO3_FILENAME ./mptrac/data/gozcards_HNO3.nc CLIM_OH_FILENAME ./mptrac/data/clams_radical_species_vmr.nc CLIM_H2O2_FILENAME ./mptrac/data/cams_H2O2.nc CLIM_HO2_FILENAME ./mptrac/data/clams_radical_species_vmr.nc CLIM_O1D_FILENAME ./mptrac/data/clams_radical_species_vmr.nc CLIM_CCL4_TIMESERIES ./mptrac/data/noaa_gml_ccl4.tab CLIM_CCL3F_TIMESERIES ./mptrac/data/noaa_gml_cfc11.tab CLIM_CCL2F2_TIMESERIES ./mptrac/data/noaa_gml_cfc12.tab CLIM_N2O_TIMESERIES ./mptrac/data/noaa_gml_n2o.tab CLIM_SF6_TIMESERIES ./mptrac/data/noaa_gml_sf6.tab" - [ $meteo = "erai" ] && metbase="METBASE meteo/erai_pck/ei DT_MET 21600 DT_MOD 360 MET_TYPE 2" - [ $meteo = "era5" ] && metbase="METBASE meteo/era5_pck/era5 DT_MET 3600 DT_MOD 180 MET_TYPE 2" - [ $cache = "1" ] && metbase+=" MET_CACHE 1" - [ $phys = "full" ] \ - && param="CONV_CAPE 0.0 BOUND_P0 1e100 BOUND_P1 -1e100 BOUND_LAT0 -90 BOUND_LAT1 90 BOUND_DPS 150 MIXING_TROP 0.1 MIXING_STRAT 0.1" \ - || param="TURB_DX_TROP 0 TURB_DX_STRAT 0 TURB_DZ_TROP 0 TURB_DZ_STRAT 0 TURB_MESOX 0 TURB_MESOZ 0" + [ "$meteo" = "erai" ] && metbase="METBASE meteo/erai_pck/ei DT_MET 21600 DT_MOD 360 MET_TYPE 2" + [ "$meteo" = "era5" ] && metbase="METBASE meteo/era5_pck/era5 DT_MET 3600 DT_MOD 180 MET_TYPE 2" + [ "$cache" = "1" ] && metbase+=" MET_CACHE 1" + if [ "$phys" = "full" ] ; then + param="CONV_CAPE 0.0 BOUND_P0 1e100 BOUND_P1 -1e100 BOUND_LAT0 -90 BOUND_LAT1 90 BOUND_DPS 150 MIXING_TROP 0.1 MIXING_STRAT 0.1" + else + param="TURB_DX_TROP 0 TURB_DX_STRAT 0 TURB_DZ_TROP 0 TURB_DZ_STRAT 0 TURB_MESOX 0 TURB_MESOZ 0" + fi t0=$($trac/src/time2jsec 2017 1 1 0 0 0 0) t1=$($trac/src/time2jsec 2017 1 2 0 0 0 0) @@ -110,7 +111,7 @@ for np in $(echo $npmin $npmax $npfac | awk '{for(np=$1; np<=$2; np*=$3) print n rm -rf $dir && mkdir -p $dir || exit # Create init file... - $trac/src/atm_init - $dir/atm_init.tab $qnt $clim ATM_TYPE 1 \ + $trac/src/atm_init - $dir/atm_init.tab $qnt ATM_TYPE 1 \ INIT_T0 $t0 INIT_T1 $t0 \ INIT_ULON 360 INIT_ULAT 180 \ INIT_Z0 30 INIT_Z1 30 INIT_UZ 60 \ @@ -118,8 +119,7 @@ for np in $(echo $npmin $npmax $npfac | awk '{for(np=$1; np<=$2; np*=$3) print n # Create data directories for tasks... for task in $(seq ${SLURM_NTASKS}) ; do - mkdir -p $dir/$task || exit - cp $dir/atm_init.tab $dir/$task/ || exit + mkdir -p $dir/$task && cp $dir/atm_init.tab $dir/$task/ || exit echo "$dir/$task" >> $dir/dirlist done @@ -144,7 +144,7 @@ for np in $(echo $npmin $npmax $npfac | awk '{for(np=$1; np<=$2; np*=$3) print n done # Compare grid files... - echo -e "\nGrid differences..." + echo -e "\nGrid differences..." paste $dir/1/grid_2017_01_02_00_00.tab reference/$meteo/phys_${phys}/$np/1/grid_2017_01_02_00_00.tab | awk '{ if(NF>0 && $1!="#") { cmax=NF/2 diff --git a/projects/benchmarking/benchmark.sh b/projects/benchmarking/benchmark.sh index 38d71419c..443d61191 100755 --- a/projects/benchmarking/benchmark.sh +++ b/projects/benchmarking/benchmark.sh @@ -25,9 +25,8 @@ parameter choices: runtime: specify maximum runtime (e.g. 00:30:00) - mptrac: specify whether MPTRAC should be recompiled + mptrac: specify whether MPTRAC repo should be updated avail = use available copy of MPTRAC as is - clone = remove existing copy of MPTRAC and make new clone from git repository pull = update existing copy of MPTRAC via pull request to git repository libs: specify whether libraries should be recompiled @@ -95,17 +94,15 @@ cache=${18} # Update MPTRAC repository... echo -e "\nUpdating MPTRAC repository..." -if [ $mptrac = "clone" ] ; then - rm -rf mptrac ; git clone https://github.com/slcs-jsc/mptrac.git || exit -elif [ $mptrac = "pull" ] ; then - cd mptrac && git pull || exit ; cd - +if [ "$mptrac" = "pull" ] ; then + git pull || exit else echo "Use as is!" fi # Compile libraries... echo -e "\nCompile libraries..." -if [ $libs = "login" ] ; then +if [ "$libs" = "login" ] ; then cd ../../libs && ./build.sh -a ; cd - else echo "Compile on compute node or assume already available!" @@ -113,36 +110,40 @@ fi # Get meteo data... echo -e "\nDownloading meteo data..." -[ -s meteo ] \ - || wget --mirror --no-parent --no-host-directories --execute robots=off --reject="index.html*" --cut-dirs=5 https://datapub.fz-juelich.de/slcs/mptrac/data/projects/benchmarking/meteo/ \ - && echo "Already available!" +if [ -s meteo ] ; then + echo "Already available!" +else + wget --mirror --no-parent --no-host-directories --execute robots=off --reject="index.html*" --cut-dirs=5 https://datapub.fz-juelich.de/slcs/mptrac/data/projects/benchmarking/meteo/ +fi du -h meteo || exit # Get reference data... echo -e "\nDownloading reference data..." -[ -s reference ] \ - || wget --mirror --no-parent --no-host-directories --execute robots=off --reject="index.html*" --cut-dirs=5 https://datapub.fz-juelich.de/slcs/mptrac/data/projects/benchmarking/reference/ \ - && echo "Already available!" -du -h meteo || exit +if [ -s reference ] ; then + echo "Already available!" +else + wget --mirror --no-parent --no-host-directories --execute robots=off --reject="index.html*" --cut-dirs=5 https://datapub.fz-juelich.de/slcs/mptrac/data/projects/benchmarking/reference/ +fi +du -h reference || exit # Execute batch job... echo -e "\nExecute batch job..." ntasks=$(echo $nodes $ntasks_per_node | awk '{print $1*$2}') slurmset="--account=$account --time=$runtime --nodes=$nodes --ntasks=$ntasks --ntasks-per-node=$ntasks_per_node --cpus-per-task=$cpus_per_task --disable-dcgm --hint=nomultithread" -[ $gpu != "0" ] && slurmset+=" --gres=gpu:1" -if [ $system = "jwb" ] ; then +[ "$gpu" != "0" ] && slurmset+=" --gres=gpu:1" +if [ "$system" = "jwb" ] ; then slurmset+=" --partition=booster" -elif [ $system = "jwc_cpu" ] ; then +elif [ "$system" = "jwc_cpu" ] ; then slurmset+=" --partition=batch" -elif [ $system = "jwc_gpu" ] ; then +elif [ "$system" = "jwc_gpu" ] ; then slurmset+=" --partition=gpus" -elif [ $system = "jrc_cpu" ] ; then +elif [ "$system" = "jrc_cpu" ] ; then slurmset+=" --partition=dc-cpu" -elif [ $system = "jrc_gpu" ] ; then +elif [ "$system" = "jrc_gpu" ] ; then slurmset+=" --partition=dc-gpu" -elif [ $system = "jrc_gh200" ] ; then +elif [ "$system" = "jrc_gh200" ] ; then slurmset+=" --partition=dc-gh" -elif [ $system = "jrc_h100" ] ; then +elif [ "$system" = "jrc_h100" ] ; then slurmset+=" --partition=dc-h100" else echo "error: system \"$system\" unknown!"