From 87b9539ee4c280f5262c92db18eae2b07cc53b67 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 24 Feb 2023 09:17:29 -0700 Subject: [PATCH 01/95] updates for gust --- machines/cmake_macros/gust.cmake | 3 +- machines/config_machines.xml | 60 +++++++++++++------------------- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/machines/cmake_macros/gust.cmake b/machines/cmake_macros/gust.cmake index 5332d19c..5809f297 100644 --- a/machines/cmake_macros/gust.cmake +++ b/machines/cmake_macros/gust.cmake @@ -7,4 +7,5 @@ set(PNETCDF_PATH "$ENV{PNETCDF}") # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml #string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") -#string(APPEND CPPDEFS " -DNO_SHR_VMATH ") \ No newline at end of file +#string(APPEND CPPDEFS " -DNO_SHR_VMATH ") +string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 8b5f36bf..1b2dd2aa 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1857,80 +1857,70 @@ This allows using a different mpirun command to launch unit tests - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/perl - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/env_modules_python.py - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/sh - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/csh - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod perl - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod python + $ENV{LMOD_ROOT}/lmod/init/perl + $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py + $ENV{LMOD_ROOT}/lmod/init/sh + $ENV{LMOD_ROOT}/lmod/init/csh + $ENV{LMOD_ROOT}/lmod/libexec/lmod perl + $ENV{LMOD_ROOT}/lmod/libexec/lmod python module module /opt/cray/pe/craype-targets/default/modulefiles + cmake - intel/2021.6.0 + intel/2021.7.1 - oneapi/2022.1.0 + oneapi/2023.0.0 - cce/15.0.0 + cce/15.0.1 gcc/12.1.0 - nvhpc/22.7 - - - cray-mpich/8.1.19 + nvhpc/23.1 - - cray-mpich/8.1.21 + + cray-mpich/8.1.24 mpi-serial/2.3.0 - - - cmake/3.23.2 - cray-libsci/22.08.1.1 + + cray-libsci/23.02.1.1 - mkl/2022.1.0 + mkl/2022.2.1 - mkl/2022.1.0 + mkl/2022.2.1 - netcdf/4.9.0 + netcdf/4.9.1 hdf5 netcdf - netcdf-mpi/4.9.0 + netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 - - parallelio/2.5.9-debug - esmf/8.4.0b20-debug - - - parallelio/2.5.9 - esmf/8.4.0b20 - - - esmf/8.4.0b21-debug + + parallelio/2.5.10-debug + esmf/8.5.0b17-debug - - esmf/8.4.0b21 + + parallelio/2.5.10 + esmf/8.5.0b17 From af3e6d70eb0b048b75bf62bec14221df439205c0 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Mon, 13 Mar 2023 13:15:29 -0600 Subject: [PATCH 02/95] Update module lists for ESMF library Update Depend files for RRTMGP GPU code modified: Depends.nvhpc modified: Depends.nvhpc-gpu modified: config_machines.xml --- machines/Depends.nvhpc | 40 +++++++++++++++++++++++++++++++++++ machines/Depends.nvhpc-gpu | 41 ++++++++++++++++++++++++++++++++++++ machines/config_machines.xml | 19 ++++++++++------- 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 812904f7..4b46e93a 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -3,5 +3,45 @@ REDUCED_OPT_OBJS=\ esm.o +RRTMGP_OBJS=\ +mo_rte_util_array.o \ +mo_rte_sw.o \ +mo_rte_lw.o \ +mo_cloud_optics.o \ +mo_gas_optics_rrtmgp.o \ +mo_gas_concentrations.o \ +mo_fluxes_broadband_kernels.o \ +mo_fluxes_byband_kernels.o \ +mo_rrtmgp_util_reorder_kernels.o \ +mo_compute_bc.o\ +mo_fluxes_byband.o\ +mo_fluxes_bygpoint.o\ +mo_heating_rates.o\ +mo_rrtmgp_clr_all_sky.o\ +mo_cloud_sampling.o\ +mo_solar_variability.o\ +mo_gas_optics.o\ +mo_rrtmgp_constants.o\ +mo_rrtmgp_util_reorder.o\ +mo_rrtmgp_util_string.o\ +mo_fluxes.o\ +mo_optical_props.o\ +mo_rte_config.o\ +mo_rte_kind.o\ +mo_source_functions.o\ +mo_optical_props_kernels.o\ +mo_rte_solver_kernels.o\ +mo_optical_props_kernels.o\ +mo_rte_solver_kernels.o\ +rrtmgp_driver.o + +RRTMGP_NOOPT=\ +mo_gas_optics_kernels.o + $(REDUCED_OPT_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< + +$(RRTMGP_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $< +$(RRTMGP_NOOPT): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $< diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu index 4817ee0e..97648428 100644 --- a/machines/Depends.nvhpc-gpu +++ b/machines/Depends.nvhpc-gpu @@ -10,7 +10,48 @@ wv_saturation.o \ macrop_driver.o \ shr_spfn_mod.o +RRTMGP_OBJS=\ +mo_rte_util_array.o \ +mo_rte_sw.o \ +mo_rte_lw.o \ +mo_cloud_optics.o \ +mo_gas_optics_rrtmgp.o \ +mo_gas_concentrations.o \ +mo_fluxes_broadband_kernels.o \ +mo_fluxes_byband_kernels.o \ +mo_rrtmgp_util_reorder_kernels.o \ +mo_compute_bc.o\ +mo_fluxes_byband.o\ +mo_fluxes_bygpoint.o\ +mo_heating_rates.o\ +mo_rrtmgp_clr_all_sky.o\ +mo_cloud_sampling.o\ +mo_solar_variability.o\ +mo_gas_optics.o\ +mo_rrtmgp_constants.o\ +mo_rrtmgp_util_reorder.o\ +mo_rrtmgp_util_string.o\ +mo_fluxes.o\ +mo_optical_props.o\ +mo_rte_config.o\ +mo_rte_kind.o\ +mo_source_functions.o\ +mo_optical_props_kernels.o\ +mo_rte_solver_kernels.o\ +mo_optical_props_kernels.o\ +mo_rte_solver_kernels.o\ +rrtmgp_driver.o + +RRTMGP_NOOPT=\ +mo_gas_optics_kernels.o + ifeq ($(DEBUG),FALSE) $(PUMAS_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel $< + $(RRTMGP_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -Mnoopenmp $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $< + $(RRTMGP_NOOPT): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -O0 $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $< endif diff --git a/machines/config_machines.xml b/machines/config_machines.xml index b9e229bf..882e5328 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -484,17 +484,17 @@ This allows using a different mpirun command to launch unit tests netcdf/4.7.4 - openmpi/4.1.1 + openmpi/4.1.4 netcdf-mpi/4.8.1 - pnetcdf/1.12.2 + pnetcdf/1.12.3 netcdf/4.8.1 - openmpi/4.1.1 + openmpi/4.1.4 netcdf-mpi/4.8.1 - pnetcdf/1.12.2 + pnetcdf/1.12.3 cuda/11.4.0 @@ -519,11 +519,11 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.0b08_casper-ncdfio-openmpi-g + esmf-8.4.1b01-ncdfio-openmpi-g /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.0b08_casper-ncdfio-openmpi-O + esmf-8.4.1b01-ncdfio-openmpi-O /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ @@ -537,10 +537,10 @@ This allows using a different mpirun command to launch unit tests ncarcompilers/0.5.0 - pio/2.5.7 + pio/2.5.10 - pio/2.5.7d + pio/2.5.10d @@ -550,6 +550,9 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg $ENV{NETCDF} + + openacc + ON SUMMARY From 3f1301f3c309cff76cc1fb3302878fc002be96ed Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 21 Mar 2023 11:16:40 -0600 Subject: [PATCH 03/95] update ESMF to v8.4.1 modified: config_machines.xml --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 882e5328..a56cb3a2 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -519,11 +519,11 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1b01-ncdfio-openmpi-g + esmf-8.4.1_casper-ncdfio-openmpi-g /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1b01-ncdfio-openmpi-O + esmf-8.4.1_casper-ncdfio-openmpi-O /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ From 9beb292b88638a37a80797775ee9e1cbfa11f07d Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 22 Mar 2023 00:17:31 -0600 Subject: [PATCH 04/95] Add Jim's changes for Gust's modules from https://github.com/jedwards4b/ccs_config_cesm/commit/87b9539ee4c280f5262c92db18eae2b07cc53b67 Use nvhpc/22.11, cray-mpich/8.1.21, and esmf/8.4.1.b02 modified: machines/cmake_macros/gust.cmake modified: machines/config_machines.xml --- machines/cmake_macros/gust.cmake | 2 +- machines/config_machines.xml | 60 +++++++++++++------------------- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/machines/cmake_macros/gust.cmake b/machines/cmake_macros/gust.cmake index 5332d19c..0bb6f4f2 100644 --- a/machines/cmake_macros/gust.cmake +++ b/machines/cmake_macros/gust.cmake @@ -7,4 +7,4 @@ set(PNETCDF_PATH "$ENV{PNETCDF}") # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml #string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") -#string(APPEND CPPDEFS " -DNO_SHR_VMATH ") \ No newline at end of file +string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/config_machines.xml b/machines/config_machines.xml index b9e229bf..ec188bcf 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1857,80 +1857,70 @@ This allows using a different mpirun command to launch unit tests - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/perl - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/env_modules_python.py - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/sh - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/csh - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod perl - /glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod python + $ENV{LMOD_ROOT}/lmod/init/perl + $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py + $ENV{LMOD_ROOT}/lmod/init/sh + $ENV{LMOD_ROOT}/lmod/init/csh + $ENV{LMOD_ROOT}/lmod/libexec/lmod perl + $ENV{LMOD_ROOT}/lmod/libexec/lmod python module module /opt/cray/pe/craype-targets/default/modulefiles + cmake - intel/2021.6.0 + intel/2021.7.1 - oneapi/2022.1.0 + oneapi/2023.0.0 - cce/15.0.0 + cce/15.0.1 gcc/12.1.0 - nvhpc/22.7 - - - cray-mpich/8.1.19 + nvhpc/22.11 - - cray-mpich/8.1.21 + + cray-mpich/8.1.21 mpi-serial/2.3.0 - - - cmake/3.23.2 - cray-libsci/22.08.1.1 + + cray-libsci/23.02.1.1 - mkl/2022.1.0 + mkl/2022.2.1 - mkl/2022.1.0 + mkl/2022.2.1 - netcdf/4.9.0 + netcdf/4.9.1 hdf5 netcdf - netcdf-mpi/4.9.0 + netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 - - parallelio/2.5.9-debug - esmf/8.4.0b20-debug - - - parallelio/2.5.9 - esmf/8.4.0b20 - - - esmf/8.4.0b21-debug + + parallelio/2.5.10-debug + esmf/8.4.1b02-debug - - esmf/8.4.0b21 + + parallelio/2.5.10 + esmf/8.4.1b02 From 4134af3498e13e9b63db40dbd61f77ff9a2e2506 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 22 Mar 2023 00:39:23 -0600 Subject: [PATCH 05/95] Add Jim's gpu_improvement branch at https://github.com/jedwards4b/ccs_config_cesm/compare/52c06b3..fbc05d6 Update settings on Gust and Casper modified: Depends.nvhpc deleted: Depends.nvhpc-gpu deleted: cmake_macros/nvhpc-gpu.cmake deleted: cmake_macros/nvhpc-gpu_casper.cmake modified: cmake_macros/nvhpc.cmake modified: cmake_macros/nvhpc_casper.cmake deleted: cmake_macros/pgi-gpu.cmake deleted: cmake_macros/pgi-gpu_casper.cmake modified: config_batch.xml modified: config_machines.xml deleted: mpi_run_gpu.casper --- machines/Depends.nvhpc | 16 ++++- machines/Depends.nvhpc-gpu | 16 ----- machines/cmake_macros/nvhpc-gpu.cmake | 46 ------------ machines/cmake_macros/nvhpc-gpu_casper.cmake | 15 ---- machines/cmake_macros/nvhpc.cmake | 18 +++++ machines/cmake_macros/nvhpc_casper.cmake | 1 + machines/cmake_macros/pgi-gpu.cmake | 46 ------------ machines/cmake_macros/pgi-gpu_casper.cmake | 15 ---- machines/config_batch.xml | 24 ++----- machines/config_machines.xml | 73 ++++++-------------- machines/mpi_run_gpu.casper | 6 -- 11 files changed, 59 insertions(+), 217 deletions(-) delete mode 100644 machines/Depends.nvhpc-gpu delete mode 100644 machines/cmake_macros/nvhpc-gpu.cmake delete mode 100644 machines/cmake_macros/nvhpc-gpu_casper.cmake delete mode 100644 machines/cmake_macros/pgi-gpu.cmake delete mode 100644 machines/cmake_macros/pgi-gpu_casper.cmake delete mode 100755 machines/mpi_run_gpu.casper diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 812904f7..2269cd31 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -3,5 +3,19 @@ REDUCED_OPT_OBJS=\ esm.o +PUMAS_OBJS=\ +micro_mg1_0.o \ +micro_pumas_v1.o \ +micro_pumas_data.o \ +micro_pumas_utils.o \ +micro_pumas_cam.o \ +wv_sat_methods.o \ +wv_saturation.o \ +macrop_driver.o \ +shr_spfn_mod.o + $(REDUCED_OPT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< + +$(PUMAS_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $< diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu deleted file mode 100644 index 4817ee0e..00000000 --- a/machines/Depends.nvhpc-gpu +++ /dev/null @@ -1,16 +0,0 @@ -# -PUMAS_OBJS=\ -micro_mg1_0.o \ -micro_pumas_v1.o \ -micro_pumas_data.o \ -micro_pumas_utils.o \ -micro_pumas_cam.o \ -wv_sat_methods.o \ -wv_saturation.o \ -macrop_driver.o \ -shr_spfn_mod.o - -ifeq ($(DEBUG),FALSE) - $(PUMAS_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel $< -endif diff --git a/machines/cmake_macros/nvhpc-gpu.cmake b/machines/cmake_macros/nvhpc-gpu.cmake deleted file mode 100644 index 679e81b2..00000000 --- a/machines/cmake_macros/nvhpc-gpu.cmake +++ /dev/null @@ -1,46 +0,0 @@ -string(APPEND CFLAGS " -gopt -time") -if (compile_threaded) - string(APPEND CFLAGS " -mp") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI") -set(CXX_LINKER "CXX") -set(FC_AUTO_R8 "-r8") -string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee") -if (compile_threaded) - string(APPEND FFLAGS " -mp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") -endif() -if (COMP_NAME STREQUAL datm) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dlnd) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL drof) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dwav) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dice) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL docn) - string(APPEND FFLAGS " -Mnovect") -endif() -set(FFLAGS_NOOPT "-O0") -set(FIXEDFLAGS "-Mfixed") -set(FREEFLAGS "-Mfree") -set(HAS_F2008_CONTIGUOUS "FALSE") -set(LDFLAGS "-time -Wl,--allow-multiple-definition") -if (compile_threaded) - string(APPEND LDFLAGS " -mp") -endif() -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") -set(SCC "nvc") -set(SCXX "nvc++") -set(SFC "nvfortran") diff --git a/machines/cmake_macros/nvhpc-gpu_casper.cmake b/machines/cmake_macros/nvhpc-gpu_casper.cmake deleted file mode 100644 index ed2adbc6..00000000 --- a/machines/cmake_macros/nvhpc-gpu_casper.cmake +++ /dev/null @@ -1,15 +0,0 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O -tp=skylake -Mnofma") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O -tp=skylake -Mnofma") -endif() -string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS") -if (NOT DEBUG) - string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -gpu=cc70,lineinfo,nofma -Minfo=accel") -endif() -string(APPEND SLIBS " -llapack -lblas") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -ldl") -endif() -string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake index 3e1b4516..1e6036d9 100644 --- a/machines/cmake_macros/nvhpc.cmake +++ b/machines/cmake_macros/nvhpc.cmake @@ -49,3 +49,21 @@ set(MPIFC "mpif90") set(SCC "nvc") set(SCXX "nvc++") set(SFC "nvfortran") +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc) + string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp) + string(APPEND GPUFLAGS " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined) + string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc) + string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp) + string(APPEND GPUFLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined) + string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel") +endif() diff --git a/machines/cmake_macros/nvhpc_casper.cmake b/machines/cmake_macros/nvhpc_casper.cmake index f3eb207d..08bc6bbe 100644 --- a/machines/cmake_macros/nvhpc_casper.cmake +++ b/machines/cmake_macros/nvhpc_casper.cmake @@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial) string(APPEND SLIBS " -ldl") endif() string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") +message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}") diff --git a/machines/cmake_macros/pgi-gpu.cmake b/machines/cmake_macros/pgi-gpu.cmake deleted file mode 100644 index 85657479..00000000 --- a/machines/cmake_macros/pgi-gpu.cmake +++ /dev/null @@ -1,46 +0,0 @@ -string(APPEND CFLAGS " -gopt -time") -if (compile_threaded) - string(APPEND CFLAGS " -mp") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI") -set(CXX_LINKER "CXX") -set(FC_AUTO_R8 "-r8") -string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee") -if (compile_threaded) - string(APPEND FFLAGS " -mp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") -endif() -if (COMP_NAME STREQUAL datm) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dlnd) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL drof) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dwav) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dice) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL docn) - string(APPEND FFLAGS " -Mnovect") -endif() -set(FFLAGS_NOOPT "-O0") -set(FIXEDFLAGS "-Mfixed") -set(FREEFLAGS "-Mfree") -set(HAS_F2008_CONTIGUOUS "FALSE") -set(LDFLAGS "-time -Wl,--allow-multiple-definition") -if (compile_threaded) - string(APPEND LDFLAGS " -mp") -endif() -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") -set(SCC "pgcc") -set(SCXX "pgc++") -set(SFC "pgf95") diff --git a/machines/cmake_macros/pgi-gpu_casper.cmake b/machines/cmake_macros/pgi-gpu_casper.cmake deleted file mode 100644 index 600521a1..00000000 --- a/machines/cmake_macros/pgi-gpu_casper.cmake +++ /dev/null @@ -1,15 +0,0 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O -tp=skylake -Mnofma") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O -tp=skylake -Mnofma") -endif() -string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS") -if (NOT DEBUG) - string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel") -endif() -string(APPEND SLIBS " -llapack -lblas") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -ldl") -endif() -string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 4ea11977..9d93418d 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -226,31 +226,17 @@ qsub - - - - - -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - -l gpu_type=v100 - - - - - - -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - -l gpu_type=v100 - - + + -l gpu_type=$GPU_TYPE + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - + -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 diff --git a/machines/config_machines.xml b/machines/config_machines.xml index ec188bcf..f326c8d1 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -410,18 +410,19 @@ This allows using a different mpirun command to launch unit tests NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs casper* LINUX - pgi,intel,nvhpc,pgi-gpu,nvhpc-gpu + nvhpc,intel openmpi /glade/scratch/$USER $ENV{CESMDATAROOT}/inputdata /glade/p/cgd/tss/CTSM_datm_forcing_data $CIME_OUTPUT_ROOT/archive/$CASE $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc 8 pbs ASAP/CISL 36 - 8 + 8 36 TRUE @@ -450,56 +451,30 @@ This allows using a different mpirun command to launch unit tests ncarenv/1.3 cmake/3.18.2 - - pgi/20.4 - - - pgi/20.4 - nvhpc/22.2 - - nvhpc/22.2 - intel/19.1.1 mkl/2020.0.1 - - openmpi/4.1.0 - netcdf-mpi/4.8.0 - pnetcdf/1.12.2 + + cuda/11.6 - - netcdf/4.8.0 + + cuda/11.6 - - openmpi/4.1.0 - netcdf-mpi/4.7.4 - pnetcdf/1.12.2 - cuda/11.0.3 - - - netcdf/4.7.4 + + cuda/11.6 - openmpi/4.1.1 + openmpi/4.1.4 netcdf-mpi/4.8.1 - pnetcdf/1.12.2 + pnetcdf/1.12.3 netcdf/4.8.1 - - openmpi/4.1.1 - netcdf-mpi/4.8.1 - pnetcdf/1.12.2 - cuda/11.4.0 - - - netcdf/4.8.1 - openmpi/4.1.1 netcdf-mpi/4.8.1 @@ -517,30 +492,22 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ esmf-8.4.0b08_casper-ncdfio-openmpi-O - + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.0b08_casper-ncdfio-openmpi-g + esmf-8.4.1_casper-ncdfio-openmpi-g - + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.0b08_casper-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.4.0b08_casper-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b11_casper-ncdfio-openmpi-O + esmf-8.4.1_casper-ncdfio-openmpi-O ncarcompilers/0.5.0 - - pio/2.5.7 + + pio/2.5.10 - - pio/2.5.7d + + pio/2.5.10d @@ -577,7 +544,7 @@ This allows using a different mpirun command to launch unit tests /glade/p/cgd/tss/CTSM_datm_forcing_data $CIME_OUTPUT_ROOT/archive/$CASE $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc.cheyenne + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc 8 pbs cseg diff --git a/machines/mpi_run_gpu.casper b/machines/mpi_run_gpu.casper deleted file mode 100755 index ade12d50..00000000 --- a/machines/mpi_run_gpu.casper +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -unset CUDA_VISIBLE_DEVICES -let dev_id=$OMPI_COMM_WORLD_LOCAL_RANK%{{ ngpus_per_node }} -export ACC_DEVICE_NUM=$dev_id -export CUDA_VISIBLE_DEVICES=$dev_id -exec $* From 3b7a18255b1cb9c26d7c938d2a70b6f6eb07f5fe Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 22 Mar 2023 22:57:38 -0600 Subject: [PATCH 06/95] Add max_cputasks_per_gpu_node, gpu_type and gpu_offload options for Casper and Gust modified: machines/config_machines.xml --- machines/config_machines.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index f326c8d1..1dee63e9 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -424,6 +424,9 @@ This allows using a different mpirun command to launch unit tests 36 8 36 + 36 + none,v100 + none,openacc,openmp,combined TRUE mpirun @@ -1815,6 +1818,9 @@ This allows using a different mpirun command to launch unit tests cseg 128 128 + 64 + none,a100 + none,openacc,openmp,combined TRUE mpiexec From ee726b5d640950786b84497ba032781c3d009bc9 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Thu, 23 Mar 2023 15:08:39 -0600 Subject: [PATCH 07/95] update batch script template for Gust GPU node modified: config_batch.xml --- machines/config_batch.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 9d93418d..27c64bf0 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -476,10 +476,17 @@ qsub + + -l gpu_type=$GPU_TYPE + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 + main bigcpu From 1f5eb30b532ea1ac0f418f067f9b02ec80d2174d Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Thu, 23 Mar 2023 15:15:42 -0600 Subject: [PATCH 08/95] typo fix for mismatched tags for GPU options modified: machines/config_machines.xml --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 1dee63e9..389de064 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -426,7 +426,7 @@ This allows using a different mpirun command to launch unit tests 36 36 none,v100 - none,openacc,openmp,combined + none,openacc,openmp,combined TRUE mpirun @@ -1820,7 +1820,7 @@ This allows using a different mpirun command to launch unit tests 128 64 none,a100 - none,openacc,openmp,combined + none,openacc,openmp,combined TRUE mpiexec From 8649a95a7daa2bd91fc3ea54e5b3ce5c7be19a67 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Thu, 23 Mar 2023 22:23:02 -0600 Subject: [PATCH 09/95] Add missing "MAX_GPUS_PER_NODE" modified: machines/config_machines.xml --- machines/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 389de064..9c6ac5f4 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1817,6 +1817,7 @@ This allows using a different mpirun command to launch unit tests pbs cseg 128 + 4 128 64 none,a100 From a9b08c993035197a591461edda46155182777c5d Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 24 Mar 2023 07:58:21 -0600 Subject: [PATCH 10/95] updates for ncarenv/23.03 --- machines/config_machines.xml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 301afb3b..91ca3801 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1866,28 +1866,29 @@ This allows using a different mpirun command to launch unit tests module module + ncarenv/23.03 - /opt/cray/pe/craype-targets/default/modulefiles + cmake - intel/2021.7.1 + intel-classic/2023.0.0 - oneapi/2023.0.0 + intel/2023.0.0 cce/15.0.1 - gcc/12.1.0 + gcc/12.2.0 nvhpc/23.1 - cray-mpich/8.1.24 + cray-mpich/8.1.25 mpi-serial/2.3.0 @@ -1898,12 +1899,12 @@ This allows using a different mpirun command to launch unit tests - mkl/2022.2.1 + mkl/2023.0.0 - - mkl/2022.2.1 + mkl/2023.0.0 + netcdf/4.9.1 @@ -1916,11 +1917,11 @@ This allows using a different mpirun command to launch unit tests parallelio/2.5.10-debug - esmf/8.5.0b17-debug + esmf/8.5.0b18-debug parallelio/2.5.10 - esmf/8.5.0b17 + esmf/8.5.0b18 From e7bfbe189bd268479a3cdc45cc77f22d6ac71abd Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 24 Mar 2023 20:48:30 -0600 Subject: [PATCH 11/95] downgrade netcdf-mpi fpr nvhpc/22.11 modified: machines/config_machines.xml --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 9c6ac5f4..017916d5 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1879,13 +1879,13 @@ This allows using a different mpirun command to launch unit tests mkl/2022.2.1 - netcdf/4.9.1 + netcdf/4.9.0 hdf5 netcdf - netcdf-mpi/4.9.1 + netcdf-mpi/4.9.0 parallel-netcdf/1.12.3 From bf149677d4d1a8e0c994421ee9fc496a32993889 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 24 Mar 2023 22:59:33 -0600 Subject: [PATCH 12/95] bug fix of batch script template with gpu_enabled option modified: machines/config_batch.xml --- machines/config_batch.xml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 27c64bf0..943c2086 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -229,17 +229,11 @@ -l gpu_type=$GPU_TYPE - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - - -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 - - + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} @@ -479,13 +473,13 @@ -l gpu_type=$GPU_TYPE - + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 main From 14f605eae88cf1ccf000ef0dad3edf281f26d186 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Sat, 25 Mar 2023 00:01:26 -0600 Subject: [PATCH 13/95] remove -target=zen3 flag on gust load cuda module on Gust for a GPU run deleted: cmake_macros/nvhpc_gust.cmake modified: config_machines.xml --- machines/cmake_macros/nvhpc_gust.cmake | 3 --- machines/config_machines.xml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 machines/cmake_macros/nvhpc_gust.cmake diff --git a/machines/cmake_macros/nvhpc_gust.cmake b/machines/cmake_macros/nvhpc_gust.cmake deleted file mode 100644 index 46113731..00000000 --- a/machines/cmake_macros/nvhpc_gust.cmake +++ /dev/null @@ -1,3 +0,0 @@ -if (NOT DEBUG) - string(APPEND FFLAGS " -target=zen3") -endif() diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 017916d5..d0959a84 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1875,6 +1875,10 @@ This allows using a different mpirun command to launch unit tests mkl/2022.2.1 + + cuda/11.7.1 + + mkl/2022.2.1 From 59086afcd63f4a773a1cc374ad328e7403aeb947 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Sat, 25 Mar 2023 20:15:45 -0600 Subject: [PATCH 14/95] update to nvhpc/23.1 and related modules modified: machines/config_machines.xml --- machines/config_machines.xml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index d0959a84..81e8f2f2 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1858,10 +1858,10 @@ This allows using a different mpirun command to launch unit tests gcc/12.1.0 - nvhpc/22.11 + nvhpc/23.1 - cray-mpich/8.1.21 + cray-mpich/8.1.24 mpi-serial/2.3.0 @@ -1874,31 +1874,28 @@ This allows using a different mpirun command to launch unit tests mkl/2022.2.1 - cuda/11.7.1 - mkl/2022.2.1 netcdf/4.9.0 - hdf5 netcdf - netcdf-mpi/4.9.0 + netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 parallelio/2.5.10-debug - esmf/8.4.1b02-debug + esmf/8.5.0b17-debug parallelio/2.5.10 - esmf/8.4.1b02 + esmf/8.5.0b17 From 918e3a8dd9617c29363969ec4e6c2f2fdea15d2c Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Sun, 26 Mar 2023 13:28:06 -0600 Subject: [PATCH 15/95] turn off MPS on Gust modified: config_batch.xml --- machines/config_batch.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 943c2086..2cabfab8 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -479,7 +479,9 @@ -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} + + main From 731e7bceaf6ba2954c08d36fc1035b1d0949405a Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 31 Mar 2023 10:59:55 -0600 Subject: [PATCH 16/95] update to ncarenv/23.03 --- machines/cmake_macros/gust.cmake | 7 +++++-- machines/config_machines.xml | 26 ++++++++++---------------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/machines/cmake_macros/gust.cmake b/machines/cmake_macros/gust.cmake index 5809f297..db5dbb41 100644 --- a/machines/cmake_macros/gust.cmake +++ b/machines/cmake_macros/gust.cmake @@ -1,3 +1,6 @@ +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=c89 ") +endif() if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() @@ -5,7 +8,7 @@ set(NETCDF_PATH "$ENV{NETCDF}") set(PIO_FILESYSTEM_HINTS "lustre") set(PNETCDF_PATH "$ENV{PNETCDF}") # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml -#string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") -string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") +string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") +#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") #string(APPEND CPPDEFS " -DNO_SHR_VMATH ") string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 91ca3801..f316c1ae 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1835,7 +1835,7 @@ This allows using a different mpirun command to launch unit tests NCAR AMD EPYC test system 16 CPU nodes 2 GPU nodes gu.*.hpc.ucar.edu CNL - intel,gnu,cray,nvhpc,oneapi + intel,gnu,cray,nvhpc,intel-oneapi,intel-classic mpich /glade/gust/scratch/$ENV{USER} /glade/p/cesmdata/inputdata @@ -1868,16 +1868,17 @@ This allows using a different mpirun command to launch unit tests ncarenv/23.03 - cmake - - intel-classic/2023.0.0 - - intel/2023.0.0 + + intel-oneapi/2023.0.0 + + + intel-classic/2023.0.0 + cce/15.0.1 @@ -1893,18 +1894,9 @@ This allows using a different mpirun command to launch unit tests mpi-serial/2.3.0 - + cray-libsci/23.02.1.1 - - - mkl/2023.0.0 - - - mkl/2023.0.0 - - netcdf/4.9.1 @@ -1915,10 +1907,12 @@ This allows using a different mpirun command to launch unit tests netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 + parallelio/2.5.10-debug esmf/8.5.0b18-debug + parallelio/2.5.10 esmf/8.5.0b18 From 584cbfb37f8d829302b977cbf808b6f81281b107 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 4 Apr 2023 09:18:50 -0600 Subject: [PATCH 17/95] remove _CLM from not_compset for ne16 grid --- modelgrid_aliases_nuopc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index a22307d6..c27eda55 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1067,7 +1067,7 @@ gx3v7 - + ne16np4.pg3 ne16np4.pg3 ne16np4.pg3 From 64938671caf19521140edc930efbf0e33f7e30a1 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 5 Apr 2023 12:28:50 -0600 Subject: [PATCH 18/95] Revert "update ESMF to v8.4.1" as it fails on Casper at runtime This reverts commit 3f1301f3c309cff76cc1fb3302878fc002be96ed. modified: machines/config_machines.xml --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index a56cb3a2..882e5328 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -519,11 +519,11 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1_casper-ncdfio-openmpi-g + esmf-8.4.1b01-ncdfio-openmpi-g /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1_casper-ncdfio-openmpi-O + esmf-8.4.1b01-ncdfio-openmpi-O /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ From f2f633ad1b06b500a0f953e0ccc8501b7ae41313 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Thu, 6 Apr 2023 09:44:24 -0600 Subject: [PATCH 19/95] minor update modified: config_machines.xml --- machines/config_machines.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 882e5328..dfa84fd9 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -550,13 +550,13 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg $ENV{NETCDF} - - openacc - ON SUMMARY + + openacc + -1 From 9aa3df5b963040fc671eb9f597ca4fa77a2d0a94 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Thu, 6 Apr 2023 09:47:39 -0600 Subject: [PATCH 20/95] Update compiler flags for nvhpc compiler Fix a typo for PUMAS in the Depends.intel modified: Depends.intel modified: Depends.nvhpc modified: Depends.nvhpc-gpu --- machines/Depends.intel | 2 +- machines/Depends.nvhpc | 57 ++++++++++++++++++++++++++++++++++---- machines/Depends.nvhpc-gpu | 43 ++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 9 deletions(-) diff --git a/machines/Depends.intel b/machines/Depends.intel index b024db74..0f80d21c 100644 --- a/machines/Depends.intel +++ b/machines/Depends.intel @@ -28,7 +28,7 @@ kissvec.o PUMAS_MG_OBJS=\ micro_mg1_0.o \ -micro_mg3_0.o \ +micro_pumas_v1.o \ micro_pumas_data.o \ micro_pumas_utils.o \ wv_sat_methods.o diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 4b46e93a..7b190cc0 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -1,8 +1,44 @@ # 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized +# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test # +# Apply a different optimization level consistent with the Intel compiler +PERFOBJS=\ +prim_advection_mod.o \ +edge_mod.o \ +derivative_mod.o \ +bndry_mod.o \ +prim_advance_mod.o + REDUCED_OPT_OBJS=\ +SatellitePhenologyMod.o \ esm.o +REDUCED_PRECISION_OBJS=\ +shr_wv_sat_mod.o + +SHR_RANDNUM_FORT_OBJS=\ +kissvec_mod.o \ +mersennetwister_mod.o \ +dSFMT_interface.o \ +shr_RandNum_mod.o + +SHR_RANDNUM_C_OBJS=\ +dSFMT.o \ +dSFMT_utils.o \ +kissvec.o + +# Apply GPU flags to the following code +PUMAS_OBJS=\ +micro_mg1_0.o \ +micro_pumas_v1.o \ +micro_pumas_data.o \ +micro_pumas_utils.o \ +micro_pumas_cam.o \ +wv_sat_methods.o \ +wv_saturation.o \ +macrop_driver.o \ +shr_spfn_mod.o + RRTMGP_OBJS=\ mo_rte_util_array.o \ mo_rte_sw.o \ @@ -38,10 +74,21 @@ rrtmgp_driver.o RRTMGP_NOOPT=\ mo_gas_optics_kernels.o -$(REDUCED_OPT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< - -$(RRTMGP_OBJS): %.o: %.F90 +ifeq ($(DEBUG),FALSE) + $(PERFOBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $< + $(REDUCED_OPT_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< + $(REDUCED_PRECISION_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $< + $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $< +# $(SHR_RANDNUM_C_OBJS): %.o: %.c +# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $< + $(PUMAS_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $< + $(RRTMGP_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $< -$(RRTMGP_NOOPT): %.o: %.F90 + $(RRTMGP_NOOPT): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $< +endif diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu index 97648428..f8eaf17b 100644 --- a/machines/Depends.nvhpc-gpu +++ b/machines/Depends.nvhpc-gpu @@ -1,4 +1,33 @@ +# 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized +# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test # +# Apply a different optimization level consistent with the Intel compiler +PERFOBJS=\ +prim_advection_mod.o \ +edge_mod.o \ +derivative_mod.o \ +bndry_mod.o \ +prim_advance_mod.o + +REDUCED_OPT_OBJS=\ +SatellitePhenologyMod.o \ +esm.o + +REDUCED_PRECISION_OBJS=\ +shr_wv_sat_mod.o + +SHR_RANDNUM_FORT_OBJS=\ +kissvec_mod.o \ +mersennetwister_mod.o \ +dSFMT_interface.o \ +shr_RandNum_mod.o + +SHR_RANDNUM_C_OBJS=\ +dSFMT.o \ +dSFMT_utils.o \ +kissvec.o + +# Apply GPU flags to the following code PUMAS_OBJS=\ micro_mg1_0.o \ micro_pumas_v1.o \ @@ -46,12 +75,20 @@ RRTMGP_NOOPT=\ mo_gas_optics_kernels.o ifeq ($(DEBUG),FALSE) + $(PERFOBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $< + $(REDUCED_OPT_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< + $(REDUCED_PRECISION_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $< + $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $< +# $(SHR_RANDNUM_C_OBJS): %.o: %.c +# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $< $(PUMAS_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -acc -gpu=cc70,lineinfo,nofma,flushz,fastmath -Minfo=accel $< $(RRTMGP_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -Mnoopenmp $< - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $< $(RRTMGP_NOOPT): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -O0 $< - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $< endif From d0093a5611bcca461fdf5a08831241cf49bc2d92 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 11 Apr 2023 15:39:28 -0600 Subject: [PATCH 21/95] Update module versions for intel compiler modified: machines/config_batch.xml modified: machines/config_machines.xml --- machines/config_batch.xml | 2 +- machines/config_machines.xml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 2cabfab8..430b41d5 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -473,7 +473,7 @@ -l gpu_type=$GPU_TYPE - + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 81e8f2f2..a5785add 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1846,10 +1846,10 @@ This allows using a different mpirun command to launch unit tests - intel/2021.7.1 + intel/2023.0.0 - oneapi/2023.0.0 + intel-oneapi/2023.0.0 cce/15.0.1 @@ -1861,7 +1861,7 @@ This allows using a different mpirun command to launch unit tests nvhpc/23.1 - cray-mpich/8.1.24 + cray-mpich/8.1.25 mpi-serial/2.3.0 @@ -1872,16 +1872,16 @@ This allows using a different mpirun command to launch unit tests - mkl/2022.2.1 + mkl/2023.0.0 cuda/11.7.1 - mkl/2022.2.1 + mkl/2023.0.0 - netcdf/4.9.0 + netcdf/4.9.1 hdf5 @@ -1891,11 +1891,11 @@ This allows using a different mpirun command to launch unit tests parallelio/2.5.10-debug - esmf/8.5.0b17-debug + esmf/8.5.0b19-debug parallelio/2.5.10 - esmf/8.5.0b17 + esmf/8.5.0b19 From 7224a3757a86d32e8a24998ca529db6cce721689 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 18 Apr 2023 13:10:06 -0600 Subject: [PATCH 22/95] load ncarenv and ncarcompilers manually modified: config_machines.xml --- machines/config_machines.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 26a3948f..eda8c6ca 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1844,7 +1844,9 @@ This allows using a different mpirun command to launch unit tests /opt/cray/pe/craype-targets/default/modulefiles cmake - + + ncarenv/23.03 + intel/2023.0.0 @@ -1897,6 +1899,9 @@ This allows using a different mpirun command to launch unit tests parallelio/2.5.10 esmf/8.5.0b19 + + ncarcompilers/0.8.0 + From 28e1e8b14b2dab5396ef3dd911d9e3717815ff0e Mon Sep 17 00:00:00 2001 From: Francis Vitt Date: Tue, 18 Apr 2023 13:54:00 -0700 Subject: [PATCH 23/95] update esmf lib for pleiades --- machines/config_machines.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index d7a625d1..386a6178 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -2818,7 +2818,7 @@ This allows using a different mpirun command to launch unit tests 100000 16 256M - /home6/fvitt/esmf-8_2_0/lib/libO/Linux.intel.64.mpt.default/esmf.mk + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk @@ -2872,7 +2872,7 @@ This allows using a different mpirun command to launch unit tests 100000 16 256M - /home6/fvitt/esmf-8_2_0/lib/libO/Linux.intel.64.mpt.default/esmf.mk + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk @@ -2926,7 +2926,7 @@ This allows using a different mpirun command to launch unit tests 100000 16 256M - /home6/fvitt/esmf-8_2_0/lib/libO/Linux.intel.64.mpt.default/esmf.mk + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk @@ -2980,7 +2980,7 @@ This allows using a different mpirun command to launch unit tests 100000 16 256M - /home6/fvitt/esmf-8_2_0/lib/libO/Linux.intel.64.mpt.default/esmf.mk + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk From 822f7b71191052eb6f592cafad4157b621b82c5f Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 25 Apr 2023 15:18:33 -0600 Subject: [PATCH 24/95] update the ncarenv and cesmdev module on Gust modified: machines/config_machines.xml --- machines/config_machines.xml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index eda8c6ca..98dfead3 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1840,12 +1840,10 @@ This allows using a different mpirun command to launch unit tests module module - - /opt/cray/pe/craype-targets/default/modulefiles - cmake - - - ncarenv/23.03 + + cesmdev/1.0 + ncarenv/23.04 + cmake/3.25.2 intel/2023.0.0 From 3fed003177fe0afbd859155c8dd2bde7cfb5dd79 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 2 May 2023 11:32:25 -0600 Subject: [PATCH 25/95] updates for gust --- machines/cmake_macros/intel_gust.cmake | 5 +++++ machines/config_machines.xml | 25 +++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/machines/cmake_macros/intel_gust.cmake b/machines/cmake_macros/intel_gust.cmake index f3943efe..1edce09d 100644 --- a/machines/cmake_macros/intel_gust.cmake +++ b/machines/cmake_macros/intel_gust.cmake @@ -4,3 +4,8 @@ string(APPEND FFLAGS " -march=core-avx2") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=c89 ") +endif() +set(SCC icx) +set(SFC ifort) \ No newline at end of file diff --git a/machines/config_machines.xml b/machines/config_machines.xml index f316c1ae..704eced0 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1866,56 +1866,61 @@ This allows using a different mpirun command to launch unit tests module module - ncarenv/23.03 - - cmake + cesmdev/1.0 + ncarenv/23.04 + intel/2023.0.0 + mkl intel-oneapi/2023.0.0 + mkl intel-classic/2023.0.0 + mkl cce/15.0.1 + cray-libsci/23.02.1.1 gcc/12.2.0 + cray-libsci/23.02.1.1 nvhpc/23.1 + + ncarcompilers/0.8.0 + cmake + cray-mpich/8.1.25 mpi-serial/2.3.0 - - cray-libsci/23.02.1.1 - + netcdf/4.9.1 - hdf5 - netcdf netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 parallelio/2.5.10-debug - esmf/8.5.0b18-debug + esmf/8.5.0b19-debug parallelio/2.5.10 - esmf/8.5.0b18 + esmf/8.5.0b19 From d7f5c3434c40f33bd89f78ca03f3c101a86c8b93 Mon Sep 17 00:00:00 2001 From: alperaltuntas Date: Wed, 3 May 2023 12:37:56 -0600 Subject: [PATCH 26/95] add new MOM6 grid called tx2_3v2 --- component_grids_nuopc.xml | 6 ++++++ maps_nuopc.xml | 12 ++++++++++++ modelgrid_aliases_nuopc.xml | 13 +++++++++++++ 3 files changed, 31 insertions(+) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 84df9901..cc0bb19d 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -359,6 +359,12 @@ tx0.66v1 is tripole v1 0.66-deg MOM6 grid: Experimental for MOM6 experiments + + 540 480 + $DIN_LOC_ROOT/share/meshes/tx2_3v2_230415_ESMFmesh.nc + tx2_3v2 is tripole v2 2/3-deg MOM6 grid: + Experimental for MOM6 experiments + 1440 1080 $DIN_LOC_ROOT/share/meshes/tx0.25v1_190204_ESMFmesh.nc diff --git a/maps_nuopc.xml b/maps_nuopc.xml index 726afd03..8f507d14 100644 --- a/maps_nuopc.xml +++ b/maps_nuopc.xml @@ -82,10 +82,18 @@ cpl/gridmaps/r05/map_r05_to_tx0.66v1_e1000r300_190314.nc cpl/gridmaps/r05/map_r05_to_tx0.66v1_e1000r300_190314.nc + + cpl/gridmaps/r05/map_r05_to_tx2_3_nnsm_e1000r1000_230415.nc + cpl/gridmaps/r05/map_r05_to_tx2_3_nnsm_e1000r1000_230415.nc + cpl/gridmaps/rJRA025/map_JRA025m_to_tx0.66v1_nnsm_e333r100_190910.nc cpl/gridmaps/rJRA025/map_JRA025m_to_tx0.66v1_nnsm_e333r100_190910.nc + + cpl/gridmaps/rJRA025/map_jra_to_tx2_3_nnsm_e333r100_230415.nc + cpl/gridmaps/rJRA025/map_jra_to_tx2_3_nnsm_e333r100_230415.nc + @@ -132,6 +140,10 @@ cpl/gridmaps/gland4km/map_gland4km_to_tx0.66v1_nnsm_e1000r300_190314.nc cpl/gridmaps/gland4km/map_gland4km_to_tx0.66v1_nnsm_e1000r300_190314.nc + + cpl/gridmaps/gland4km/map_gland4km_to_tx2_3_nnsm_e1000r300_230415.nc + cpl/gridmaps/gland4km/map_gland4km_to_tx2_3_nnsm_e1000r300_230415.nc + diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index c27eda55..7770aef4 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -349,6 +349,13 @@ JRA025 + + TL319 + TL319 + tx2_3v2 + JRA025 + + TL319 TL319 @@ -417,6 +424,12 @@ tx0.66v1 + + 0.9x1.25 + 0.9x1.25 + tx2_3v2 + + T62 T62 From 4c10c5d25053776982062f13920e708473e43d20 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 3 May 2023 14:59:24 -0600 Subject: [PATCH 27/95] more gust mods for serial builds --- machines/cmake_macros/gnu.cmake | 6 +++++- machines/cmake_macros/gust.cmake | 3 --- machines/cmake_macros/nvhpc_gust.cmake | 3 +++ machines/config_machines.xml | 3 ++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/machines/cmake_macros/gnu.cmake b/machines/cmake_macros/gnu.cmake index 621740c5..bea6c8e9 100644 --- a/machines/cmake_macros/gnu.cmake +++ b/machines/cmake_macros/gnu.cmake @@ -1,4 +1,8 @@ -string(APPEND CFLAGS " -std=gnu99") +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=gnu89") +else() + string(APPEND CFLAGS " -std=gnu99") +endif() if (compile_threaded) string(APPEND CFLAGS " -fopenmp") endif() diff --git a/machines/cmake_macros/gust.cmake b/machines/cmake_macros/gust.cmake index db5dbb41..2772a168 100644 --- a/machines/cmake_macros/gust.cmake +++ b/machines/cmake_macros/gust.cmake @@ -1,6 +1,3 @@ -if (COMP_NAME STREQUAL mpi-serial) - string(APPEND CFLAGS " -std=c89 ") -endif() if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/machines/cmake_macros/nvhpc_gust.cmake b/machines/cmake_macros/nvhpc_gust.cmake index 46113731..b7fdbd69 100644 --- a/machines/cmake_macros/nvhpc_gust.cmake +++ b/machines/cmake_macros/nvhpc_gust.cmake @@ -1,3 +1,6 @@ +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=c89 ") +endif() if (NOT DEBUG) string(APPEND FFLAGS " -target=zen3") endif() diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 446acdbc..59262837 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1837,7 +1837,7 @@ This allows using a different mpirun command to launch unit tests CNL intel,gnu,cray,nvhpc,intel-oneapi,intel-classic mpich - /glade/gust/scratch/$ENV{USER} + $ENV{SCRATCH} /glade/p/cesmdata/inputdata /glade/p/cesmdata/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE @@ -1869,6 +1869,7 @@ This allows using a different mpirun command to launch unit tests cesmdev/1.0 ncarenv/23.04 + craype intel/2023.0.0 From 21b8a3847e6b4ad68a5ef7569cb70d1774975a7e Mon Sep 17 00:00:00 2001 From: alperaltuntas Date: Thu, 4 May 2023 09:01:51 -0600 Subject: [PATCH 28/95] add ne30pg3_t232 resolution --- modelgrid_aliases_nuopc.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 7770aef4..f84d8706 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -888,6 +888,13 @@ tx0.66v1 + + ne30np4.pg3 + ne30np4.pg3 + tx2_3v2 + tx2_3v2 + + ne30np4 1.9x2.5 From 6df7e064dd5a823e0b2b9450f3e34582586860ef Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 10 May 2023 13:14:33 -0600 Subject: [PATCH 29/95] update esmf build on gust --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 3b6d8f66..af25be63 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1919,12 +1919,12 @@ This allows using a different mpirun command to launch unit tests parallelio/2.5.10-debug - esmf/8.5.0b19-debug + esmf/8.5.0b21-debug parallelio/2.5.10 - esmf/8.5.0b19 + esmf/8.5.0b21 From 47e03035e9f93b8134408bac8f81bc349c44dafe Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 1 Jun 2023 12:56:30 -0600 Subject: [PATCH 30/95] port to new ncar system derecho --- machines/cmake_macros/derecho.cmake | 11 +++ machines/cmake_macros/intel_derecho.cmake | 11 +++ machines/cmake_macros/nvhpc_derecho.cmake | 6 ++ machines/config_batch.xml | 11 +++ machines/config_machines.xml | 111 ++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 machines/cmake_macros/derecho.cmake create mode 100644 machines/cmake_macros/intel_derecho.cmake create mode 100644 machines/cmake_macros/nvhpc_derecho.cmake diff --git a/machines/cmake_macros/derecho.cmake b/machines/cmake_macros/derecho.cmake new file mode 100644 index 00000000..2772a168 --- /dev/null +++ b/machines/cmake_macros/derecho.cmake @@ -0,0 +1,11 @@ +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +set(NETCDF_PATH "$ENV{NETCDF}") +set(PIO_FILESYSTEM_HINTS "lustre") +set(PNETCDF_PATH "$ENV{PNETCDF}") +# If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml +string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") +#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") +#string(APPEND CPPDEFS " -DNO_SHR_VMATH ") +string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/cmake_macros/intel_derecho.cmake b/machines/cmake_macros/intel_derecho.cmake new file mode 100644 index 00000000..1edce09d --- /dev/null +++ b/machines/cmake_macros/intel_derecho.cmake @@ -0,0 +1,11 @@ +set(CONFIG_ARGS "--host=cray") +string(APPEND CFLAGS " -march=core-avx2") +string(APPEND FFLAGS " -march=core-avx2") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_SLASHPROC") +endif() +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=c89 ") +endif() +set(SCC icx) +set(SFC ifort) \ No newline at end of file diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake new file mode 100644 index 00000000..b7fdbd69 --- /dev/null +++ b/machines/cmake_macros/nvhpc_derecho.cmake @@ -0,0 +1,6 @@ +if (COMP_NAME STREQUAL mpi-serial) + string(APPEND CFLAGS " -std=c89 ") +endif() +if (NOT DEBUG) + string(APPEND FFLAGS " -target=zen3") +endif() diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 4ea11977..c4b309b5 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -376,6 +376,17 @@ + + qsub + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} + + + main + + + sbatch diff --git a/machines/config_machines.xml b/machines/config_machines.xml index af25be63..622afe9f 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1284,6 +1284,117 @@ This allows using a different mpirun command to launch unit tests + + NCAR AMD EPYC system + de.*.hpc.ucar.edu + CNL + intel,gnu,cray,nvhpc,intel-oneapi,intel-classic + mpich + $ENV{SCRATCH} + $ENV{CESMDATAROOT}/inputdata + $ENV{CESMDATAROOT}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/ccsm_baselines + $ENV{CESMDATAROOT}/cprnc/cprnc + 16 + pbs + cseg + 128 + 128 + TRUE + + mpiexec + + --label + --line-buffer + -n {{ total_tasks }} + + + + $ENV{LMOD_ROOT}/lmod/init/perl + $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py + $ENV{LMOD_ROOT}/lmod/init/sh + $ENV{LMOD_ROOT}/lmod/init/csh + $ENV{LMOD_ROOT}/lmod/libexec/lmod perl + $ENV{LMOD_ROOT}/lmod/libexec/lmod python + module + module + + cesmdev/1.0 + ncarenv/23.06 + + craype + + + intel/2023.0.0 + mkl + + + intel-oneapi/2023.0.0 + mkl + + + intel-classic/2023.0.0 + mkl + + + cce/15.0.1 + cray-libsci/23.02.1.1 + + + gcc/12.2.0 + cray-libsci/23.02.1.1 + + + nvhpc/23.1 + + + ncarcompilers/1.0.0 + cmake + + + cray-mpich/8.1.25 + + + mpi-serial/2.3.0 + + + + netcdf/4.9.2 + + + + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 + + + + parallelio/2.6.0-debug + esmf/8.5.0b23-debug + + + + parallelio/2.6.0 + esmf/8.5.0b23 + + + + + 64M + hybrid + + + + + + ON + SUMMARY + + + PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM LINUX From ebf97c0ebb973fdf89849796ce9da9ccdbb84c97 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 12 Jul 2023 09:40:55 -0600 Subject: [PATCH 31/95] updates for ncar derecho system --- machines/config_machines.xml | 2 ++ machines/config_pio.xml | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 622afe9f..70633579 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1382,6 +1382,8 @@ This allows using a different mpirun command to launch unit tests 64M hybrid + + memhooks - + + + + main diff --git a/machines/config_machines.xml b/machines/config_machines.xml index e6bfd54f..58e5b657 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1272,7 +1272,11 @@ This allows using a different mpirun command to launch unit tests pbs cseg 128 + 4 128 + 64 + none,a100 + none,openacc,openmp,combined TRUE mpiexec @@ -1958,21 +1962,6 @@ This allows using a different mpirun command to launch unit tests module module -<<<<<<< HEAD - - cesmdev/1.0 - ncarenv/23.04 - cmake/3.25.2 - - - intel/2023.0.0 - - - intel-oneapi/2023.0.0 - - - cce/15.0.1 -======= cesmdev/1.0 ncarenv/23.04 @@ -1993,7 +1982,6 @@ This allows using a different mpirun command to launch unit tests cce/15.0.1 cray-libsci/23.02.1.1 ->>>>>>> main gcc/12.2.0 @@ -2002,58 +1990,22 @@ This allows using a different mpirun command to launch unit tests nvhpc/23.1 -<<<<<<< HEAD -======= ncarcompilers/0.8.0 cmake ->>>>>>> main cray-mpich/8.1.25 mpi-serial/2.3.0 -<<<<<<< HEAD - - cray-libsci/23.02.1.1 - - - - mkl/2023.0.0 - - - cuda/11.7.1 - - - mkl/2023.0.0 - -======= ->>>>>>> main netcdf/4.9.1 + -<<<<<<< HEAD - hdf5 - netcdf - netcdf-mpi/4.9.1 - parallel-netcdf/1.12.3 - - - parallelio/2.5.10-debug - esmf/8.5.0b19-debug - - - parallelio/2.5.10 - esmf/8.5.0b19 - - - ncarcompilers/0.8.0 -======= netcdf-mpi/4.9.1 parallel-netcdf/1.12.3 @@ -2066,7 +2018,6 @@ This allows using a different mpirun command to launch unit tests parallelio/2.5.10 esmf/8.5.0b21 ->>>>>>> main From 7215ebb98c39afa0747ca912554d238a2a0fc459 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 4 Aug 2023 22:58:33 -0600 Subject: [PATCH 37/95] minor update --- machines/cmake_macros/nvhpc_derecho.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake index b7fdbd69..ab1d0d57 100644 --- a/machines/cmake_macros/nvhpc_derecho.cmake +++ b/machines/cmake_macros/nvhpc_derecho.cmake @@ -4,3 +4,4 @@ endif() if (NOT DEBUG) string(APPEND FFLAGS " -target=zen3") endif() +message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}") From 3eb4cdacd69fcb260297ab4cbd379c0e4ce3c009 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Sun, 6 Aug 2023 22:28:16 -0600 Subject: [PATCH 38/95] update GPU settings for Derecho --- machines/config_batch.xml | 4 ++-- machines/config_machines.xml | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index f40caac8..39d5ca3a 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -363,11 +363,11 @@ -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=490GB:ngpus={{ ngpus_per_node }} diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 58e5b657..fa6f6812 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1324,6 +1324,7 @@ This allows using a different mpirun command to launch unit tests nvhpc/23.1 + ncarcompilers/1.0.0 cmake @@ -1335,6 +1336,10 @@ This allows using a different mpirun command to launch unit tests mpi-serial/2.3.0 + + cuda/11.7.1 + + netcdf/4.9.2 @@ -1352,7 +1357,7 @@ This allows using a different mpirun command to launch unit tests parallelio/2.6.0 esmf/8.5.0b23 - + @@ -1371,6 +1376,9 @@ This allows using a different mpirun command to launch unit tests ON SUMMARY + + openacc + From 9e77b246cb365a0a2344a9c709bb480753cef6b9 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Sun, 6 Aug 2023 23:39:17 -0600 Subject: [PATCH 39/95] remove unused flag --- machines/config_batch.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 39d5ca3a..0b0508c4 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -358,9 +358,6 @@ qsub - - -l gpu_type=$GPU_TYPE - -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB From 75c3bfb854e7eff1ab0b0f1cb99864b8f04b0a67 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Mon, 7 Aug 2023 11:23:14 -0600 Subject: [PATCH 40/95] remove PBS -V option --- machines/config_batch.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 0b0508c4..b5efa560 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -156,7 +156,6 @@ -r {{ rerunnable }} -j oe - -V From d1b8e681a8ef14d3eeeeba040b35c3b9f77caa06 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 8 Aug 2023 09:34:49 -0600 Subject: [PATCH 41/95] Add "get_local_rank" as an attribute --- machines/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index fa6f6812..76d35388 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1277,6 +1277,7 @@ This allows using a different mpirun command to launch unit tests 64 none,a100 none,openacc,openmp,combined + get_local_rank TRUE mpiexec From d5da373b99d7709c6bcc4a1292a8d351538d8d1f Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 9 Aug 2023 09:17:53 -0600 Subject: [PATCH 42/95] Update memory request on Derecho --- machines/config_batch.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index b5efa560..17122838 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -359,11 +359,11 @@ qsub -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=490GB:ngpus={{ ngpus_per_node }} + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }} From 8241e8c0167fab28a836062e6447cf04a7b5277f Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 9 Aug 2023 10:28:24 -0600 Subject: [PATCH 43/95] Turn on MPS for Derecho --- machines/config_batch.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 17122838..fef0ba1f 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -363,9 +363,7 @@ -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }} - - + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 main From 5fbfdc25ba8e06c944912419ba7a32655e415628 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 9 Aug 2023 11:39:27 -0600 Subject: [PATCH 44/95] Change baseline path from ccsm to cesm. --- machines/config_machines.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 06aef907..3c3adf7e 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1294,7 +1294,7 @@ This allows using a different mpirun command to launch unit tests $ENV{CESMDATAROOT}/inputdata $ENV{CESMDATAROOT}/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/ccsm_baselines + $ENV{CESMDATAROOT}/cesm_baselines $ENV{CESMDATAROOT}/cprnc/cprnc 16 pbs @@ -1370,12 +1370,12 @@ This allows using a different mpirun command to launch unit tests parallelio/2.6.0-debug - esmf/8.5.0b23-debug + esmf/8.5.0 parallelio/2.6.0 - esmf/8.5.0b23 + esmf/8.5.0 From 81c5b99a1dea97fe3fe89589bb6b2f3c67294b4d Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 9 Aug 2023 11:52:17 -0600 Subject: [PATCH 45/95] Update ESMF on derecho to 8.6.0b03 --- machines/config_machines.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 3c3adf7e..504f97c6 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1370,12 +1370,12 @@ This allows using a different mpirun command to launch unit tests parallelio/2.6.0-debug - esmf/8.5.0 + esmf/8.6.0b03 parallelio/2.6.0 - esmf/8.5.0 + esmf/8.6.0b03 From 560c043bee42c48429fd5b25d1e9f6593dc9649a Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 9 Aug 2023 12:00:56 -0600 Subject: [PATCH 46/95] remove duplicated lines --- machines/Depends.nvhpc | 2 -- 1 file changed, 2 deletions(-) diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 15675846..b51e3c90 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -67,8 +67,6 @@ mo_rte_kind.o\ mo_source_functions.o\ mo_optical_props_kernels.o\ mo_rte_solver_kernels.o\ -mo_optical_props_kernels.o\ -mo_rte_solver_kernels.o\ rrtmgp_driver.o\ mo_gas_optics_kernels.o From e084ecf9df052c71ac11e6924b84aab34308f330 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 15 Sep 2023 11:05:14 -0600 Subject: [PATCH 47/95] Turn off FMA for intel compiler on Derecho --- machines/cmake_macros/intel_derecho.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machines/cmake_macros/intel_derecho.cmake b/machines/cmake_macros/intel_derecho.cmake index 1edce09d..6d454625 100644 --- a/machines/cmake_macros/intel_derecho.cmake +++ b/machines/cmake_macros/intel_derecho.cmake @@ -1,6 +1,6 @@ set(CONFIG_ARGS "--host=cray") -string(APPEND CFLAGS " -march=core-avx2") -string(APPEND FFLAGS " -march=core-avx2") +string(APPEND CFLAGS " -qopt-report -march=core-avx2 -no-fma") +string(APPEND FFLAGS " -qopt-report -march=core-avx2 -no-fma") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() @@ -8,4 +8,4 @@ if (COMP_NAME STREQUAL mpi-serial) string(APPEND CFLAGS " -std=c89 ") endif() set(SCC icx) -set(SFC ifort) \ No newline at end of file +set(SFC ifort) From 8b3f25838a33581ef9fb6eb862f096c547618ca3 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 15 Sep 2023 13:27:54 -0600 Subject: [PATCH 48/95] Set nofma by default for intel and nvhpc compilers --- machines/cmake_macros/intel.cmake | 8 ++++---- machines/cmake_macros/intel_aws-hpc6a.cmake | 4 ++-- machines/cmake_macros/intel_bluewaters.cmake | 4 ++-- machines/cmake_macros/intel_casper.cmake | 4 ++-- machines/cmake_macros/intel_cheyenne.cmake | 4 ++-- machines/cmake_macros/intel_derecho.cmake | 4 ++-- machines/cmake_macros/intel_stampede2-knl.cmake | 4 ++-- machines/cmake_macros/intel_stampede2-skx.cmake | 4 ++-- machines/cmake_macros/intel_zeus.cmake | 8 ++++---- machines/cmake_macros/nvhpc.cmake | 2 +- machines/cmake_macros/nvhpc_casper.cmake | 4 ++-- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/machines/cmake_macros/intel.cmake b/machines/cmake_macros/intel.cmake index 7328c85e..96cad2a2 100644 --- a/machines/cmake_macros/intel.cmake +++ b/machines/cmake_macros/intel.cmake @@ -3,10 +3,10 @@ if (compile_threaded) string(APPEND CFLAGS " -qopenmp") endif() if (NOT DEBUG) - string(APPEND CFLAGS " -O2 -debug minimal") + string(APPEND CFLAGS " -O2 -debug minimal -no-fma") endif() if (DEBUG) - string(APPEND CFLAGS " -O0 -g") + string(APPEND CFLAGS " -O0 -g -no-fma") endif() string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DCPRINTEL") string(APPEND CXX_LDFLAGS " -cxxlib") @@ -17,10 +17,10 @@ if (compile_threaded) string(APPEND FFLAGS " -qopenmp") endif() if (DEBUG) - string(APPEND FFLAGS " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") + string(APPEND FFLAGS " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -no-fma") endif() if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal") + string(APPEND FFLAGS " -O2 -debug minimal -no-fma") endif() set(FFLAGS_NOOPT "-O0") set(FIXEDFLAGS "-fixed") diff --git a/machines/cmake_macros/intel_aws-hpc6a.cmake b/machines/cmake_macros/intel_aws-hpc6a.cmake index 0e4d2815..2c7921c6 100644 --- a/machines/cmake_macros/intel_aws-hpc6a.cmake +++ b/machines/cmake_macros/intel_aws-hpc6a.cmake @@ -1,5 +1,5 @@ -string(APPEND CFLAGS " -qopt-report -march=core-avx2 -mtune=core-avx2 -no-fma") -string(APPEND FFLAGS " -qopt-report -march=core-avx2 -mtune=core-avx2 -no-fma") +string(APPEND CFLAGS " -qopt-report -march=core-avx2 -mtune=core-avx2") +string(APPEND FFLAGS " -qopt-report -march=core-avx2 -mtune=core-avx2") if (DEBUG) string(APPEND CMAKE_OPTS " -DPIO_ENABLE_LOGGING=ON") endif() diff --git a/machines/cmake_macros/intel_bluewaters.cmake b/machines/cmake_macros/intel_bluewaters.cmake index 29b1da06..3cb50907 100644 --- a/machines/cmake_macros/intel_bluewaters.cmake +++ b/machines/cmake_macros/intel_bluewaters.cmake @@ -1,3 +1,3 @@ set(HAS_F2008_CONTIGUOUS "FALSE") -string(APPEND FFLAGS " -dynamic -mkl=sequential -no-fma") -string(APPEND CFLAGS " -dynamic -mkl=sequential -no-fma") +string(APPEND FFLAGS " -dynamic -mkl=sequential") +string(APPEND CFLAGS " -dynamic -mkl=sequential") diff --git a/machines/cmake_macros/intel_casper.cmake b/machines/cmake_macros/intel_casper.cmake index 08617ce8..b10bf277 100644 --- a/machines/cmake_macros/intel_casper.cmake +++ b/machines/cmake_macros/intel_casper.cmake @@ -1,5 +1,5 @@ -string(APPEND CFLAGS " -qopt-report -xCORE_AVX2 -no-fma") -string(APPEND FFLAGS " -qopt-report -xCORE_AVX2 -no-fma") +string(APPEND CFLAGS " -qopt-report -xCORE_AVX2") +string(APPEND FFLAGS " -qopt-report -xCORE_AVX2") if (DEBUG) string(APPEND CMAKE_OPTS " -DPIO_ENABLE_LOGGING=ON") endif() diff --git a/machines/cmake_macros/intel_cheyenne.cmake b/machines/cmake_macros/intel_cheyenne.cmake index 08617ce8..b10bf277 100644 --- a/machines/cmake_macros/intel_cheyenne.cmake +++ b/machines/cmake_macros/intel_cheyenne.cmake @@ -1,5 +1,5 @@ -string(APPEND CFLAGS " -qopt-report -xCORE_AVX2 -no-fma") -string(APPEND FFLAGS " -qopt-report -xCORE_AVX2 -no-fma") +string(APPEND CFLAGS " -qopt-report -xCORE_AVX2") +string(APPEND FFLAGS " -qopt-report -xCORE_AVX2") if (DEBUG) string(APPEND CMAKE_OPTS " -DPIO_ENABLE_LOGGING=ON") endif() diff --git a/machines/cmake_macros/intel_derecho.cmake b/machines/cmake_macros/intel_derecho.cmake index 6d454625..7efd2ee3 100644 --- a/machines/cmake_macros/intel_derecho.cmake +++ b/machines/cmake_macros/intel_derecho.cmake @@ -1,6 +1,6 @@ set(CONFIG_ARGS "--host=cray") -string(APPEND CFLAGS " -qopt-report -march=core-avx2 -no-fma") -string(APPEND FFLAGS " -qopt-report -march=core-avx2 -no-fma") +string(APPEND CFLAGS " -qopt-report -march=core-avx2") +string(APPEND FFLAGS " -qopt-report -march=core-avx2") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/machines/cmake_macros/intel_stampede2-knl.cmake b/machines/cmake_macros/intel_stampede2-knl.cmake index 6cdd9fe9..7836ea9d 100644 --- a/machines/cmake_macros/intel_stampede2-knl.cmake +++ b/machines/cmake_macros/intel_stampede2-knl.cmake @@ -1,5 +1,5 @@ -string(APPEND CFLAGS " -xCOMMON-AVX512 -no-fma") -string(APPEND FFLAGS " -xCOMMON-AVX512 -no-fma") +string(APPEND CFLAGS " -xCOMMON-AVX512") +string(APPEND FFLAGS " -xCOMMON-AVX512") if (MPILIB STREQUAL mpi-serial) string(APPEND FFLAGS " -mcmodel medium") endif() diff --git a/machines/cmake_macros/intel_stampede2-skx.cmake b/machines/cmake_macros/intel_stampede2-skx.cmake index 6cdd9fe9..7836ea9d 100644 --- a/machines/cmake_macros/intel_stampede2-skx.cmake +++ b/machines/cmake_macros/intel_stampede2-skx.cmake @@ -1,5 +1,5 @@ -string(APPEND CFLAGS " -xCOMMON-AVX512 -no-fma") -string(APPEND FFLAGS " -xCOMMON-AVX512 -no-fma") +string(APPEND CFLAGS " -xCOMMON-AVX512") +string(APPEND FFLAGS " -xCOMMON-AVX512") if (MPILIB STREQUAL mpi-serial) string(APPEND FFLAGS " -mcmodel medium") endif() diff --git a/machines/cmake_macros/intel_zeus.cmake b/machines/cmake_macros/intel_zeus.cmake index 281ae04b..be4fc013 100644 --- a/machines/cmake_macros/intel_zeus.cmake +++ b/machines/cmake_macros/intel_zeus.cmake @@ -2,14 +2,14 @@ set(AR "xiar") set(ARFLAGS "cru") if (MPILIB STREQUAL mpi-serial) string(APPEND FFLAGS " -mcmodel medium") - string(APPEND FFLAGS " -mkl -xSKYLAKE-AVX512 -qopt-zmm-usage=high -no-fma") - string(APPEND CFLAGS " -mkl -xSKYLAKE-AVX512 -qopt-zmm-usage=high -no-fma") + string(APPEND FFLAGS " -mkl -xSKYLAKE-AVX512 -qopt-zmm-usage=high") + string(APPEND CFLAGS " -mkl -xSKYLAKE-AVX512 -qopt-zmm-usage=high") string(APPEND LDFLAGS " -mkl -lstdc++") string(APPEND SLIBS " -lstdc++") endif() if (MPILIB STREQUAL impi) - string(APPEND FFLAGS " -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=high -no-fma") - string(APPEND CFLAGS " -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=high -no-fma") + string(APPEND FFLAGS " -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=high") + string(APPEND CFLAGS " -mkl=cluster -xSKYLAKE-AVX512 -qopt-zmm-usage=high") string(APPEND LDFLAGS " -mkl=cluster") string(APPEND SLIBS " -lstdc++") endif() diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake index 1e6036d9..778746fa 100644 --- a/machines/cmake_macros/nvhpc.cmake +++ b/machines/cmake_macros/nvhpc.cmake @@ -3,7 +3,7 @@ if (compile_threaded) string(APPEND CFLAGS " -mp") endif() if (NOT DEBUG) - string(APPEND CFLAGS " -O ") + string(APPEND CFLAGS " -O -Mnofma") string(APPEND FFLAGS " -O -Mnofma") endif() diff --git a/machines/cmake_macros/nvhpc_casper.cmake b/machines/cmake_macros/nvhpc_casper.cmake index 08bc6bbe..c789907f 100644 --- a/machines/cmake_macros/nvhpc_casper.cmake +++ b/machines/cmake_macros/nvhpc_casper.cmake @@ -1,8 +1,8 @@ if (NOT DEBUG) - string(APPEND CFLAGS " -O -tp=skylake -Mnofma") + string(APPEND CFLAGS " -O -tp=skylake") endif() if (NOT DEBUG) - string(APPEND FFLAGS " -O -tp=skylake -Mnofma") + string(APPEND FFLAGS " -O -tp=skylake") endif() string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS") if (NOT DEBUG) From 9aa019b990ec292212636daa2f3f4be00b1456c2 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 15 Sep 2023 13:42:19 -0600 Subject: [PATCH 49/95] address Jim's comments --- machines/cmake_macros/intel.cmake | 10 ++++++---- machines/cmake_macros/nvhpc.cmake | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/machines/cmake_macros/intel.cmake b/machines/cmake_macros/intel.cmake index 96cad2a2..3c25605f 100644 --- a/machines/cmake_macros/intel.cmake +++ b/machines/cmake_macros/intel.cmake @@ -3,11 +3,12 @@ if (compile_threaded) string(APPEND CFLAGS " -qopenmp") endif() if (NOT DEBUG) - string(APPEND CFLAGS " -O2 -debug minimal -no-fma") + string(APPEND CFLAGS " -O2 -debug minimal") endif() if (DEBUG) - string(APPEND CFLAGS " -O0 -g -no-fma") + string(APPEND CFLAGS " -O0 -g") endif() +string(APPEND CFLAGS " -no-fma") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DCPRINTEL") string(APPEND CXX_LDFLAGS " -cxxlib") set(CXX_LINKER "FORTRAN") @@ -17,11 +18,12 @@ if (compile_threaded) string(APPEND FFLAGS " -qopenmp") endif() if (DEBUG) - string(APPEND FFLAGS " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -no-fma") + string(APPEND FFLAGS " -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created") endif() if (NOT DEBUG) - string(APPEND FFLAGS " -O2 -debug minimal -no-fma") + string(APPEND FFLAGS " -O2 -debug minimal") endif() +string(APPEND FFLAGS " -no-fma") set(FFLAGS_NOOPT "-O0") set(FIXEDFLAGS "-fixed") set(FREEFLAGS "-free") diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake index 778746fa..d541457e 100644 --- a/machines/cmake_macros/nvhpc.cmake +++ b/machines/cmake_macros/nvhpc.cmake @@ -3,9 +3,11 @@ if (compile_threaded) string(APPEND CFLAGS " -mp") endif() if (NOT DEBUG) - string(APPEND CFLAGS " -O -Mnofma") - string(APPEND FFLAGS " -O -Mnofma") + string(APPEND CFLAGS " -O") + string(APPEND FFLAGS " -O") endif() +string(APPEND CFLAGS " -Mnofma") +string(APPEND FFLAGS " -Mnofma") string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI") set(CXX_LINKER "CXX") From d14773a2d25159f205ef46507faf101506c9e0a5 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Mon, 18 Sep 2023 14:59:47 -0600 Subject: [PATCH 50/95] Add "-V" option back to avoid a python issue on Cheyenne --- machines/config_batch.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index fef0ba1f..d449a349 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -156,6 +156,7 @@ -r {{ rerunnable }} -j oe + -V From fd6608bc6dd66ecc62cf3c3570305d204b55b93f Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Wed, 27 Sep 2023 07:58:19 -0600 Subject: [PATCH 51/95] add grid alias for ne3_ne3_mg37 --- modelgrid_aliases_nuopc.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index f84d8706..1d82934f 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -839,6 +839,13 @@ gx3v7 + + ne3np4 + ne3np4 + ne3np4 + gx3v7 + + ne5np4 ne5np4 From 6ed448535cb8c7506073c9e0041fb4f5f029dff4 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Wed, 27 Sep 2023 11:14:54 -0600 Subject: [PATCH 52/95] add ne3np4 mesh and domain files for scam --- component_grids_nuopc.xml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index cc0bb19d..108b9c43 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -91,6 +91,13 @@ + + 488 1 + $DIN_LOC_ROOT/share/domains/domain.lnd.ne3np4_gx3v7.230718.nc + $DIN_LOC_ROOT/share/meshes/ne3np4_ESMFmesh_c230714_cdf5.nc + ne3np4 is Spectral Elem 10-deg grid: + EXPERIMENTAL FVM physics grid + 486 1 $DIN_LOC_ROOT/share/meshes/ne3pg3_ESMFmesh_c221214_cdf5.nc @@ -99,6 +106,8 @@ 1352 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne5np4_gx3v7.140810.nc $DIN_LOC_ROOT/share/meshes/ne5np4_esmf_mesh_c210121.nc ne5np4 is Spectral Elem 6-deg grid: For ultra-low resolution spectral element grid testing @@ -111,6 +120,8 @@ 1350 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne5np4.pg3_gx3v7.170605.nc $DIN_LOC_ROOT/share/meshes/ne5pg3_ESMFmesh_cdf5_c20210118.nc ne5np4 is Spectral Elem 6-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid @@ -123,35 +134,47 @@ 13826 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne16np4_gx1v7.171018.nc $DIN_LOC_ROOT/share/meshes/ne16np4_scrip_171002_ESMFmesh.nc ne16np4 is Spectral Elem 2-deg grid: For low resolution spectral element grid testing 13824 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne16pg3_gx1v7.171003.nc $DIN_LOC_ROOT/share/meshes/ne16pg3_ESMFmesh_cdf5_c20211018.nc ne16np4.pg3 is a Spectral Elem 2-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 48602 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30_gx1v7.171003.nc $DIN_LOC_ROOT/share/meshes/ne30np4_ESMFmesh_cdf5_c20211018.nc ne30np4 is Spectral Elem 1-deg grid: 21600 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg2_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne30pg2_ESMFmesh_cdf5_c20211018.nc ne30np4.pg2 is a Spectral Elem 1-deg grid with a 2x2 FVM physics grid: EXPERIMENTAL FVM physics grid 48600 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg3_gx1v7.170605.nc $DIN_LOC_ROOT/share/meshes/ne30pg3_ESMFmesh_cdf5_c20211018.nc ne30np4.pg3 is a Spectral Elem ne30 grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 86400 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg4_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne30pg4_ESMFmesh_cdf5_c20211018.nc ne30np4.pg4 is a Spectral Elem 1-deg grid with a 4x4 FVM physics grid: EXPERIMENTAL FVM physics grid @@ -163,18 +186,24 @@ 86400 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg2_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg2_ESMFmesh_cdf5_c20211018.nc ne60np4.pg2 is a Spectral Elem 0.5-deg grid with a 2x2 FVM physics grid: EXPERIMENTAL FVM physics grid 194400 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg3_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg3_ESMFmesh_cdf5_c20211018.nc ne60np4.pg3 is a Spectral Elem 0.5-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 345600 1 + + $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg4_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg4_ESMFmesh_cdf5_c20211018.nc ne60np4.pg4 is a Spectral Elem 0.5-deg grid with a 4x4 FVM physics grid: EXPERIMENTAL FVM physics grid From 980862e0ee4bd4c51faf665d487dcfd36f704be2 Mon Sep 17 00:00:00 2001 From: Brian Eaton Date: Tue, 3 Oct 2023 11:33:51 -0400 Subject: [PATCH 53/95] build RRTMGP code w/ -fno-openmp option --- machines/Depends.gnu | 12 ++++++++++++ machines/Depends.intel | 12 +++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/machines/Depends.gnu b/machines/Depends.gnu index 2d532472..aa04911d 100644 --- a/machines/Depends.gnu +++ b/machines/Depends.gnu @@ -1,2 +1,14 @@ geopk.o:geopk.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fcray-pointer $< + +RRTMGP_OBJS=\ +mo_fluxes_byband.o mo_rrtmgp_clr_all_sky.o \ +mo_zenith_angle_spherical_correction.o mo_gas_concentrations.o \ +mo_aerosol_optics_rrtmgp_merra.o mo_cloud_optics_rrtmgp.o \ +mo_gas_optics_rrtmgp.o mo_gas_optics_rrtmgp_kernels.o \ +mo_rte_sw.o mo_rte_lw.o \ +mo_rte_util_array_validation.o mo_rte_util_array.o \ +mo_fluxes_broadband_kernels.o + +$(RRTMGP_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fno-openmp $< diff --git a/machines/Depends.intel b/machines/Depends.intel index 0f80d21c..fb0c15a3 100644 --- a/machines/Depends.intel +++ b/machines/Depends.intel @@ -33,6 +33,14 @@ micro_pumas_data.o \ micro_pumas_utils.o \ wv_sat_methods.o +RRTMGP_OBJS=\ +mo_fluxes_byband.o mo_rrtmgp_clr_all_sky.o \ +mo_zenith_angle_spherical_correction.o mo_gas_concentrations.o \ +mo_aerosol_optics_rrtmgp_merra.o mo_cloud_optics_rrtmgp.o \ +mo_gas_optics_rrtmgp.o mo_gas_optics_rrtmgp_kernels.o \ +mo_rte_sw.o mo_rte_lw.o \ +mo_rte_util_array_validation.o mo_rte_util_array.o \ +mo_fluxes_broadband_kernels.o ifeq ($(DEBUG),FALSE) $(PERFOBJS): %.o: %.F90 @@ -47,5 +55,7 @@ ifeq ($(DEBUG),FALSE) $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $< $(PUMAS_MG_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-fma -ftz -no-prec-sqrt -qoverride-limits -no-inline-max-total-size -inline-factor=200 -qopt-report=5 $< - endif + +$(RRTMGP_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fno-openmp $< From c87730c8d49a2b04a0877daa14ca6874fd89213f Mon Sep 17 00:00:00 2001 From: Brian Eaton Date: Thu, 12 Oct 2023 17:44:14 -0400 Subject: [PATCH 54/95] add comments; change intel option; add cray --- machines/Depends.cray | 14 ++++++++++++++ machines/Depends.gnu | 2 ++ machines/Depends.intel | 4 +++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/machines/Depends.cray b/machines/Depends.cray index c337bde0..68679f02 100644 --- a/machines/Depends.cray +++ b/machines/Depends.cray @@ -1,4 +1,18 @@ NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o +# RRTMGP contains openmp directives for running on GPUs. These directives need to be +# disabled to allow building CAM with threading on CPUs enabled. +RRTMGP_OBJS=\ +mo_fluxes_byband.o mo_rrtmgp_clr_all_sky.o \ +mo_zenith_angle_spherical_correction.o mo_gas_concentrations.o \ +mo_aerosol_optics_rrtmgp_merra.o mo_cloud_optics_rrtmgp.o \ +mo_gas_optics_rrtmgp.o mo_gas_optics_rrtmgp_kernels.o \ +mo_rte_sw.o mo_rte_lw.o \ +mo_rte_util_array_validation.o mo_rte_util_array.o \ +mo_fluxes_broadband_kernels.o + $(NOOPTOBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FFLAGS_NOOPT) $(FREEFLAGS) $< + +$(RRTMGP_OBJS): %.o: %.F90 + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -h noomp $< diff --git a/machines/Depends.gnu b/machines/Depends.gnu index aa04911d..88c35fa5 100644 --- a/machines/Depends.gnu +++ b/machines/Depends.gnu @@ -1,6 +1,8 @@ geopk.o:geopk.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fcray-pointer $< +# RRTMGP contains openmp directives for running on GPUs. These directives need to be +# disabled to allow building CAM with threading on CPUs enabled. RRTMGP_OBJS=\ mo_fluxes_byband.o mo_rrtmgp_clr_all_sky.o \ mo_zenith_angle_spherical_correction.o mo_gas_concentrations.o \ diff --git a/machines/Depends.intel b/machines/Depends.intel index fb0c15a3..8bbc3415 100644 --- a/machines/Depends.intel +++ b/machines/Depends.intel @@ -33,6 +33,8 @@ micro_pumas_data.o \ micro_pumas_utils.o \ wv_sat_methods.o +# RRTMGP contains openmp directives for running on GPUs. These directives need to be +# disabled to allow building CAM with threading on CPUs enabled. RRTMGP_OBJS=\ mo_fluxes_byband.o mo_rrtmgp_clr_all_sky.o \ mo_zenith_angle_spherical_correction.o mo_gas_concentrations.o \ @@ -58,4 +60,4 @@ ifeq ($(DEBUG),FALSE) endif $(RRTMGP_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fno-openmp $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qno-openmp $< From 6f1058e03508f1b20fd4b3647f3af3e1e0504f94 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 12 Oct 2023 16:27:36 -0600 Subject: [PATCH 55/95] support for job priority on derecho --- machines/config_batch.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index d449a349..cb56c334 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -358,6 +358,9 @@ qsub + + -l job_priority=$JOB_PRIORITY + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB From 974a1a7736eb75d10f811e8cdd9906f455edbb55 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 17 Oct 2023 09:42:54 -0600 Subject: [PATCH 56/95] update pio module on derecho --- machines/config_machines.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 794b7608..fa3a4492 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1351,13 +1351,13 @@ This allows using a different mpirun command to launch unit tests - parallelio/2.6.0-debug - esmf/8.6.0b03-debug + parallelio/2.6.2-debug + esmf/8.6.0b04-debug - parallelio/2.6.0 - esmf/8.6.0b03 + parallelio/2.6.2 + esmf/8.6.0b04 From d68bbe63497c1ec6809f66ad9839351d1326f938 Mon Sep 17 00:00:00 2001 From: alperaltuntas Date: Wed, 18 Oct 2023 10:19:26 -0600 Subject: [PATCH 57/95] update ROF2OCN_LIQ_RMAPNAME and GLC2OCN_LIQ_RMAPNAME maps for t232 MOM6 grid. --- maps_nuopc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maps_nuopc.xml b/maps_nuopc.xml index 8f507d14..c9e6b607 100644 --- a/maps_nuopc.xml +++ b/maps_nuopc.xml @@ -83,7 +83,7 @@ cpl/gridmaps/r05/map_r05_to_tx0.66v1_e1000r300_190314.nc - cpl/gridmaps/r05/map_r05_to_tx2_3_nnsm_e1000r1000_230415.nc + cpl/gridmaps/r05/map_r05_to_tx2_3_nnsm_e250r250_230914.nc cpl/gridmaps/r05/map_r05_to_tx2_3_nnsm_e1000r1000_230415.nc @@ -141,7 +141,7 @@ cpl/gridmaps/gland4km/map_gland4km_to_tx0.66v1_nnsm_e1000r300_190314.nc - cpl/gridmaps/gland4km/map_gland4km_to_tx2_3_nnsm_e1000r300_230415.nc + cpl/gridmaps/gland4km/map_gland4km_to_tx2_3_nnsm_e250r250_230914.nc cpl/gridmaps/gland4km/map_gland4km_to_tx2_3_nnsm_e1000r300_230415.nc From 699fb71d048b2a6b349fdb9ad95283da3a7e9713 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 15 Nov 2023 10:42:11 -0700 Subject: [PATCH 58/95] update modules for derecho --- machines/config_machines.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/machines/config_machines.xml b/machines/config_machines.xml index fa3a4492..71730466 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -1260,7 +1260,7 @@ This allows using a different mpirun command to launch unit tests NCAR AMD EPYC system de.*.hpc.ucar.edu CNL - intel,gnu,cray,nvhpc,intel-oneapi,intel-classic + intel,gnu,nvhpc,intel-oneapi,intel-classic mpich $ENV{SCRATCH} $ENV{CESMDATAROOT}/inputdata @@ -1298,20 +1298,20 @@ This allows using a different mpirun command to launch unit tests module cesmdev/1.0 - ncarenv/23.06 + ncarenv/23.09 craype - intel/2023.0.0 + intel/2023.2.1 mkl - intel-oneapi/2023.0.0 + intel-oneapi/2023.2.1 mkl - intel-classic/2023.0.0 + intel-classic/2023.2.1 mkl @@ -1323,7 +1323,7 @@ This allows using a different mpirun command to launch unit tests cray-libsci/23.02.1.1 - nvhpc/23.5 + nvhpc/23.7 @@ -1331,14 +1331,14 @@ This allows using a different mpirun command to launch unit tests cmake - cray-mpich/8.1.25 + cray-mpich/8.1.27 mpi-serial/2.3.0 - cuda/11.7.1 + cuda/12.2.1 From 97233831d24697ce35bf3399a2dfdb4beb04f8bd Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Sun, 26 Nov 2023 10:31:23 -0700 Subject: [PATCH 59/95] scam se updates add ne4 grid --- component_grids_nuopc.xml | 6 ++++++ modelgrid_aliases_nuopc.xml | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 108b9c43..b64ea428 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -104,6 +104,12 @@ ne3np4 is Spectral Elem 10-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid + + 866 1 + $DIN_LOC_ROOT/share/domains/domain.lnd.ne4np4_oQU240.160614.nc + ne4np4 is Spectral Elem 10-deg grid: + EXPERIMENTAL FVM physics grid + 1352 1 diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 1d82934f..22bb6d83 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -846,6 +846,13 @@ gx3v7 + + ne4np4 + ne4np4 + ne4np4 + gx3v7 + + ne5np4 ne5np4 From 3b191e41e0851c4cc23c1e29726cb9f7049fd0f4 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 30 Nov 2023 08:12:54 -0700 Subject: [PATCH 60/95] increase derecho walltime to 12hrs --- machines/config_batch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index cb56c334..087480af 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -370,7 +370,7 @@ -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 - main + main From 734fbce2ec10ebeba2252af388e7a320721133f0 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 6 Dec 2023 12:47:08 -0700 Subject: [PATCH 61/95] Add pFUnit support on derecho with the intel compiler I built and installed pFUnit as follows: git clone git@github.com:Goddard-Fortran-Ecosystem/pFUnit.git cd pFUnit git describe --tags export PFUNIT=$CESMDATAROOT/tools/pFUnit/pFUnit4.8.0_derecho_Intel2023.2.1_noMPI_noOpenMP mkdir build-dir cd build-dir /glade/work/sacks/ctsm_code/ctsm/cime/CIME/scripts/configure --mpilib mpi-serial source ./.env_mach_specific.sh cmake -DSKIP_MPI=YES -DSKIP_OPENMP=YES -DCMAKE_INSTALL_PREFIX=$PFUNIT .. make -j 8 make tests make install Note that this used the following modules: Currently Loaded Modules: 1) cesmdev/1.0 (H,S) 3) craype/2.7.23 5) mkl/2023.2.0 7) cmake/3.26.3 9) hdf5/1.12.2 11) parallelio/2.6.2 2) ncarenv/23.09 (S) 4) intel/2023.2.1 6) ncarcompilers/1.0.0 8) mpi-serial/2.3.0 10) netcdf/4.9.2 12) esmf/8.6.0b04 --- machines/cmake_macros/intel_derecho.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/machines/cmake_macros/intel_derecho.cmake b/machines/cmake_macros/intel_derecho.cmake index 7efd2ee3..bdc0366f 100644 --- a/machines/cmake_macros/intel_derecho.cmake +++ b/machines/cmake_macros/intel_derecho.cmake @@ -7,5 +7,8 @@ endif() if (COMP_NAME STREQUAL mpi-serial) string(APPEND CFLAGS " -std=c89 ") endif() +if (MPILIB STREQUAL mpi-serial AND NOT compile_threaded) + set(PFUNIT_PATH "$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit4.8.0_derecho_Intel2023.2.1_noMPI_noOpenMP") +endif() set(SCC icx) set(SFC ifort) From ffd10a28ef5c5a56870a7b2414f913c154a4c2b4 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 7 Dec 2023 08:17:12 -0700 Subject: [PATCH 62/95] new config_machines.xml file structure --- machines/aleph/config_machines.xml | 64 + machines/athena/config_machines.xml | 85 + machines/aws-hpc6a/config_machines.xml | 50 + machines/casper/config_machines.xml | 127 + machines/centos7-linux/config_machines.xml | 59 + machines/cheyenne/config_machines.xml | 288 ++ machines/coeus/config_machines.xml | 57 + machines/config_machines.xml | 3705 +---------------- machines/constance/config_machines.xml | 94 + machines/container/config_machines.xml | 37 + machines/daint/config_machines.xml | 28 + machines/derecho/config_machines.xml | 125 + machines/eastwind/config_machines.xml | 55 + machines/euler2/config_machines.xml | 62 + machines/euler3/config_machines.xml | 66 + machines/euler4/config_machines.xml | 51 + machines/frontera/config_machines.xml | 89 + machines/gaea/config_machines.xml | 63 + .../greenplanet-sib29/config_machines.xml | 78 + .../greenplanet-sky24/config_machines.xml | 80 + machines/gust/config_machines.xml | 111 + machines/homebrew/config_machines.xml | 49 + machines/izumi/config_machines.xml | 134 + machines/lawrencium-lr3/config_machines.xml | 53 + machines/lobata/config_machines.xml | 45 + machines/lonestar5/config_machines.xml | 56 + machines/melvin/config_machines.xml | 71 + machines/modex/config_machines.xml | 57 + machines/olympus/config_machines.xml | 45 + machines/perlmutter/config_machines.xml | 95 + machines/pleiades-bro/config_machines.xml | 54 + machines/pleiades-has/config_machines.xml | 53 + machines/pleiades-ivy/config_machines.xml | 53 + machines/pleiades-san/config_machines.xml | 54 + machines/sandia-srn-sems/config_machines.xml | 53 + machines/sandiatoss3/config_machines.xml | 64 + machines/stampede2-knl/config_machines.xml | 57 + machines/stampede2-skx/config_machines.xml | 77 + machines/theia/config_machines.xml | 55 + machines/theta/config_machines.xml | 88 + machines/thunder/config_machines.xml | 74 + machines/ubuntu-latest/config_machines.xml | 32 + machines/zeus/config_machines.xml | 80 + 43 files changed, 3095 insertions(+), 3678 deletions(-) create mode 100644 machines/aleph/config_machines.xml create mode 100644 machines/athena/config_machines.xml create mode 100644 machines/aws-hpc6a/config_machines.xml create mode 100644 machines/casper/config_machines.xml create mode 100644 machines/centos7-linux/config_machines.xml create mode 100644 machines/cheyenne/config_machines.xml create mode 100644 machines/coeus/config_machines.xml create mode 100644 machines/constance/config_machines.xml create mode 100644 machines/container/config_machines.xml create mode 100644 machines/daint/config_machines.xml create mode 100644 machines/derecho/config_machines.xml create mode 100644 machines/eastwind/config_machines.xml create mode 100644 machines/euler2/config_machines.xml create mode 100644 machines/euler3/config_machines.xml create mode 100644 machines/euler4/config_machines.xml create mode 100644 machines/frontera/config_machines.xml create mode 100644 machines/gaea/config_machines.xml create mode 100644 machines/greenplanet-sib29/config_machines.xml create mode 100644 machines/greenplanet-sky24/config_machines.xml create mode 100644 machines/gust/config_machines.xml create mode 100644 machines/homebrew/config_machines.xml create mode 100644 machines/izumi/config_machines.xml create mode 100644 machines/lawrencium-lr3/config_machines.xml create mode 100644 machines/lobata/config_machines.xml create mode 100644 machines/lonestar5/config_machines.xml create mode 100644 machines/melvin/config_machines.xml create mode 100644 machines/modex/config_machines.xml create mode 100644 machines/olympus/config_machines.xml create mode 100644 machines/perlmutter/config_machines.xml create mode 100644 machines/pleiades-bro/config_machines.xml create mode 100644 machines/pleiades-has/config_machines.xml create mode 100644 machines/pleiades-ivy/config_machines.xml create mode 100644 machines/pleiades-san/config_machines.xml create mode 100644 machines/sandia-srn-sems/config_machines.xml create mode 100644 machines/sandiatoss3/config_machines.xml create mode 100644 machines/stampede2-knl/config_machines.xml create mode 100644 machines/stampede2-skx/config_machines.xml create mode 100644 machines/theia/config_machines.xml create mode 100644 machines/theta/config_machines.xml create mode 100644 machines/thunder/config_machines.xml create mode 100644 machines/ubuntu-latest/config_machines.xml create mode 100644 machines/zeus/config_machines.xml diff --git a/machines/aleph/config_machines.xml b/machines/aleph/config_machines.xml new file mode 100644 index 00000000..9d2d2daf --- /dev/null +++ b/machines/aleph/config_machines.xml @@ -0,0 +1,64 @@ + + XC50 SkyLake, os is CNL, 40 pes/node, batch system is PBSPro + CNL + intel,gnu,cray + mpt,mpi-serial + /proj/$ENV{USER} + $ENV{DIN_LOC_ROOT} + $DIN_LOC_ROOT + ${CIME_OUTPUT_ROOT}/archive/$CASE + ${CIME_OUTPUT_ROOT}/cesm_baselines + 8 + pbs + @ pusan.ac.kr + 40 + 40 + + aprun + + -j {{ hyperthreading }} + -n {{ total_tasks }} + -N $MAX_MPITASKS_PER_NODE + -S {{ tasks_per_numa }} + -d $ENV{OMP_NUM_THREADS} + --mpmd-env OMP_NUM_THREADS=$OMP_NUM_THREADS + + + + /opt/modules/default/init/perl.pm + /opt/modules/default/init/python.py + /opt/modules/default/init/sh + /opt/modules/default/init/csh + /opt/modules/default/bin/modulecmd perl + /opt/modules/default/bin/modulecmd python + module + module + + craype-x86-skylake + PrgEnv-pgi + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + cray-netcdf + cray-hdf5 + cray-parallel-netcdf + papi + + + PrgEnv-intel + craype-x86-skylake + craype-hugepages2M + perftools-base/7.0.4 + cray-netcdf/4.6.1.3 + cray-hdf5/1.10.2.0 + cray-parallel-netcdf/1.11.1.1 + papi/5.6.0.4 + gridftp/6.0 + cray-python/3.6.5.1 + + + + 256M + /home/jedwards/workflow/CESM_postprocessing + + diff --git a/machines/athena/config_machines.xml b/machines/athena/config_machines.xml new file mode 100644 index 00000000..651172a0 --- /dev/null +++ b/machines/athena/config_machines.xml @@ -0,0 +1,85 @@ + + + CMCC IBM iDataPlex, os is Linux, 16 pes/node, batch system is LSFd mpich + .*.cluster.net + LINUX + intel,intel15 + mpich2 + /work/$USER/CESM2 + /users/home/dp16116/CESM2/inputdata + $DIN_LOC_ROOT/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/ccsm_baselines + /users/home/dp16116/CESM2/cesm2.0.1/cime/tools/cprnc/cprnc + /usr/lib64/perl5:/usr/share/perl5 + 8 + lsf + + 30 + 15 + FALSE + + mpirun_Impi5 + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + /usr/share/Modules/init/csh + /usr/share/Modules/init/sh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + + + ANACONDA2/python2.7 + INTEL/intel_xe_2015.3.187 + SZIP/szip-2.1_int15 + + + ESMF/esmf-6.3.0rp1-intelmpi-64-g_int15 + + + ESMF/esmf-6.3.0rp1-intelmpi-64-O_int15 + + + ESMF/esmf-6.3.0rp1-mpiuni-64-g_int15 + + + ESMF/esmf-6.3.0rp1-mpiuni-64-O_int15 + + + HDF5/hdf5-1.8.15-patch1 + NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1 + + + HDF5/hdf5-1.8.15-patch1_parallel + NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1_parallel + PARALLEL_NETCDF/parallel-netcdf-1.6.1 + + + CMAKE/cmake-3.3.0-rc1 + + + INTEL/intel_xe_2013.5.192 + INTEL/intel_xe_2013 + HDF5/hdf5-1.8.10-patch1 + INTEL/intel_xe_2015.3.187 + + + + 256M + + + gpfs + on + snb + lsf + 1 + on + on + /users/home/models/nemo/xios-cmip6/intel_xe_2013 + + diff --git a/machines/aws-hpc6a/config_machines.xml b/machines/aws-hpc6a/config_machines.xml new file mode 100644 index 00000000..faa6f097 --- /dev/null +++ b/machines/aws-hpc6a/config_machines.xml @@ -0,0 +1,50 @@ + + AWS HPC6a (96-core AMD) Nodes + regular-dy-computehpc6a-.* + LINUX + intel + impi + /scratch/$USER + /scratch/inputdata + $DIN_LOC_ROOT/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc + 8 + slurm + cseg + + 96 + 96 + FALSE + + scontrol show hostnames $SLURM_JOB_NODELIST > hostfile ; mpirun -f hostfile + + -n {{ total_tasks }} + + + + env -u I_MPI_OFI_PROVIDER mpirun + + -n {{ total_tasks }} + + + + + 256M + /scratch/$USER + icc + icpc + ifort + ifort + /opt/ncar/software + /opt/ncar/esmf/lib/esmf.mk + + + ON + SUMMARY + + + -1 + + diff --git a/machines/casper/config_machines.xml b/machines/casper/config_machines.xml new file mode 100644 index 00000000..276d69d7 --- /dev/null +++ b/machines/casper/config_machines.xml @@ -0,0 +1,127 @@ + + NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs + casper* + LINUX + nvhpc,intel + openmpi + /glade/scratch/$USER + $ENV{CESMDATAROOT}/inputdata + /glade/p/cgd/tss/CTSM_datm_forcing_data + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc + 8 + pbs + ASAP/CISL + 36 + 8 + 36 + 36 + none,v100 + none,openacc,openmp,combined + TRUE + + mpirun + + -np {{ total_tasks }} + + + + mpirun + + -np {{ total_tasks }} + + + + /glade/u/apps/dav/opt/lmod/7.7.29/init/perl + /glade/u/apps/dav/opt/lmod/7.7.29/init/env_modules_python.py + /glade/u/apps/dav/opt/lmod/7.7.29/init/sh + /glade/u/apps/dav/opt/lmod/7.7.29/init/csh + /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod perl + /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod python + module + module + + + ncarenv/1.3 + cmake/3.18.2 + + + nvhpc/22.2 + + + intel/19.1.1 + mkl/2020.0.1 + + + cuda/11.6 + + + openmpi/4.1.4 + netcdf-mpi/4.8.1 + pnetcdf/1.12.3 + + + netcdf/4.8.1 + + + openmpi/4.1.1 + netcdf-mpi/4.8.1 + pnetcdf/1.12.2 + + + netcdf/4.7.4 + + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.4.0b08_casper-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.4.0b08_casper-ncdfio-openmpi-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ + esmf-8.4.1b01-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ + esmf-8.4.1b01-ncdfio-openmpi-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.4.0b08_casper-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b11_casper-ncdfio-openmpi-O + + + ncarcompilers/0.5.0 + + + pio/2.5.10 + + + pio/2.5.10d + + + + /glade/u/apps/dav/modulefiles/default/compilers:/glade/u/apps/dav/modulefiles/default/idep + 256M + /glade/scratch/$USER + /glade/p/cesmdata/cseg + $ENV{NETCDF} + + + ON + SUMMARY + + + openacc + + + -1 + + diff --git a/machines/centos7-linux/config_machines.xml b/machines/centos7-linux/config_machines.xml new file mode 100644 index 00000000..3579632d --- /dev/null +++ b/machines/centos7-linux/config_machines.xml @@ -0,0 +1,59 @@ + + + Example port to centos7 linux system with gcc, netcdf, pnetcdf and mpich + using modules from http://www.admin-magazine.com/HPC/Articles/Environment-Modules + + regex.expression.matching.your.machine + LINUX + https://howto.get.out + gnu + mpich + none + + $ENV{HOME}/cesm/scratch + $ENV{HOME}/cesm/inputdata + $ENV{HOME}/cesm/inputdata/lmwg + $ENV{HOME}/cesm/archive/$CASE + $ENV{HOME}/cesm/cesm_baselines + $ENV{HOME}/cesm/tools/cime/tools/cprnc/cprnc + make + 8 + none + me@my.address + 8 + 8 + FALSE + + mpiexec + + -np {{ total_tasks }} + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + /usr/share/Modules/init/csh + /usr/share/Modules/init/sh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + lang/python/3.7.0 + + + compiler/gnu/8.2.0 + mpi/3.3/gnu-8.2.0 + tool/netcdf/4.7.4/gcc-8.2.0 + tool/parallel-netcdf/1.12.1/mpich + + + + 256M + $ENV{HOME}/ESMF_8_2_0b22/lib/libg/Linux.gfortran.64.mvapich2.default/esmf.mk + + + -1 + + diff --git a/machines/cheyenne/config_machines.xml b/machines/cheyenne/config_machines.xml new file mode 100644 index 00000000..5a17247b --- /dev/null +++ b/machines/cheyenne/config_machines.xml @@ -0,0 +1,288 @@ + + NCAR SGI platform, os is Linux, 36 pes/node, batch system is PBS + .*.?cheyenne\d?.ucar.edu + + MPT: Launcher network accept (MPI_LAUNCH_TIMEOUT) timed out + 10 + LINUX + intel,gnu,nvhpc,pgi + mpt,openmpi + openmpi,mpt + openmpi,mpt + mpt,openmpi + /glade/scratch/$USER + $ENV{CESMDATAROOT}/inputdata + /glade/p/cgd/tss/CTSM_datm_forcing_data + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc + 8 + pbs + cseg + + 36 + 36 + TRUE + + + mpiexec_mpt + + -p "%g:" + -np {{ total_tasks }} + + omplace -tm open64 + + + + mpirun `hostname` + + -np {{ total_tasks }} + + omplace -tm open64 + + + + mpiexec_mpt + + -p "%g:" + -np {{ total_tasks }} + + omplace -tm open64 -vv + + + + mpirun `hostname` + + -np {{ total_tasks }} + + + + mpirun + + -np {{ total_tasks }} + --tag-output + + + + + /opt/sgi/mpt/mpt-2.15/bin/mpirun $ENV{UNIT_TEST_HOST} -np 1 + + + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/perl + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/env_modules_python.py + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/csh + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/sh + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/libexec/lmod perl + /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/libexec/lmod python + module + module + + + ncarenv/1.3 + python/3.7.9 + cmake/3.22.0 + + + intel/19.1.1 + esmf_libs + mkl + + + gnu/10.1.0 + openblas/0.3.9 + + + pgi/20.4 + + + nvhpc/22.2 + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-mpt-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-mpt-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-openmpi-O + + + mpi-serial/2.3.0 + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-mpiuni-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ + esmf-8.5.0-ncdfio-mpiuni-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-mpt-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-mpt-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-openmpi-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-mpiuni-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ + esmf-8.5.0-ncdfio-mpiuni-O + + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-mpt-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-mpt-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-mpt-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-mpt-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-openmpi-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-openmpi-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-openmpi-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-mpiuni-g + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ + esmf-8.2.0b23-ncdfio-mpiuni-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-mpiuni-O + + + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 + esmf-8.5.0-ncdfio-mpiuni-O + + + mpt/2.25 + netcdf-mpi/4.9.0 + pnetcdf/1.12.3 + + + mpt/2.22 + netcdf-mpi/4.8.1 + pnetcdf/1.12.2 + + + mpt/2.22 + netcdf-mpi/4.7.4 + pnetcdf/1.12.1 + + + mpt/2.25 + netcdf-mpi/4.9.0 + pnetcdf/1.12.3 + + + openmpi/4.1.4 + netcdf-mpi/4.9.0 + pnetcdf/1.12.3 + + + openmpi/4.1.4 + netcdf-mpi/4.9.0 + pnetcdf/1.12.3 + + + openmpi/4.0.5 + netcdf-mpi/4.7.4 + + + openmpi/4.1.4 + netcdf-mpi/4.9.0 + pnetcdf/1.12.3 + + + ncarcompilers/0.5.0 + + + netcdf/4.9.0 + + + netcdf/4.9.0 + + + netcdf/4.9.0 + + + netcdf/4.9.0 + + + pio/2.5.10 + + + pio/2.5.10d + + + + 1024M + /glade/scratch/$USER + 16 + + + + ON + SUMMARY + /glade/work/turuncu/FV3GFS/benchmark-inputs/2012010100/gfs/fcst + /glade/work/turuncu/FV3GFS/fix_am + /glade/work/turuncu/FV3GFS/addon + PASSIVE + true + + + false + + + /glade/scratch/$USER + + + -1 + + diff --git a/machines/coeus/config_machines.xml b/machines/coeus/config_machines.xml new file mode 100644 index 00000000..7e2cb629 --- /dev/null +++ b/machines/coeus/config_machines.xml @@ -0,0 +1,57 @@ + + + Portland State University Coeus Cluster Dec 2019 CentOS 7 + + (login[1,2].cluster|compute[0-9]*.cluster) + LINUX + gnu + mvapich2 + none + $ENV{CESMDATAROOT}/$USER + $ENV{CESMDATAROOT}/inputdata + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + /vol/apps/hpc/src/cesm-2.1.0/cime/tools/cprnc/cprnc + make + 8 + slurm + oit-rc-groups@pdx.edu + 40 + 20 + FALSE + + srun + + --ntasks={{ total_tasks }} + --cpu_bind=sockets --cpu_bind=verbose + --kill-on-bad-exit + + + + + + /usr/share/Modules/init/csh + /usr/share/Modules/init/sh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + + + gcc-6.3.0 + mvapich2-2.2-psm/gcc-6.3.0 + General/netcdf/4.4.1.1/gcc-6.3.0 + Python/2.7.13/gcc-6.3.0 + + + + 256M + /vol/apps/hpc/stow/netcdf/4.4.1.1/gcc-6.3.0/ + + + -1 + + + diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 71730466..2c0a8479 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -48,3686 +48,35 @@ This allows using a different mpirun command to launch unit tests --> - - - XC50 SkyLake, os is CNL, 40 pes/node, batch system is PBSPro - .*eth\d - CNL - intel,gnu,cray - mpt,mpi-serial - /proj/$ENV{USER} - $ENV{DIN_LOC_ROOT} - $DIN_LOC_ROOT - ${CIME_OUTPUT_ROOT}/archive/$CASE - ${CIME_OUTPUT_ROOT}/cesm_baselines - 8 - pbs - @ pusan.ac.kr - 40 - 40 - - aprun - - -j {{ hyperthreading }} - -n {{ total_tasks }} - -N $MAX_MPITASKS_PER_NODE - -S {{ tasks_per_numa }} - -d $ENV{OMP_NUM_THREADS} - --mpmd-env OMP_NUM_THREADS=$OMP_NUM_THREADS - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - craype-x86-skylake - PrgEnv-pgi - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - cray-netcdf - cray-hdf5 - cray-parallel-netcdf - papi - - - PrgEnv-intel - craype-x86-skylake - craype-hugepages2M - perftools-base/7.0.4 - cray-netcdf/4.6.1.3 - cray-hdf5/1.10.2.0 - cray-parallel-netcdf/1.11.1.1 - papi/5.6.0.4 - gridftp/6.0 - cray-python/3.6.5.1 - - - - 256M - /home/jedwards/workflow/CESM_postprocessing - - + + + .*eth\d + .*.cluster.net + regular-dy-computehpc6a-.* + $ENV{NCAR_HOST}:casper + .*.?cheyenne\d?.ucar.edu + (login[1,2].cluster|compute[0-9]*.cluster) + $ENV{NCAR_HOST}:derecho + .*frontera + gplogin\d.gp.local + gu.*.hpc.ucar.edu + ^i.*\.ucar\.edu + lobata + .*ls5\.tacc\.utexas\.edu + (melvin|watson) + $ENV{NERSC_HOST}:perlmutter + (s999964|climate|penn) + (skybridge|chama)-login + .*stampede2 + swan.* + tfe + theta.* + .*.thunder.ucar.edu + $ENV{CIME_TEST_PLATFORM}:ubuntu-latest + (login[1,2]-ib|n[0-9][0-9][0-9]-ib) + - - CMCC IBM iDataPlex, os is Linux, 16 pes/node, batch system is LSFd mpich - .*.cluster.net - LINUX - intel,intel15 - mpich2 - /work/$USER/CESM2 - /users/home/dp16116/CESM2/inputdata - $DIN_LOC_ROOT/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/ccsm_baselines - /users/home/dp16116/CESM2/cesm2.0.1/cime/tools/cprnc/cprnc - /usr/lib64/perl5:/usr/share/perl5 - 8 - lsf - - 30 - 15 - FALSE - - mpirun_Impi5 - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - - - ANACONDA2/python2.7 - INTEL/intel_xe_2015.3.187 - SZIP/szip-2.1_int15 - - - ESMF/esmf-6.3.0rp1-intelmpi-64-g_int15 - - - ESMF/esmf-6.3.0rp1-intelmpi-64-O_int15 - - - ESMF/esmf-6.3.0rp1-mpiuni-64-g_int15 - - - ESMF/esmf-6.3.0rp1-mpiuni-64-O_int15 - - - HDF5/hdf5-1.8.15-patch1 - NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1 - - - HDF5/hdf5-1.8.15-patch1_parallel - NETCDF/netcdf-C_4.3.3.1-F_4.4.2_C++_4.2.1_parallel - PARALLEL_NETCDF/parallel-netcdf-1.6.1 - - - CMAKE/cmake-3.3.0-rc1 - - - INTEL/intel_xe_2013.5.192 - INTEL/intel_xe_2013 - HDF5/hdf5-1.8.10-patch1 - INTEL/intel_xe_2015.3.187 - - - - 256M - - - gpfs - on - snb - lsf - 1 - on - on - /users/home/models/nemo/xios-cmip6/intel_xe_2013 - - - - AWS HPC6a (96-core AMD) Nodes - regular-dy-computehpc6a-.* - LINUX - intel - impi - /scratch/$USER - /scratch/inputdata - $DIN_LOC_ROOT/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc - 8 - slurm - cseg - - 96 - 96 - FALSE - - scontrol show hostnames $SLURM_JOB_NODELIST > hostfile ; mpirun -f hostfile - - -n {{ total_tasks }} - - - - env -u I_MPI_OFI_PROVIDER mpirun - - -n {{ total_tasks }} - - - - - 256M - /scratch/$USER - icc - icpc - ifort - ifort - /opt/ncar/software - /opt/ncar/esmf/lib/esmf.mk - - - ON - SUMMARY - - - -1 - - - - - ORNL XE6, os is CNL, 32 pes/node, batch system is PBS - h2o - CNL - intel,pgi,cray,gnu - mpich - banu - /scratch/sciteam/$USER - $ENV{CESMDATAROOT}/inputdata - $ENV{CESMDATAROOT}/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/ccsm_baselines - $ENV{CESMDATAROOT}/tools/cprnc - 8 - pbs - cseg - 32 - 16 - TRUE - - aprun - - -n {{ total_tasks }} - - -N $MAX_MPITASKS_PER_NODE - -d $ENV{OMP_NUM_THREADS} - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/3.2.10.3/bin/modulecmd perl - /opt/modules/3.2.10.3/bin/modulecmd python - module - module - - PrgEnv-pgi - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - pgi - cray - intel - cray-netcdf - gcc - - - PrgEnv-intel - intel - intel/18.0.3.222 - - gcc - - - PrgEnv-pgi - pgi pgi/18.7.0 - - - PrgEnv-gnu - gcc gcc/6.3.0 - - - PrgEnv-cray - cce cce/8.5.8 - - - papi/5.5.1.1 - cray-mpich cray-mpich/7.7.1 - cray-libsci cray-libsci/18.04.1 - torque/6.0.4 - - - cray-hdf5-parallel/1.10.2.0 - cray-netcdf-hdf5parallel/4.6.1.0 - cray-parallel-netcdf/1.8.1.3 - - - cray-netcdf/4.6.1.0 - - - cmake/3.1.3 - darshan - /sw/modulefiles/CESM - CESM-ENV - - - - 64M - $ENV{HOME}/bin:$ENV{PATH} - - - - - - Example port to centos7 linux system with gcc, netcdf, pnetcdf and mpich - using modules from http://www.admin-magazine.com/HPC/Articles/Environment-Modules - - regex.expression.matching.your.machine - LINUX - https://howto.get.out - gnu - mpich - none - - $ENV{HOME}/cesm/scratch - $ENV{HOME}/cesm/inputdata - $ENV{HOME}/cesm/inputdata/lmwg - $ENV{HOME}/cesm/archive/$CASE - $ENV{HOME}/cesm/cesm_baselines - $ENV{HOME}/cesm/tools/cime/tools/cprnc/cprnc - make - 8 - none - me@my.address - 8 - 8 - FALSE - - mpiexec - - -np {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - lang/python/3.7.0 - - - compiler/gnu/8.2.0 - mpi/3.3/gnu-8.2.0 - tool/netcdf/4.7.4/gcc-8.2.0 - tool/parallel-netcdf/1.12.1/mpich - - - - 256M - $ENV{HOME}/ESMF_8_2_0b22/lib/libg/Linux.gfortran.64.mvapich2.default/esmf.mk - - - -1 - - - - - NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs - casper* - LINUX - nvhpc,intel - openmpi - /glade/scratch/$USER - $ENV{CESMDATAROOT}/inputdata - /glade/p/cgd/tss/CTSM_datm_forcing_data - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc - 8 - pbs - ASAP/CISL - 36 - 8 - 36 - 36 - none,v100 - none,openacc,openmp,combined - TRUE - - mpirun - - -np {{ total_tasks }} - - - - mpirun - - -np {{ total_tasks }} - - - - /glade/u/apps/dav/opt/lmod/7.7.29/init/perl - /glade/u/apps/dav/opt/lmod/7.7.29/init/env_modules_python.py - /glade/u/apps/dav/opt/lmod/7.7.29/init/sh - /glade/u/apps/dav/opt/lmod/7.7.29/init/csh - /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod perl - /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod python - module - module - - - ncarenv/1.3 - cmake/3.18.2 - - - nvhpc/22.2 - - - intel/19.1.1 - mkl/2020.0.1 - - - cuda/11.6 - - - openmpi/4.1.4 - netcdf-mpi/4.8.1 - pnetcdf/1.12.3 - - - netcdf/4.8.1 - - - openmpi/4.1.1 - netcdf-mpi/4.8.1 - pnetcdf/1.12.2 - - - netcdf/4.7.4 - - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.4.0b08_casper-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.4.0b08_casper-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1b01-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ - esmf-8.4.1b01-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.4.0b08_casper-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b11_casper-ncdfio-openmpi-O - - - ncarcompilers/0.5.0 - - - pio/2.5.10 - - - pio/2.5.10d - - - - /glade/u/apps/dav/modulefiles/default/compilers:/glade/u/apps/dav/modulefiles/default/idep - 256M - /glade/scratch/$USER - /glade/p/cesmdata/cseg - $ENV{NETCDF} - - - ON - SUMMARY - - - openacc - - - -1 - - - - - NCAR SGI platform, os is Linux, 36 pes/node, batch system is PBS - .*.?cheyenne\d?.ucar.edu - - MPT: Launcher network accept (MPI_LAUNCH_TIMEOUT) timed out - 10 - LINUX - intel,gnu,nvhpc,pgi - mpt,openmpi - openmpi,mpt - openmpi,mpt - mpt,openmpi - /glade/scratch/$USER - $ENV{CESMDATAROOT}/inputdata - /glade/p/cgd/tss/CTSM_datm_forcing_data - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc - 8 - pbs - cseg - - 36 - 36 - TRUE - - - mpiexec_mpt - - -p "%g:" - -np {{ total_tasks }} - - omplace -tm open64 - - - - mpirun `hostname` - - -np {{ total_tasks }} - - omplace -tm open64 - - - - mpiexec_mpt - - -p "%g:" - -np {{ total_tasks }} - - omplace -tm open64 -vv - - - - mpirun `hostname` - - -np {{ total_tasks }} - - - - mpirun - - -np {{ total_tasks }} - --tag-output - - - - - /opt/sgi/mpt/mpt-2.15/bin/mpirun $ENV{UNIT_TEST_HOST} -np 1 - - - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/perl - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/env_modules_python.py - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/csh - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/init/sh - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/libexec/lmod perl - /glade/u/apps/ch/opt/lmod/7.5.3/lmod/lmod/libexec/lmod python - module - module - - - ncarenv/1.3 - python/3.7.9 - cmake/3.22.0 - - - intel/19.1.1 - esmf_libs - mkl - - - gnu/10.1.0 - openblas/0.3.9 - - - pgi/20.4 - - - nvhpc/22.2 - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-mpt-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-mpt-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-openmpi-O - - - mpi-serial/2.3.0 - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-mpiuni-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ - esmf-8.5.0-ncdfio-mpiuni-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-mpt-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-mpt-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-mpiuni-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/gnu/10.1.0/ - esmf-8.5.0-ncdfio-mpiuni-O - - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-mpt-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-mpt-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-mpt-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-mpt-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-openmpi-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-openmpi-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-mpiuni-g - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/ - esmf-8.2.0b23-ncdfio-mpiuni-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-mpiuni-O - - - /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2 - esmf-8.5.0-ncdfio-mpiuni-O - - - mpt/2.25 - netcdf-mpi/4.9.0 - pnetcdf/1.12.3 - - - mpt/2.22 - netcdf-mpi/4.8.1 - pnetcdf/1.12.2 - - - mpt/2.22 - netcdf-mpi/4.7.4 - pnetcdf/1.12.1 - - - mpt/2.25 - netcdf-mpi/4.9.0 - pnetcdf/1.12.3 - - - openmpi/4.1.4 - netcdf-mpi/4.9.0 - pnetcdf/1.12.3 - - - openmpi/4.1.4 - netcdf-mpi/4.9.0 - pnetcdf/1.12.3 - - - openmpi/4.0.5 - netcdf-mpi/4.7.4 - - - openmpi/4.1.4 - netcdf-mpi/4.9.0 - pnetcdf/1.12.3 - - - ncarcompilers/0.5.0 - - - netcdf/4.9.0 - - - netcdf/4.9.0 - - - netcdf/4.9.0 - - - netcdf/4.9.0 - - - pio/2.5.10 - - - pio/2.5.10d - - - - 1024M - /glade/scratch/$USER - 16 - - - - ON - SUMMARY - /glade/work/turuncu/FV3GFS/benchmark-inputs/2012010100/gfs/fcst - /glade/work/turuncu/FV3GFS/fix_am - /glade/work/turuncu/FV3GFS/addon - PASSIVE - true - - - false - - - /glade/scratch/$USER - - - -1 - - - - - - Portland State University Coeus Cluster Dec 2019 CentOS 7 - - (login[1,2].cluster|compute[0-9]*.cluster) - LINUX - gnu - mvapich2 - none - $ENV{CESMDATAROOT}/$USER - $ENV{CESMDATAROOT}/inputdata - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - /vol/apps/hpc/src/cesm-2.1.0/cime/tools/cprnc/cprnc - make - 8 - slurm - oit-rc-groups@pdx.edu - 40 - 20 - FALSE - - srun - - --ntasks={{ total_tasks }} - --cpu_bind=sockets --cpu_bind=verbose - --kill-on-bad-exit - - - - - - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - - - gcc-6.3.0 - mvapich2-2.2-psm/gcc-6.3.0 - General/netcdf/4.4.1.1/gcc-6.3.0 - Python/2.7.13/gcc-6.3.0 - - - - 256M - /vol/apps/hpc/stow/netcdf/4.4.1.1/gcc-6.3.0/ - - - -1 - - - - - PNL Haswell cluster, OS is Linux, batch system is SLURM - LINUX - intel,pgi - mvapich2,openmpi,intelmpi,mvapich - /pic/scratch/$USER - /pic/scratch/tcraig/IRESM/inputdata - /pic/scratch/tcraig/IRESM/inputdata/atm/datm7 - /pic/scratch/$USER/cases/archive/$CASE - /pic/scratch/tcraig/IRESM/ccsm_baselines - /people/tcraig/bin/cprnc - 8 - slurm - tcraig -at- ucar.edu - 24 - 24 - FALSE - - srun - - --mpi=none - --ntasks={{ total_tasks }} - --cpu_bind=sockets --cpu_bind=verbose - --kill-on-bad-exit - - - - srun - - --ntasks={{ total_tasks }} - --cpu_bind=sockets --cpu_bind=verbose - --kill-on-bad-exit - - - - mpirun - - -n {{ total_tasks }} - - - - mpirun - - -n {{ total_tasks }} - - - - /share/apps/modules/Modules/3.2.10/init/perl.pm - /etc/profile.d/modules.csh - /etc/profile.d/modules.sh - /share/apps/modules/Modules/3.2.10/bin/modulecmd perl - module - module - - - - - perl/5.20.0 - cmake/2.8.12 - - - intel/15.0.1 - netcdf/4.3.2 - mkl/15.0.1 - - - pgi/14.10 - netcdf/4.3.2 - - - mvapich2/2.1 - - - mvapich2/2.1 - - - intelmpi/5.0.1.035 - - - openmpi/1.8.3 - - - - 64M - - - $MLIB_LIB - /share/apps/netcdf/4.3.2/intel/15.0.1 - - - /share/apps/netcdf/4.3.2/pgi/14.10 - - - - - - Containerized development environment (Docker/Singularity) for CESM w/ GNU compilers - - LINUX - gnu - mpich - $ENV{HOME}/scratch - $ENV{CESMDATAROOT}/inputdata - $DIN_LOC_ROOT/atm/datm7 - $ENV{HOME}/archive/$CASE - make - 4 - none - cgd - 4 - 4 - FALSE - - mpiexec - - -n {{ total_tasks }} - - - - - - /usr/local - /usr/local - /usr/lib64 - /usr/lib64 - - - -1 - - - - - - - - NERSC XC40 Haswell, os is CNL, 32 pes/node, batch system is Slurm - cori - CNL - intel,gnu,cray - mpt - $ENV{SCRATCH} - /project/projectdirs/ccsm1/inputdata - /project/projectdirs/ccsm1/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /project/projectdirs/ccsm1/ccsm_baselines - /project/projectdirs/ccsm1/tools/cprnc.corip1/cprnc - 8 - slurm - cseg - 64 - 32 - - srun - - --label - -n {{ total_tasks }} - -c {{ srun_binding }} - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - intel - cce - cray-parallel-netcdf - cray-parallel-hdf5 - pmi - cray-libsci - cray-mpich2 - cray-mpich - cray-netcdf - cray-hdf5 - cray-netcdf-hdf5parallel - craype-sandybridge - craype-ivybridge - craype - - - - PrgEnv-intel - intel intel/19.1.3.304 - /global/project/projectdirs/ccsm1/modulefiles/cori - - - esmf/7.1.0r-defio-intel18.0.1.163-mpi-O-cori-haswell - - - esmf/7.1.0r-netcdf-intel18.0.1.163-mpiuni-O-haswell - - - - PrgEnv-cray - cce cce/10.0.3 - - - PrgEnv-gnu - gcc gcc/10.1.0 - - - cray-memkind - craype craype/2.7.2 - - - cray-libsci/20.09.1 - - - cray-mpich/7.7.16 - - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - cray-hdf5/1.12.0.0 - cray-netcdf/4.7.4.0 - - - cray-netcdf-hdf5parallel/4.7.4.0 - cray-hdf5-parallel/1.12.0.0 - cray-parallel-netcdf/1.12.1.0 - - - cmake/3.21.3 - - - - 256M - spread - threads - - - - - - - NERSC XC* KNL, os is CNL, 68 pes/node, batch system is Slurm - CNL - intel,gnu,cray - mpt - $ENV{SCRATCH} - /project/projectdirs/ccsm1/inputdata - /project/projectdirs/ccsm1/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /project/projectdirs/ccsm1/ccsm_baselines - /project/projectdirs/ccsm1/tools/cprnc.corip1/cprnc - 8 - slurm - cseg - 256 - 64 - 68 - - srun - - --label - -n {{ total_tasks }} - -c {{ srun_binding }} --cpu_bind=cores - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - craype-mic-knl - craype-haswell - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - intel - cce - cray-parallel-netcdf - cray-parallel-hdf5 - pmi - cray-libsci - cray-mpich2 - cray-mpich - cray-netcdf - cray-hdf5 - cray-netcdf-hdf5parallel - - - - PrgEnv-intel - intel intel/19.1.3.304 - /global/project/projectdirs/ccsm1/modulefiles/cori - - - esmf/7.1.0r-defio-intel18.0.1.163-mpi-O-cori-knl - - - esmf/7.1.0r-netcdf-intel18.0.1.163-mpiuni-O-knl - - - - PrgEnv-cray - cce cce/10.0.3 - - - PrgEnv-gnu - gcc gcc/10.1.0 - - - cray-memkind - craype craype/2.7.2 - craype-mic-knl - - - cray-libsci/20.09.1 - - - cray-mpich/7.7.16 - - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - cray-hdf5/1.12.0.0 - cray-netcdf/4.7.4.0 - - - cray-netcdf-hdf5parallel/4.7.4.0 - cray-hdf5-parallel/1.12.0.0 - cray-parallel-netcdf/1.12.1.0 - - - - 256M - spread - threads - - - - - CSCS Cray XC50, os is SUSE SLES, 12 pes/node, batch system is SLURM - CNL - pgi,cray,gnu - mpich - /scratch/snx3000/$USER - /project/s824/cesm_inputdata - /project/s824/cesm_inputdata/atm/datm7 - /project/s824/$USER/archive/$CASE - /project/s824/ccsm_baselines - /project/s824/cesm_tools/ccsm_cprnc/cprnc - 12 - slurm - edouard.davin -at- env.ethz.ch - 12 - 12 - - srun - - -n {{ total_tasks }} - -d $ENV{OMP_NUM_THREADS} - - - - - 64M - - - - - NCAR AMD EPYC system - de.*.hpc.ucar.edu - CNL - intel,gnu,nvhpc,intel-oneapi,intel-classic - mpich - $ENV{SCRATCH} - $ENV{CESMDATAROOT}/inputdata - $ENV{CESMDATAROOT}/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/cprnc/cprnc - 16 - pbs - cseg - 128 - 4 - 128 - 64 - none,a100 - none,openacc,openmp,combined - get_local_rank - TRUE - - mpiexec - - --label - --line-buffer - -n {{ total_tasks }} - - - - $ENV{LMOD_ROOT}/lmod/init/perl - $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py - $ENV{LMOD_ROOT}/lmod/init/sh - $ENV{LMOD_ROOT}/lmod/init/csh - $ENV{LMOD_ROOT}/lmod/libexec/lmod perl - $ENV{LMOD_ROOT}/lmod/libexec/lmod python - module - module - - cesmdev/1.0 - ncarenv/23.09 - - craype - - - intel/2023.2.1 - mkl - - - intel-oneapi/2023.2.1 - mkl - - - intel-classic/2023.2.1 - mkl - - - cce/15.0.1 - cray-libsci/23.02.1.1 - - - gcc/12.2.0 - cray-libsci/23.02.1.1 - - - nvhpc/23.7 - - - - ncarcompilers/1.0.0 - cmake - - - cray-mpich/8.1.27 - - - mpi-serial/2.3.0 - - - - cuda/12.2.1 - - - - netcdf/4.9.2 - - - - netcdf-mpi/4.9.2 - parallel-netcdf/1.12.3 - - - - parallelio/2.6.2-debug - esmf/8.6.0b04-debug - - - - parallelio/2.6.2 - esmf/8.6.0b04 - - - - - 64M - hybrid - - memhooks - - - - - - ON - SUMMARY - - - openacc - - - - - PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM - LINUX - pgi,intel - mvapich2,mvapich - /lustre/$USER - /lustre/tcraig/IRESM/inputdata - /lustre/tcraig/IRESM/inputdata/atm/datm7 - /lustre/$USER/archive/$CASE - /lustre/tcraig/IRESM/ccsm_baselines - /lustre/tcraig/IRESM/tools/cprnc/cprnc - 8 - slurm - tcraig -at- ucar.edu - 12 - 12 - - srun - - --ntasks={{ total_tasks }} - --cpu_bind=sockets - --cpu_bind=verbose - --kill-on-bad-exit - - - - srun - - --mpi=none - --ntasks={{ total_tasks }} - --cpu_bind=sockets - --cpu_bind=verbose - --kill-on-bad-exit - - - - /etc/profile.d/modules.perl - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /share/apps/modules/Modules/3.2.7/bin/modulecmd perl - module - module - - - perl/5.20.7 - cmake/3.0.0 - pgi/15.5 - mpi/mvapich2/1.5.1p1/pgi11.3 - netcdf/4.1.2/pgi - - - - 64M - - - - - Euler II Linux Cluster ETH, 24 pes/node, InfiniBand, XeonE5_2680v3, batch system LSF - LINUX - intel,pgi - openmpi,mpich - /cluster/work/climate/$USER - /cluster/work/climate/cesm/inputdata - /cluster/work/climate/cesm/inputdata/atm/datm7 - /cluster/work/climate/$USER/archive/$CASE - /cluster/work/climate/cesm/ccsm_baselines - /cluster/work/climate/cesm/tools/cprnc/cprnc - 1 - lsf - urs.beyerle -at- env.ethz.ch - 24 - 24 - - mpirun - - -hostfile $ENV{PBS_JOBID} - -ppn $MAX_MPITASKS_PER_NODE - -n {{ total_tasks }} - - - - mpirun - - - - - /cluster/apps/modules/init/python.py - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /cluster/apps/modules/bin/modulecmd python - module - module - - - - - new - - - intel/2018.1 - - - netcdf/4.3.1 - - - pgi/14.1 - - - mvapich2/1.8.1 - - - open_mpi/1.6.5 - - - - 64M - - - - - Euler III Linux Cluster ETH, 4 pes/node, Ethernet, XeonE3_1585Lv5, batch system LSF - LINUX - intel,pgi - openmpi,mpich - /cluster/work/climate/$USER - /cluster/work/climate/cesm/inputdata - /cluster/work/climate/cesm/inputdata/atm/datm7 - /cluster/work/climate/$USER/archive/$CASE - /cluster/work/climate/cesm/ccsm_baselines - /cluster/work/climate/cesm/tools/cprnc/cprnc - 1 - lsf - urs.beyerle -at- env.ethz.ch - 4 - 4 - - mpirun - - -hostfile $ENV{PBS_JOBID} - -ppn $MAX_MPITASKS_PER_NODE - -n {{ total_tasks }} - - - - mpirun - - - - - /cluster/apps/modules/init/python.py - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /cluster/apps/modules/bin/modulecmd python - module - module - - - - - new - - - interconnect/ethernet - - - intel/2018.1 - - - netcdf/4.3.1 - - - pgi/14.1 - - - mvapich2/1.8.1 - - - open_mpi/1.6.5 - - - - 64M - - - - - Euler IV Linux Cluster ETH, 36 pes/node, InfiniBand, XeonGold_6150, batch system LSF - LINUX - intel,pgi - openmpi,mpich - /cluster/work/climate/$USER - /cluster/work/climate/cesm/inputdata - /cluster/work/climate/cesm/inputdata/atm/datm7 - /cluster/work/climate/$USER/archive/$CASE - /cluster/work/climate/cesm/ccsm_baselines - /cluster/work/climate/cesm/tools/cprnc/cprnc - 1 - lsf - urs.beyerle -at- env.ethz.ch - 36 - 36 - - mpirun - - -hostfile $ENV{PBS_JOBID} - -ppn $MAX_MPITASKS_PER_NODE - -n {{ total_tasks }} - - - - mpirun - - - - - /cluster/apps/modules/init/python.py - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /cluster/apps/modules/bin/modulecmd python - module - module - - - - - new - - - intel/2018.1 - - - - 64M - - - - - Intel Xeon Cascade Lake,56 cores, batch system is SLURM - .*frontera - LINUX - intel - impi,mvapich - ATM20005 - $ENV{SCRATCH} - /work/02503/edwardsj/CESM/inputdata/ - /work/02503/edwardsj/CESM/inputdata/lmwg - $ENV{SCRATCH}/archive/$CASE - /work/02503/edwardsj/CESM/cesm_baselines - /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc - 4 - slurm - cseg - 112 - 56 - - sbcast ${EXEROOT}/cesm.exe /tmp/cesm.exe; ibrun - - -n {{ total_tasks }} - - /tmp/cesm.exe - - - ibrun - - -n {{ total_tasks }} - - - - /opt/apps/lmod/lmod/init/perl - /opt/apps/lmod/lmod/init/env_modules_python.py - /opt/apps/lmod/lmod/init/sh - /opt/apps/lmod/lmod/init/csh - /opt/apps/lmod/lmod/libexec/lmod perl - /opt/apps/lmod/lmod/libexec/lmod python - module - module - - - TACC - /scratch1/projects/compilers/modulefiles - intel/22.2.0 - cmake/3.24.2 - - - impi/22.2.0 - pnetcdf/1.11.2 - parallel-netcdf/4.6.2 - - - impi - mvapich2-x/2.3 - pnetcdf/1.11.2 - parallel-netcdf/4.6.2 - - - netcdf/4.6.2 - - - - 256M - 20 - /scratch3/projects/csa_gettelman/esmf-8.4.1b02_intel22.2.0_impi22.2.0/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk - ON - SUMMARY - /scratch3/projects/csa_gettelman/intel22.2.0_impi22.2.0/pio2_5_10_avx512 - - - -prepend-rank - 4 - 3 - 3 - 3 - 2 - enable - - - - spread - hybrid - 0 - 1 - 4096 - 4096 - mvapich2_ssh - - - - - NOAA XE6, os is CNL, 24 pes/node, batch system is PBS - CNL - pgi - mpich - /lustre/fs/scratch/Julio.T.Bacmeister - /lustre/fs/scratch/Julio.T.Bacmeister/inputdata - /lustre/fs/scratch/Julio.T.Bacmeister/inputdata - /lustre/fs/scratch/Julio.T.Bacmeister/archive/$CASE - UNSET - UNSET - 8 - pbs - julio -at- ucar.edu - 24 - 24 - - aprun - - -j {{ hyperthreading }} - -n {{ total_tasks }} - -S {{ tasks_per_numa }} - -N $MAX_MPITASKS_PER_NODE - -d $ENV{OMP_NUM_THREADS} - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/csh - /opt/modules/default/init/sh - /opt/modules/default/bin/modulecmd perl - module - module - - PrgEnv-pgi - PrgEnv-cray - PrgEnv-gnu - pgi - cray - - - PrgEnv-pgi - pgi pgi/12.5.0 - - - PrgEnv-gnu - torque - - - PrgEnv-cray/4.0.36 - cce/8.0.2 - - - torque/4.1.3 - netcdf-hdf5parallel/4.2.0 - parallel-netcdf/1.2.0 - - - - 64M - 1 - - - - - UCI Linux Cluster; 16 pes/node, batch system is slurm - LINUX - - - intel - openmpi - - - - - - /DFS-L/SCRATCH/moore/$USER/cesm_runs - /DFS-L/DATA/moore/cesm/inputdata - /DFS-L/DATA/moore/cesm/inputdata - $CIME_OUTPUT_ROOT/archive/$CASE - /DFS-L/DATA/moore/cesm/baselines - - /DFS-L/DATA/moore/cesm/tools/cprnc/cprnc - gmake - 16 - slurm - mlevy@ucar.edu - 16 - 16 - FALSE - - - - mpirun - - -np {{ total_tasks }} - - - - mpirun - - -np {{ total_tasks }} - - - - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/libexec/lmod perl - /usr/share/lmod/lmod/libexec/lmod python - module - module - - - - - intel/2018.3 - netcdf/4.7.0 - - - openmpi/3.1.6 - pnetcdf/1.10.0 - - - - - 256M - 16 - - - - -1 - - - - - UCI Linux Cluster; 40 pes/node, batch system is slurm - gplogin\d.gp.local - LINUX - - - intel - openmpi - - - - - - /DFS-L/SCRATCH/moore/$USER/cesm_runs - /DFS-L/DATA/moore/cesm/inputdata - /DFS-L/DATA/moore/cesm/inputdata - $CIME_OUTPUT_ROOT/archive/$CASE - /DFS-L/DATA/moore/cesm/baselines - - /DFS-L/DATA/moore/cesm/tools/cprnc/cprnc - gmake - 16 - slurm - mlevy@ucar.edu - 40 - 40 - FALSE - - - - mpirun - - -np {{ total_tasks }} - - - - mpirun - - -np {{ total_tasks }} - - - - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/libexec/lmod perl - /usr/share/lmod/lmod/libexec/lmod python - module - module - - - - - intel/2018.3 - netcdf/4.7.0 - - - openmpi/3.1.6 - pnetcdf/1.10.0 - - - - - 256M - 16 - - - - -1 - - - - - NCAR AMD EPYC test system 16 CPU nodes 2 GPU nodes - gu.*.hpc.ucar.edu - CNL - intel,gnu,cray,nvhpc,intel-oneapi,intel-classic - mpich - $ENV{SCRATCH} - /glade/p/cesmdata/inputdata - /glade/p/cesmdata/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /glade/p/cesmdata/ccsm_baselines - /glade/p/cesmdata/cprnc/cprnc - 16 - pbs - cseg - 128 - 4 - 128 - 64 - none,a100 - none,openacc,openmp,combined - TRUE - - mpiexec - - --label - -n {{ total_tasks }} - - - - $ENV{LMOD_ROOT}/lmod/init/perl - $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py - $ENV{LMOD_ROOT}/lmod/init/sh - $ENV{LMOD_ROOT}/lmod/init/csh - $ENV{LMOD_ROOT}/lmod/libexec/lmod perl - $ENV{LMOD_ROOT}/lmod/libexec/lmod python - module - module - - cesmdev/1.0 - ncarenv/23.04 - - craype - - - intel/2023.0.0 - mkl - - - intel-oneapi/2023.0.0 - mkl - - - intel-classic/2023.0.0 - mkl - - - cce/15.0.1 - cray-libsci/23.02.1.1 - - - gcc/12.2.0 - cray-libsci/23.02.1.1 - - - nvhpc/23.1 - - - ncarcompilers/0.8.0 - cmake - - - cray-mpich/8.1.25 - - - mpi-serial/2.3.0 - - - - netcdf/4.9.1 - - - - netcdf-mpi/4.9.1 - parallel-netcdf/1.12.3 - - - - parallelio/2.5.10-debug - esmf/8.5.0b21-debug - - - - parallelio/2.5.10 - esmf/8.5.0b21 - - - - - 64M - hybrid - - - *:romio_cb_read=enable:romio_cb_write=enable:striping_factor=2 - - - - ON - SUMMARY - - - - - NCAR CGD Linux Cluster 48 pes/node, batch system is PBS - ^h.*\.cgd\.ucar\.edu - LINUX - intel,pgi,nag,gnu - mvapich2,openmpi - /scratch/cluster/$USER - /fs/cgd/csm/inputdata - /project/tss - /scratch/cluster/$USER/archive/$CASE - /fs/cgd/csm/ccsm_baselines - /fs/cgd/csm/tools/cime/tools/cprnc/cprnc - gmake --output-sync - 4 - pbs - cseg - 48 - 48 - - mpiexec - - --machinefile $ENV{PBS_NODEFILE} - -n {{ total_tasks }} - - - - mpiexec - - -n {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - - - compiler/intel/18.0.3 - tool/netcdf/4.6.1/intel - - - mpi/intel/mvapich2-2.3rc2-intel-18.0.3 - - - compiler/pgi/18.1 - tool/netcdf/4.6.1/pgi - - - compiler/nag/6.2 - tool/netcdf/4.6.1/nag - - - mpi/nag/mvapich2-2.3rc2 - - - mpi/nag/openmpi-3.1.0 - - - compiler/gnu/8.1.0 - tool/netcdf/4.6.1/gcc - - - mpi/gcc/openmpi-3.1.0a - - - mpi/gcc/mvapich2-2.3rc2-qlc - - - - 64M - - $ENV{PATH}:/cluster/torque/bin - /home/dunlap/ESMF-INSTALL/8.0.0bs16/lib/libg/Linux.intel.64.mvapich2.default/esmf.mk - - - -1 - - - - - - - Customize these fields as appropriate for your system, - particularly changing MAX_TASKS_PER_NODE and MAX_MPITASKS_PER_NODE to the - number of cores on your machine. You may also want to change - instances of '$ENV{HOME}/projects' to your desired directory - organization. You can use this in either of two ways: (1) - Without making any changes, by adding `--machine homebrew` to - create_newcase or create_test (2) Copying this into a - config_machines.xml file in your personal .cime directory and - then changing the machine name (MACH="homebrew") to - your machine name and the NODENAME_REGEX to something matching - your machine's hostname. With (2), you should not need the - `--machine` argument, because the machine should be determined - automatically. However, with (2), you will also need to copy the - homebrew-specific settings in config_compilers.xml into a - config_compilers.xml file in your personal .cime directory, again - changing the machine name (MACH="homebrew") to your machine name. - - - something.matching.your.machine.hostname - Darwin - gnu - mpich - $ENV{HOME}/projects/scratch - $ENV{HOME}/projects/cesm-inputdata - $ENV{HOME}/projects/ptclm-data - $ENV{HOME}/projects/scratch/archive/$CASE - $ENV{HOME}/projects/baselines - $ENV{HOME}/cesm/tools/cprnc/cprnc - make - 4 - none - __YOUR_NAME_HERE__ - 8 - 4 - - mpirun - - -np {{ total_tasks }} - -prepend-rank - - - - - /usr/local - - - - - NCAR CGD Linux Cluster 48 pes/node, batch system is PBS - ^i.*\.ucar\.edu - LINUX - intel,nag,gnu - mvapich2,openmpi - /scratch/cluster/$USER - /fs/cgd/csm/inputdata - /project/tss - /scratch/cluster/$USER/archive/$CASE - /fs/cgd/csm/ccsm_baselines - /fs/cgd/csm/tools/cime/tools/cprnc/cprnc - gmake --output-sync - 4 - pbs - cseg - 48 - 48 - - mpiexec - - --machinefile $ENV{PBS_NODEFILE} - -n {{ total_tasks }} - - - - mpiexec - - -n {{ total_tasks }} - --tag-output - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - lang/python/3.7.0 - /fs/cgd/data0/modules/modulefiles - - - compiler/gnu/9.3.0 - tool/netcdf/4.7.4/gnu/9.3.0 - - - openmpi/4.0.3/gnu/9.3.0 - - - mpi/2.3.3/gnu/9.3.0 - - - compiler/intel/20.0.1 - tool/netcdf/4.7.4/intel/20.0.1 - - - mpi/2.3.3/intel/20.0.1 - - - compiler/nag/6.2-8.1.0 - tool/netcdf/c4.6.1-f4.4.4/nag-gnu/6.2-8.1.0 - - - mpi/2.3.3/nag/6.2 - - - mpi - - - esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mvapich2-O - mvapich2/2.3.3/gnu/9.3.0/pio/2_5_10 - - - esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mvapich2-g - mvapich2/2.3.3/gnu/9.3.0/pio/2_5_10 - - - esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mpiuni-O - mpi-serial/2.3.0/gnu/9.3.0/pio/2_5_10 - - - esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mpiuni-g - mpi-serial/2.3.0/gnu/9.3.0/pio/2_5_10 - - - - esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mvapich2-O - mvapich2/2.3.3/nag/6.2/pio/2_5_10 - - - esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mvapich2-g - mvapich2/2.3.3/nag/6.2/pio/2_5_10 - - - esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mpiuni-g - mpi-serial/2.3.0/nag/6.2/pio/2_5_10 - - - esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mpiuni-O - mpi-serial/2.3.0/nag/6.2/pio/2_5_10 - - - - esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mpiuni-g - mpi-serial/2.3.0/intel/20.0.1/pio/2_5_10 - - - esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mpiuni-O - mpi-serial/2.3.0/intel/20.0.1/pio/2_5_10 - - - esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mvapich2-g - mvapich2/2.3.3/intel/20.0.1/pio/2_5_10 - - - esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mvapich2-O - mvapich2/2.3.3/intel/20.0.1/pio/2_5_10 - - - - 64M - - $ENV{PATH}:/cluster/torque/bin - - - -1 - - - - - NCAR SGI test platform, os is Linux, 36 pes/node, batch system is PBS - .*.laramie.ucar.edu - LINUX - intel,gnu - mpt - /picnic/scratch/$USER - $ENV{CESMDATAROOT}/inputdata - $ENV{CESMDATAROOT}/lmwg - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc - 8 - pbs - cseg - - 36 - 36 - FALSE - - mpiexec_mpt - - -p "%g:" - omplace - - - - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/init/perl - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/init/env_modules_python.py - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/init/csh - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/init/sh - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/libexec/lmod perl - /picnic/u/apps/la/opt/lmod/8.1.7/lmod/lmod/libexec/lmod python - module - module - - - ncarenv/1.3 - cmake/3.16.4 - - - intel/19.0.5 - mkl - - - gnu/9.1.0 - openblas/0.3.6 - - - mpt/2.21 - netcdf-mpi/4.7.3 - - - pnetcdf/1.12.1 - pio/2.4.4 - - - openmpi/3.1.4 - netcdf-mpi/4.7.3 - - - ncarcompilers/0.5.0 - - - netcdf/4.7.3 - - - - 256M - 16 - - - - - Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM - LINUX - intel - openmpi - /global/scratch/$ENV{USER} - /global/scratch/$ENV{USER}/cesm_input_datasets/ - /global/scratch/$ENV{USER}/cesm_input_datasets/atm/datm7 - $CIME_OUTPUT_ROOT/cesm_archive/$CASE - $CIME_OUTPUT_ROOT/cesm_baselines - /$CIME_OUTPUT_ROOT/cesm_tools/cprnc/cprnc - 4 - slurm - rgknox at lbl dot gov and glemieux at lbl dot gov - 16 - 16 - TRUE - - mpirun - - -np {{ total_tasks }} - -npernode $MAX_MPITASKS_PER_NODE - - - - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /usr/Modules/init/perl.pm - /usr/Modules/python.py - module - module - /usr/Modules/bin/modulecmd perl - /usr/Modules/bin/modulecmd python - - - cmake - perl xml-libxml switch python/3.6 - - - intel/2016.4.072 - mkl - - - netcdf/4.4.1.1-intel-s - - - openmpi - netcdf/4.4.1.1-intel-p - - - - - - FATES development machine at LBNL, System76 Thelio Massive Workstation Pop!_OS 20.04 - lobata - LINUX - gnu - openmpi - $ENV{HOME}/scratch/ - /data/cesmdataroot/inputdata - /data/cesmdataroot/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{HOME}/scratch/ctsm-baselines - /home/glemieux/Repos/cime/tools/cprnc/cprnc - make - 16 - none - glemieux at lbl dot gov - 4 - 4 - FALSE - - mpirun - - -np {{ total_tasks }} - --map-by ppr:{{ tasks_per_node }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread - - - - /usr/share/modules/init/python.py - /usr/share/modules/init/perl.pm - /usr/share/modules/init/sh - /usr/share/modules/init/csh - /usr/bin/modulecmd python - /usr/bin/modulecmd perl - module - module - - - hdf5 - netcdf-c - netcdf-fortran - esmf - - - - - - - Lonestar5 cluster at TACC, OS is Linux (intel), batch system is SLURM - .*ls5\.tacc\.utexas\.edu - LINUX - intel - mpich - $ENV{SCRATCH} - /work/02503/edwardsj/CESM/inputdata - /work/02503/edwardsj/CESM/inputdata/lmwg - $CIME_OUTPUT_ROOT/cesm_archive/$CASE - /work/02503/edwardsj/CESM/cesm_baselines - /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc - 4 - slurm - cseg - 48 - 24 - FALSE - - srun - - --ntasks={{ total_tasks }} - - - - - /opt/apps/lmod/lmod/init/perl - /opt/apps/lmod/lmod/init/env_modules_python.py - /opt/apps/lmod/lmod/init/sh - /opt/apps/lmod/lmod/init/csh - /opt/apps/lmod/lmod/libexec/lmod perl - /opt/apps/lmod/lmod/libexec/lmod python - module - module - - - - cmake - - - intel/18.0.2 - - - netcdf/4.6.2 - - - cray_mpich - - - pnetcdf/1.8.0 - parallel-netcdf/4.6.2 - - - - - - - Linux workstation for Jenkins testing - (melvin|watson) - LINUX - sonproxy.sandia.gov:80 - gnu - openmpi - /sems-data-store/ACME/timings - $ENV{HOME}/acme/scratch - /sems-data-store/ACME/inputdata - /sems-data-store/ACME/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /sems-data-store/ACME/baselines - /sems-data-store/ACME/cprnc/build/cprnc - make - 32 - acme_developer - none - jgfouca at sandia dot gov - 64 - 64 - - mpirun - - -np {{ total_tasks }} - --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread - - - - /usr/share/Modules/init/python.py - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/bin/modulecmd python - /usr/bin/modulecmd perl - module - module - - - sems-env - acme-env - sems-git - sems-python/2.7.9 - sems-cmake/2.8.12 - - - sems-gcc/5.3.0 - - - sems-intel/16.0.3 - - - sems-netcdf/4.4.1/exo - acme-pfunit/3.2.8/base - - - sems-openmpi/1.8.7 - sems-netcdf/4.4.1/exo_parallel - - - - $ENV{SEMS_NETCDF_ROOT} - 64M - spread - threads - - - $ENV{SEMS_NETCDF_ROOT} - - - - - ANL IBM BG/Q, os is BGP, 16 pes/node, batch system is cobalt - .*.fst.alcf.anl.gov - BGQ - ibm - ibm - /projects/$PROJECT/usr/$ENV{USER} - /projects/ccsm/inputdata - /projects/ccsm/inputdata/atm/datm7 - /projects/$PROJECT/usr/$USER/archive/$CASE - /projects/ccsm/ccsm_baselines/ - /projects/ccsm/tools/cprnc/cprnc - 4 - cobalt - cseg - 64 - 8 - TRUE - - /usr/bin/runjob - - --label short - - --ranks-per-node $MAX_MPITASKS_PER_NODE - - --np {{ total_tasks }} - --block $COBALT_PARTNAME --envs OMP_WAIT_POLICY=active --envs BG_SMP_FAST_WAKEUP=yes $LOCARGS - --envs BG_THREADLAYOUT=1 - --envs OMP_STACKSIZE=32M - --envs OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} - - - - /etc/profile.d/00softenv.csh - /etc/profile.d/00softenv.sh - soft - soft - - +mpiwrapper-xl - @ibm-compilers-2015-02 - +cmake - +python - - - - 10000 - FALSE - 64M - /soft/libraries/hdf5/1.8.14/cnk-xl/current - - - - - Medium sized linux cluster at BNL, torque scheduler. - LINUX - gnu - openmpi,mpi-serial - /data/$ENV{USER} - /data/Model_Data/cesm_input_datasets/ - /data/Model_Data/cesm_input_datasets/atm/datm7 - $CIME_OUTPUT_ROOT/cesm_archive/$CASE - $CIME_OUTPUT_ROOT/cesm_baselines - /data/software/cesm_tools/cprnc/cprnc - 4 - pbs - rgknox at lbl dot gov and sserbin at bnl gov - 12 - 12 - 12 - FALSE - - mpirun - - -np {{ total_tasks }} - -npernode $MAX_TASKS_PER_NODE - - - - /etc/profile.d/modules.sh - /etc/profile.d/modules.csh - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - module - module - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - - - perl/5.22.1 - libxml2/2.9.2 - maui/3.3.1 - python/2.7.13 - - - gcc/5.4.0 - gfortran/5.4.0 - hdf5/1.8.19fates - netcdf/4.4.1.1-gnu540-fates - openmpi/2.1.1-gnu540 - - - openmpi/2.1.1-gnu540 - - - - /data/software/hdf5/1.8.19fates - /data/software/netcdf/4.4.1.1-gnu540-fates - - - - - PNL cluster, os is Linux (pgi), batch system is SLURM - LINUX - pgi - mpich - /pic/scratch/$USER - /pic/scratch/tcraig/IRESM/inputdata - /pic/scratch/tcraig/IRESM/inputdata/atm/datm7 - /pic/scratch/$USER/archive/$CASE - /pic/scratch/tcraig/IRESM/ccsm_baselines - /pic/scratch/tcraig/IRESM/tools/cprnc/cprnc - 8 - slurm - tcraig -at- ucar.edu - 32 - 32 - FALSE - - mpiexec_mpt - - --mpi=none - -n={{ total_tasks }} - --kill-on-bad-exit - - - - /share/apps/modules/Modules/3.2.7/init/perl.pm - /share/apps/modules/Modules/3.2.7/init/csh - /share/apps/modules/Modules/3.2.7/init/sh - /share/apps/modules/Modules/3.2.7/bin/modulecmd perl - module - module - - - precision/i4 - pgi/11.8 - mvapich2/1.7 - netcdf/4.1.3 - - - - 64M - - - - - NERSC EX AMD EPYC, os is CNL, 64 pes/node, batch system is Slurm - $ENV{NERSC_HOST}:perlmutter - CNL - gnu,cray,nvidia,aocc - mpich - mp9_g - $ENV{SCRATCH} - /global/cfs/cdirs/ccsm1/inputdata - /global/cfs/cdirs/ccsm1/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /global/cfs/cdirs/ccsm1/ccsm_baselines - /global/cfs/cdirs/ccsm1/tools/cprnc.perlmutter/cprnc - 8 - slurm - cseg - 128 - 4 - 64 - TRUE - - srun - - --label - -n {{ total_tasks }} - -c {{ srun_binding }} - - - - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/libexec/lmod perl - /usr/share/lmod/lmod/libexec/lmod python - module - module - - PrgEnv-nvidia - PrgEnv-cray - PrgEnv-aocc - PrgEnv-gnu - nvidia - cce - gnu - aocc - cray-parallel-netcdf - cray-hdf5-parallel - cray-libsci - cray-mpich - cray-hdf5 - cray-netcdf-hdf5parallel - cray-netcdf - craype - - - - PrgEnv-cray - cce cce/12.0.3 - - - PrgEnv-gnu - gcc gcc/11.2.0 - - - craype craype/2.7.10 - - - cray-libsci/21.08.1.2 - - - cray-mpich/8.1.9 - - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - cray-hdf5/1.12.0.7 - cray-netcdf/4.7.4.7 - - - cray-hdf5-parallel/1.12.0.7 - cray-netcdf-hdf5parallel/4.7.4.7 - cray-parallel-netcdf/1.12.1.7 - - - cmake/3.20.5 - - - - 256M - - - - - - NASA/AMES Linux Cluster, Linux (ia64), 2.4 GHz Broadwell Intel Xeon E5-2680v4 processors, 28 pes/node (two 14-core processors) and 128 GB of memory/node, batch system is PBS - LINUX - intel - mpt - /nobackup/$USER - /nobackup/fvitt/csm/inputdata - /nobackup/fvitt/csm/inputdata/atm/datm7 - /nobackup/$USER/archive/$CASE - /nobackup/fvitt/cesm_baselines - /u/fvitt/bin/cprnc - 8 - pbs - fvitt -at- ucar.edu - 28 - 28 - TRUE - - mpiexec_mpt - - -n {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/share/Modules/init/python.py - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - nas - pkgsrc - python3 - comp-intel/2020.4.304 - mpi-hpe/mpt.2.25 - szip/2.1.1 - hdf4/4.2.12 - hdf5/1.8.18_mpt - netcdf/4.4.1.1_mpt - - - - 1024 - 100000 - 16 - 256M - /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk - - - - - NASA/AMES Linux Cluster, Linux (ia64), 2.5 GHz Haswell Intel Xeon E5-2680v3 processors, 24 pes/node (two 12-core processors) and 128 GB of memory/node, batch system is PBS - LINUX - intel - mpt - /nobackup/$USER - /nobackup/fvitt/csm/inputdata - /nobackup/fvitt/csm/inputdata/atm/datm7 - /nobackup/$USER/archive/$CASE - /nobackup/fvitt/cesm_baselines - /u/fvitt/bin/cprnc - 8 - pbs - fvitt -at- ucar.edu - 24 - 24 - TRUE - - mpiexec_mpt - - -n {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/share/Modules/init/python.py - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - nas - pkgsrc - python3 - comp-intel/2020.4.304 - mpi-hpe/mpt.2.25 - szip/2.1.1 - hdf4/4.2.12 - hdf5/1.8.18_mpt - netcdf/4.4.1.1_mpt - - - - 1024 - 100000 - 16 - 256M - /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk - - - - - NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.6 GHz Sandy Bridge processors, 16 cores/node and 32 GB of memory, batch system is PBS - LINUX - intel - mpt - /nobackup/$USER - /nobackup/fvitt/csm/inputdata - /nobackup/fvitt/csm/inputdata/atm/datm7 - /nobackup/$USER/archive/$CASE - /nobackup/fvitt/cesm_baselines - /u/fvitt/bin/cprnc - 8 - pbs - fvitt -at- ucar.edu - 16 - 16 - TRUE - - mpiexec_mpt - - -n {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/share/Modules/init/python.py - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - nas - pkgsrc - python3 - comp-intel/2020.4.304 - mpi-hpe/mpt.2.25 - szip/2.1.1 - hdf4/4.2.12 - hdf5/1.8.18_mpt - netcdf/4.4.1.1_mpt - - - - 1024 - 100000 - 16 - 256M - /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk - - - - - NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.8 GHz Ivy Bridge processors, 20 cores/node and 3.2 GB of memory per core, batch system is PBS - LINUX - intel - mpich - /nobackup/$USER - /nobackup/fvitt/csm/inputdata - /nobackup/fvitt/csm/inputdata/atm/datm7 - /nobackup/$USER/archive/$CASE - /nobackup/fvitt/cesm_baselines - /u/fvitt/bin/cprnc - 8 - pbs - fvitt -at- ucar.edu - 20 - 20 - TRUE - - mpiexec_mpt - - -n {{ total_tasks }} - - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/share/Modules/init/python.py - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - nas - pkgsrc - python3 - comp-intel/2020.4.304 - mpi-hpe/mpt.2.25 - szip/2.1.1 - hdf4/4.2.12 - hdf5/1.8.18_mpt - netcdf/4.4.1.1_mpt - - - - 1024 - 100000 - 16 - 256M - /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk - - - - - Linux workstation at Sandia on SRN with SEMS TPL modules - (s999964|climate|penn) - LINUX - wwwproxy.sandia.gov:80 - gnu - openmpi - /sems-data-store/ACME/timings - $ENV{HOME}/acme/scratch - /sems-data-store/ACME/inputdata - /sems-data-store/ACME/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /sems-data-store/ACME/baselines - /sems-data-store/ACME/cprnc/build/cprnc - make - 32 - acme_developer - none - jgfouca at sandia dot gov - 64 - 64 - - mpirun - - -np {{ total_tasks }} - - - - /usr/share/Modules/init/python.py - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/bin/modulecmd python - /usr/bin/modulecmd perl - module - module - - - sems-env - sems-git - sems-python/2.7.9 - sems-gcc/5.1.0 - sems-openmpi/1.8.7 - sems-cmake/2.8.12 - sems-netcdf/4.3.2/parallel - - - - $ENV{SEMS_NETCDF_ROOT} - $ENV{SEMS_NETCDF_ROOT} - - - - - SNL clust - (skybridge|chama)-login - LINUX - wwwproxy.sandia.gov:80 - intel - openmpi - /projects/ccsm/timings - /gscratch/$USER/acme_scratch/$MACH - /projects/ccsm/inputdata - /projects/ccsm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /projects/ccsm/ccsm_baselines - /projects/ccsm/cprnc/build/cprnc_wrap - 8 - acme_integration - slurm - jgfouca at sandia dot gov - 16 - 16 - TRUE - - - mpirun - - -np {{ total_tasks }} - -npernode $MAX_MPITASKS_PER_NODE - - - - /usr/share/Modules/init/python.py - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/bin/modulecmd python - /usr/bin/modulecmd perl - module - module - - - sems-env - sems-git - sems-python/2.7.9 - gnu/4.9.2 - intel/intel-15.0.3.187 - libraries/intel-mkl-15.0.2.164 - libraries/intel-mkl-15.0.2.164 - - - openmpi-intel/1.8 - sems-hdf5/1.8.12/parallel - sems-netcdf/4.3.2/parallel - sems-hdf5/1.8.12/base - sems-netcdf/4.3.2/base - - - - $ENV{SEMS_NETCDF_ROOT} - 64M - - - $ENV{SEMS_NETCDF_ROOT} - - - - - Intel Xeon Platinum 8160 ("Skylake"),48 cores on two sockets (24 cores/socket) , batch system is SLURM - .*stampede2 - LINUX - intel - impi,mvapich2 - $ENV{SCRATCH} - /work/02503/edwardsj/CESM/inputdata - /work/02503/edwardsj/CESM/inputdata/lmwg - $ENV{WORK}/archive/$CASE - /work/02503/edwardsj/CESM/cesm_baselines - /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc - 4 - slurm - cseg - 96 - 48 - - ibrun - - -n {{ total_tasks }} - - - - ibrun - - -n {{ total_tasks }} - - - - /opt/apps/lmod/lmod/init/perl - /opt/apps/lmod/lmod/init/env_modules_python.py - /opt/apps/lmod/lmod/init/sh - /opt/apps/lmod/lmod/init/csh - /opt/apps/lmod/lmod/libexec/lmod perl - /opt/apps/lmod/lmod/libexec/lmod python - module - module - - - TACC - python/2.7.13 - intel/18.0.2 - cmake/3.16.1 - - - mvapich2/2.3.1 - pnetcdf/1.11 - parallel-netcdf/4.6.2 - - - mvapich2 - impi/18.0.2 - pnetcdf/1.11 - parallel-netcdf/4.6.2 - - - netcdf/4.3.3.1 - - - - 256M - - - /work/01118/tg803972/stampede2/ESMF-INSTALL/8.0.0bs38/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk - - - ON - SUMMARY - /work/06242/tg855414/stampede2/FV3GFS/benchmark-inputs/2012010100/gfs/fcst - /work/06242/tg855414/stampede2/FV3GFS/fix_am - /work/06242/tg855414/stampede2/FV3GFS/addon - - - - - - Intel Xeon Phi 7250 ("Knights Landing") , batch system is SLURM - LINUX - intel - impi,mvapich2 - $ENV{SCRATCH} - /work/02503/edwardsj/CESM/inputdata - /work/02503/edwardsj/CESM/inputdata/lmwg - $ENV{WORK}/archive/$CASE - /work/02503/edwardsj/CESM/cesm_baselines - /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc - 4 - slurm - cseg - 256 - 64 - - ibrun - - - ibrun - - - /opt/apps/lmod/lmod/init/perl - /opt/apps/lmod/lmod/init/env_modules_python.py - /opt/apps/lmod/lmod/init/sh - /opt/apps/lmod/lmod/init/csh - /opt/apps/lmod/lmod/libexec/lmod perl - /opt/apps/lmod/lmod/libexec/lmod python - module - module - - - TACC - python/2.7.13 - intel/18.0.2 - cmake/3.16.1 - - - mvapich2/2.3.1 - pnetcdf/1.11 - parallel-netcdf/4.6.2 - - - mvapich2 - impi/18.0.2 - pnetcdf/1.11 - parallel-netcdf/4.6.2 - - - netcdf/4.3.3.1 - - - - 256M - - - - - Cray test platform - swan.* - CNL - cray, intel - mpt - - /lus/scratch/$USER - /lus/scratch/$USER/inputdata - /lus/scratch/$USER/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lus/scratch/$USER/cesm/baselines - /lus/scratch/$USER/cesm/tools/cprnc/cprnc - 8 - pbs - jedwards - 64 - 64 - FALSE - - aprun - - -n {{ total_tasks }} - -N {{ tasks_per_node }} - --cc depth -d $OMP_NUM_THREADS - -e OMP_STACKSIZE=64M - -e OMP_NUM_THREADS=$OMP_NUM_THREADS - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - PrgEnv-cray/6.0.10 - cce cce/12.0.3 - - - perftools-base - craype craype/2.7.10 - - - cray-libsci/20.09.1 - - - cray-mpich/7.7.18 - - - cray-netcdf-hdf5parallel - cray-hdf5-parallel - cray-parallel-netcdf - - - - ON - SUMMARY - /home/users/p62939/esmf/lib/libg/Unicos.cce.64.mpi.default/esmf.mk - - - - - theia - tfe - LINUX - intel - impi - nems - - /scratch4/NCEPDEV/nems/noscrub/$USER/cimecases - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/inputdata - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/BASELINES - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/tools/cprnc - make - 8 - slurm - cseg - 24 - 24 - TRUE - - srun - - -n $TOTALPES - - - - - - - /apps/lmod/lmod/init/sh - /apps/lmod/lmod/init/csh - module - module - /apps/lmod/lmod/libexec/lmod python - - - intel/15.1.133 - impi/5.1.1.109 - netcdf/4.3.0 - pnetcdf - /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles - yaml-cpp - esmf/8.0.0bs29g - - - - ON - SUMMARY - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/benchmark-inputs/2012010100/gfs/fcst - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/fix_am - /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/addon - - - - - ALCF Cray XC* KNL, os is CNL, 64 pes/node, batch system is cobalt - theta.* - CNL - intel,gnu,cray - mpt - CESM_Highres_Testing - /projects/CESM_Highres_Testing/cesm/scratch/$USER - /projects/CESM_Highres_Testing/cesm/inputdata - /projects/CESM_Highres_Testing/cesm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /projects/CESM_Highres_Testing/cesm/baselines - /projects/CESM_Highres_Testing/cesm/tools/cprnc/cprnc - 8 - cobalt_theta - cseg - 64 - 64 - TRUE - - aprun - - -n {{ total_tasks }} - -N {{ tasks_per_node }} - --cc depth -d $OMP_NUM_THREADS - -e OMP_STACKSIZE=64M - -e OMP_NUM_THREADS=$OMP_NUM_THREADS - - - - /opt/modules/default/init/perl.pm - /opt/modules/default/init/python.py - /opt/modules/default/init/sh - /opt/modules/default/init/csh - /opt/modules/default/bin/modulecmd perl - /opt/modules/default/bin/modulecmd python - module - module - - craype-mic-knl - PrgEnv-intel - PrgEnv-cray - PrgEnv-gnu - intel - cce - cray-parallel-netcdf - cray-hdf5-parallel - pmi - cray-libsci - cray-mpich - cray-netcdf - cray-hdf5 - cray-netcdf-hdf5parallel - craype - papi - - - - PrgEnv-intel/6.0.4 - intel intel/18.0.0.128 - cray-libsci - - - - PrgEnv-cray/6.0.4 - cce cce/8.7.0 - - - PrgEnv-gnu/6.0.4 - gcc gcc/7.3.0 - - - papi/5.6.0.1 - craype craype/2.5.14 - - - cray-libsci/18.04.1 - - - cray-mpich/7.7.0 - - - cray-netcdf-hdf5parallel/4.4.1.1.6 - cray-hdf5-parallel/1.10.1.1 - cray-parallel-netcdf/1.8.1.3 - - - - - - NCAR ARM platform, os is Linux, 64/128 pes/node, batch system is SLURM - .*.thunder.ucar.edu - LINUX - - armgcc,gnu,arm - openmpi - /glade/scratch/$USER - $ENV{CESMDATAROOT}/inputdata - $DIN_LOC_ROOT/CTSM_datm_forcing_data - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cprnc/cprnc - 16 - slurm - cseg - 64 - 128 - - mpiexec - - --tag-output - -np {{ total_tasks }} - - - - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/perl - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/env_modules_python.py - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/csh - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/sh - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod perl - /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod python - module - module - - - ncarenv/1.3 - cmake/3.14.4 - - - arm/19.3 - - - armgcc/8.2.0 - - - gnu/9.1.0 - openblas/0.3.6 - esmf_libs/8.0.0 - - - - ncarcompilers/0.5.0 - - - openmpi/4.0.3 - netcdf-mpi/4.7.1 - pnetcdf/1.12.1 - - - netcdf/4.7.1 - - - esmf-8.0.0-ncdfio-uni-g - - - esmf-8.0.0-ncdfio-uni-O - - - - 256M - $ENV{NETCDF} - - - - - - used for github testing - - $ENV{CIME_TEST_PLATFORM}:ubuntu-latest - LINUX - - gnu - openmpi - none - - $ENV{HOME}/cesm/scratch - $ENV{HOME}/cesm/inputdata - $ENV{HOME}/cesm/inputdata/lmwg - $ENV{HOME}/cesm/archive/$CASE - $ENV{HOME}/cesm/cesm_baselines - - make - 8 - none - jedwards - 4 - 4 - FALSE - - mpiexec - - -n {{ total_tasks }} - - - - - - - CMCC Lenovo ThinkSystem SD530, os is Linux, 36 pes/node, batch system is LSF - (login[1,2]-ib|n[0-9][0-9][0-9]-ib) - LINUX - intel - impi,mpi-serial - R000 - /work/$ENV{DIVISION}/$ENV{USER}/CESM2 - $ENV{CESMDATAROOT}/inputdata - $DIN_LOC_ROOT/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - $ENV{CESMDATAROOT}/ccsm_baselines - $ENV{CESMDATAROOT}/cesm2_tools/cprnc/cprnc - /usr/lib64/perl5:/usr/share/perl5 - 8 - lsf - cmcc - 72 - 36 - TRUE - - mpirun - - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/csh - /usr/share/Modules/init/sh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - - - intel20.1/20.1.217 - intel20.1/szip/2.1.1 - cmake/3.17.3 - curl/7.70.0 - - - intel20.1/hdf5/1.12.0 - intel20.1/netcdf/C_4.7.4-F_4.5.3_CXX_4.3.1 - - - impi20.1/19.7.217 - impi20.1/hdf5/1.12.0 - impi20.1/netcdf/C_4.7.4-F_4.5.3_CXX_4.3.1 - impi20.1/parallel-netcdf/1.12.1 - - - impi20.1/esmf/8.1.1-intelmpi-64-g - - - impi20.1/esmf/8.1.1-intelmpi-64-O - - - intel20.1/esmf/8.1.1-mpiuni-64-g - - - intel20.1/esmf/8.1.1-mpiuni-64-O - - - - /data/inputs/CESM/xios-2.5 - - - 1 - gpfs - 0 - 60 - skx - skx_avx512 - lsf - 1 - {{ num_nodes }} - - ${EXEROOT}/cesm.exe diff --git a/machines/constance/config_machines.xml b/machines/constance/config_machines.xml new file mode 100644 index 00000000..df1459e9 --- /dev/null +++ b/machines/constance/config_machines.xml @@ -0,0 +1,94 @@ + + PNL Haswell cluster, OS is Linux, batch system is SLURM + LINUX + intel,pgi + mvapich2,openmpi,intelmpi,mvapich + /pic/scratch/$USER + /pic/scratch/tcraig/IRESM/inputdata + /pic/scratch/tcraig/IRESM/inputdata/atm/datm7 + /pic/scratch/$USER/cases/archive/$CASE + /pic/scratch/tcraig/IRESM/ccsm_baselines + /people/tcraig/bin/cprnc + 8 + slurm + tcraig -at- ucar.edu + 24 + 24 + FALSE + + srun + + --mpi=none + --ntasks={{ total_tasks }} + --cpu_bind=sockets --cpu_bind=verbose + --kill-on-bad-exit + + + + srun + + --ntasks={{ total_tasks }} + --cpu_bind=sockets --cpu_bind=verbose + --kill-on-bad-exit + + + + mpirun + + -n {{ total_tasks }} + + + + mpirun + + -n {{ total_tasks }} + + + + /share/apps/modules/Modules/3.2.10/init/perl.pm + /etc/profile.d/modules.csh + /etc/profile.d/modules.sh + /share/apps/modules/Modules/3.2.10/bin/modulecmd perl + module + module + + + + + perl/5.20.0 + cmake/2.8.12 + + + intel/15.0.1 + netcdf/4.3.2 + mkl/15.0.1 + + + pgi/14.10 + netcdf/4.3.2 + + + mvapich2/2.1 + + + mvapich2/2.1 + + + intelmpi/5.0.1.035 + + + openmpi/1.8.3 + + + + 64M + + + $MLIB_LIB + /share/apps/netcdf/4.3.2/intel/15.0.1 + + + /share/apps/netcdf/4.3.2/pgi/14.10 + + + diff --git a/machines/container/config_machines.xml b/machines/container/config_machines.xml new file mode 100644 index 00000000..cb7cf245 --- /dev/null +++ b/machines/container/config_machines.xml @@ -0,0 +1,37 @@ + + + Containerized development environment (Docker/Singularity) for CESM w/ GNU compilers + + LINUX + gnu + mpich + $ENV{HOME}/scratch + $ENV{CESMDATAROOT}/inputdata + $DIN_LOC_ROOT/atm/datm7 + $ENV{HOME}/archive/$CASE + make + 4 + none + cgd + 4 + 4 + FALSE + + mpiexec + + -n {{ total_tasks }} + + + + + + /usr/local + /usr/local + /usr/lib64 + /usr/lib64 + + + -1 + + + diff --git a/machines/daint/config_machines.xml b/machines/daint/config_machines.xml new file mode 100644 index 00000000..ad36a913 --- /dev/null +++ b/machines/daint/config_machines.xml @@ -0,0 +1,28 @@ + + CSCS Cray XC50, os is SUSE SLES, 12 pes/node, batch system is SLURM + CNL + pgi,cray,gnu + mpich + /scratch/snx3000/$USER + /project/s824/cesm_inputdata + /project/s824/cesm_inputdata/atm/datm7 + /project/s824/$USER/archive/$CASE + /project/s824/ccsm_baselines + /project/s824/cesm_tools/ccsm_cprnc/cprnc + 12 + slurm + edouard.davin -at- env.ethz.ch + 12 + 12 + + srun + + -n {{ total_tasks }} + -d $ENV{OMP_NUM_THREADS} + + + + + 64M + + diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml new file mode 100644 index 00000000..0589ece8 --- /dev/null +++ b/machines/derecho/config_machines.xml @@ -0,0 +1,125 @@ + + NCAR AMD EPYC system + $ENV{NCAR_HOST}:derecho + CNL + intel,gnu,nvhpc,intel-oneapi,intel-classic + mpich + $ENV{SCRATCH} + $ENV{CESMDATAROOT}/inputdata + $ENV{CESMDATAROOT}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/cprnc/cprnc + 16 + pbs + cseg + 128 + 4 + 128 + 64 + none,a100 + none,openacc,openmp,combined + get_local_rank + TRUE + + mpiexec + + --label + --line-buffer + -n {{ total_tasks }} + + + + $ENV{LMOD_ROOT}/lmod/init/perl + $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py + $ENV{LMOD_ROOT}/lmod/init/sh + $ENV{LMOD_ROOT}/lmod/init/csh + $ENV{LMOD_ROOT}/lmod/libexec/lmod perl + $ENV{LMOD_ROOT}/lmod/libexec/lmod python + module + module + + cesmdev/1.0 + ncarenv/23.09 + + craype + + + intel/2023.2.1 + mkl + + + intel-oneapi/2023.2.1 + mkl + + + intel-classic/2023.2.1 + mkl + + + cce/15.0.1 + cray-libsci/23.02.1.1 + + + gcc/12.2.0 + cray-libsci/23.02.1.1 + + + nvhpc/23.7 + + + + ncarcompilers/1.0.0 + cmake + + + cray-mpich/8.1.27 + + + mpi-serial/2.3.0 + + + + cuda/12.2.1 + + + + netcdf/4.9.2 + + + + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 + + + + parallelio/2.6.2-debug + esmf/8.6.0b04-debug + + + + parallelio/2.6.2 + esmf/8.6.0b04 + + + + + 64M + hybrid + + memhooks + + + + + + ON + SUMMARY + + + openacc + + diff --git a/machines/eastwind/config_machines.xml b/machines/eastwind/config_machines.xml new file mode 100644 index 00000000..e43aaf70 --- /dev/null +++ b/machines/eastwind/config_machines.xml @@ -0,0 +1,55 @@ + + PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM + LINUX + pgi,intel + mvapich2,mvapich + /lustre/$USER + /lustre/tcraig/IRESM/inputdata + /lustre/tcraig/IRESM/inputdata/atm/datm7 + /lustre/$USER/archive/$CASE + /lustre/tcraig/IRESM/ccsm_baselines + /lustre/tcraig/IRESM/tools/cprnc/cprnc + 8 + slurm + tcraig -at- ucar.edu + 12 + 12 + + srun + + --ntasks={{ total_tasks }} + --cpu_bind=sockets + --cpu_bind=verbose + --kill-on-bad-exit + + + + srun + + --mpi=none + --ntasks={{ total_tasks }} + --cpu_bind=sockets + --cpu_bind=verbose + --kill-on-bad-exit + + + + /etc/profile.d/modules.perl + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /share/apps/modules/Modules/3.2.7/bin/modulecmd perl + module + module + + + perl/5.20.7 + cmake/3.0.0 + pgi/15.5 + mpi/mvapich2/1.5.1p1/pgi11.3 + netcdf/4.1.2/pgi + + + + 64M + + diff --git a/machines/euler2/config_machines.xml b/machines/euler2/config_machines.xml new file mode 100644 index 00000000..4d26fbd5 --- /dev/null +++ b/machines/euler2/config_machines.xml @@ -0,0 +1,62 @@ + + Euler II Linux Cluster ETH, 24 pes/node, InfiniBand, XeonE5_2680v3, batch system LSF + LINUX + intel,pgi + openmpi,mpich + /cluster/work/climate/$USER + /cluster/work/climate/cesm/inputdata + /cluster/work/climate/cesm/inputdata/atm/datm7 + /cluster/work/climate/$USER/archive/$CASE + /cluster/work/climate/cesm/ccsm_baselines + /cluster/work/climate/cesm/tools/cprnc/cprnc + 1 + lsf + urs.beyerle -at- env.ethz.ch + 24 + 24 + + mpirun + + -hostfile $ENV{PBS_JOBID} + -ppn $MAX_MPITASKS_PER_NODE + -n {{ total_tasks }} + + + + mpirun + + + + + /cluster/apps/modules/init/python.py + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /cluster/apps/modules/bin/modulecmd python + module + module + + + + + new + + + intel/2018.1 + + + netcdf/4.3.1 + + + pgi/14.1 + + + mvapich2/1.8.1 + + + open_mpi/1.6.5 + + + + 64M + + diff --git a/machines/euler3/config_machines.xml b/machines/euler3/config_machines.xml new file mode 100644 index 00000000..712fa2d9 --- /dev/null +++ b/machines/euler3/config_machines.xml @@ -0,0 +1,66 @@ + + + Euler III Linux Cluster ETH, 4 pes/node, Ethernet, XeonE3_1585Lv5, batch system LSF + LINUX + intel,pgi + openmpi,mpich + /cluster/work/climate/$USER + /cluster/work/climate/cesm/inputdata + /cluster/work/climate/cesm/inputdata/atm/datm7 + /cluster/work/climate/$USER/archive/$CASE + /cluster/work/climate/cesm/ccsm_baselines + /cluster/work/climate/cesm/tools/cprnc/cprnc + 1 + lsf + urs.beyerle -at- env.ethz.ch + 4 + 4 + + mpirun + + -hostfile $ENV{PBS_JOBID} + -ppn $MAX_MPITASKS_PER_NODE + -n {{ total_tasks }} + + + + mpirun + + + + + /cluster/apps/modules/init/python.py + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /cluster/apps/modules/bin/modulecmd python + module + module + + + + + new + + + interconnect/ethernet + + + intel/2018.1 + + + netcdf/4.3.1 + + + pgi/14.1 + + + mvapich2/1.8.1 + + + open_mpi/1.6.5 + + + + 64M + + diff --git a/machines/euler4/config_machines.xml b/machines/euler4/config_machines.xml new file mode 100644 index 00000000..7a58b7a0 --- /dev/null +++ b/machines/euler4/config_machines.xml @@ -0,0 +1,51 @@ + + + Euler IV Linux Cluster ETH, 36 pes/node, InfiniBand, XeonGold_6150, batch system LSF + LINUX + intel,pgi + openmpi,mpich + /cluster/work/climate/$USER + /cluster/work/climate/cesm/inputdata + /cluster/work/climate/cesm/inputdata/atm/datm7 + /cluster/work/climate/$USER/archive/$CASE + /cluster/work/climate/cesm/ccsm_baselines + /cluster/work/climate/cesm/tools/cprnc/cprnc + 1 + lsf + urs.beyerle -at- env.ethz.ch + 36 + 36 + + mpirun + + -hostfile $ENV{PBS_JOBID} + -ppn $MAX_MPITASKS_PER_NODE + -n {{ total_tasks }} + + + + mpirun + + + + + /cluster/apps/modules/init/python.py + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /cluster/apps/modules/bin/modulecmd python + module + module + + + + + new + + + intel/2018.1 + + + + 64M + + diff --git a/machines/frontera/config_machines.xml b/machines/frontera/config_machines.xml new file mode 100644 index 00000000..fba36580 --- /dev/null +++ b/machines/frontera/config_machines.xml @@ -0,0 +1,89 @@ + + Intel Xeon Cascade Lake,56 cores, batch system is SLURM + LINUX + intel + impi,mvapich + ATM20005 + $ENV{SCRATCH} + /work/02503/edwardsj/CESM/inputdata/ + /work/02503/edwardsj/CESM/inputdata/lmwg + $ENV{SCRATCH}/archive/$CASE + /work/02503/edwardsj/CESM/cesm_baselines + /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc + 4 + slurm + cseg + 112 + 56 + + sbcast ${EXEROOT}/cesm.exe /tmp/cesm.exe; ibrun + + -n {{ total_tasks }} + + /tmp/cesm.exe + + + ibrun + + -n {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + TACC + /scratch1/projects/compilers/modulefiles + intel/22.2.0 + cmake/3.24.2 + + + impi/22.2.0 + pnetcdf/1.11.2 + parallel-netcdf/4.6.2 + + + impi + mvapich2-x/2.3 + pnetcdf/1.11.2 + parallel-netcdf/4.6.2 + + + netcdf/4.6.2 + + + + 256M + 20 + /scratch3/projects/csa_gettelman/esmf-8.4.1b02_intel22.2.0_impi22.2.0/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk + ON + SUMMARY + /scratch3/projects/csa_gettelman/intel22.2.0_impi22.2.0/pio2_5_10_avx512 + + + -prepend-rank + 4 + 3 + 3 + 3 + 2 + enable + + + + spread + hybrid + 0 + 1 + 4096 + 4096 + mvapich2_ssh + + diff --git a/machines/gaea/config_machines.xml b/machines/gaea/config_machines.xml new file mode 100644 index 00000000..c9182491 --- /dev/null +++ b/machines/gaea/config_machines.xml @@ -0,0 +1,63 @@ + + NOAA XE6, os is CNL, 24 pes/node, batch system is PBS + CNL + pgi + mpich + /lustre/fs/scratch/Julio.T.Bacmeister + /lustre/fs/scratch/Julio.T.Bacmeister/inputdata + /lustre/fs/scratch/Julio.T.Bacmeister/inputdata + /lustre/fs/scratch/Julio.T.Bacmeister/archive/$CASE + UNSET + UNSET + 8 + pbs + julio -at- ucar.edu + 24 + 24 + + aprun + + -j {{ hyperthreading }} + -n {{ total_tasks }} + -S {{ tasks_per_numa }} + -N $MAX_MPITASKS_PER_NODE + -d $ENV{OMP_NUM_THREADS} + + + + /opt/modules/default/init/perl.pm + /opt/modules/default/init/csh + /opt/modules/default/init/sh + /opt/modules/default/bin/modulecmd perl + module + module + + PrgEnv-pgi + PrgEnv-cray + PrgEnv-gnu + pgi + cray + + + PrgEnv-pgi + pgi pgi/12.5.0 + + + PrgEnv-gnu + torque + + + PrgEnv-cray/4.0.36 + cce/8.0.2 + + + torque/4.1.3 + netcdf-hdf5parallel/4.2.0 + parallel-netcdf/1.2.0 + + + + 64M + 1 + + diff --git a/machines/greenplanet-sib29/config_machines.xml b/machines/greenplanet-sib29/config_machines.xml new file mode 100644 index 00000000..7fadfc50 --- /dev/null +++ b/machines/greenplanet-sib29/config_machines.xml @@ -0,0 +1,78 @@ + + UCI Linux Cluster; 16 pes/node, batch system is slurm + LINUX + + + intel + openmpi + + + + + + /DFS-L/SCRATCH/moore/$USER/cesm_runs + /DFS-L/DATA/moore/cesm/inputdata + /DFS-L/DATA/moore/cesm/inputdata + $CIME_OUTPUT_ROOT/archive/$CASE + /DFS-L/DATA/moore/cesm/baselines + + /DFS-L/DATA/moore/cesm/tools/cprnc/cprnc + gmake + 16 + slurm + mlevy@ucar.edu + 16 + 16 + FALSE + + + + mpirun + + -np {{ total_tasks }} + + + + mpirun + + -np {{ total_tasks }} + + + + /usr/share/lmod/lmod/init/perl + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/libexec/lmod perl + /usr/share/lmod/lmod/libexec/lmod python + module + module + + + + + intel/2018.3 + netcdf/4.7.0 + + + openmpi/3.1.6 + pnetcdf/1.10.0 + + + + + 256M + 16 + + + + -1 + + diff --git a/machines/greenplanet-sky24/config_machines.xml b/machines/greenplanet-sky24/config_machines.xml new file mode 100644 index 00000000..35d1ccf4 --- /dev/null +++ b/machines/greenplanet-sky24/config_machines.xml @@ -0,0 +1,80 @@ + + UCI Linux Cluster; 40 pes/node, batch system is slurm + gplogin\d.gp.local + LINUX + + + intel + openmpi + + + + + + /DFS-L/SCRATCH/moore/$USER/cesm_runs + /DFS-L/DATA/moore/cesm/inputdata + /DFS-L/DATA/moore/cesm/inputdata + $CIME_OUTPUT_ROOT/archive/$CASE + /DFS-L/DATA/moore/cesm/baselines + + /DFS-L/DATA/moore/cesm/tools/cprnc/cprnc + gmake + 16 + slurm + mlevy@ucar.edu + 40 + 40 + FALSE + + + + mpirun + + -np {{ total_tasks }} + + + + mpirun + + -np {{ total_tasks }} + + + + /usr/share/lmod/lmod/init/perl + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/libexec/lmod perl + /usr/share/lmod/lmod/libexec/lmod python + module + module + + + + + intel/2018.3 + netcdf/4.7.0 + + + openmpi/3.1.6 + pnetcdf/1.10.0 + + + + + 256M + 16 + + + + -1 + + + diff --git a/machines/gust/config_machines.xml b/machines/gust/config_machines.xml new file mode 100644 index 00000000..b88172df --- /dev/null +++ b/machines/gust/config_machines.xml @@ -0,0 +1,111 @@ + + NCAR AMD EPYC test system 16 CPU nodes 2 GPU nodes + gu.*.hpc.ucar.edu + CNL + intel,gnu,cray,nvhpc,intel-oneapi,intel-classic + mpich + $ENV{SCRATCH} + /glade/p/cesmdata/inputdata + /glade/p/cesmdata/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /glade/p/cesmdata/ccsm_baselines + /glade/p/cesmdata/cprnc/cprnc + 16 + pbs + cseg + 128 + 4 + 128 + 64 + none,a100 + none,openacc,openmp,combined + TRUE + + mpiexec + + --label + -n {{ total_tasks }} + + + + $ENV{LMOD_ROOT}/lmod/init/perl + $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py + $ENV{LMOD_ROOT}/lmod/init/sh + $ENV{LMOD_ROOT}/lmod/init/csh + $ENV{LMOD_ROOT}/lmod/libexec/lmod perl + $ENV{LMOD_ROOT}/lmod/libexec/lmod python + module + module + + cesmdev/1.0 + ncarenv/23.04 + + craype + + + intel/2023.0.0 + mkl + + + intel-oneapi/2023.0.0 + mkl + + + intel-classic/2023.0.0 + mkl + + + cce/15.0.1 + cray-libsci/23.02.1.1 + + + gcc/12.2.0 + cray-libsci/23.02.1.1 + + + nvhpc/23.1 + + + ncarcompilers/0.8.0 + cmake + + + cray-mpich/8.1.25 + + + mpi-serial/2.3.0 + + + + netcdf/4.9.1 + + + + netcdf-mpi/4.9.1 + parallel-netcdf/1.12.3 + + + + parallelio/2.5.10-debug + esmf/8.5.0b21-debug + + + + parallelio/2.5.10 + esmf/8.5.0b21 + + + + + 64M + hybrid + + + *:romio_cb_read=enable:romio_cb_write=enable:striping_factor=2 + + + + ON + SUMMARY + + diff --git a/machines/homebrew/config_machines.xml b/machines/homebrew/config_machines.xml new file mode 100644 index 00000000..17f76234 --- /dev/null +++ b/machines/homebrew/config_machines.xml @@ -0,0 +1,49 @@ + + + + Customize these fields as appropriate for your system, + particularly changing MAX_TASKS_PER_NODE and MAX_MPITASKS_PER_NODE to the + number of cores on your machine. You may also want to change + instances of '$ENV{HOME}/projects' to your desired directory + organization. You can use this in either of two ways: (1) + Without making any changes, by adding `--machine homebrew` to + create_newcase or create_test (2) Copying this into a + config_machines.xml file in your personal .cime directory and + then changing the machine name (MACH="homebrew") to + your machine name and the NODENAME_REGEX to something matching + your machine's hostname. With (2), you should not need the + `--machine` argument, because the machine should be determined + automatically. However, with (2), you will also need to copy the + homebrew-specific settings in config_compilers.xml into a + config_compilers.xml file in your personal .cime directory, again + changing the machine name (MACH="homebrew") to your machine name. + + + something.matching.your.machine.hostname + Darwin + gnu + mpich + $ENV{HOME}/projects/scratch + $ENV{HOME}/projects/cesm-inputdata + $ENV{HOME}/projects/ptclm-data + $ENV{HOME}/projects/scratch/archive/$CASE + $ENV{HOME}/projects/baselines + $ENV{HOME}/cesm/tools/cprnc/cprnc + make + 4 + none + __YOUR_NAME_HERE__ + 8 + 4 + + mpirun + + -np {{ total_tasks }} + -prepend-rank + + + + + /usr/local + + diff --git a/machines/izumi/config_machines.xml b/machines/izumi/config_machines.xml new file mode 100644 index 00000000..8cab2739 --- /dev/null +++ b/machines/izumi/config_machines.xml @@ -0,0 +1,134 @@ + + NCAR CGD Linux Cluster 48 pes/node, batch system is PBS + ^i.*\.ucar\.edu + LINUX + intel,nag,gnu + mvapich2,openmpi + /scratch/cluster/$USER + /fs/cgd/csm/inputdata + /project/tss + /scratch/cluster/$USER/archive/$CASE + /fs/cgd/csm/ccsm_baselines + /fs/cgd/csm/tools/cime/tools/cprnc/cprnc + gmake --output-sync + 4 + pbs + cseg + 48 + 48 + + mpiexec + + --machinefile $ENV{PBS_NODEFILE} + -n {{ total_tasks }} + + + + mpiexec + + -n {{ total_tasks }} + --tag-output + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + /usr/share/Modules/init/csh + /usr/share/Modules/init/sh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + lang/python/3.7.0 + /fs/cgd/data0/modules/modulefiles + + + compiler/gnu/9.3.0 + tool/netcdf/4.7.4/gnu/9.3.0 + + + openmpi/4.0.3/gnu/9.3.0 + + + mpi/2.3.3/gnu/9.3.0 + + + compiler/intel/20.0.1 + tool/netcdf/4.7.4/intel/20.0.1 + + + mpi/2.3.3/intel/20.0.1 + + + compiler/nag/6.2-8.1.0 + tool/netcdf/c4.6.1-f4.4.4/nag-gnu/6.2-8.1.0 + + + mpi/2.3.3/nag/6.2 + + + mpi + + + esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mvapich2-O + mvapich2/2.3.3/gnu/9.3.0/pio/2_5_10 + + + esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mvapich2-g + mvapich2/2.3.3/gnu/9.3.0/pio/2_5_10 + + + esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mpiuni-O + mpi-serial/2.3.0/gnu/9.3.0/pio/2_5_10 + + + esmfpkgs/gfortran/9.3.0/esmf-8.5.0-ncdfio-mpiuni-g + mpi-serial/2.3.0/gnu/9.3.0/pio/2_5_10 + + + + esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mvapich2-O + mvapich2/2.3.3/nag/6.2/pio/2_5_10 + + + esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mvapich2-g + mvapich2/2.3.3/nag/6.2/pio/2_5_10 + + + esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mpiuni-g + mpi-serial/2.3.0/nag/6.2/pio/2_5_10 + + + esmfpkgs/nag/6.2/esmf-8.5.0-ncdfio-mpiuni-O + mpi-serial/2.3.0/nag/6.2/pio/2_5_10 + + + + esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mpiuni-g + mpi-serial/2.3.0/intel/20.0.1/pio/2_5_10 + + + esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mpiuni-O + mpi-serial/2.3.0/intel/20.0.1/pio/2_5_10 + + + esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mvapich2-g + mvapich2/2.3.3/intel/20.0.1/pio/2_5_10 + + + esmfpkgs/intel/20.0.1/esmf-8.5.0-ncdfio-mvapich2-O + mvapich2/2.3.3/intel/20.0.1/pio/2_5_10 + + + + 64M + + $ENV{PATH}:/cluster/torque/bin + + + -1 + + + diff --git a/machines/lawrencium-lr3/config_machines.xml b/machines/lawrencium-lr3/config_machines.xml new file mode 100644 index 00000000..68af79ed --- /dev/null +++ b/machines/lawrencium-lr3/config_machines.xml @@ -0,0 +1,53 @@ + + + Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM + LINUX + intel + openmpi + /global/scratch/$ENV{USER} + /global/scratch/$ENV{USER}/cesm_input_datasets/ + /global/scratch/$ENV{USER}/cesm_input_datasets/atm/datm7 + $CIME_OUTPUT_ROOT/cesm_archive/$CASE + $CIME_OUTPUT_ROOT/cesm_baselines + /$CIME_OUTPUT_ROOT/cesm_tools/cprnc/cprnc + 4 + slurm + rgknox at lbl dot gov and glemieux at lbl dot gov + 16 + 16 + TRUE + + mpirun + + -np {{ total_tasks }} + -npernode $MAX_MPITASKS_PER_NODE + + + + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /usr/Modules/init/perl.pm + /usr/Modules/python.py + module + module + /usr/Modules/bin/modulecmd perl + /usr/Modules/bin/modulecmd python + + + cmake + perl xml-libxml switch python/3.6 + + + intel/2016.4.072 + mkl + + + netcdf/4.4.1.1-intel-s + + + openmpi + netcdf/4.4.1.1-intel-p + + + + diff --git a/machines/lobata/config_machines.xml b/machines/lobata/config_machines.xml new file mode 100644 index 00000000..ea1678d4 --- /dev/null +++ b/machines/lobata/config_machines.xml @@ -0,0 +1,45 @@ + + FATES development machine at LBNL, System76 Thelio Massive Workstation Pop!_OS 20.04 + lobata + LINUX + gnu + openmpi + $ENV{HOME}/scratch/ + /data/cesmdataroot/inputdata + /data/cesmdataroot/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{HOME}/scratch/ctsm-baselines + /home/glemieux/Repos/cime/tools/cprnc/cprnc + make + 16 + none + glemieux at lbl dot gov + 4 + 4 + FALSE + + mpirun + + -np {{ total_tasks }} + --map-by ppr:{{ tasks_per_node }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread + + + + /usr/share/modules/init/python.py + /usr/share/modules/init/perl.pm + /usr/share/modules/init/sh + /usr/share/modules/init/csh + /usr/bin/modulecmd python + /usr/bin/modulecmd perl + module + module + + + hdf5 + netcdf-c + netcdf-fortran + esmf + + + + diff --git a/machines/lonestar5/config_machines.xml b/machines/lonestar5/config_machines.xml new file mode 100644 index 00000000..db9f9040 --- /dev/null +++ b/machines/lonestar5/config_machines.xml @@ -0,0 +1,56 @@ + + + Lonestar5 cluster at TACC, OS is Linux (intel), batch system is SLURM + .*ls5\.tacc\.utexas\.edu + LINUX + intel + mpich + $ENV{SCRATCH} + /work/02503/edwardsj/CESM/inputdata + /work/02503/edwardsj/CESM/inputdata/lmwg + $CIME_OUTPUT_ROOT/cesm_archive/$CASE + /work/02503/edwardsj/CESM/cesm_baselines + /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc + 4 + slurm + cseg + 48 + 24 + FALSE + + srun + + --ntasks={{ total_tasks }} + + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + + cmake + + + intel/18.0.2 + + + netcdf/4.6.2 + + + cray_mpich + + + pnetcdf/1.8.0 + parallel-netcdf/4.6.2 + + + + diff --git a/machines/melvin/config_machines.xml b/machines/melvin/config_machines.xml new file mode 100644 index 00000000..02b90343 --- /dev/null +++ b/machines/melvin/config_machines.xml @@ -0,0 +1,71 @@ + + Linux workstation for Jenkins testing + (melvin|watson) + LINUX + sonproxy.sandia.gov:80 + gnu + openmpi + /sems-data-store/ACME/timings + $ENV{HOME}/acme/scratch + /sems-data-store/ACME/inputdata + /sems-data-store/ACME/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /sems-data-store/ACME/baselines + /sems-data-store/ACME/cprnc/build/cprnc + make + 32 + acme_developer + none + jgfouca at sandia dot gov + 64 + 64 + + mpirun + + -np {{ total_tasks }} + --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread + + + + /usr/share/Modules/init/python.py + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/bin/modulecmd python + /usr/bin/modulecmd perl + module + module + + + sems-env + acme-env + sems-git + sems-python/2.7.9 + sems-cmake/2.8.12 + + + sems-gcc/5.3.0 + + + sems-intel/16.0.3 + + + sems-netcdf/4.4.1/exo + acme-pfunit/3.2.8/base + + + sems-openmpi/1.8.7 + sems-netcdf/4.4.1/exo_parallel + + + + $ENV{SEMS_NETCDF_ROOT} + 64M + spread + threads + + + $ENV{SEMS_NETCDF_ROOT} + + + diff --git a/machines/modex/config_machines.xml b/machines/modex/config_machines.xml new file mode 100644 index 00000000..e3d64699 --- /dev/null +++ b/machines/modex/config_machines.xml @@ -0,0 +1,57 @@ + + Medium sized linux cluster at BNL, torque scheduler. + LINUX + gnu + openmpi,mpi-serial + /data/$ENV{USER} + /data/Model_Data/cesm_input_datasets/ + /data/Model_Data/cesm_input_datasets/atm/datm7 + $CIME_OUTPUT_ROOT/cesm_archive/$CASE + $CIME_OUTPUT_ROOT/cesm_baselines + /data/software/cesm_tools/cprnc/cprnc + 4 + pbs + rgknox at lbl dot gov and sserbin at bnl gov + 12 + 12 + 12 + FALSE + + mpirun + + -np {{ total_tasks }} + -npernode $MAX_TASKS_PER_NODE + + + + /etc/profile.d/modules.sh + /etc/profile.d/modules.csh + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + module + module + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + + + perl/5.22.1 + libxml2/2.9.2 + maui/3.3.1 + python/2.7.13 + + + gcc/5.4.0 + gfortran/5.4.0 + hdf5/1.8.19fates + netcdf/4.4.1.1-gnu540-fates + openmpi/2.1.1-gnu540 + + + openmpi/2.1.1-gnu540 + + + + /data/software/hdf5/1.8.19fates + /data/software/netcdf/4.4.1.1-gnu540-fates + + diff --git a/machines/olympus/config_machines.xml b/machines/olympus/config_machines.xml new file mode 100644 index 00000000..db7da51c --- /dev/null +++ b/machines/olympus/config_machines.xml @@ -0,0 +1,45 @@ + + PNL cluster, os is Linux (pgi), batch system is SLURM + LINUX + pgi + mpich + /pic/scratch/$USER + /pic/scratch/tcraig/IRESM/inputdata + /pic/scratch/tcraig/IRESM/inputdata/atm/datm7 + /pic/scratch/$USER/archive/$CASE + /pic/scratch/tcraig/IRESM/ccsm_baselines + /pic/scratch/tcraig/IRESM/tools/cprnc/cprnc + 8 + slurm + tcraig -at- ucar.edu + 32 + 32 + FALSE + + mpiexec_mpt + + --mpi=none + -n={{ total_tasks }} + --kill-on-bad-exit + + + + /share/apps/modules/Modules/3.2.7/init/perl.pm + /share/apps/modules/Modules/3.2.7/init/csh + /share/apps/modules/Modules/3.2.7/init/sh + /share/apps/modules/Modules/3.2.7/bin/modulecmd perl + module + module + + + precision/i4 + pgi/11.8 + mvapich2/1.7 + netcdf/4.1.3 + + + + 64M + + + diff --git a/machines/perlmutter/config_machines.xml b/machines/perlmutter/config_machines.xml new file mode 100644 index 00000000..358f0176 --- /dev/null +++ b/machines/perlmutter/config_machines.xml @@ -0,0 +1,95 @@ + + NERSC EX AMD EPYC, os is CNL, 64 pes/node, batch system is Slurm + $ENV{NERSC_HOST}:perlmutter + CNL + gnu,cray,nvidia,aocc + mpich + mp9_g + $ENV{SCRATCH} + /global/cfs/cdirs/ccsm1/inputdata + /global/cfs/cdirs/ccsm1/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /global/cfs/cdirs/ccsm1/ccsm_baselines + /global/cfs/cdirs/ccsm1/tools/cprnc.perlmutter/cprnc + 8 + slurm + cseg + 128 + 4 + 64 + TRUE + + srun + + --label + -n {{ total_tasks }} + -c {{ srun_binding }} + + + + /usr/share/lmod/lmod/init/perl + /usr/share/lmod/lmod/init/env_modules_python.py + /usr/share/lmod/lmod/init/sh + /usr/share/lmod/lmod/init/csh + /usr/share/lmod/lmod/libexec/lmod perl + /usr/share/lmod/lmod/libexec/lmod python + module + module + + PrgEnv-nvidia + PrgEnv-cray + PrgEnv-aocc + PrgEnv-gnu + nvidia + cce + gnu + aocc + cray-parallel-netcdf + cray-hdf5-parallel + cray-libsci + cray-mpich + cray-hdf5 + cray-netcdf-hdf5parallel + cray-netcdf + craype + + + + PrgEnv-cray + cce cce/12.0.3 + + + PrgEnv-gnu + gcc gcc/11.2.0 + + + craype craype/2.7.10 + + + cray-libsci/21.08.1.2 + + + cray-mpich/8.1.9 + + + cray-netcdf-hdf5parallel + cray-hdf5-parallel + cray-parallel-netcdf + cray-hdf5/1.12.0.7 + cray-netcdf/4.7.4.7 + + + cray-hdf5-parallel/1.12.0.7 + cray-netcdf-hdf5parallel/4.7.4.7 + cray-parallel-netcdf/1.12.1.7 + + + cmake/3.20.5 + + + + 256M + + + diff --git a/machines/pleiades-bro/config_machines.xml b/machines/pleiades-bro/config_machines.xml new file mode 100644 index 00000000..333e06d4 --- /dev/null +++ b/machines/pleiades-bro/config_machines.xml @@ -0,0 +1,54 @@ + + NASA/AMES Linux Cluster, Linux (ia64), 2.4 GHz Broadwell Intel Xeon E5-2680v4 processors, 28 pes/node (two 14-core processors) and 128 GB of memory/node, batch system is PBS + LINUX + intel + mpt + /nobackup/$USER + /nobackup/fvitt/csm/inputdata + /nobackup/fvitt/csm/inputdata/atm/datm7 + /nobackup/$USER/archive/$CASE + /nobackup/fvitt/cesm_baselines + /u/fvitt/bin/cprnc + 8 + pbs + fvitt -at- ucar.edu + 28 + 28 + TRUE + + mpiexec_mpt + + -n {{ total_tasks }} + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/share/Modules/init/python.py + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + nas + pkgsrc + python3 + comp-intel/2020.4.304 + mpi-hpe/mpt.2.25 + szip/2.1.1 + hdf4/4.2.12 + hdf5/1.8.18_mpt + netcdf/4.4.1.1_mpt + + + + 1024 + 100000 + 16 + 256M + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk + + + diff --git a/machines/pleiades-has/config_machines.xml b/machines/pleiades-has/config_machines.xml new file mode 100644 index 00000000..9c274c94 --- /dev/null +++ b/machines/pleiades-has/config_machines.xml @@ -0,0 +1,53 @@ + + NASA/AMES Linux Cluster, Linux (ia64), 2.5 GHz Haswell Intel Xeon E5-2680v3 processors, 24 pes/node (two 12-core processors) and 128 GB of memory/node, batch system is PBS + LINUX + intel + mpt + /nobackup/$USER + /nobackup/fvitt/csm/inputdata + /nobackup/fvitt/csm/inputdata/atm/datm7 + /nobackup/$USER/archive/$CASE + /nobackup/fvitt/cesm_baselines + /u/fvitt/bin/cprnc + 8 + pbs + fvitt -at- ucar.edu + 24 + 24 + TRUE + + mpiexec_mpt + + -n {{ total_tasks }} + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/share/Modules/init/python.py + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + nas + pkgsrc + python3 + comp-intel/2020.4.304 + mpi-hpe/mpt.2.25 + szip/2.1.1 + hdf4/4.2.12 + hdf5/1.8.18_mpt + netcdf/4.4.1.1_mpt + + + + 1024 + 100000 + 16 + 256M + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk + + diff --git a/machines/pleiades-ivy/config_machines.xml b/machines/pleiades-ivy/config_machines.xml new file mode 100644 index 00000000..cf231792 --- /dev/null +++ b/machines/pleiades-ivy/config_machines.xml @@ -0,0 +1,53 @@ + + NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.8 GHz Ivy Bridge processors, 20 cores/node and 3.2 GB of memory per core, batch system is PBS + LINUX + intel + mpich + /nobackup/$USER + /nobackup/fvitt/csm/inputdata + /nobackup/fvitt/csm/inputdata/atm/datm7 + /nobackup/$USER/archive/$CASE + /nobackup/fvitt/cesm_baselines + /u/fvitt/bin/cprnc + 8 + pbs + fvitt -at- ucar.edu + 20 + 20 + TRUE + + mpiexec_mpt + + -n {{ total_tasks }} + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/share/Modules/init/python.py + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + nas + pkgsrc + python3 + comp-intel/2020.4.304 + mpi-hpe/mpt.2.25 + szip/2.1.1 + hdf4/4.2.12 + hdf5/1.8.18_mpt + netcdf/4.4.1.1_mpt + + + + 1024 + 100000 + 16 + 256M + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk + + diff --git a/machines/pleiades-san/config_machines.xml b/machines/pleiades-san/config_machines.xml new file mode 100644 index 00000000..8f484578 --- /dev/null +++ b/machines/pleiades-san/config_machines.xml @@ -0,0 +1,54 @@ + + NASA/AMES Linux Cluster, Linux (ia64), Altix ICE, 2.6 GHz Sandy Bridge processors, 16 cores/node and 32 GB of memory, batch system is PBS + LINUX + intel + mpt + /nobackup/$USER + /nobackup/fvitt/csm/inputdata + /nobackup/fvitt/csm/inputdata/atm/datm7 + /nobackup/$USER/archive/$CASE + /nobackup/fvitt/cesm_baselines + /u/fvitt/bin/cprnc + 8 + pbs + fvitt -at- ucar.edu + 16 + 16 + TRUE + + mpiexec_mpt + + -n {{ total_tasks }} + + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/share/Modules/init/python.py + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + nas + pkgsrc + python3 + comp-intel/2020.4.304 + mpi-hpe/mpt.2.25 + szip/2.1.1 + hdf4/4.2.12 + hdf5/1.8.18_mpt + netcdf/4.4.1.1_mpt + + + + 1024 + 100000 + 16 + 256M + /home6/fvitt/ESMFv8.4.1/lib/libO/Linux.intel.64.mpt.default/esmf.mk + + + diff --git a/machines/sandia-srn-sems/config_machines.xml b/machines/sandia-srn-sems/config_machines.xml new file mode 100644 index 00000000..1aa7a487 --- /dev/null +++ b/machines/sandia-srn-sems/config_machines.xml @@ -0,0 +1,53 @@ + + + Linux workstation at Sandia on SRN with SEMS TPL modules + (s999964|climate|penn) + LINUX + wwwproxy.sandia.gov:80 + gnu + openmpi + /sems-data-store/ACME/timings + $ENV{HOME}/acme/scratch + /sems-data-store/ACME/inputdata + /sems-data-store/ACME/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /sems-data-store/ACME/baselines + /sems-data-store/ACME/cprnc/build/cprnc + make + 32 + acme_developer + none + jgfouca at sandia dot gov + 64 + 64 + + mpirun + + -np {{ total_tasks }} + + + + /usr/share/Modules/init/python.py + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/bin/modulecmd python + /usr/bin/modulecmd perl + module + module + + + sems-env + sems-git + sems-python/2.7.9 + sems-gcc/5.1.0 + sems-openmpi/1.8.7 + sems-cmake/2.8.12 + sems-netcdf/4.3.2/parallel + + + + $ENV{SEMS_NETCDF_ROOT} + $ENV{SEMS_NETCDF_ROOT} + + diff --git a/machines/sandiatoss3/config_machines.xml b/machines/sandiatoss3/config_machines.xml new file mode 100644 index 00000000..6db2fe9f --- /dev/null +++ b/machines/sandiatoss3/config_machines.xml @@ -0,0 +1,64 @@ + + SNL clust + (skybridge|chama)-login + LINUX + wwwproxy.sandia.gov:80 + intel + openmpi + /projects/ccsm/timings + /gscratch/$USER/acme_scratch/$MACH + /projects/ccsm/inputdata + /projects/ccsm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /projects/ccsm/ccsm_baselines + /projects/ccsm/cprnc/build/cprnc_wrap + 8 + acme_integration + slurm + jgfouca at sandia dot gov + 16 + 16 + TRUE + + + mpirun + + -np {{ total_tasks }} + -npernode $MAX_MPITASKS_PER_NODE + + + + /usr/share/Modules/init/python.py + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/bin/modulecmd python + /usr/bin/modulecmd perl + module + module + + + sems-env + sems-git + sems-python/2.7.9 + gnu/4.9.2 + intel/intel-15.0.3.187 + libraries/intel-mkl-15.0.2.164 + libraries/intel-mkl-15.0.2.164 + + + openmpi-intel/1.8 + sems-hdf5/1.8.12/parallel + sems-netcdf/4.3.2/parallel + sems-hdf5/1.8.12/base + sems-netcdf/4.3.2/base + + + + $ENV{SEMS_NETCDF_ROOT} + 64M + + + $ENV{SEMS_NETCDF_ROOT} + + diff --git a/machines/stampede2-knl/config_machines.xml b/machines/stampede2-knl/config_machines.xml new file mode 100644 index 00000000..c72ab44b --- /dev/null +++ b/machines/stampede2-knl/config_machines.xml @@ -0,0 +1,57 @@ + + Intel Xeon Phi 7250 ("Knights Landing") , batch system is SLURM + LINUX + intel + impi,mvapich2 + $ENV{SCRATCH} + /work/02503/edwardsj/CESM/inputdata + /work/02503/edwardsj/CESM/inputdata/lmwg + $ENV{WORK}/archive/$CASE + /work/02503/edwardsj/CESM/cesm_baselines + /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc + 4 + slurm + cseg + 256 + 64 + + ibrun + + + ibrun + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + TACC + python/2.7.13 + intel/18.0.2 + cmake/3.16.1 + + + mvapich2/2.3.1 + pnetcdf/1.11 + parallel-netcdf/4.6.2 + + + mvapich2 + impi/18.0.2 + pnetcdf/1.11 + parallel-netcdf/4.6.2 + + + netcdf/4.3.3.1 + + + + 256M + + diff --git a/machines/stampede2-skx/config_machines.xml b/machines/stampede2-skx/config_machines.xml new file mode 100644 index 00000000..ee7e7109 --- /dev/null +++ b/machines/stampede2-skx/config_machines.xml @@ -0,0 +1,77 @@ + + + Intel Xeon Platinum 8160 ("Skylake"),48 cores on two sockets (24 cores/socket) , batch system is SLURM + .*stampede2 + LINUX + intel + impi,mvapich2 + $ENV{SCRATCH} + /work/02503/edwardsj/CESM/inputdata + /work/02503/edwardsj/CESM/inputdata/lmwg + $ENV{WORK}/archive/$CASE + /work/02503/edwardsj/CESM/cesm_baselines + /work/02503/edwardsj/CESM/cime/tools/cprnc/cprnc + 4 + slurm + cseg + 96 + 48 + + ibrun + + -n {{ total_tasks }} + + + + ibrun + + -n {{ total_tasks }} + + + + /opt/apps/lmod/lmod/init/perl + /opt/apps/lmod/lmod/init/env_modules_python.py + /opt/apps/lmod/lmod/init/sh + /opt/apps/lmod/lmod/init/csh + /opt/apps/lmod/lmod/libexec/lmod perl + /opt/apps/lmod/lmod/libexec/lmod python + module + module + + + TACC + python/2.7.13 + intel/18.0.2 + cmake/3.16.1 + + + mvapich2/2.3.1 + pnetcdf/1.11 + parallel-netcdf/4.6.2 + + + mvapich2 + impi/18.0.2 + pnetcdf/1.11 + parallel-netcdf/4.6.2 + + + netcdf/4.3.3.1 + + + + 256M + + + /work/01118/tg803972/stampede2/ESMF-INSTALL/8.0.0bs38/lib/libO/Linux.intel.64.intelmpi.default/esmf.mk + + + ON + SUMMARY + /work/06242/tg855414/stampede2/FV3GFS/benchmark-inputs/2012010100/gfs/fcst + /work/06242/tg855414/stampede2/FV3GFS/fix_am + /work/06242/tg855414/stampede2/FV3GFS/addon + + + + diff --git a/machines/theia/config_machines.xml b/machines/theia/config_machines.xml new file mode 100644 index 00000000..dfe8594b --- /dev/null +++ b/machines/theia/config_machines.xml @@ -0,0 +1,55 @@ + + theia + tfe + LINUX + intel + impi + nems + + /scratch4/NCEPDEV/nems/noscrub/$USER/cimecases + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/inputdata + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/BASELINES + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/cesmdataroot/tools/cprnc + make + 8 + slurm + cseg + 24 + 24 + TRUE + + srun + + -n $TOTALPES + + + + + + + /apps/lmod/lmod/init/sh + /apps/lmod/lmod/init/csh + module + module + /apps/lmod/lmod/libexec/lmod python + + + intel/15.1.133 + impi/5.1.1.109 + netcdf/4.3.0 + pnetcdf + /scratch4/NCEPDEV/nems/noscrub/emc.nemspara/soft/modulefiles + yaml-cpp + esmf/8.0.0bs29g + + + + ON + SUMMARY + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/benchmark-inputs/2012010100/gfs/fcst + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/fix_am + /scratch4/NCEPDEV/nems/noscrub/Rocky.Dunlap/INPUTDATA/addon + + diff --git a/machines/theta/config_machines.xml b/machines/theta/config_machines.xml new file mode 100644 index 00000000..72eaa884 --- /dev/null +++ b/machines/theta/config_machines.xml @@ -0,0 +1,88 @@ + + ALCF Cray XC* KNL, os is CNL, 64 pes/node, batch system is cobalt + theta.* + CNL + intel,gnu,cray + mpt + CESM_Highres_Testing + /projects/CESM_Highres_Testing/cesm/scratch/$USER + /projects/CESM_Highres_Testing/cesm/inputdata + /projects/CESM_Highres_Testing/cesm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /projects/CESM_Highres_Testing/cesm/baselines + /projects/CESM_Highres_Testing/cesm/tools/cprnc/cprnc + 8 + cobalt_theta + cseg + 64 + 64 + TRUE + + aprun + + -n {{ total_tasks }} + -N {{ tasks_per_node }} + --cc depth -d $OMP_NUM_THREADS + -e OMP_STACKSIZE=64M + -e OMP_NUM_THREADS=$OMP_NUM_THREADS + + + + /opt/modules/default/init/perl.pm + /opt/modules/default/init/python.py + /opt/modules/default/init/sh + /opt/modules/default/init/csh + /opt/modules/default/bin/modulecmd perl + /opt/modules/default/bin/modulecmd python + module + module + + craype-mic-knl + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + intel + cce + cray-parallel-netcdf + cray-hdf5-parallel + pmi + cray-libsci + cray-mpich + cray-netcdf + cray-hdf5 + cray-netcdf-hdf5parallel + craype + papi + + + + PrgEnv-intel/6.0.4 + intel intel/18.0.0.128 + cray-libsci + + + + PrgEnv-cray/6.0.4 + cce cce/8.7.0 + + + PrgEnv-gnu/6.0.4 + gcc gcc/7.3.0 + + + papi/5.6.0.1 + craype craype/2.5.14 + + + cray-libsci/18.04.1 + + + cray-mpich/7.7.0 + + + cray-netcdf-hdf5parallel/4.4.1.1.6 + cray-hdf5-parallel/1.10.1.1 + cray-parallel-netcdf/1.8.1.3 + + + diff --git a/machines/thunder/config_machines.xml b/machines/thunder/config_machines.xml new file mode 100644 index 00000000..6be7e1cd --- /dev/null +++ b/machines/thunder/config_machines.xml @@ -0,0 +1,74 @@ + + NCAR ARM platform, os is Linux, 64/128 pes/node, batch system is SLURM + .*.thunder.ucar.edu + LINUX + + armgcc,gnu,arm + openmpi + /glade/scratch/$USER + $ENV{CESMDATAROOT}/inputdata + $DIN_LOC_ROOT/CTSM_datm_forcing_data + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cprnc/cprnc + 16 + slurm + cseg + 64 + 128 + + mpiexec + + --tag-output + -np {{ total_tasks }} + + + + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/perl + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/env_modules_python.py + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/csh + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/init/sh + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod perl + /glade/u/apps/th/opt/lmod/8.1.7/lmod/lmod/libexec/lmod python + module + module + + + ncarenv/1.3 + cmake/3.14.4 + + + arm/19.3 + + + armgcc/8.2.0 + + + gnu/9.1.0 + openblas/0.3.6 + esmf_libs/8.0.0 + + + + ncarcompilers/0.5.0 + + + openmpi/4.0.3 + netcdf-mpi/4.7.1 + pnetcdf/1.12.1 + + + netcdf/4.7.1 + + + esmf-8.0.0-ncdfio-uni-g + + + esmf-8.0.0-ncdfio-uni-O + + + + 256M + $ENV{NETCDF} + + diff --git a/machines/ubuntu-latest/config_machines.xml b/machines/ubuntu-latest/config_machines.xml new file mode 100644 index 00000000..e393914b --- /dev/null +++ b/machines/ubuntu-latest/config_machines.xml @@ -0,0 +1,32 @@ + + + used for github testing + + $ENV{CIME_TEST_PLATFORM}:ubuntu-latest + LINUX + + gnu + openmpi + none + + $ENV{HOME}/cesm/scratch + $ENV{HOME}/cesm/inputdata + $ENV{HOME}/cesm/inputdata/lmwg + $ENV{HOME}/cesm/archive/$CASE + $ENV{HOME}/cesm/cesm_baselines + + make + 8 + none + jedwards + 4 + 4 + FALSE + + mpiexec + + -n {{ total_tasks }} + + + + diff --git a/machines/zeus/config_machines.xml b/machines/zeus/config_machines.xml new file mode 100644 index 00000000..f67a92ab --- /dev/null +++ b/machines/zeus/config_machines.xml @@ -0,0 +1,80 @@ + + + CMCC Lenovo ThinkSystem SD530, os is Linux, 36 pes/node, batch system is LSF + (login[1,2]-ib|n[0-9][0-9][0-9]-ib) + LINUX + intel + impi,mpi-serial + R000 + /work/$ENV{DIVISION}/$ENV{USER}/CESM2 + $ENV{CESMDATAROOT}/inputdata + $DIN_LOC_ROOT/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/ccsm_baselines + $ENV{CESMDATAROOT}/cesm2_tools/cprnc/cprnc + /usr/lib64/perl5:/usr/share/perl5 + 8 + lsf + cmcc + 72 + 36 + TRUE + + mpirun + + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + /usr/share/Modules/init/csh + /usr/share/Modules/init/sh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + + + intel20.1/20.1.217 + intel20.1/szip/2.1.1 + cmake/3.17.3 + curl/7.70.0 + + + intel20.1/hdf5/1.12.0 + intel20.1/netcdf/C_4.7.4-F_4.5.3_CXX_4.3.1 + + + impi20.1/19.7.217 + impi20.1/hdf5/1.12.0 + impi20.1/netcdf/C_4.7.4-F_4.5.3_CXX_4.3.1 + impi20.1/parallel-netcdf/1.12.1 + + + impi20.1/esmf/8.1.1-intelmpi-64-g + + + impi20.1/esmf/8.1.1-intelmpi-64-O + + + intel20.1/esmf/8.1.1-mpiuni-64-g + + + intel20.1/esmf/8.1.1-mpiuni-64-O + + + + /data/inputs/CESM/xios-2.5 + + + 1 + gpfs + 0 + 60 + skx + skx_avx512 + lsf + 1 + {{ num_nodes }} + + From 38290f495ab29108007666d582225b0da7999198 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Mon, 11 Dec 2023 10:48:50 -0700 Subject: [PATCH 63/95] remove NODENAME_REGEX from individual machines files --- machines/athena/config_machines.xml | 1 - machines/aws-hpc6a/config_machines.xml | 1 - machines/casper/config_machines.xml | 1 - machines/centos7-linux/config_machines.xml | 4 +++- machines/cheyenne/config_machines.xml | 1 - machines/coeus/config_machines.xml | 1 - machines/derecho/config_machines.xml | 1 - machines/greenplanet-sky24/config_machines.xml | 1 - machines/gust/config_machines.xml | 1 - machines/homebrew/config_machines.xml | 4 +++- machines/izumi/config_machines.xml | 1 - machines/lobata/config_machines.xml | 1 - machines/lonestar5/config_machines.xml | 2 -- machines/melvin/config_machines.xml | 1 - machines/perlmutter/config_machines.xml | 1 - machines/sandia-srn-sems/config_machines.xml | 1 - machines/sandiatoss3/config_machines.xml | 1 - machines/stampede2-skx/config_machines.xml | 1 - machines/theia/config_machines.xml | 1 - machines/theta/config_machines.xml | 1 - machines/thunder/config_machines.xml | 1 - machines/ubuntu-latest/config_machines.xml | 1 - machines/userdefined_laptop_template/config_machines.xml | 3 ++- machines/zeus/config_machines.xml | 1 - 24 files changed, 8 insertions(+), 25 deletions(-) diff --git a/machines/athena/config_machines.xml b/machines/athena/config_machines.xml index 651172a0..9e9b5e30 100644 --- a/machines/athena/config_machines.xml +++ b/machines/athena/config_machines.xml @@ -1,7 +1,6 @@ CMCC IBM iDataPlex, os is Linux, 16 pes/node, batch system is LSFd mpich - .*.cluster.net LINUX intel,intel15 mpich2 diff --git a/machines/aws-hpc6a/config_machines.xml b/machines/aws-hpc6a/config_machines.xml index faa6f097..6d4b9bfc 100644 --- a/machines/aws-hpc6a/config_machines.xml +++ b/machines/aws-hpc6a/config_machines.xml @@ -1,6 +1,5 @@ AWS HPC6a (96-core AMD) Nodes - regular-dy-computehpc6a-.* LINUX intel impi diff --git a/machines/casper/config_machines.xml b/machines/casper/config_machines.xml index 276d69d7..0cc9917f 100644 --- a/machines/casper/config_machines.xml +++ b/machines/casper/config_machines.xml @@ -1,6 +1,5 @@ NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs - casper* LINUX nvhpc,intel openmpi diff --git a/machines/centos7-linux/config_machines.xml b/machines/centos7-linux/config_machines.xml index 3579632d..786fb1be 100644 --- a/machines/centos7-linux/config_machines.xml +++ b/machines/centos7-linux/config_machines.xml @@ -3,7 +3,9 @@ Example port to centos7 linux system with gcc, netcdf, pnetcdf and mpich using modules from http://www.admin-magazine.com/HPC/Articles/Environment-Modules - regex.expression.matching.your.machine + LINUX https://howto.get.out gnu diff --git a/machines/cheyenne/config_machines.xml b/machines/cheyenne/config_machines.xml index 5a17247b..c84c7502 100644 --- a/machines/cheyenne/config_machines.xml +++ b/machines/cheyenne/config_machines.xml @@ -1,6 +1,5 @@ NCAR SGI platform, os is Linux, 36 pes/node, batch system is PBS - .*.?cheyenne\d?.ucar.edu MPT: Launcher network accept (MPI_LAUNCH_TIMEOUT) timed out diff --git a/machines/coeus/config_machines.xml b/machines/coeus/config_machines.xml index 7e2cb629..f66ff9da 100644 --- a/machines/coeus/config_machines.xml +++ b/machines/coeus/config_machines.xml @@ -2,7 +2,6 @@ Portland State University Coeus Cluster Dec 2019 CentOS 7 - (login[1,2].cluster|compute[0-9]*.cluster) LINUX gnu mvapich2 diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 0589ece8..633fe74d 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -1,6 +1,5 @@ NCAR AMD EPYC system - $ENV{NCAR_HOST}:derecho CNL intel,gnu,nvhpc,intel-oneapi,intel-classic mpich diff --git a/machines/greenplanet-sky24/config_machines.xml b/machines/greenplanet-sky24/config_machines.xml index 35d1ccf4..9b46fb06 100644 --- a/machines/greenplanet-sky24/config_machines.xml +++ b/machines/greenplanet-sky24/config_machines.xml @@ -1,6 +1,5 @@ UCI Linux Cluster; 40 pes/node, batch system is slurm - gplogin\d.gp.local LINUX diff --git a/machines/gust/config_machines.xml b/machines/gust/config_machines.xml index b88172df..7998a3d3 100644 --- a/machines/gust/config_machines.xml +++ b/machines/gust/config_machines.xml @@ -1,6 +1,5 @@ NCAR AMD EPYC test system 16 CPU nodes 2 GPU nodes - gu.*.hpc.ucar.edu CNL intel,gnu,cray,nvhpc,intel-oneapi,intel-classic mpich diff --git a/machines/homebrew/config_machines.xml b/machines/homebrew/config_machines.xml index 17f76234..95adb826 100644 --- a/machines/homebrew/config_machines.xml +++ b/machines/homebrew/config_machines.xml @@ -19,7 +19,9 @@ changing the machine name (MACH="homebrew") to your machine name. - something.matching.your.machine.hostname + Darwin gnu mpich diff --git a/machines/izumi/config_machines.xml b/machines/izumi/config_machines.xml index 8cab2739..1a89600d 100644 --- a/machines/izumi/config_machines.xml +++ b/machines/izumi/config_machines.xml @@ -1,6 +1,5 @@ NCAR CGD Linux Cluster 48 pes/node, batch system is PBS - ^i.*\.ucar\.edu LINUX intel,nag,gnu mvapich2,openmpi diff --git a/machines/lobata/config_machines.xml b/machines/lobata/config_machines.xml index ea1678d4..bdca2528 100644 --- a/machines/lobata/config_machines.xml +++ b/machines/lobata/config_machines.xml @@ -1,6 +1,5 @@ FATES development machine at LBNL, System76 Thelio Massive Workstation Pop!_OS 20.04 - lobata LINUX gnu openmpi diff --git a/machines/lonestar5/config_machines.xml b/machines/lonestar5/config_machines.xml index db9f9040..6c9f368f 100644 --- a/machines/lonestar5/config_machines.xml +++ b/machines/lonestar5/config_machines.xml @@ -1,7 +1,5 @@ - Lonestar5 cluster at TACC, OS is Linux (intel), batch system is SLURM - .*ls5\.tacc\.utexas\.edu LINUX intel mpich diff --git a/machines/melvin/config_machines.xml b/machines/melvin/config_machines.xml index 02b90343..4439d06b 100644 --- a/machines/melvin/config_machines.xml +++ b/machines/melvin/config_machines.xml @@ -1,6 +1,5 @@ Linux workstation for Jenkins testing - (melvin|watson) LINUX sonproxy.sandia.gov:80 gnu diff --git a/machines/perlmutter/config_machines.xml b/machines/perlmutter/config_machines.xml index 358f0176..a89159b8 100644 --- a/machines/perlmutter/config_machines.xml +++ b/machines/perlmutter/config_machines.xml @@ -1,6 +1,5 @@ NERSC EX AMD EPYC, os is CNL, 64 pes/node, batch system is Slurm - $ENV{NERSC_HOST}:perlmutter CNL gnu,cray,nvidia,aocc mpich diff --git a/machines/sandia-srn-sems/config_machines.xml b/machines/sandia-srn-sems/config_machines.xml index 1aa7a487..8ead7d8f 100644 --- a/machines/sandia-srn-sems/config_machines.xml +++ b/machines/sandia-srn-sems/config_machines.xml @@ -1,7 +1,6 @@ Linux workstation at Sandia on SRN with SEMS TPL modules - (s999964|climate|penn) LINUX wwwproxy.sandia.gov:80 gnu diff --git a/machines/sandiatoss3/config_machines.xml b/machines/sandiatoss3/config_machines.xml index 6db2fe9f..0885838a 100644 --- a/machines/sandiatoss3/config_machines.xml +++ b/machines/sandiatoss3/config_machines.xml @@ -1,6 +1,5 @@ SNL clust - (skybridge|chama)-login LINUX wwwproxy.sandia.gov:80 intel diff --git a/machines/stampede2-skx/config_machines.xml b/machines/stampede2-skx/config_machines.xml index ee7e7109..21d5c853 100644 --- a/machines/stampede2-skx/config_machines.xml +++ b/machines/stampede2-skx/config_machines.xml @@ -1,7 +1,6 @@ Intel Xeon Platinum 8160 ("Skylake"),48 cores on two sockets (24 cores/socket) , batch system is SLURM - .*stampede2 LINUX intel impi,mvapich2 diff --git a/machines/theia/config_machines.xml b/machines/theia/config_machines.xml index dfe8594b..2add8964 100644 --- a/machines/theia/config_machines.xml +++ b/machines/theia/config_machines.xml @@ -1,6 +1,5 @@ theia - tfe LINUX intel impi diff --git a/machines/theta/config_machines.xml b/machines/theta/config_machines.xml index 72eaa884..a245e621 100644 --- a/machines/theta/config_machines.xml +++ b/machines/theta/config_machines.xml @@ -1,6 +1,5 @@ ALCF Cray XC* KNL, os is CNL, 64 pes/node, batch system is cobalt - theta.* CNL intel,gnu,cray mpt diff --git a/machines/thunder/config_machines.xml b/machines/thunder/config_machines.xml index 6be7e1cd..03263adb 100644 --- a/machines/thunder/config_machines.xml +++ b/machines/thunder/config_machines.xml @@ -1,6 +1,5 @@ NCAR ARM platform, os is Linux, 64/128 pes/node, batch system is SLURM - .*.thunder.ucar.edu LINUX armgcc,gnu,arm diff --git a/machines/ubuntu-latest/config_machines.xml b/machines/ubuntu-latest/config_machines.xml index e393914b..61158dd4 100644 --- a/machines/ubuntu-latest/config_machines.xml +++ b/machines/ubuntu-latest/config_machines.xml @@ -2,7 +2,6 @@ used for github testing - $ENV{CIME_TEST_PLATFORM}:ubuntu-latest LINUX gnu diff --git a/machines/userdefined_laptop_template/config_machines.xml b/machines/userdefined_laptop_template/config_machines.xml index 2d1005c5..e8b42c9d 100644 --- a/machines/userdefined_laptop_template/config_machines.xml +++ b/machines/userdefined_laptop_template/config_machines.xml @@ -5,7 +5,8 @@ desired layout (change '${HOME}/projects' to your prefered location). --> __USEFUL_DESCRIPTION__ - something.matching.your.machine.hostname + Darwin gnu mpich diff --git a/machines/zeus/config_machines.xml b/machines/zeus/config_machines.xml index f67a92ab..914ea9d9 100644 --- a/machines/zeus/config_machines.xml +++ b/machines/zeus/config_machines.xml @@ -1,7 +1,6 @@ CMCC Lenovo ThinkSystem SD530, os is Linux, 36 pes/node, batch system is LSF - (login[1,2]-ib|n[0-9][0-9][0-9]-ib) LINUX intel impi,mpi-serial From f42c53f29ce6de81b47b8434cce8459b7bafdf20 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Mon, 11 Dec 2023 11:41:11 -0700 Subject: [PATCH 64/95] replaces ncar_updates_231206 --- machines/casper/config_machines.xml | 51 ++++++++++++++--------------- machines/cmake_macros/derecho.cmake | 3 +- machines/config_batch.xml | 2 +- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/machines/casper/config_machines.xml b/machines/casper/config_machines.xml index 0cc9917f..afd46f00 100644 --- a/machines/casper/config_machines.xml +++ b/machines/casper/config_machines.xml @@ -31,45 +31,45 @@ -np {{ total_tasks }} - - /glade/u/apps/dav/opt/lmod/7.7.29/init/perl - /glade/u/apps/dav/opt/lmod/7.7.29/init/env_modules_python.py - /glade/u/apps/dav/opt/lmod/7.7.29/init/sh - /glade/u/apps/dav/opt/lmod/7.7.29/init/csh - /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod perl - /glade/u/apps/dav/opt/lmod/7.7.29/libexec/lmod python + + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/perl + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/env_modules_python.py + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/sh + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/init/csh + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/libexec/lmod perl + /glade/u/apps/casper/23.10/spack/opt/spack/lmod/8.7.24/gcc/7.5.0/m4jx/lmod/lmod/libexec/lmod python module module - ncarenv/1.3 - cmake/3.18.2 + ncarenv/23.10 + cmake/3.26.3 - nvhpc/22.2 + nvhpc/23.7 - intel/19.1.1 - mkl/2020.0.1 + intel/2023.2.1 + mkl/2023.2.0 - cuda/11.6 + cuda/12.2.1 - openmpi/4.1.4 - netcdf-mpi/4.8.1 - pnetcdf/1.12.3 + openmpi/4.1.6 + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 - netcdf/4.8.1 + netcdf/4.9.2 - openmpi/4.1.1 - netcdf-mpi/4.8.1 - pnetcdf/1.12.2 + openmpi/4.1.6 + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 - netcdf/4.7.4 + netcdf/4.9.2 @@ -99,11 +99,10 @@ ncarcompilers/0.5.0 - - pio/2.5.10 - - - pio/2.5.10d + + parallelio/2.6.2 + esmf/8.5.0 + ncarcompilers/1.0.0 diff --git a/machines/cmake_macros/derecho.cmake b/machines/cmake_macros/derecho.cmake index 2772a168..9b20dea6 100644 --- a/machines/cmake_macros/derecho.cmake +++ b/machines/cmake_macros/derecho.cmake @@ -1,11 +1,10 @@ if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() +set(MPI_SERIAL_PATH "$ENV{NCAR_ROOT_MPI_SERIAL}") set(NETCDF_PATH "$ENV{NETCDF}") set(PIO_FILESYSTEM_HINTS "lustre") set(PNETCDF_PATH "$ENV{PNETCDF}") # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") -#string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}") -#string(APPEND CPPDEFS " -DNO_SHR_VMATH ") string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 087480af..0b2caa48 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -235,7 +235,7 @@ -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=300GB:ngpus={{ ngpus_per_node }} From 3cd3a88ab1f3874e46e2673612b21dab355aef89 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 15 Dec 2023 07:19:52 -0700 Subject: [PATCH 65/95] add develop queue --- machines/config_batch.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index cb56c334..481083b3 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -370,7 +370,8 @@ -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 - main + main + develop From cc3dcc4368b07a8af3032d04db8eea83c50f3ac3 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 20 Dec 2023 10:53:24 -0700 Subject: [PATCH 66/95] use the system-wide mpi wrapper script to handle MPI-only and hybrid MPI/OpenMP configurations automatically --- machines/derecho/config_machines.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 633fe74d..974208d3 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -21,12 +21,7 @@ get_local_rank TRUE - mpiexec - - --label - --line-buffer - -n {{ total_tasks }} - + mpibind $ENV{LMOD_ROOT}/lmod/init/perl From 30a8f2885c63d68d92c81b19cd2372cab8f2886d Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 5 Jan 2024 09:22:48 -0700 Subject: [PATCH 67/95] add back the MPI debug flags and updated temporary mpibind script path --- machines/derecho/config_machines.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 974208d3..56c56ca8 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -21,7 +21,12 @@ get_local_rank TRUE - mpibind + /glade/work/rory/mpibind/mpibind + + --label + --line-buffer + -- + $ENV{LMOD_ROOT}/lmod/init/perl From 8793c520d224c133116a74b9429c49aba9308e73 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Fri, 5 Jan 2024 11:04:03 -0700 Subject: [PATCH 68/95] update to env script --- machines/derecho/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 56c56ca8..6f9d6fd6 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -21,7 +21,7 @@ get_local_rank TRUE - /glade/work/rory/mpibind/mpibind + mpibind --label --line-buffer From 6643b3ac3580ba56dfc5011b7b550d679c4c5078 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Tue, 9 Jan 2024 21:17:52 -0700 Subject: [PATCH 69/95] update gpu wrapper script on Derecho --- machines/derecho/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 6f9d6fd6..4d757cfb 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -18,7 +18,7 @@ 64 none,a100 none,openacc,openmp,combined - get_local_rank + set_gpu_rank TRUE mpibind From 9421e54d89ec570496f15cd9321835c5fb4e1947 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sat, 13 Jan 2024 07:38:14 -0700 Subject: [PATCH 70/95] workaround mpi-serial debug issue on derecho --- machines/derecho/config_machines.xml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 6f9d6fd6..02737765 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -27,7 +27,7 @@ --line-buffer -- - + $ENV{LMOD_ROOT}/lmod/init/perl $ENV{LMOD_ROOT}/lmod/init/env_modules_python.py @@ -74,15 +74,13 @@ cray-mpich/8.1.27 - - mpi-serial/2.3.0 - cuda/12.2.1 + mpi-serial/2.3.0 netcdf/4.9.2 @@ -91,14 +89,19 @@ parallel-netcdf/1.12.3 - - parallelio/2.6.2-debug - esmf/8.6.0b04-debug + + parallelio/2.6.2 + esmf/8.6.0 - + parallelio/2.6.2 - esmf/8.6.0b04 + esmf/8.6.0 + + + + parallelio/2.6.2-debug + esmf/8.6.0-debug From 9b3d428a8eeada21efcdbf27e1a83ec0544a21f9 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 30 Jan 2024 16:40:36 -0700 Subject: [PATCH 71/95] Fix esmf debug module setting --- machines/derecho/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 8fc5eddd..cd6d59d1 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -99,7 +99,7 @@ esmf/8.6.0 - + parallelio/2.6.2-debug esmf/8.6.0-debug From d1ff6e01e6fcdfc8f21e53c96c32dc908a2dd4e2 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Wed, 7 Feb 2024 13:44:50 -0700 Subject: [PATCH 72/95] Update the filenames for RRTMGP v1.7 --- machines/Depends.nvhpc | 43 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index b51e3c90..0771da43 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -40,35 +40,24 @@ macrop_driver.o \ shr_spfn_mod.o RRTMGP_OBJS=\ -mo_rte_util_array.o \ -mo_rte_sw.o \ -mo_rte_lw.o \ -mo_cloud_optics.o \ -mo_gas_optics_rrtmgp.o \ +rrtmgp_allsky.o \ +rrtmgp_rfmip_lw.o \ +rrtmgp_rfmip_sw.o \ +mo_fluxes_byband.o \ +mo_zenith_angle_spherical_correction.o \ +mo_rrtmgp_clr_all_sky.o \ mo_gas_concentrations.o \ +mo_aerosol_optics_rrtmgp_merra.o \ +mo_cloud_optics_rrtmgp.o \ +mo_gas_optics_rrtmgp.o \ +mo_gas_optics_rrtmgp_kernels.o \ +mo_rte_lw.o \ +mo_rte_sw.o \ +mo_rte_util_array_validation.o \ +mo_rte_util_array.o \ mo_fluxes_broadband_kernels.o \ -mo_fluxes_byband_kernels.o \ -mo_rrtmgp_util_reorder_kernels.o \ -mo_compute_bc.o\ -mo_fluxes_byband.o\ -mo_fluxes_bygpoint.o\ -mo_heating_rates.o\ -mo_rrtmgp_clr_all_sky.o\ -mo_cloud_sampling.o\ -mo_solar_variability.o\ -mo_gas_optics.o\ -mo_rrtmgp_constants.o\ -mo_rrtmgp_util_reorder.o\ -mo_rrtmgp_util_string.o\ -mo_fluxes.o\ -mo_optical_props.o\ -mo_rte_config.o\ -mo_rte_kind.o\ -mo_source_functions.o\ -mo_optical_props_kernels.o\ -mo_rte_solver_kernels.o\ -rrtmgp_driver.o\ -mo_gas_optics_kernels.o +mo_rte_solver_kernels.o \ +mo_optical_props_kernels.o ifeq ($(DEBUG),FALSE) $(PERFOBJS): %.o: %.F90 From e4dcc2c9bc97db5b3847b4466b73e08c0f22fdf1 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 1 Mar 2024 14:04:29 -0700 Subject: [PATCH 73/95] change the queue used for st_archive on derecho --- machines/config_batch.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index ac63ec53..290ca327 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -370,8 +370,8 @@ -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 + develop main - develop From 4b14fda6a3646f8e1b4934649034ca4d9435b899 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 8 Mar 2024 08:51:09 -0700 Subject: [PATCH 74/95] remove -V option due to issue #4594 --- machines/Depends.cray | 2 +- machines/config_batch.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/machines/Depends.cray b/machines/Depends.cray index 68679f02..362847c1 100644 --- a/machines/Depends.cray +++ b/machines/Depends.cray @@ -1,4 +1,4 @@ -NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o +NOOPTOBJS= ice_boundary.o dyn_comp.o unicon.o SnowHydrologyMod.o # RRTMGP contains openmp directives for running on GPUs. These directives need to be # disabled to allow building CAM with threading on CPUs enabled. diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 290ca327..02e5b82e 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -156,7 +156,7 @@ -r {{ rerunnable }} -j oe - -V + From bd805a28a821dfea13a4d52847800cec10ba1153 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 8 Mar 2024 09:10:46 -0700 Subject: [PATCH 75/95] add -V to all non-ncar PBS, remove retired systems --- machines/config_batch.xml | 125 ++++---------------------------------- 1 file changed, 11 insertions(+), 114 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 02e5b82e..837d9e88 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -156,7 +156,6 @@ -r {{ rerunnable }} -j oe - @@ -185,6 +184,8 @@ -l nodes={{ num_nodes }} -q iccp + -V + iccp @@ -210,19 +211,6 @@ - - - (\d+.bw)$ - - -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}:xe - -S {{ shell }} - - - normal - debug - - - qsub @@ -237,47 +225,13 @@ -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=300GB:ngpus={{ ngpus_per_node }} - casper - casper - - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} - - - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} - - - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} - - - - -S {{ shell }} - -l select=1:mpiprocs={{ total_tasks }}:ompthreads={{ thread_count }} - - - - regular - - - regular - premium - share - economy - - - squeue @@ -311,39 +265,6 @@ - - sbatch - - --time $JOB_WALLCLOCK_TIME - -q $JOB_QUEUE - --account $PROJECT - - - -C haswell - - - regular - - - - - - sbatch - - --time $JOB_WALLCLOCK_TIME - -q $JOB_QUEUE - --account $PROJECT - - - -C knl,quad,cache - -S 2 - - - regular - - - - sbatch @@ -450,6 +371,7 @@ -d $RUNDIR -o $RUNDIR/$CASE.out -S /bin/bash + -V debug @@ -508,28 +430,13 @@ - - - - -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} - -S {{ shell }} - - - short - medium - long - verylong - overnight - monster - - - - + qsub (\d+.izumi.cgd.ucar.edu)$ -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} -S {{ shell }} + -V short @@ -541,16 +448,6 @@ - - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} - - - regular - - - sbatch @@ -578,17 +475,12 @@ - - - default - - - -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} -S {{ shell }} + -V batch @@ -632,6 +524,7 @@ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=bro -l place=scatter:excl -S {{ shell }} + -V normal @@ -647,6 +540,7 @@ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=has -l place=scatter:excl -S {{ shell }} + -V normal @@ -662,6 +556,7 @@ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=ivy -l place=scatter:excl -S {{ shell }} + -V normal @@ -677,6 +572,7 @@ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:model=san -l place=scatter:excl -S {{ shell }} + -V normal @@ -713,6 +609,7 @@ -l nodes={{ num_nodes }} + -V default From 11e3bab69e2d67f3b3dc633cf11448a16cd0c222 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Tue, 12 Mar 2024 21:57:18 -0600 Subject: [PATCH 76/95] trim changes to ccs_config --- component_grids_nuopc.xml | 26 -------------------------- modelgrid_aliases_nuopc.xml | 7 ------- 2 files changed, 33 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index b64ea428..a49144b6 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -104,12 +104,6 @@ ne3np4 is Spectral Elem 10-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid - - 866 1 - $DIN_LOC_ROOT/share/domains/domain.lnd.ne4np4_oQU240.160614.nc - ne4np4 is Spectral Elem 10-deg grid: - EXPERIMENTAL FVM physics grid - 1352 1 @@ -126,8 +120,6 @@ 1350 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne5np4.pg3_gx3v7.170605.nc $DIN_LOC_ROOT/share/meshes/ne5pg3_ESMFmesh_cdf5_c20210118.nc ne5np4 is Spectral Elem 6-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid @@ -140,47 +132,35 @@ 13826 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne16np4_gx1v7.171018.nc $DIN_LOC_ROOT/share/meshes/ne16np4_scrip_171002_ESMFmesh.nc ne16np4 is Spectral Elem 2-deg grid: For low resolution spectral element grid testing 13824 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne16pg3_gx1v7.171003.nc $DIN_LOC_ROOT/share/meshes/ne16pg3_ESMFmesh_cdf5_c20211018.nc ne16np4.pg3 is a Spectral Elem 2-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 48602 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30_gx1v7.171003.nc $DIN_LOC_ROOT/share/meshes/ne30np4_ESMFmesh_cdf5_c20211018.nc ne30np4 is Spectral Elem 1-deg grid: 21600 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg2_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne30pg2_ESMFmesh_cdf5_c20211018.nc ne30np4.pg2 is a Spectral Elem 1-deg grid with a 2x2 FVM physics grid: EXPERIMENTAL FVM physics grid 48600 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg3_gx1v7.170605.nc $DIN_LOC_ROOT/share/meshes/ne30pg3_ESMFmesh_cdf5_c20211018.nc ne30np4.pg3 is a Spectral Elem ne30 grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 86400 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne30np4.pg4_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne30pg4_ESMFmesh_cdf5_c20211018.nc ne30np4.pg4 is a Spectral Elem 1-deg grid with a 4x4 FVM physics grid: EXPERIMENTAL FVM physics grid @@ -192,24 +172,18 @@ 86400 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg2_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg2_ESMFmesh_cdf5_c20211018.nc ne60np4.pg2 is a Spectral Elem 0.5-deg grid with a 2x2 FVM physics grid: EXPERIMENTAL FVM physics grid 194400 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg3_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg3_ESMFmesh_cdf5_c20211018.nc ne60np4.pg3 is a Spectral Elem 0.5-deg grid with a 3x3 FVM physics grid: EXPERIMENTAL FVM physics grid 345600 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne60np4.pg4_gx1v7.170628.nc $DIN_LOC_ROOT/share/meshes/ne60pg4_ESMFmesh_cdf5_c20211018.nc ne60np4.pg4 is a Spectral Elem 0.5-deg grid with a 4x4 FVM physics grid: EXPERIMENTAL FVM physics grid diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 22bb6d83..1d82934f 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -846,13 +846,6 @@ gx3v7 - - ne4np4 - ne4np4 - ne4np4 - gx3v7 - - ne5np4 ne5np4 From 764a1aa8666d41f9fde15f95542c9cd0144131a2 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Tue, 12 Mar 2024 22:02:11 -0600 Subject: [PATCH 77/95] remove unneeded domain file --- component_grids_nuopc.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index a49144b6..c87674c0 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -106,8 +106,6 @@ 1352 1 - - $DIN_LOC_ROOT/share/domains/domain.lnd.ne5np4_gx3v7.140810.nc $DIN_LOC_ROOT/share/meshes/ne5np4_esmf_mesh_c210121.nc ne5np4 is Spectral Elem 6-deg grid: For ultra-low resolution spectral element grid testing From 0b004bbc2b341b62bc232207e7629af1394545f0 Mon Sep 17 00:00:00 2001 From: John Truesdale Date: Tue, 12 Mar 2024 22:05:23 -0600 Subject: [PATCH 78/95] clean up support comment for new ne3np4 grid --- component_grids_nuopc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index c87674c0..275139e4 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -93,10 +93,10 @@ 488 1 + $DIN_LOC_ROOT/share/domains/domain.lnd.ne3np4_gx3v7.230718.nc $DIN_LOC_ROOT/share/meshes/ne3np4_ESMFmesh_c230714_cdf5.nc ne3np4 is Spectral Elem 10-deg grid: - EXPERIMENTAL FVM physics grid 486 1 From 707c59db4065e299aebc92423a57aa25a46f992f Mon Sep 17 00:00:00 2001 From: adamrher Date: Wed, 13 Mar 2024 11:42:31 -0600 Subject: [PATCH 79/95] add polarcar grid, fix mpasa15-3conus naming issues --- component_grids_nuopc.xml | 6 ++++++ modelgrid_aliases_mct.xml | 8 ++++---- modelgrid_aliases_nuopc.xml | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 275139e4..0023fc26 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -249,6 +249,12 @@ ne0np4.ARCTICGRIS.ne30x8 is a Spectral Elem 1-deg grid with a 1/8 deg refined region over Greenland: Test support only + + 186194 1 + /glade/work/aherring/grids/var-res/ne0np4.POLARRES.ne30x4/grids/polarcap/POLARCAP_ne30x4_np4_MESH_cdf5.nc + ne0np4.POLARCAP.ne30x4 is a Spectral Elem 1-deg grid with a 1/4 deg refined region over the Arctic and Antarctica: + Test support only + diff --git a/modelgrid_aliases_mct.xml b/modelgrid_aliases_mct.xml index 74ea6c06..890b2b86 100644 --- a/modelgrid_aliases_mct.xml +++ b/modelgrid_aliases_mct.xml @@ -1137,10 +1137,10 @@ gx1v7 - - mpasa15-3 - mpasa15-3 - mpasa15-3 + + mpasa15-3conus + mpasa15-3conus + mpasa15-3conus gx1v7 diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 1d82934f..f11675cd 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1245,6 +1245,13 @@ tx0.1v2 + + ne0np4.POLARCAP.ne30x4 + ne0np4.POLARCAP.ne30x4 + ne0np4.POLARCAP.ne30x4 + tx0.1v2 + + @@ -1296,18 +1303,11 @@ gx1v7 - - mpasa15-3 - mpasa15-3 - mpasa15-3 - gx1v7 - - - - mpasa15-3 - mpasa15-3 - mpasa15-3 - gx1v7 + + mpasa15-3conus + mpasa15-3conus + mpasa15-3conus + tx0.1v3 From ed817ff8e6c78fca48fb2ac34ef5dffc51caa69a Mon Sep 17 00:00:00 2001 From: adamrher Date: Wed, 13 Mar 2024 12:42:58 -0600 Subject: [PATCH 80/95] fix typo --- modelgrid_aliases_nuopc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index f11675cd..3296525e 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1245,7 +1245,7 @@ tx0.1v2 - + ne0np4.POLARCAP.ne30x4 ne0np4.POLARCAP.ne30x4 ne0np4.POLARCAP.ne30x4 From 1f836f04ea45226a0fd87aa6322ed0afd72071fa Mon Sep 17 00:00:00 2001 From: mvertens Date: Mon, 1 Apr 2024 13:38:14 +0200 Subject: [PATCH 81/95] changes needed for adding dglc component (#159) Co-authored-by: Mariana Vertenstein --- modelgrid_aliases_nuopc.xml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 1d82934f..3a257d88 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -16,6 +16,7 @@ r05 null gris4 + gris4 gris4 null ww3a @@ -187,7 +188,7 @@ gx3v7 - + T31 T31 gx3v7 @@ -195,7 +196,7 @@ gx3v7 - + T31 T31 gx3v7 @@ -203,7 +204,7 @@ gx3v7 - + T31 T31 gx3v7 @@ -211,7 +212,7 @@ gx3v7 - + T31 T31 gx3v7 @@ -545,7 +546,7 @@ tn0.25v3 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -553,7 +554,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -561,7 +562,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -569,7 +570,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -577,7 +578,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -585,7 +586,7 @@ gx1v7 - + 0.9x1.25 0.9x1.25 gx1v7 @@ -684,7 +685,7 @@ gx1v7 - + 1.9x2.5 1.9x2.5 gx1v7 @@ -692,13 +693,20 @@ gx1v7 - + 1.9x2.5 1.9x2.5 gx1v7 gris4 gx1v7 + + 1.9x2.5 + 1.9x2.5 + gx1v7 + ais8 + gx1v7 + 1.9x2.5 @@ -801,7 +809,7 @@ gx3v7 - + 10x15 10x15 10x15 @@ -809,7 +817,7 @@ gx3v7 - + 10x15 10x15 10x15 From 2e409ba1feda667824f32055beae2409450dde29 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 2 Apr 2024 08:44:15 -0500 Subject: [PATCH 82/95] update modules and batch for tacc frontera --- machines/config_batch.xml | 2 +- machines/frontera/config_machines.xml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/machines/config_batch.xml b/machines/config_batch.xml index 837d9e88..1f1052fa 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -356,7 +356,7 @@ development - normal + normal large diff --git a/machines/frontera/config_machines.xml b/machines/frontera/config_machines.xml index fba36580..3c4a324b 100644 --- a/machines/frontera/config_machines.xml +++ b/machines/frontera/config_machines.xml @@ -41,22 +41,22 @@ TACC /scratch1/projects/compilers/modulefiles - intel/22.2.0 + cmake/3.24.2 - impi/22.2.0 - pnetcdf/1.11.2 - parallel-netcdf/4.6.2 + impi/19.0.9 + pnetcdf/1.12.3 + parallel-netcdf/4.9.0 impi mvapich2-x/2.3 - pnetcdf/1.11.2 - parallel-netcdf/4.6.2 + pnetcdf/1.12.3 + parallel-netcdf/4.9.0 - netcdf/4.6.2 + netcdf/4.9.0 From 39c3ee3c310b3e7e158ee2957c17993b590c2897 Mon Sep 17 00:00:00 2001 From: Katetc Date: Wed, 3 Apr 2024 16:36:41 -0600 Subject: [PATCH 83/95] Adding glc supporing grid aliases for NUOPC --- modelgrid_aliases_nuopc.xml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 3a257d88..5b49f06e 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -431,6 +431,20 @@ tx2_3v2 + + 0.9x1.25 + 0.9x1.25 + tx2_3v2 + gris4 + + + + 0.9x1.25 + 0.9x1.25 + tx2_3v2 + ais8:gris4 + + T62 T62 @@ -896,6 +910,22 @@ gx1v7 + + ne30np4.pg3 + ne30np4.pg3 + gx1v7 + gris4 + gx1v7 + + + + ne30np4.pg3 + ne30np4.pg3 + gx1v7 + ais8:gris4 + gx1v7 + + ne30np4.pg3 ne30np4.pg3 @@ -910,6 +940,14 @@ tx2_3v2 + + ne30np4.pg3 + ne30np4.pg3 + tx2_3v2 + gris4 + tx2_3v2 + + ne30np4 1.9x2.5 From 3f4f942dcfa986a0520adc9f193907db0c9e0834 Mon Sep 17 00:00:00 2001 From: Katetc Date: Thu, 4 Apr 2024 15:45:01 -0600 Subject: [PATCH 84/95] Adding ais to workhorse grid for testing options --- modelgrid_aliases_nuopc.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 5b49f06e..7bc4fae4 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -940,7 +940,7 @@ tx2_3v2 - + ne30np4.pg3 ne30np4.pg3 tx2_3v2 @@ -948,6 +948,14 @@ tx2_3v2 + + ne30np4.pg3 + ne30np4.pg3 + tx2_3v2 + ais8:gris4 + tx2_3v2 + + ne30np4 1.9x2.5 From ea9589c56072a6bc92a228d15caed7067f1a3802 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Mon, 8 Apr 2024 16:10:48 -0600 Subject: [PATCH 85/95] update the GPU settings on Derecho --- machines/Depends.nvhpc | 3 ++- machines/derecho/config_machines.xml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 0771da43..f94ee156 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -33,6 +33,7 @@ micro_mg1_0.o \ micro_pumas_v1.o \ micro_pumas_data.o \ micro_pumas_utils.o \ +pumas_stochastic_collect_tau.o \ micro_pumas_cam.o \ wv_sat_methods.o \ wv_saturation.o \ @@ -73,5 +74,5 @@ ifeq ($(DEBUG),FALSE) $(PUMAS_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $< $(RRTMGP_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $< endif diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index cd6d59d1..12446f37 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -64,7 +64,7 @@ cray-libsci/23.02.1.1 - nvhpc/23.7 + nvhpc/24.3 @@ -122,6 +122,6 @@ SUMMARY - openacc + -lcuda -lcudart From d0bfcd0a5789b5a96ca9b8f22f80b7126c1b2126 Mon Sep 17 00:00:00 2001 From: Jian Sun Date: Mon, 8 Apr 2024 16:16:01 -0600 Subject: [PATCH 86/95] bring back accidentally deleted option --- machines/Depends.nvhpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index f94ee156..e1c9b4e7 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -74,5 +74,5 @@ ifeq ($(DEBUG),FALSE) $(PUMAS_OBJS): %.o: %.F90 $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $< $(RRTMGP_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $< endif From 3909054c767330412cccbb499cf084db70e7e5f8 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 9 Apr 2024 10:14:14 -0600 Subject: [PATCH 87/95] Revert "add polarcap grid, fix mpasa15-3 naming issues" --- component_grids_nuopc.xml | 6 ------ modelgrid_aliases_mct.xml | 8 ++++---- modelgrid_aliases_nuopc.xml | 24 ++++++++++++------------ 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 0023fc26..275139e4 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -249,12 +249,6 @@ ne0np4.ARCTICGRIS.ne30x8 is a Spectral Elem 1-deg grid with a 1/8 deg refined region over Greenland: Test support only - - 186194 1 - /glade/work/aherring/grids/var-res/ne0np4.POLARRES.ne30x4/grids/polarcap/POLARCAP_ne30x4_np4_MESH_cdf5.nc - ne0np4.POLARCAP.ne30x4 is a Spectral Elem 1-deg grid with a 1/4 deg refined region over the Arctic and Antarctica: - Test support only - diff --git a/modelgrid_aliases_mct.xml b/modelgrid_aliases_mct.xml index 890b2b86..74ea6c06 100644 --- a/modelgrid_aliases_mct.xml +++ b/modelgrid_aliases_mct.xml @@ -1137,10 +1137,10 @@ gx1v7 - - mpasa15-3conus - mpasa15-3conus - mpasa15-3conus + + mpasa15-3 + mpasa15-3 + mpasa15-3 gx1v7 diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 0d53481b..7bc4fae4 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1299,13 +1299,6 @@ tx0.1v2 - - ne0np4.POLARCAP.ne30x4 - ne0np4.POLARCAP.ne30x4 - ne0np4.POLARCAP.ne30x4 - tx0.1v2 - - @@ -1357,11 +1350,18 @@ gx1v7 - - mpasa15-3conus - mpasa15-3conus - mpasa15-3conus - tx0.1v3 + + mpasa15-3 + mpasa15-3 + mpasa15-3 + gx1v7 + + + + mpasa15-3 + mpasa15-3 + mpasa15-3 + gx1v7 From a61edd7a1c7e22e959b5bd017c076c43ee527de1 Mon Sep 17 00:00:00 2001 From: adamrher Date: Tue, 9 Apr 2024 12:28:39 -0600 Subject: [PATCH 88/95] redo polarcap/mpas grid fix PR --- component_grids_nuopc.xml | 7 +++++++ modelgrid_aliases_mct.xml | 8 ++++---- modelgrid_aliases_nuopc.xml | 25 ++++++++++++------------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/component_grids_nuopc.xml b/component_grids_nuopc.xml index 275139e4..97b3fc9e 100644 --- a/component_grids_nuopc.xml +++ b/component_grids_nuopc.xml @@ -250,6 +250,13 @@ Test support only + + 186194 1 + $DIN_LOC_ROOT/share/meshes/POLARCAP_ne30x4_np4_ESMFmesh_cdf5_c20240222.nc + ne0np4.POLARCAP.ne30x4 is a Spectral Elem 1-deg grid with a 1/4 deg refined region over the Arctic and Antarctica: + Test support only + + 2562 1 diff --git a/modelgrid_aliases_mct.xml b/modelgrid_aliases_mct.xml index 74ea6c06..890b2b86 100644 --- a/modelgrid_aliases_mct.xml +++ b/modelgrid_aliases_mct.xml @@ -1137,10 +1137,10 @@ gx1v7 - - mpasa15-3 - mpasa15-3 - mpasa15-3 + + mpasa15-3conus + mpasa15-3conus + mpasa15-3conus gx1v7 diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 7bc4fae4..8a8ff706 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1299,6 +1299,13 @@ tx0.1v2 + + ne0np4.POLARCAP.ne30x4 + ne0np4.POLARCAP.ne30x4 + ne0np4.POLARCAP.ne30x4 + tx0.1v2 + + @@ -1350,19 +1357,11 @@ gx1v7 - - mpasa15-3 - mpasa15-3 - mpasa15-3 - gx1v7 - - - - mpasa15-3 - mpasa15-3 - mpasa15-3 - gx1v7 - + + mpasa15-3conus + mpasa15-3conus + mpasa15-3conus + tx0.1v3 mpasa7p5 From 97c3baee2fd2e62b45bc7f662c328e714582ad3b Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Mon, 15 Apr 2024 06:42:20 -0600 Subject: [PATCH 89/95] update to model_grid_aliases --- modelgrid_aliases_nuopc.xml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 0ee170f4..7a417db7 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -4,27 +4,28 @@ null + null + null - null - null + null - null - r05 + null gris4 - ww3a + gris4 + gris4 + null + r05 rx1 r05 - ww3a - r05 r05 - gris4 - gris4 - + null + ww3a + ww3a wtnx1v4 From b213d357159b6d654c5ba25efe089fa576ed41f0 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 22 Apr 2024 15:45:52 -0600 Subject: [PATCH 90/95] Add missing --- modelgrid_aliases_nuopc.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index 8a8ff706..0d53481b 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -1362,6 +1362,7 @@ mpasa15-3conus mpasa15-3conus tx0.1v3 + mpasa7p5 From b3bdf13dc9bed8eef3f4eff7624836ef75794df9 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 23 Apr 2024 13:43:33 -0600 Subject: [PATCH 91/95] update python version on izumi --- machines/izumi/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/izumi/config_machines.xml b/machines/izumi/config_machines.xml index 1a89600d..10a0e078 100644 --- a/machines/izumi/config_machines.xml +++ b/machines/izumi/config_machines.xml @@ -40,7 +40,7 @@ module - lang/python/3.7.0 + lang/python/3.11.5 /fs/cgd/data0/modules/modulefiles From fc67b1cbf334ae414f9104afae68187bcc4a9f4b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 25 Apr 2024 07:11:07 -0600 Subject: [PATCH 92/95] fix build using nvhpc on derecho --- machines/cmake_macros/derecho.cmake | 4 ++++ machines/cmake_macros/nvhpc_derecho.cmake | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/machines/cmake_macros/derecho.cmake b/machines/cmake_macros/derecho.cmake index 9b20dea6..0dd51b4a 100644 --- a/machines/cmake_macros/derecho.cmake +++ b/machines/cmake_macros/derecho.cmake @@ -5,6 +5,10 @@ set(MPI_SERIAL_PATH "$ENV{NCAR_ROOT_MPI_SERIAL}") set(NETCDF_PATH "$ENV{NETCDF}") set(PIO_FILESYSTEM_HINTS "lustre") set(PNETCDF_PATH "$ENV{PNETCDF}") +if(DEFINED ENV{PIO}) + set(PIO_LIBDIR "$ENV{PIO}/lib") + set(PIO_INCDIR "$ENV{PIO}/include") +endif() # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") string(APPEND CPPDEFS " -DHAVE_GETTID") diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake index ab1d0d57..f0b3cd87 100644 --- a/machines/cmake_macros/nvhpc_derecho.cmake +++ b/machines/cmake_macros/nvhpc_derecho.cmake @@ -2,6 +2,6 @@ if (COMP_NAME STREQUAL mpi-serial) string(APPEND CFLAGS " -std=c89 ") endif() if (NOT DEBUG) - string(APPEND FFLAGS " -target=zen3") + string(APPEND FFLAGS " -tp=zen3") endif() message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}") From 7c0692dbd18c2cf554593c85a76a96a44753aff2 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Thu, 9 May 2024 22:16:12 +0200 Subject: [PATCH 93/95] added new config_machine files --- machines/betzy/config_machines.xml | 84 ++++++++++++++++++++++++++++++ machines/fram/config_machines.xml | 61 ++++++++++++++++++++++ machines/saga/config_machines.xml | 47 +++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 machines/betzy/config_machines.xml create mode 100644 machines/fram/config_machines.xml create mode 100644 machines/saga/config_machines.xml diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml new file mode 100644 index 00000000..d59bc150 --- /dev/null +++ b/machines/betzy/config_machines.xml @@ -0,0 +1,84 @@ + + BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM + .*.?betzy\d?.sigma2.no + LINUX + intel + openmpi,impi + /cluster/work/users/$USER/noresm + /cluster/shared/noresm/inputdata + /cluster/shared/noresm/inputdata/atm/datm7 + /cluster/work/users/$USER/archive/$CASE + /cluster/shared/noresm/noresm_baselines + /cluster/shared/noresm/tools/cprnc/cprnc + 8 + slurm + noresmCommunity + 128 + 128 + TRUE + + + + + srun + + + $ENV{LMOD_PKG}/init/perl + $ENV{LMOD_PKG}/init/env_modules_python.py + $ENV{LMOD_PKG}/init/csh + $ENV{LMOD_PKG}/init/sh + $ENV{LMOD_PKG}/libexec/lmod perl + $ENV{LMOD_PKG}/libexec/lmod python + module + module + + + StdEnv + /cluster/shared/noresm/eb_mods/modules/all + ESMF/8.4.1-iomkl-2021b-ParallelIO-2.5.10 + CMake/3.21.1-GCCcore-11.2.0 + Python/3.9.6-GCCcore-11.2.0 + + ParMETIS/4.0.3-iompi-2021b + + + + StdEnv + /cluster/shared/noresm/eb_mods/modules/all + ESMF/8.4.1-intel-2021b-ParallelIO-2.5.10 + CMake/3.21.1-GCCcore-11.2.0 + Python/3.9.6-GCCcore-11.2.0 + ParMETIS/4.0.3-iimpi-2021b + + + + $ENV{EBROOTESMF}/lib/esmf.mk + ON + SUMMARY + mlx5_0:1 + 64M + 5 + 2 + $ENV{EBROOTPARALLELIO}/lib + $ENV{EBROOTPARALLELIO}/include + pnetcdf,netcdf,netcdf4p,netcdf4c + 1 + self,vader + 1 + 1 + ^fca + 95 + 8 + ompio + 1048576 + 8 + ^lockedfile,individual + lustre + on + + + + + -1 + + diff --git a/machines/fram/config_machines.xml b/machines/fram/config_machines.xml new file mode 100644 index 00000000..16835595 --- /dev/null +++ b/machines/fram/config_machines.xml @@ -0,0 +1,61 @@ + + Lenovo NeXtScale M5, 32-way nodes, dual 16-core Xeon E5-2683@2.10GHz, 64 GiB per node, os is Linux, batch system is SLURM + $HOSTNAME + LINUX + intel + impi, mpi-serial + /cluster/work/users/$USER/noresm + /cluster/shared/noresm/inputdata + /cluster/shared/noresm/inputdata/atm/datm7 + /cluster/work/users/$USER/archive/$CASE + UNSET + /cluster/shared/noresm/tools/cprnc/cprnc + 8 + slurm + noresmCommunity + 32 + 32 + TRUE + + + + + mpirun + + + $ENV{LMOD_PKG}/init/perl + $ENV{LMOD_PKG}/init/env_modules_python.py + $ENV{LMOD_PKG}/init/csh + $ENV{LMOD_PKG}/init/sh + $ENV{LMOD_PKG}/libexec/lmod perl + $ENV{LMOD_PKG}/libexec/lmod python + module + module + + + StdEnv + /cluster/shared/noresm/eb_mods/modules/all/ + CMake/3.23.1-GCCcore-11.3.0 + Python/3.10.4-GCCcore-11.3.0 + XML-LibXML/2.0207-GCCcore-11.3.0 + ESMF/8.4.1-intel-2022a-ParallelIO-2.5.10 + + + + + 256M + lustre + on + $ENV{EBROOTESMF}/lib/esmf.mk + $ENV{EBROOTESMF}/lib + ON + SUMMARY + 2 + $ENV{EBROOTPARALLELIO}/lib + $ENV{EBROOTPARALLELIO}/include + pnetcdf,netcdf,netcdf4p,netcdf4c + + + -1 + + diff --git a/machines/saga/config_machines.xml b/machines/saga/config_machines.xml new file mode 100644 index 00000000..378ab9e9 --- /dev/null +++ b/machines/saga/config_machines.xml @@ -0,0 +1,47 @@ + + Hewlett Packard Enterprise - CentOS Linux release 7.6.1810 (Core) + $HOSTNAME + LINUX + intel + impi + /cluster/work/users/$USER/noresm + /cluster/shared/noresm/inputdata + /cluster/shared/noresm/inputdata/atm/datm7 + /cluster/work/users/$USER/archive/$CASE + UNSET + UNSET + 8 + slurm + noresmCommunity + 40 + 40 + TRUE + + mpirun + + + $ENV{LMOD_PKG}/init/env_modules_python.py + $ENV{LMOD_PKG}/init/sh + $ENV{LMOD_PKG}/libexec/lmod python + module + + + StdEnv + CMake/3.22.1-GCCcore-11.2.0 + XML-LibXML/2.0207-GCCcore-11.2.0 + ESMF/8.2.0-intel-2021b + + + + 256M + lustre + on + $ENV{EBROOTESMF}/lib/esmf.mk + $ENV{EBROOTESMF}/lib + ON + SUMMARY + + + -1 + + From 52a69cb2c5c4dfbcb3e49f0def27106fa924a385 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 10 May 2024 09:54:51 +0200 Subject: [PATCH 94/95] addition of betzy, fram and saga config_machines.xml files --- machines/betzy/config_machines.xml | 1 - machines/fram/config_machines.xml | 1 - machines/saga/config_machines.xml | 1 - 3 files changed, 3 deletions(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index d59bc150..d96cb97c 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -1,6 +1,5 @@ BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM - .*.?betzy\d?.sigma2.no LINUX intel openmpi,impi diff --git a/machines/fram/config_machines.xml b/machines/fram/config_machines.xml index 16835595..762f3681 100644 --- a/machines/fram/config_machines.xml +++ b/machines/fram/config_machines.xml @@ -1,6 +1,5 @@ Lenovo NeXtScale M5, 32-way nodes, dual 16-core Xeon E5-2683@2.10GHz, 64 GiB per node, os is Linux, batch system is SLURM - $HOSTNAME LINUX intel impi, mpi-serial diff --git a/machines/saga/config_machines.xml b/machines/saga/config_machines.xml index 378ab9e9..fa3592c1 100644 --- a/machines/saga/config_machines.xml +++ b/machines/saga/config_machines.xml @@ -1,6 +1,5 @@ Hewlett Packard Enterprise - CentOS Linux release 7.6.1810 (Core) - $HOSTNAME LINUX intel impi From 6074740c51d9856da46f445bfe57dc73700a0ca1 Mon Sep 17 00:00:00 2001 From: Mariana Vertenstein Date: Fri, 10 May 2024 10:32:21 +0200 Subject: [PATCH 95/95] updated modelgrid_aliases_nuopc.xml to noresm --- modelgrid_aliases_nuopc.xml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/modelgrid_aliases_nuopc.xml b/modelgrid_aliases_nuopc.xml index eb704799..ca2cf642 100644 --- a/modelgrid_aliases_nuopc.xml +++ b/modelgrid_aliases_nuopc.xml @@ -4,28 +4,27 @@ null - null - null - + null + null null - null - gris4 - gris4 - gris4 - null r05 + gris4 + ww3a + rx1 r05 + ww3a + r05 r05 - null - ww3a - ww3a + gris4 + gris4 + wtnx1v4