From 9bdf11e1aebfa7e2cec833ef606c6edda310abdb Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Fri, 27 Sep 2024 12:33:11 +0200 Subject: [PATCH 1/7] add login3 modules --- machines/betzy/config_machines.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 64ac4c06..45c3f989 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -33,12 +33,13 @@ StdEnv - /cluster/shared/noresm/eb_mods/modules/all - ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10 + cluster/projects/nn9560k/mdeb/updt_mods/modules/all + ESMF/8.6.1-iomkl-2022a-ParallelIO-2.6.3 Python/3.11.3-GCCcore-12.3.0 CMake/3.26.3-GCCcore-12.3.0 ParMETIS/4.0.3-iompi-2022a - git/2.38.1-GCCcore-12.2.0-nodocs + git/2.41.0-GCCcore-12.3.0-nodocs + XML-LibXML/2.0209-GCCcore-12.3.0 From 8db70f89de6552811c9ceb7c141b472fa3d3ff2a Mon Sep 17 00:00:00 2001 From: mvdebolskiy <80036033+mvdebolskiy@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:05:25 +0300 Subject: [PATCH 2/7] fix typo --- machines/betzy/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 45c3f989..cf6fc775 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -33,7 +33,7 @@ StdEnv - cluster/projects/nn9560k/mdeb/updt_mods/modules/all + /cluster/projects/nn9560k/mdeb/updt_mods/modules/all ESMF/8.6.1-iomkl-2022a-ParallelIO-2.6.3 Python/3.11.3-GCCcore-12.3.0 CMake/3.26.3-GCCcore-12.3.0 From c74127cf61f72afebddd3f7db9770c2194581895 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Wed, 16 Oct 2024 14:09:56 +0200 Subject: [PATCH 3/7] swtich to 2022a --- machines/betzy/config_machines.xml | 6 +++--- machines/betzy/intel-oneapi_betzy.cmake | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 7a69ee42..8c8c7db7 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -19,7 +19,7 @@ - srun + srun --kill-on-bad-exit --label $ENV{LMOD_PKG}/init/perl @@ -33,8 +33,8 @@ StdEnv - /cluster/projects/nn9560k/mdeb/updt_mods/modules/all - ESMF/8.6.1-iomkl-2022a-ParallelIO-2.6.3 + /cluster/shared/noresm/eb_mods/modules/all + ESMF/8.4.2-iomkl-2022a-ParallelIO-2.5.10 Python/3.11.3-GCCcore-12.3.0 CMake/3.26.3-GCCcore-12.3.0 ParMETIS/4.0.3-iompi-2022a diff --git a/machines/betzy/intel-oneapi_betzy.cmake b/machines/betzy/intel-oneapi_betzy.cmake index fb169c4f..c4d24ac9 100644 --- a/machines/betzy/intel-oneapi_betzy.cmake +++ b/machines/betzy/intel-oneapi_betzy.cmake @@ -10,7 +10,7 @@ if(DEFINED ENV{PIO}) set(PIO_INCDIR "$ENV{PIO}/include") endif() -string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") +#string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") string(APPEND CPPDEFS " -DHAVE_GETTID") if (COMP_WAV STREQUAL ww3) From aae3f73ae8f20259d38d0725a70a8c2fc02d1a16 Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Wed, 16 Oct 2024 16:11:40 +0200 Subject: [PATCH 4/7] swap hpcx to 2.20 --- machines/betzy/config_machines.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 8c8c7db7..02ccf841 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -40,6 +40,7 @@ ParMETIS/4.0.3-iompi-2022a git/2.41.0-GCCcore-12.3.0-nodocs XML-LibXML/2.0209-GCCcore-12.3.0 + hpcx/2.20 From 5d96eed7145f906fd1087186430b0ea2d1a31e9a Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Thu, 17 Oct 2024 13:33:23 +0200 Subject: [PATCH 5/7] remove oneapi and add it to the default intel --- machines/betzy/config_machines.xml | 26 ++------------------------ machines/betzy/intel_betzy.cmake | 13 +++++++++++-- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 02ccf841..325fbd31 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -1,7 +1,7 @@ BullSequana XH2000 AMD® Epyc™ "Rome" 2.2GHz, 128-way nodes, os is Linux, batch system is SLURM LINUX - intel-oneapi,intel + intel openmpi,impi /cluster/work/users/$USER/noresm /cluster/shared/noresm/inputdata @@ -30,7 +30,7 @@ $ENV{LMOD_PKG}/libexec/lmod python module module - + StdEnv /cluster/shared/noresm/eb_mods/modules/all @@ -42,28 +42,6 @@ XML-LibXML/2.0209-GCCcore-12.3.0 hpcx/2.20 - - - StdEnv - git/2.41.0-GCCcore-12.3.0-nodocs - /cluster/shared/noresm/eb_mods/modules/all - ESMF/8.4.1-iomkl-2021b-ParallelIO-2.5.10 - CMake/3.21.1-GCCcore-11.2.0 - Python/3.9.6-GCCcore-11.2.0 - ParMETIS/4.0.3-iompi-2021b - XML-LibXML/2.0209-GCCcore-12.3.0 - - - - StdEnv - git/2.41.0-GCCcore-12.3.0-nodocs - /cluster/shared/noresm/eb_mods/modules/all - ESMF/8.4.1-intel-2021b-ParallelIO-2.5.10 - CMake/3.21.1-GCCcore-11.2.0 - Python/3.9.6-GCCcore-11.2.0 - ParMETIS/4.0.3-iimpi-2021b - XML-LibXML/2.0209-GCCcore-12.3.0 - $ENV{EBROOTESMF}/lib/esmf.mk diff --git a/machines/betzy/intel_betzy.cmake b/machines/betzy/intel_betzy.cmake index 529eb456..c4d24ac9 100644 --- a/machines/betzy/intel_betzy.cmake +++ b/machines/betzy/intel_betzy.cmake @@ -1,14 +1,23 @@ set(PIO_FILESYSTEM_HINTS "lustre") - +string(APPEND CFLAGS " -qopt-report -march=core-avx2") +string(APPEND FFLAGS " -qopt-report -march=core-avx2") set(NETCDF_C_PATH $ENV{EBROOTNETCDF}) set(NETCDF_FORTRAN_PATH $ENV{EBROOTNETCDFMINFORTRAN}) set(PNETCDF_PATH $ENV{PNETCDF}) +if(DEFINED ENV{PIO}) + set(PIO_LIBDIR "$ENV{PIO}/lib") + set(PIO_INCDIR "$ENV{PIO}/include") +endif() + +#string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}") +string(APPEND CPPDEFS " -DHAVE_GETTID") + if (COMP_WAV STREQUAL ww3) if (MPILIB STREQUAL openmpi) string(APPEND SLIBS " -l:libparmetis.a -lmetis ") endif() -endif() +endif() if (MPILIB STREQUAL impi) set(MPICC "mpiicc") From d996995af8fc8aaea0ea6dd555ba69783f9c660f Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Thu, 17 Oct 2024 17:52:56 +0200 Subject: [PATCH 6/7] add cprnc comaptible with 2022a --- machines/betzy/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/betzy/config_machines.xml b/machines/betzy/config_machines.xml index 325fbd31..d756b205 100644 --- a/machines/betzy/config_machines.xml +++ b/machines/betzy/config_machines.xml @@ -8,7 +8,7 @@ /cluster/shared/noresm/inputdata/atm/datm7 /cluster/work/users/$USER/archive/$CASE /cluster/shared/noresm/noresm_baselines - /cluster/shared/noresm/tools/cprnc/cprnc + /cluster/shared/noresm/tools/cprnc-iompi-2022a/bin/cprnc 8 slurm noresmCommunity From 052cf1d85ef45b3205d2f871b0969c7331ed735c Mon Sep 17 00:00:00 2001 From: mvdebolskiy Date: Sat, 19 Oct 2024 10:57:35 +0200 Subject: [PATCH 7/7] fix memory --- machines/betzy/config_batch.xml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/machines/betzy/config_batch.xml b/machines/betzy/config_batch.xml index 7ce79de5..6c43299a 100644 --- a/machines/betzy/config_batch.xml +++ b/machines/betzy/config_batch.xml @@ -8,7 +8,6 @@ --ntasks={{ total_tasks }} --export=ALL --switches=1 - --exclude=b3355,b3356,b3357,b3359,b3379,b3382,b3383 --partition=normal @@ -19,11 +18,11 @@ --partition=preproc - --mem=16G + --mem-per-cpu=1900M - - normal - preproc - devel - + + normal + preproc + devel +