Merge pull request #6 from IngoBethke/main

Updated libraries for NorCPM to 2023b on Betzy and fixed various issues to make NorCPM work with the new library versions

assim/enkf_cmip6_i1/micom_init/Makefile.config.betzy

@@ -6,10 +6,10 @@ TARGET=micom_init
 #MYCPPDEFS="-DMPI -DICEDYN -DLEVITUS2X -DTRC -DIDLAGE -DPNETCDF"
 
 # Fortran compiler
-FC=mpif90
+FC=mpifort
 
 # C compiler
-CC=mpicc
+CC=icc
 
 # Linker
 LD=$(FC)

assim/enkf_cmip6_i1/shared/make.inc.betzy

@@ -5,8 +5,8 @@ INCS = $(INC_NETCDF) $(INC_FFTW)
 LIBS = -mkl=cluster $(LIB_LAPACK) $(LIB_NETCDF) $(LIB_FFTW)
 
 ifeq ($(MPI),YES)
-        CF90 = mpif90
-        CC = mpicc
+        CF90 = mpifort
+        CC = icc
 	PARO =
 	CPPFLAGS = -DQMPI 
 else

assim/enkf_noresm2_oda/EnKF/MODEL.CPP

@@ -8,3 +8,4 @@
 #define EXPCOV
 #undef CHECK_SOLUTION
 #undef X4SVD
+#define INTEL_COMPILER

assim/enkf_noresm2_oda/EnKF/order.c

@@ -23,7 +23,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <math.h>
-#include "cfortran.h"
+#include <cfortran.h>
 
 typedef struct {
     int index;

assim/enkf_noresm2_oda/shared/make.inc.betzy

@@ -5,18 +5,19 @@ INCS = $(INC_NETCDF) $(INC_FFTW)
 LIBS = -qmkl=cluster $(LIB_LAPACK) $(LIB_NETCDF) $(LIB_FFTW)
 
 ifeq ($(MPI),YES)
-        CF90 = mpif90
-        CC = mpicc
+        CF90 = mpifort
+        CC = icc
 	PARO =
 	CPPFLAGS = -DQMPI 
+	CPP = cpp -P -traditional-cpp
 else
 	CF90 = ifort
 	CC = icc
 	PAR0 = 
+	CPP = cpp -P -traditional-cpp
 endif
 CF77 = $(CF90)
 LD = $(CF90) 
-CPP = cpp -P -traditional-cpp
 
 CPPARCH = -DIA32 -DFFTW -DNOMPIR8
 CPPFLAGS += $(CPPARCH) -DF90_NOFLUSH
@@ -27,13 +28,14 @@ OPTO_NOAVX2 = -O2 -align -fp-model source
 #OPTO_NOAVX2 = -g -traceback -O2 -align -fp-model source  
 OPTO = $(OPTO_NOAVX2) -xavx2 -no-fma #-fast -fastsse 
 ARCHO= -mcmodel=medium
-INLO = -ip
+#INLO = -ip
 DIVO = 
 DEBUG_FLAGS = 
 
-FFLAGS_NOAVX2 = $(SIZEO) $(OPTO_NOAVX2) $(ARCHO) $(PARO) $(INLO) $(DIVO) $(DEBUG_FLAGS) $(INCS)
-FFLAGS = $(SIZEO) $(OPTO) $(ARCHO) $(PARO) $(INLO) $(DIVO) $(DEBUG_FLAGS) $(INCS)
-CFLAGS = $(FFLAGS)
+CFLAGS_NOAVX2 = -O2 $(ARCHO) $(PARO) $(INLO) $(DIVO) $(DEBUG_FLAGS) $(INCS)
+CFLAGS = -O2 $(ARCHO) $(PARO) $(INLO) $(DIVO) $(DEBUG_FLAGS) $(INCS)
+FFLAGS_NOAVX2 = $(CFLAGS_NOAVX2) $(SIZEO) $(OPTO_NOAVX2) 
+FFLAGS = $(CFLAGS) $(SIZEO) $(OPTO)
 LINKFLAGS = $(SIZEO) $(OPTO_NOAVX2) $(ARCHO) $(PARO) $(INLO) $(DIVO) $(DEBUG_FLAGS) 
 
 # language-specific flags

model/noresm1/models/ocn/micom/bld/micom_dimensions

@@ -1,14 +1,33 @@
 #!/bin/bash
 #
-# The purpose of this script is to generate a MICOM module file,
+# ------------------------------------------------------------------------------
+# Copyright (C) 2009-2020 Mats Bentsen
+#
+# This file is part of BLOM.
+#
+# BLOM is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# BLOM is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for
+# more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with BLOM. If not, see <https://www.gnu.org/licenses/>.
+# ------------------------------------------------------------------------------
+#
+# The purpose of this script is to generate a BLOM module file,
 # dimensions.F, containing all dimension parameters and grid
 # partitioning information. The necessary information is taken from
 # patch.input files for the appropriate processor count.
 #
 set -e
 
 sn=`basename $0`
-usage="Usage: $sn -n pes [-k kdm] [-d directory] [-s]\nPurpose: write dimensions.F containing all dimension parameters and grid\npartitioning information for MICOM.\nOptions:\n -n pes\t\tnumber of processing elements.\n -k kdm\t\tvertical dimension (default kdm=1).\n -d directory\tdirectory to search for patch.input files (default\n\t\tcurrent working directory).\n -s\t\tdo not write dimensions.F."
+usage="Usage: $sn -n pes [-k kdm] [-d directory] [-s]\nPurpose: write dimensions.F containing all dimension parameters and grid\npartitioning information for BLOM.\nOptions:\n -n pes\t\tnumber of processing elements.\n -k kdm\t\tvertical dimension (default kdm=1).\n -d directory\tdirectory to search for patch.input files (default\n\t\tcurrent working directory).\n -s\t\tdo not write dimensions.F."
 
 #
 # Get options
@@ -86,24 +105,21 @@ fi
 #
 l=3
 m=0
-i=0
-j=0
+ispt=()
+iipe=()
 while [ $m -lt $mpe ]; do
   n=0
   while [ $n -lt $npe ]; do
-    while read -d " " i0_pe[i]; do
-      i0_pe[i]=$((${i0_pe[i]}-1))
-      i=$((i+1))
-      n=$((n+1))
-    done <<< ${line[l]#*=}" "
+    read -a tmp <<< ${line[l]#*=}
+    ispt+=(${tmp[@]})
+    n=$((n+${#tmp[@]}))
     l=$((l+1))
   done
   n=0
   while [ $n -lt $npe ]; do
-    while read -d " " ii_pe[j]; do
-      j=$((j+1))
-      n=$((n+1))
-    done <<< ${line[l]#*=}" "
+    read -a tmp <<< ${line[l]#*=}
+    iipe+=(${tmp[@]})
+    n=$((n+${#tmp[@]}))
     l=$((l+1))
   done
   m=$((m+1))
@@ -114,18 +130,19 @@ done
 #
 l=$((l+1))
 m=0
+jspt=()
 while [ $m -lt $mpe ]; do
-  while read -d " " j0_pe[m]; do
-    j0_pe[m]=$((${j0_pe[m]}-1))
-    m=$((m+1))
-  done <<< ${line[l]#*=}" "
+  read -a tmp <<< ${line[l]#*=}
+  jspt+=(${tmp[@]})
+  m=$((m+${#tmp[@]}))
   l=$((l+1))
 done
 m=0
+jjpe=()
 while [ $m -lt $mpe ]; do
-  while read -d " " jj_pe[m]; do
-    m=$((m+1))
-  done <<< ${line[l]#*=}" "
+  read -a tmp <<< ${line[l]#*=}
+  jjpe+=(${tmp[@]})
+  m=$((m+${#tmp[@]}))
   l=$((l+1))
 done
 
@@ -169,74 +186,78 @@ echo -n "      data i0_pe/" >> dimensions.F
 P=72
 i=0
 while [ $i -lt $((npe*mpe)) ]; do
-  if [ $((P+${#i0_pe[i]}+1)) -gt 72 ]; then
+  i0_pe=$((${ispt[i]}-1))
+  P=$((P+${#i0_pe}+1))
+  if [ $P -gt 72 ]; then
     echo -ne "\n     .  " >> dimensions.F
-    P=8
+    P=$((8+${#i0_pe}+1))
   fi
-  echo -n "${i0_pe[i]}" >> dimensions.F
-  if [ $i -lt $((npe*mpe-1)) ]; then
+  echo -n "$i0_pe" >> dimensions.F
+  i=$((i+1))
+  if [ $i -lt $((npe*mpe)) ]; then
     echo -n "," >> dimensions.F
   else
     echo "/" >> dimensions.F
   fi
-  P=$((P+${#i0_pe[i]}+1))
-  i=$((i+1))
 done
 
 echo -n "      data ii_pe/" >> dimensions.F
 P=72
 i=0
 while [ $i -lt $((npe*mpe)) ]; do
-  if [ $((P+${#ii_pe[i]}+1)) -gt 72 ]; then
+  ii_pe=${iipe[i]}
+  P=$((P+${#ii_pe}+1))
+  if [ $P -gt 72 ]; then
     echo -ne "\n     .  " >> dimensions.F
-    P=8
+    P=$((8+${#ii_pe}+1))
   fi
-  echo -n "${ii_pe[i]}" >> dimensions.F
-  if [ $i -lt $((npe*mpe-1)) ]; then
+  echo -n "$ii_pe" >> dimensions.F
+  i=$((i+1))
+  if [ $i -lt $((npe*mpe)) ]; then
     echo -n "," >> dimensions.F
   else
     echo "/" >> dimensions.F
   fi
-  P=$((P+${#ii_pe[i]}+1))
-  i=$((i+1))
 done
 
 echo -n "      data j0_pe/" >> dimensions.F
 P=72
 i=0
 while [ $i -lt $((npe*mpe)) ]; do
   n=$((i/npe))
-  if [ $((P+${#j0_pe[n]}+1)) -gt 72 ]; then
+  j0_pe=$((${jspt[n]}-1))
+  P=$((P+${#j0_pe}+1))
+  if [ $P -gt 72 ]; then
     echo -ne "\n     .  " >> dimensions.F
-    P=8
+    P=$((8+${#j0_pe}+1))
   fi
-  echo -n "${j0_pe[n]}" >> dimensions.F
-  if [ $i -lt $((npe*mpe-1)) ]; then
+  echo -n "$j0_pe" >> dimensions.F
+  i=$((i+1))
+  if [ $i -lt $((npe*mpe)) ]; then
     echo -n "," >> dimensions.F
   else
     echo "/" >> dimensions.F
   fi
-  P=$((P+${#j0_pe[n]}+1))
-  i=$((i+1))
 done
 
 echo -n "      data jj_pe/" >> dimensions.F
 P=72
 i=0
 while [ $i -lt $((npe*mpe)) ]; do
   n=$((i/npe))
-  if [ $((P+${#jj_pe[n]}+1)) -gt 72 ]; then
+  jj_pe=${jjpe[n]}
+  P=$((P+${#jj_pe}+1))
+  if [ $P -gt 72 ]; then
     echo -ne "\n     .  " >> dimensions.F
-    P=8
+    P=$((8+${#jj_pe}+1))
   fi
-  echo -n "${jj_pe[n]}" >> dimensions.F
-  if [ $i -lt $((npe*mpe-1)) ]; then
+  echo -n "$jj_pe" >> dimensions.F
+  i=$((i+1))
+  if [ $i -lt $((npe*mpe)) ]; then
     echo -n "," >> dimensions.F
   else
     echo "/" >> dimensions.F
   fi
-  P=$((P+${#jj_pe[n]}+1))
-  i=$((i+1))
 done
 
 cat <<EOF >>dimensions.F

model/noresm1/scripts/ccsm_utils/Machines/Macros.betzy

@@ -48,7 +48,7 @@ endif
 
 # Note that NETCDF_DIR is obtained from invoking modules
 FC            := mpifort -g -traceback
-CC            := mpicc -g -traceback
+CC            := icc -g -traceback
 NETCDF_PATH   := $(EBROOTNETCDF)
 INC_NETCDF    := $(EBROOTNETCDFMINFORTRAN)/include
 LIB_NETCDF    := $(EBROOTNETCDFMINFORTRAN)/lib

model/noresm1/scripts/ccsm_utils/Machines/env_machopts.betzy

@@ -17,10 +17,7 @@ endif
 
 module --quiet restore system
 module load StdEnv
-module load netCDF-Fortran/4.6.0-iompi-2022a
-module load NCO/5.1.9-iomkl-2022a
-module load CMake/3.23.1-GCCcore-11.3.0
-module load Python/3.10.4-GCCcore-11.3.0
+module load NCO/5.1.9-iomkl-2022a XML-LibXML/2.0209-GCCcore-12.3.0 CMake/3.27.6-GCCcore-13.2.0 Python/3.11.5-GCCcore-13.2.0 netCDF-Fortran/4.6.1-iompi-2023b iomkl/2023b 
 
 #-------------------------------------------------------------------------------
 # Runtime environment variables

model/noresm1/scripts/ccsm_utils/Machines/mkbatch.betzy

@@ -75,7 +75,7 @@ setenv I_MPI_EXTRA_FILESYSTEM on
 setenv I_MPI_EXTRA_FILESYSTEM_LIST lustre
 
 if (\$USE_MPISERIAL == "FALSE") then
-   srun ./ccsm.exe >&! ccsm.log.\$LID
+   srun --kill-on-bad-exit --label ./ccsm.exe >&! ccsm.log.\$LID
 else
          ./ccsm.exe >&! ccsm.log.\$LID
 endif

setup/noresm1/settings/norcpm-cf-system1_assimtest_19811115.sh

@@ -21,7 +21,7 @@
 : ${STOP_OPTION:=nmonths} # units for run length specification STOP_N 
 : ${STOP_N:=1} # run continuesly for this length 
 : ${RESTART:=2} # restart this many times  
-: ${WALLTIME:='24:00:00'}
+: ${WALLTIME:='01:00:00'}
 : ${PECOUNT:=L} # T=32, S=64, M=96, L=128, X1=502
 : ${ACCOUNT:=nn9039k}
 : ${MAX_PARALLEL_STARCHIVE:=30}

setup/noresm1/settings/setmach.sh

@@ -36,10 +36,7 @@ betzy)
   source /cluster/installations/lmod/lmod/init/sh
   module --quiet restore system
   module load StdEnv
-  module load netCDF-Fortran/4.6.0-iompi-2022a
-  module load NCO/5.1.9-iomkl-2022a
-  module load CMake/3.23.1-GCCcore-11.3.0
-  module load Python/3.10.4-GCCcore-11.3.0
+  module load NCO/5.1.9-iomkl-2022a XML-LibXML/2.0209-GCCcore-12.3.0 CMake/3.27.6-GCCcore-13.2.0 Python/3.11.5-GCCcore-13.2.0 netCDF-Fortran/4.6.1-iompi-2023b iomkl/2023b 
   export MKL_DEBUG_CPU_TYPE=5
   ulimit -s unlimited
   ;; 

setup/noresm2/run_ensemble.sh

@@ -82,7 +82,7 @@ do
     NODES_DA=$((NODES_TOTAL-NODES_NORESM))
     head -$((NODES_TOTAL-NODES_NORESM)) hostfile > $ANALYSISROOT/hostfile_da 
   fi  
-  sed -i -e "s%<executable>srun.*%<executable>srun -n $NTASKS_NORESM -N $NODES_NORESM -F hostfile_noresm </executable>%" env_mach_specific.xml 
+  sed -i -e "s%<executable>srun.*%<executable>srun --kill-on-bad-exit --label -n $NTASKS_NORESM -N $NODES_NORESM -F hostfile_noresm </executable>%" env_mach_specific.xml 
   ./case.submit --no-batch
   while true
   do