Skip to content

Commit 43354fd

Browse files
tikk3rtikk3r
committed
Fix #51 and some BLAS issues
1 parent 9b0afa0 commit 43354fd

3 files changed

+50
-40
lines changed

‎Singularity.lofar_sksp

+25-24
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,11 @@
22
#From: tikk3r/lofar-grid-hpccloud:lofar_sksp_base
33
# If building from a base image locally enable this instead.
44
Bootstrap: localimage
5-
From: lofar_sksp_base_v3.3.1_x86-64_generic_avx512_cuda_11_3.sif
5+
From: lofar_sksp_v3.3.3_beta_x86-64_generic_avx512_cuda_11_5_base.sif
66

7-
#%environment
8-
#export PYTHONPATH=
9-
#export INSTALLDIR=/opt/lofar
10-
#. $INSTALLDIR/init.sh
11-
#source $INSTALLDIR/pyenv-py2/bin/activate
7+
%environment
8+
. $INSTALLDIR/init.sh
9+
source $INSTALLDIR/pyenv-py2/bin/activate
1210

1311
%post
1412
. $INSTALLDIR/init.sh
@@ -65,7 +63,7 @@ From: lofar_sksp_base_v3.3.1_x86-64_generic_avx512_cuda_11_3.sif
6563
#export MTUNE='cascadelake'
6664
export NOAVX512=false
6765

68-
if [ $NOAVX512 ]; then
66+
if [ $NOAVX512 = false]; then
6967
export CFLAGS="-march=${MARCH} -mtune=${MTUNE} -mno-avx512f -mno-avx512pf -mno-avx512er -mno-avx512cd -mno-avx512vl -mno-avx512bw -mno-avx512dq -mno-avx512ifma -mno-avx512vbmi"
7068
export CXXFLAGS="-march=${MARCH} -mtune=${MTUNE} -std=c++11 -mno-avx512f -mno-avx512pf -mno-avx512er -mno-avx512cd -mno-avx512vl -mno-avx512bw -mno-avx512dq -mno-avx512ifma -mno-avx512vbmi"
7169
else
@@ -332,14 +330,14 @@ From: lofar_sksp_base_v3.3.1_x86-64_generic_avx512_cuda_11_3.sif
332330
#git clone https://gitlab.com/astron-idg/idg.git src
333331
git clone https://git.astron.nl/RD/idg.git src
334332
cd src && git checkout $IDG_VERSION && echo export IDG_VERSION=$(git rev-parse --short HEAD) >> $INSTALLDIR/init.sh && mkdir build && cd build
335-
if [ $HAS_CUDA ] && [ $HAS_MKL ]; then
336-
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=ON -DBUILD_LIB_CUDA=ON -DCUDA_INCLUDE_DIR=/usr/local/cuda/include -DCMAKE_BUILD_TYPE=Debug ..
337-
elif [ ! $HAS_CUDA ] && [ $HAS_MKL ]; then
338-
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=ON -DCMAKE_BUILD_TYPE=Debug ..
339-
elif [ $HAS_CUDA ] && [ ! $HAS_MKL ]; then
340-
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=OFF -DBUILD_LIB_CUDA=ON -DCUDA_INCLUDE_DIR=/usr/local/cuda/include -DCMAKE_BUILD_TYPE=Debug ..
333+
if [ $HAS_CUDA = true ] && [ $HAS_MKL = true ]; then
334+
cmake3 -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=ON -DBUILD_LIB_CUDA=ON -DCUDA_INCLUDE_DIR=/usr/local/cuda/include -DCMAKE_BUILD_TYPE=Debug ..
335+
elif [ $HAS_CUDA = false ] && [ $HAS_MKL = true ]; then
336+
cmake3 -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=ON -DCMAKE_BUILD_TYPE=Debug ..
337+
elif [ $HAS_CUDA = false] && [ ! $HAS_MKL = false ]; then
338+
cmake3 -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBUILD_WITH_MKL=OFF -DBUILD_LIB_CUDA=ON -DCUDA_INCLUDE_DIR=/usr/local/cuda/include -DCMAKE_BUILD_TYPE=Debug -DBLAS_openblas_LIBRARY=/usr/lib64/libopenblasp.so -DBLAS_blas_LIBRARY=/usr/lib64/libopenblasp.so ..
341339
else
342-
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg ..
340+
cmake3 -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/idg -DBLAS_openblas_LIBRARY=/usr/lib64/libopenblasp.so -DBLAS_blas_LIBRARY=/usr/lib64/libopenblasp.so ..
343341
fi
344342
make -j $J
345343
make install
@@ -349,26 +347,29 @@ From: lofar_sksp_base_v3.3.1_x86-64_generic_avx512_cuda_11_3.sif
349347
#
350348
# Install DPPP
351349
#
352-
# Temporarily get rid of MKL. Throws an error about needing a not-OpenMP version of OpenBLAS and MKL's multithreaded stuff uses OpenMP from what I can tell.
353-
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
354-
export CPATH=$(echo $CPATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
355-
export CMAKE_PREFIX_PATH=$(echo $CMAKE_PREFIX_PATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
356-
357350
mkdir -p $INSTALLDIR/DPPP/build
358351
git clone https://git.astron.nl/RD/DP3.git $INSTALLDIR/DPPP/src
359352
cd $INSTALLDIR/DPPP/src
360353
git checkout ${DPPP_VERSION}
361354
echo export DPPP_VERSION=$(git rev-parse --short HEAD) >> $INSTALLDIR/init.sh
362355
cd $INSTALLDIR/DPPP/build
356+
# Temporarily get rid of MKL. Throws an error about needing a not-OpenMP version of OpenBLAS and MKL's multithreaded stuff uses OpenMP from what I can tell.
357+
if [ $HAS_MKL = true ]; then
358+
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
359+
export CPATH=$(echo $CPATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
360+
export CMAKE_PREFIX_PATH=$(echo $CMAKE_PREFIX_PATH | tr ":" "\n" | grep -v "/opt/intel" | tr "\n" ":")
361+
fi
363362
# Link to libopenblasp.so (note the p) and not libopenblas.so so we get the multi-threaded version.
364-
cmake3 -DCMAKE_CXX_FLAGS="-D_GLIB_USE_CXX_ABI=1 -DBOOST_NO_CXX11_SCOPED_ENUMS" -DCMAKE_INSTALL_PREFIX:PATH=$INSTALLDIR/DPPP -DLOFAR_STATION_RESPONSE_DIR:PATH=$INSTALLDIR/lofar/include -DLOFAR_STATION_RESPONSE_LIB:FILEPATH=$INSTALLDIR/lofar/lib/libstationresponse.so -DIDGAPI_LIBRARIES=$INSTALLDIR/idg/lib/libidg-api.so -DIDGAPI_INCLUDE_DIRS=$INSTALLDIR/idg/include -DAOFLAGGER_INCLUDE_DIR=$INSTALLDIR/aoflagger/include -DAOFLAGGER_LIB=$INSTALLDIR/aoflagger/lib/libaoflagger.so -DBLAS_openblas_LIBRARY=/usr/lib64/libopenblasp.so ../src
363+
cmake3 -DCMAKE_CXX_FLAGS="-D_GLIB_USE_CXX_ABI=1 -DBOOST_NO_CXX11_SCOPED_ENUMS" -DCMAKE_INSTALL_PREFIX:PATH=$INSTALLDIR/DPPP -DLOFAR_STATION_RESPONSE_DIR:PATH=$INSTALLDIR/lofar/include -DLOFAR_STATION_RESPONSE_LIB:FILEPATH=$INSTALLDIR/lofar/lib/libstationresponse.so -DIDGAPI_LIBRARIES=$INSTALLDIR/idg/lib/libidg-api.so -DIDGAPI_INCLUDE_DIRS=$INSTALLDIR/idg/include -DAOFLAGGER_INCLUDE_DIR=$INSTALLDIR/aoflagger/include -DAOFLAGGER_LIB=$INSTALLDIR/aoflagger/lib/libaoflagger.so -DBLAS_openblas_LIBRARY:FILEPATH=/usr/lib64/libopenblasp.so ../src
365364
$make -s -j $J && $make install
366365
cd $INSTALLDIR
367366
rm -rf $INSTALLDIR/DPPP/build
368367
rm -rf $INSTALLDIR/DPPP/src
369-
export LD_LIBRARY_PATH=/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
370-
export CPATH=/opt/intel/mkl/include:$CPATH
371-
export CMAKE_PREFIX_PATH=/opt/intel/mkl/lib/intel64:$CMAKE_PREFIX_PATH
368+
if [ $HAS_MKL = true ]; then
369+
export LD_LIBRARY_PATH=/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH
370+
export CPATH=/opt/intel/mkl/include:$CPATH
371+
export CMAKE_PREFIX_PATH=/opt/intel/mkl/lib/intel64:$CMAKE_PREFIX_PATH
372+
fi
372373

373374
############################################
374375
# Install Python packages for Python 2 now #
@@ -443,7 +444,7 @@ From: lofar_sksp_base_v3.3.1_x86-64_generic_avx512_cuda_11_3.sif
443444
export CC=`which mpicc`
444445
export CXX=`which mpic++`
445446
# TARGET_CPU is a WSClean 2.10.2 feature. Change to PORTABLE=TRUE if using and older version to avoid -march=native being triggered.
446-
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/wsclean -DTARGET_CPU=${MARCH} -DIDGAPI_LIBRARIES=$INSTALLDIR/idg/lib/libidg-api.so -DIDGAPI_INCLUDE_DIRS=$INSTALLDIR/idg/include ..
447+
cmake -DCMAKE_INSTALL_PREFIX=$INSTALLDIR/wsclean -DTARGET_CPU=${MARCH} -DIDGAPI_LIBRARIES=$INSTALLDIR/idg/lib/libidg-api.so -DIDGAPI_INCLUDE_DIRS=$INSTALLDIR/idg/include -DBLAS_openblas_LIBRARY=/usr/lib64/libopenblasp.so ..
447448
$make -j ${J}
448449
$make install
449450
cd $INSTALLDIR

‎Singularity.lofar_sksp_base

+5-5
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ From: fedora:31
2424
export WCSLIB_VERSION=6.4
2525
export DEBIAN_FRONTEND=noninteractive
2626
export HAS_CUDA=true
27-
export HAS_MKL=true
27+
export HAS_MKL=false
2828
export OMPI_ALLOW_RUN_AS_ROOT=1
2929
export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
3030

@@ -148,7 +148,7 @@ From: fedora:31
148148
export CXX=`which g++`
149149

150150

151-
if [ $HAS_MKL ]; then
151+
if [ $HAS_MKL = true]; then
152152
#
153153
# Install Intel MKL
154154
#
@@ -157,7 +157,7 @@ From: fedora:31
157157
dnf -y install intel-mkl-2020.0-088 intel-mkl-64bit-2020.0-088
158158
fi
159159

160-
if [ $HAS_CUDA ]; then
160+
if [ $HAS_CUDA = true]; then
161161
#
162162
# Install CUDA 11.3
163163
#
@@ -181,8 +181,8 @@ From: fedora:31
181181
echo export INSTALLDIR=$INSTALLDIR > $INSTALLDIR/init.sh
182182
echo export PATH=/usr/local/cuda/bin:/opt/hdf5/bin:\$PATH >> $INSTALLDIR/init.sh
183183
echo export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/opt/hdf5/lib:/opt/intel/mkl/lib/intel64/:\$LD_LIBRARY_PATH >> $INSTALLDIR/init.sh
184-
echo export HAS_CUDA=true >> $INSTALLDIR/init.sh
185-
echo export HAS_MKL=true >> $INSTALLDIR/init.sh
184+
echo export HAS_CUDA=$HAS_CUDA >> $INSTALLDIR/init.sh
185+
echo export HAS_MKL=$HAS_MKL >> $INSTALLDIR/init.sh
186186

187187
%runscript
188188
echo Base container built from LOFAR-SKSP Singularity v3.3

‎Singularity.lofar_sksp_ddf_public

+20-11
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#Bootstrap: shub
22
#From: tikk3r/lofar-grid-hpccloud:lofar_sksp
33
Bootstrap: localimage
4-
From: lofar_sksp.sif
4+
From: lofar_sksp_v3.3.3_beta_x86-64_generic_avx512_cuda_11_5.sif
55

66

77

@@ -15,6 +15,9 @@ From: lofar_sksp.sif
1515
# General environment settings.
1616
export J=4
1717
export INSTALLDIR=/opt/lofar
18+
export DDFACET_VERSION=v0.5.3.1
19+
#export DDFACET_VERSION=v0.6.0
20+
export KILLMS_VERSION=v3.0.1
1821
export PYTHON_VERSION=2.7
1922
export PYTHON_CASACORE_VERSION=v3.3.0
2023
export make=/opt/lofar/make/bin/make
@@ -93,23 +96,29 @@ From: lofar_sksp.sif
9396
#
9497
# Install DDFacet
9598
#
96-
export PATH=$INSTALLDIR/casacore/bin:$PATH
97-
export PYTHONPATH=$INSTALLDIR/python-casacore2/lib/python2.7/site-packages:$INSTALLDIR/python-casacore2/lib64/python2.7/site-packages:$PYTHONPATH
98-
export LD_LIBRARY_PATH=$INSTALLDIR/casacore/lib:$LD_LIBRARY_PATH
9999
mkdir -p $INSTALLDIR/DDFacet
100100
cd $INSTALLDIR/DDFacet
101-
git clone --single-branch -b v0.5.3.1 https://github.com/saopicc/DDFacet.git src
102-
#find . -type f -exec sed -i 's/c++14/c++11/g' {} \;
103-
# Figure out how to adapt this patch
104-
#patch $INSTALLDIR/DDFacet/src/DDFacet/Other/AsyncProcessPool.py $PATCH_DDFACET_CPUS
105-
wget https://raw.githubusercontent.com/saopicc/DDFacet/729e52675cf0dc4684881bef0c9e60e0a0b3677e/setup.py -O $INSTALLDIR/DDFacet/src/setup.py
106-
cd src && python setup.py install --prefix=$INSTALLDIR/DDFacet
101+
git clone --single-branch -b $DDFACET_VERSION https://github.com/saopicc/DDFacet.git src
102+
cd src
103+
python setup.py install --prefix=$INSTALLDIR/DDFacet
104+
#export PATH=$INSTALLDIR/casacore/bin:$PATH
105+
#export PYTHONPATH=$INSTALLDIR/python-casacore2/lib/python2.7/site-packages:$INSTALLDIR/python-casacore2/lib64/python2.7/site-packages:$PYTHONPATH
106+
#export LD_LIBRARY_PATH=$INSTALLDIR/casacore/lib:$LD_LIBRARY_PATH
107+
#mkdir -p $INSTALLDIR/DDFacet
108+
#cd $INSTALLDIR/DDFacet
109+
#git clone --single-branch -b v0.5.3.1 https://github.com/saopicc/DDFacet.git src
110+
##find . -type f -exec sed -i 's/c++14/c++11/g' {} \;
111+
## Figure out how to adapt this patch
112+
##patch $INSTALLDIR/DDFacet/src/DDFacet/Other/AsyncProcessPool.py $PATCH_DDFACET_CPUS
113+
#wget https://raw.githubusercontent.com/saopicc/DDFacet/729e52675cf0dc4684881bef0c9e60e0a0b3677e/setup.py -O $INSTALLDIR/DDFacet/src/setup.py
114+
#cd src && python setup.py install --prefix=$INSTALLDIR/DDFacet
107115

108116
#
109117
# Install DynSpecMS
110118
#
111119
cd $INSTALLDIR && git clone https://github.com/cyriltasse/DynSpecMS.git
112-
cd $INSTALLDIR/DynSpecMS && git checkout ApplyBeam
120+
#cd $INSTALLDIR && git clone https://github.com/cyriltasse/DynSpecMS.git
121+
#cd $INSTALLDIR/DynSpecMS && git checkout ApplyBeam
113122

114123
#
115124
# Install ddf-pipeline

0 commit comments

Comments
 (0)