Add CentOS to CI (#19)

Author: letmaik

.ci/azure-pipelines/matrix.yml

@@ -3,13 +3,17 @@
 
 parameters:
   templates:
-    Linux: unix.yml
+    Ubuntu: unix.yml
+    CentOS: unix.yml
     macOS: unix.yml
     Windows: windows.yml
 
-  Linux:
+  Ubuntu:
     CC: gcc-9
     FC: gfortran-9
+  CentOS:
+    CC: gcc
+    FC: gfortran
   macOS:
     CC: gcc-9
     FC: gfortran-9
@@ -31,7 +35,7 @@ parameters:
 
   # Loop/matrix parameters.
   # Note that these may be overridden by the including yml (typically azure-pipelines.yml).
-  OS_NAMES: [Windows, Linux, macOS]
+  OS_NAMES: [Windows, Ubuntu, CentOS, macOS]
   BUILD_SYSTEMS: [Make, CMake]
   BUILD_TYPES: [Debug, Release]
   # WPS in the Make variant does not support smpar out of the box, so ignore here.
@@ -60,9 +64,9 @@ jobs:
       - ${{ if eq(parameters.WATS_RUN, 'true') }}:
         - template: wats_diff.yml
           parameters:
-            # For Windows, since there is no Make-based reference, compare against Linux.
+            # For Windows, since there is no Make-based reference, compare against Ubuntu.
             ${{ if eq(OS_NAME, 'Windows') }}:
-              OS_NAME_LEFT: Linux
+              OS_NAME_LEFT: Ubuntu
             ${{ if ne(OS_NAME, 'Windows') }}:
               OS_NAME_LEFT: ${{ OS_NAME }}
             OS_NAME_RIGHT: ${{ OS_NAME }}
@@ -78,7 +82,7 @@ jobs:
     - ${{ each MODE in parameters.MODES }}:
       - template: wats_diff.yml
         parameters:
-          OS_NAME_LEFT: Linux
+          OS_NAME_LEFT: Ubuntu
           OS_NAME_RIGHT: macOS
           BUILD_SYSTEM_LEFT: Make
           BUILD_SYSTEM_RIGHT: Make

.ci/azure-pipelines/release.yml

@@ -12,7 +12,7 @@ steps:
     displayName: Create distribution package
 
 - ${{ if not(eq(parameters.OS_NAME, 'Windows')) }}:
-  - bash: bash .ci/unix/delocate.sh
+  - bash: $(run) .ci/unix/delocate.sh
     displayName: Delocate
 
   - task: ArchiveFiles@2
@@ -24,9 +24,14 @@ steps:
       archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.SourceBranchName)-$(NESTING)_nesting-$(MODE)-x64-$(OS_NAME)-$(BUILD_TYPE).tar.xz' 
     displayName: Create distribution package
 
-- pwsh: |
-    dir $(Build.ArtifactStagingDirectory) -r | % { if ($_.Name -cne $_.Name.ToLower()) { ren $_.FullName $_.Name.ToLower() } }
-  displayName: Rename to lowercase
+- bash: |
+    set -ex
+    cd "$(Build.ArtifactStagingDirectory)"
+    # CentOS packages will run on most glibc-based Linux distributions
+    find . -name '*CentOS*' -exec bash -c 'mv $0 ${0/CentOS/Linux}' {} \;
+    # Rename all files to lowercase
+    for file in $(ls); do mv ${file} $(echo ${file} | tr '[:upper:]' '[:lower:]'); done
+  displayName: Rename package files
 
 # not strictly needed
 - task: PublishBuildArtifacts@1

.ci/azure-pipelines/unix.yml

@@ -8,34 +8,30 @@ jobs:
   timeoutInMinutes: 0
 
   pool:
-    ${{ if eq(parameters.OS_NAME, 'Linux') }}:
-      vmImage: ubuntu-16.04
+    ${{ if or(eq(parameters.OS_NAME, 'Ubuntu'), eq(parameters.OS_NAME, 'CentOS')) }}:
+      vmImage: ubuntu-latest
     ${{ if eq(parameters.OS_NAME, 'macOS') }}:
-      vmImage: macos-10.13
-
-  ${{ if eq(parameters.OS_NAME, 'Linux') }}:
-    container:
-      image: ubuntu:16.04
-      options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
+      vmImage: macOS-10.14
 
   variables:
     ${{ insert }}: ${{ parameters }}
+    ${{ if eq(parameters.OS_NAME, 'Ubuntu') }}:
+      IMAGE: ubuntu:16.04
+      run: .ci/unix/run-in-docker.sh
+    ${{ if eq(parameters.OS_NAME, 'CentOS') }}:
+      IMAGE: quay.io/pypa/manylinux2010_x86_64:latest
+      run: .ci/unix/run-in-docker.sh
+    ${{ if eq(parameters.OS_NAME, 'macOS') }}:
+      run: ''
 
-  steps:
-  - ${{ if eq(parameters.OS_NAME, 'Linux') }}:
-    # https://github.com/Microsoft/azure-pipelines-agent/issues/2043#issuecomment-489692810
-    - script: |
-        /tmp/docker exec -t -u 0 ci-container \
-          sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
-      displayName: Set up sudo
-    
-  - script: .ci/unix/dump-vm-specs.sh
+  steps:    
+  - script: $(run) .ci/unix/dump-vm-specs.sh
     displayName: Dump VM specs
 
-  - script: printenv
+  - script: $(run) printenv | sort
     displayName: Dump environment variables
 
-  - ${{ if eq(parameters.OS_NAME, 'Linux') }}:
+  - ${{ if eq(parameters.OS_NAME, 'Ubuntu') }}:
     - script: |
         sudo apt-get update && sudo apt-get install -y curl
       displayName: Install curl
@@ -47,13 +43,13 @@ jobs:
       curl -L --retry 3 https://github.com/$WRF_REPO/archive/$WRF_COMMIT.tar.gz | tar xz --strip-components=1
     displayName: Download WRF
 
-  - script: ../WRF/.ci/unix/setup-dependencies.sh
+  - script: $(run) ../WRF/.ci/unix/setup-dependencies.sh
     displayName: Setup dependencies
 
-  - script: ../WRF/.ci/unix/install-wrf.sh
+  - script: $(run) ../WRF/.ci/unix/install-wrf.sh
     displayName: Install WRF
 
-  - script: .ci/unix/install-wps.sh
+  - script: $(run) .ci/unix/install-wps.sh
     displayName: Install WPS
 
   - ${{ if eq(parameters.WATS_RUN, 'true') }}:

.ci/azure-pipelines/wats_diff.yml

@@ -6,7 +6,7 @@ jobs:
   displayName: 'WATS ${{ parameters.OS_NAME_LEFT }}/${{ parameters.OS_NAME_RIGHT }} ${{ parameters.BUILD_SYSTEM_LEFT }}/${{ parameters.BUILD_SYSTEM_RIGHT }}: ${{ parameters.BUILD_TYPE }}, ${{ parameters.MODE }}'
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-latest
 
   dependsOn:
   - ${{ parameters.OS_NAME_LEFT }}_${{ parameters.BUILD_SYSTEM_LEFT }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}
@@ -19,11 +19,7 @@ jobs:
   - bash: .ci/unix/use-conda.sh
     displayName: Enable Conda
 
-  - bash: |
-      set -ex
-      mkdir wats && cd wats
-      curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz --strip-components=1
-      conda env update -n base -f environment.yml
+  - bash: .ci/unix/install-wats.sh
     displayName: Install WATS
 
   - task: DownloadBuildArtifacts@0

.ci/azure-pipelines/wats_pkg.yml

@@ -11,7 +11,7 @@ jobs:
   displayName: Publish WATS Package
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-latest
 
   dependsOn:
   - ${{ parameters.OS_NAME }}_${{ parameters.BUILD_SYSTEM }}_${{ parameters.BUILD_TYPE }}_${{ parameters.MODE }}

.ci/azure-pipelines/wats_run.yml

@@ -2,28 +2,13 @@
 # Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
 
 steps:
-- bash: .ci/unix/use-conda.sh
+- bash: $(run) .ci/unix/use-conda.sh
   displayName: Enable Conda
 
-- bash: |
-    set -ex
-    mkdir wats && cd wats
-    curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz --strip-components=1
-    conda env update -n base -f environment.yml
+- bash: $(run) .ci/unix/install-wats.sh
   displayName: Install WATS
 
-- bash: |
-    set -ex
-    if [[ $MODE == dm* ]]; then mpi_flag="--mpi"; fi
-    if [[ $OS_NAME == macOS ]]; then
-      # Work around Open MPI issue
-      # https://github.com/open-mpi/ompi/issues/6518
-      # https://github.com/open-mpi/ompi/issues/5798
-      # https://www.mail-archive.com/devel@lists.open-mpi.org/msg20760.html
-      export OMPI_MCA_btl=self,tcp
-    fi
-    if [[ $BUILD_SYSTEM == "CMake" ]]; then dir_suffix="build/install"; fi
-    python wats/wats/main.py run --mode $WATS_MODE --wrf-dir ../WRF/$dir_suffix --wps-dir ./$dir_suffix --work-dir wats_work $mpi_flag
+- bash: $(run) .ci/unix/run-wats.sh
   displayName: Run WATS
 
 - task: PublishBuildArtifacts@1

.ci/azure-pipelines/windows.yml

@@ -8,7 +8,7 @@ jobs:
   timeoutInMinutes: 0
 
   pool:
-    vmImage: windows-2019
+    vmImage: windows-latest
 
   variables:
     ${{ insert }}: ${{ parameters }}

.ci/unix/delocate.sh

@@ -10,12 +10,37 @@ cd $SCRIPTDIR/../..
 
 if [ "$(uname)" == "Darwin" ]; then
 
-    pip3 install delocate
+    pip install delocate
     delocate-listdeps --all --depending build/install
     delocate-path build/install
     delocate-listdeps --all --depending build/install
 
+elif [ "$(lsb_release -i -s)" == "CentOS" ]; then
+
+    # Assumes we're in the manylinux Docker image.
+    pys=(/opt/python/cp*)
+    # Use the newest Python available in the image (last item).
+    py=${pys[@]:(-1)}/bin/python
+
+    root_dir=$(pwd)
+    tmp_dir=$(mktemp -d)
+    cd $tmp_dir
+
+    echo "from setuptools import setup; setup(name='app', packages=['main'], package_data={'main': ['*.exe']})" > setup.py
+    ln -s $root_dir/build/install main
+    $py setup.py bdist_wheel
+
+    # CentOS uses /usr/lib64 but some manually installed dependencies end up in /usr/lib
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
+
+    auditwheel repair dist/*.whl --no-update-tags
+    cd wheelhouse
+    unzip *.whl
+
+    # /bin/cp as cp is aliased to 'cp -i' and would ask before overwriting
+    /bin/cp -r main/. $root_dir/build/install
+
 else
-    echo "Unknown OS: $(uname)"
+    echo "Unsupported OS: $(uname)"
     exit 1
 fi

.ci/unix/dump-vm-specs.sh

@@ -15,7 +15,11 @@ if [ "$(uname)" == "Darwin" ]; then
     sudo scutil --get LocalHostName || true
 elif [ "$(uname)" == "Linux" ]; then
     if [ "$(which lsb_release)" == "" ]; then
-        sudo apt install -y lsb-release
+        if [ -f /etc/redhat-release ]; then
+            sudo yum install -y redhat-lsb-core
+        else
+            sudo apt install -y lsb-release
+        fi
     fi
     lsb_release -a
     free -m

.ci/unix/install-wats.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# WRF-CMake (https://github.com/WRF-CMake/wps).
+# Copyright 2019 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+set -ex
+
+SCRIPTDIR=$(dirname "$0")
+cd $SCRIPTDIR/../..
+
+curl -L --retry 3 https://github.com/$WATS_REPO/archive/$WATS_BRANCH.tar.gz | tar xz
+mv wats-$WATS_BRANCH wats
+conda env update -n base -f wats/environment.yml

.ci/unix/install-wps.sh

@@ -38,7 +38,7 @@ elif [ $BUILD_SYSTEM == "Make" ]; then
             *) echo "Invalid: $MODE" ;;
         esac
 
-        if [ "$(lsb_release -c -s)" == "trusty" ]; then
+        if [ "$(lsb_release -c -s)" == "trusty" -o "$(lsb_release -i -s)" == "CentOS" ]; then
             export NETCDF=/usr
         else
             # Need to create symlinked folder hierarchy that WRF expects...
@@ -52,7 +52,11 @@ elif [ $BUILD_SYSTEM == "Make" ]; then
 
         ## As the zlib and PNG libraries are not in a standard path that will be checked automatically by the compiler,
         ## we include them with the JASPER include and library path
-        export JASPERLIB="/usr/lib/x86_64-linux-gnu"
+        if [ "$(lsb_release -i -s)" == "CentOS" ]; then
+            export JASPERLIB="/usr/lib64"
+        else
+            export JASPERLIB="/usr/lib/x86_64-linux-gnu"
+        fi
         export JASPERINC="/usr/include/jasper -I/usr/include"
 
     elif [ "$(uname)" == "Darwin" ]; then
@@ -69,6 +73,11 @@ elif [ $BUILD_SYSTEM == "Make" ]; then
         export JASPERLIB=$(brew --prefix jasper)/lib
         export JASPERINC=$(brew --prefix jasper)/include
 
+        # Otherwise libpng is not found.
+        # Note that LIBRARY_PATH and CPATH applies to all compiler invocations.
+        export LIBRARY_PATH=/usr/local/lib
+        export CPATH=/usr/local/include
+
     else
         echo "The environment is not recognised"
         exit 1

.ci/unix/run-in-docker.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# WRF-CMake (https://github.com/WRF-CMake/wps).
+# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+set -e
+
+SCRIPTDIR=$(dirname "$0")
+REPODIR=$(realpath $SCRIPTDIR/../..)
+# above repo root as this is where WRF is cloned into
+MOUNTDIR=$(realpath $SCRIPTDIR/../../..)
+
+container=wps-ci
+
+# Create container if it doesn't exist
+if [ ! "$(docker ps -q -f name=$container)" ]; then
+    echo "Creating Docker container $container"
+    set -x
+    docker run --name $container -t -d -v $MOUNTDIR:$MOUNTDIR -w $REPODIR -e DOCKER=1 $IMAGE
+    set +x
+    
+    echo "Installing sudo inside container"
+    if [[ $OS_NAME == CentOS ]]; then
+      docker exec $container sh -c "yum install -y sudo"
+    else
+      docker exec $container sh -c "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+    fi
+fi
+
+echo "Running inside container: $@"
+host_envs=$(env | cut -f1 -d= | sed 's/^/-e /' | grep -v -e PATH -e HOME)
+# Use login shell so that ~/.bash_profile is read.
+# use-conda.sh appends to that file to modify the PATH.
+docker exec $host_envs $container bash --login -c "$@"

.ci/unix/run-wats.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# WRF-CMake (https://github.com/WRF-CMake/wps).
+# Copyright 2018 M. Riechert and D. Meyer. Licensed under the MIT License.
+
+set -ex
+
+SCRIPTDIR=$(dirname "$0")
+cd $SCRIPTDIR/../..
+
+if [ "$(uname)" == "Linux" ]; then
+    if [ "$(lsb_release -i -s)" == "CentOS" ]; then
+        # CentOS uses /usr/lib64 but some manually installed dependencies end up in /usr/lib
+        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib
+    fi
+fi
+
+if [[ $MODE == dm* ]]; then
+    mpi_flag="--mpi"
+fi
+
+if [[ $OS_NAME == macOS ]]; then
+    # Work around Open MPI issue
+    # https://github.com/open-mpi/ompi/issues/6518
+    # https://github.com/open-mpi/ompi/issues/5798
+    # https://www.mail-archive.com/devel@lists.open-mpi.org/msg20760.html
+    export OMPI_MCA_btl=self,tcp
+    # Disable new shared memory component of Open MPI to work around issue
+    # https://github.com/open-mpi/ompi/issues/7516
+    export PMIX_MCA_gds=hash
+fi
+
+if [[ $BUILD_SYSTEM == "CMake" ]]; then
+    dir_suffix="build/install"
+fi
+
+python wats/wats/main.py run \
+    --mode $WATS_MODE \
+    --wrf-dir ../WRF/$dir_suffix \
+    --wps-dir ./$dir_suffix \
+    --work-dir wats_work \
+    $mpi_flag