Merge branch 'openPMD:dev' into add_cmake_fetchContent

.circleci/config.yml

@@ -0,0 +1,57 @@
+version: 2
+
+jobs:
+  linux-aarch64:
+    working_directory: ~/linux-aarch64-wheels
+    machine:
+      image: ubuntu-2204:current
+    # resource_class is what tells CircleCI to use an ARM worker for native arm builds
+    #   https://circleci.com/product/features/resource-classes/
+    #   https://circleci.com/docs/using-arm/
+    resource_class: arm.large
+    steps:
+      - checkout
+      - run:
+          name: Install build dependencies
+          command: |
+            sudo apt update
+            sudo apt install cmake g++ gfortran libfabric-dev libopenmpi-dev libhdf5-openmpi-dev hdf5-tools pkgconf python3 python3-setuptools
+            sudo .github/workflows/dependencies/install_spack
+            python3 -m pip install -U pip
+            python3 -m pip install -U packaging setuptools wheel
+            python3 -m pip install -U numpy
+            python3 -m pip install -U mpi4py
+            python3 -m pip install -U pandas
+            python3 -m pip install -U dask
+            python3 -m pip install -U pyarrow
+
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+            spack install
+
+            share/openPMD/download_samples.sh build
+      - run:
+          name: Build openPMD-api
+          command: |
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+
+            export CXXFLAGS="-DPYBIND11_DETAILED_ERROR_MESSAGES=1"
+
+            cmake -S . -B build \
+              -DopenPMD_USE_PYTHON=ON \
+              -DopenPMD_USE_MPI=ON    \
+              -DopenPMD_USE_HDF5=ON   \
+              -DopenPMD_USE_ADIOS2=ON \
+              -DopenPMD_USE_INVASIVE_TESTS=ON \
+              -DPython_EXECUTABLE=$(which python3)
+            cmake --build build --parallel 4
+      - run:
+          name: Test openPMD-api
+          command: |
+            eval $(spack env activate --sh .github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/)
+            ctest --test-dir build --output-on-failure
+
+workflows:
+  version: 2
+  all-tests:
+    jobs:
+      - linux-aarch64

.clang-tidy

@@ -2,5 +2,5 @@
 # FIXME: all performance-* reports
 # FIXME: all cert-* reports
 # FIXME: all bugprone-* reports
-Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-for-range-copy,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
+Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
 HeaderFilterRegex: '((^(?!\/share\/openPMD\/).*)*include\/openPMD\/.+\.hpp|src\/^(?!binding).+\.cpp$)'

.github/ci/spack-envs/gcc_py_ompi_h5_ad2_arm64/spack.yaml

@@ -0,0 +1,79 @@
+# This is a Spack environment file.
+#
+# Activating and installing this environment will provide all dependencies
+# that are needed for full-feature development.
+#   https//spack.readthedocs.io/en/latest/environments.html#anonymous-environments
+#
+spack:
+  specs:
+  - adios2
+  - hdf5
+  - openmpi
+
+  packages:
+    adios2:
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+    cmake:
+      externals:
+      - spec: cmake@3.22.1
+        prefix: /usr
+      buildable: False
+    libfabric:
+      externals:
+      - spec: libfabric@1.11.0
+        prefix: /usr
+      buildable: False
+    openmpi:
+      externals:
+      - spec: openmpi@4.1.2
+        prefix: /usr
+      buildable: False
+    perl:
+      externals:
+      - spec: perl@5.34.0
+        prefix: /usr
+      buildable: False
+    pkgconf:
+      externals:
+      - spec: pkgconf@1.8.0
+        prefix: /usr
+      buildable: False
+    python:
+      externals:
+      - spec: python@3.11.5
+        prefix: /usr
+      buildable: False
+    hdf5:
+      externals:
+      - spec: hdf5@1.10.7
+        prefix: /usr
+      buildable: False
+    all:
+      target: [aarch64]
+      variants: ~fortran
+      compiler: [gcc@11.4.0]
+
+  compilers:
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags: {}
+      modules: []
+      operating_system: ubuntu22.04
+      paths:
+        cc: /usr/bin/gcc
+        cxx: /usr/bin/g++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      spec: gcc@11.4.0
+      target: aarch64
+
+  # arm.large with 4 vCPU cores
+  #   https://circleci.com/product/features/resource-classes/
+  #   https://circleci.com/docs/using-arm/
+  config:
+    build_jobs: 4
+
+  # https://cache.spack.io
+  mirrors:
+    E4S: https://cache.e4s.io

.github/workflows/codeql.yml

@@ -48,20 +48,20 @@ jobs:
         run: $CMAKE -S . -B build
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           queries: +security-and-quality
 
       - name: Build (Py)
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
         if: ${{ matrix.language == 'python' }}
 
       - name: Build (C++)
         if: ${{ matrix.language == 'cpp' }}
         run: $CMAKE --build build -j 2
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{ matrix.language }}"

.github/workflows/dependencies/install_icc

@@ -7,9 +7,14 @@ sudo apt-get -qqq update
 sudo apt-get install g++
 #                    libopenmpi-dev
 sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
 sudo apt-get update
 sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
 #                       intel-oneapi-python

.github/workflows/dependencies/install_icx

@@ -5,17 +5,20 @@ set -eu -o pipefail
 
 # Ref.: https://github.com/rscohn2/oneapi-ci
 # intel-basekit intel-hpckit are too large in size
-wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \
-  | sudo apt-key add -
-echo "deb https://apt.repos.intel.com/oneapi all main" \
-  | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 
 sudo apt-get update
 
 sudo apt-get install -y --no-install-recommends \
     build-essential \
     cmake           \
-    intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \
+    intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \
     g++ gfortran
 #    libopenmpi-dev
 #    openmpi-bin

.github/workflows/linux.yml

@@ -37,7 +37,8 @@ jobs:
           -DopenPMD_USE_INVASIVE_TESTS=ON \
           -DCMAKE_VERBOSE_MAKEFILE=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
   clang7_nopy_ompi_h5_ad2_libcpp:
     runs-on: ubuntu-20.04
@@ -74,13 +75,15 @@ jobs:
           -DopenPMD_USE_INVASIVE_TESTS=ON \
           -DCMAKE_VERBOSE_MAKEFILE=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+
+        cd build
+        ctest --output-on-failure
 
         find . -name *.bp     | xargs -n1 -P1 -I {} rm -rf {}
         find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {}
-        ctest --test-dir build --output-on-failure
+        ctest --output-on-failure
 
-  clang7_nopy_ompi_h5_ad2_newLayout:
+  clang7_nopy_ompi_h5_ad2:
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
     steps:
@@ -94,7 +97,7 @@ jobs:
         sudo apt-get install clang-7 gfortran libopenmpi-dev python3
         sudo .github/workflows/dependencies/install_spack
     - name: Build
-      env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209}
+      env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror}
       run: |
         eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/)
         spack install
@@ -108,7 +111,8 @@ jobs:
           -DopenPMD_USE_INVASIVE_TESTS=ON \
           -DCMAKE_VERBOSE_MAKEFILE=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
 # TODO
 #  clang7_py36_nompi_h5_ad2_libstdc++
@@ -148,7 +152,7 @@ jobs:
         ctest --test-dir build --output-on-failure
 
   # ADIOS2 v2.7.1
-  clang8_py38_mpich_h5_ad2_newLayout:
+  clang8_py38_mpich_h5_ad2:
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
     steps:
@@ -162,7 +166,7 @@ jobs:
         sudo apt-get install clang-8 gfortran libmpich-dev python3
         sudo .github/workflows/dependencies/install_spack
     - name: Build
-      env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209}
+      env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror}
       run: |
         cmake --version
         mpiexec --version
@@ -179,7 +183,8 @@ jobs:
           -DopenPMD_USE_ADIOS2=ON  \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
 # TODO: (old Travis-CI coverage)
 #  clang10_py38_ompi_h5_1-10-6_ad2_release
@@ -229,7 +234,8 @@ jobs:
           -DopenPMD_USE_ADIOS2=ON \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
   gcc9_py38_pd_nompi_h5_ad2_libcpp:
     runs-on: ubuntu-20.04
@@ -251,7 +257,8 @@ jobs:
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
   musllinux_py10:
     runs-on: ubuntu-20.04
@@ -276,14 +283,15 @@ jobs:
           -DopenPMD_USE_INVASIVE_TESTS=ON \
           -DPython_EXECUTABLE=$(which python3.10)
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure
 
   conda_ompi_all:
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v3
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       name: Setup conda
       with:
         auto-update-conda: true
@@ -310,4 +318,5 @@ jobs:
           -DopenPMD_USE_ADIOS2=ON \
           -DopenPMD_USE_INVASIVE_TESTS=ON
         cmake --build build --parallel 2
-        ctest --test-dir build --output-on-failure
+        cd build
+        ctest --output-on-failure

.github/workflows/macos.yml

@@ -28,18 +28,20 @@ jobs:
         python3 -m pip install -U mpi4py numpy pandas
         set -e
     - name: Build
-      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 10.15}
+      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 11.0}
       # 10.14+ due to std::visit
       # 10.15+ due to std::filesystem in toml11
       #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
+      # 11.0+ for arm64/aarch64 (M1/M2) builds
       run: |
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
           -DopenPMD_USE_PYTHON=ON \
           -DopenPMD_USE_MPI=ON    \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_ADIOS2=ON \
-          -DopenPMD_USE_INVASIVE_TESTS=ON
+          -DopenPMD_USE_INVASIVE_TESTS=ON \
+          -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh"
         cmake --build build --parallel 3
         ctest --test-dir build --verbose
 
@@ -49,16 +51,17 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Install
-      env: {MACOSX_DEPLOYMENT_TARGET: 10.14}
+      env: {MACOSX_DEPLOYMENT_TARGET: 11.0}
       run: |
         set +e
         python3 -m pip install -U numpy pandas
         set -e
     - name: Build
-      env: {CXXFLAGS: -Werror -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.14}
+      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 11.0}
       # 10.14+ due to std::visit
-      # std::filesystem in toml11 needs macOS 10.15
+      # 10.15+ due to std::filesystem in toml11
       #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
+      # 11.0+ for arm64/aarch64 (M1/M2) builds
       run: |
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
@@ -70,6 +73,9 @@ jobs:
           -DPython_EXECUTABLE=$(which python3)
         cmake --build build --parallel 3
         ctest --test-dir build --verbose
+        rm -rf build/samples
+        share/openPMD/download_samples.sh build
+        OPENPMD2_ADIOS2_USE_GROUP_TABLE=1 ctest --test-dir build --verbose
 
 # TODO: apple_conda_ompi_all (similar to conda_ompi_all on Linux)
 #   both OpenMPI and MPICH cause startup (MPI_Init) issues on GitHub Actions