Changes for coupling with ERF using MultiBlock (Exawind#1478)

---------

Co-authored-by: Jean M. Sexton <jmsexton@lbl.gov>
Co-authored-by: Marc T. Henry de Frahan <marchdf@gmail.com>

.clang-tidy

@@ -73,7 +73,7 @@ CheckOptions:
   - { key: readability-identifier-naming.VariableCase, value: lower_case }
   - { key: readability-identifier-naming.GlobalConstantCase, value: UPPER_CASE }
   - { key: readability-identifier-naming.ClassIgnoredRegexp, value: "incflo"}
-  - { key: readability-identifier-naming.FunctionIgnoredRegexp, value: "(CheckAndSetUpDryRun|Godunov_|setVal|ComputeDt|ComputePrescribeDt|ApplyPredictor|ApplyCorrector|ApplyPrescribeStep|ApplyProjection|UpdateGradP|ReadCheckpointFile|Factory|ReadParameters|InitialProjection|InitialIterations|PrintMax|CheckForNans|InitData|Evolve|buildInfoGet|FAST_).*"}
+  - { key: readability-identifier-naming.FunctionIgnoredRegexp, value: "(CheckAndSetUpDryRun|Godunov_|setVal|ComputeDt|ComputePrescribeDt|ApplyPredictor|ApplyCorrector|ApplyPrescribeStep|ApplyProjection|UpdateGradP|ReadCheckpointFile|Factory|ReadParameters|InitialProjection|InitialIterations|PrintMax|CheckForNans|InitData|Evolve|buildInfoGet|FAST_|SetMultiBlockPointer).*"}
   - { key: readability-identifier-naming.GlobalConstantIgnoredRegexp, value: "^.*"}
   - { key: readability-identifier-naming.NamespaceIgnoredRegexp, value: "Hydro"}
   - { key: readability-identifier-naming.StructIgnoredRegexp, value: "(SC_DX|OpFM).*"}

CMakeLists.txt

@@ -3,7 +3,7 @@
 cmake_minimum_required (VERSION 3.20 FATAL_ERROR)
 project(AMR-Wind CXX C)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "")
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
 include(CMakePackageConfigHelpers)
 include(GNUInstallDirs)
 include(amr-wind-utils)
@@ -24,6 +24,10 @@ option(AMR_WIND_SAVE_GOLDS "Provide a directory in which to save golds during te
 option(AMR_WIND_ENABLE_FPE_TRAP_FOR_TESTS "Enable FPE trapping in tests" ON)
 option(AMR_WIND_TEST_WITH_PYTHON "Enable testing relying on python, like postprocessing" OFF)
 
+#Use internal AMReX and AMReX-Hydro by default
+option(AMR_WIND_USE_INTERNAL_AMREX "Use the AMReX submodule" ON)
+option(AMR_WIND_USE_INTERNAL_AMREX_HYDRO "Use the AMReX-Hydro submodule" ON)
+
 #Options for the executable
 option(AMR_WIND_ENABLE_MPI "Enable MPI" OFF)
 option(AMR_WIND_ENABLE_OPENMP "Enable OpenMP" OFF)
@@ -200,7 +204,7 @@ if(AMR_WIND_ENABLE_ASCENT)
   target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_ASCENT)
 endif()
 
-# Link with HELICS module 
+# Link with HELICS module
 if(AMR_WIND_ENABLE_HELICS)
   set(CMAKE_PREFIX_PATH ${HELICS_DIR} ${CMAKE_PREFIX_PATH})
   find_package(HELICS 3 REQUIRED)

amr-wind/CFDSim.H

@@ -9,6 +9,9 @@
 #include "amr-wind/core/MeshMap.H"
 #include "amr-wind/helics.H"
 
+#include "amr-wind/wind_energy/ABLReadERFFunction.H"
+class MultiBlockContainer;
+
 /** AMR-Wind
  *
  *  All C++ code in AMR-Wind is organized within the amr_wind namespace.
@@ -62,6 +65,12 @@ public:
     SimTime& time() { return m_time; }
     const SimTime& time() const { return m_time; }
 
+    void set_mbc(MultiBlockContainer* mbc) { m_mbc = mbc; }
+    MultiBlockContainer** mbc() { return &m_mbc; }
+
+    void set_read_erf(ReadERFFunction fcn) { m_read_erf = std::move(fcn); }
+    ReadERFFunction* get_read_erf() { return &m_read_erf; }
+
     //! Return the field repository
     FieldRepo& repo() { return m_repo; }
     FieldRepo& repo() const { return m_repo; }
@@ -170,6 +179,9 @@ private:
 
     // State of solver - know if during an overset timestep or not
     bool m_during_overset_advance{false};
+
+    MultiBlockContainer* m_mbc{nullptr};
+    ReadERFFunction m_read_erf{nullptr};
 };
 
 } // namespace amr_wind

amr-wind/CMakeLists.txt

@@ -38,7 +38,7 @@ include(AMReXBuildInfo)
 generate_buildinfo(${amr_wind_lib_name} ${CMAKE_SOURCE_DIR})
 
 # Generate AMR-Wind version header
-configure_file("${CMAKE_SOURCE_DIR}/cmake/AMRWindVersion.H.in"
+configure_file("${PROJECT_SOURCE_DIR}/cmake/AMRWindVersion.H.in"
   "${CMAKE_CURRENT_BINARY_DIR}/AMRWindVersion.H" @ONLY)
 
 target_link_libraries_system(${amr_wind_lib_name} PUBLIC AMReX::amrex AMReX-Hydro::amrex_hydro_api)

amr-wind/incflo.H

@@ -13,6 +13,8 @@
 #include "amr-wind/core/FieldRepo.H"
 #include "amr-wind/overset/OversetOps.H"
 
+#include "amr-wind/wind_energy/ABLReadERFFunction.H"
+class MultiBlockContainer;
 namespace amr_wind {
 namespace pde {
 class PDEBase;
@@ -58,6 +60,9 @@ public:
     // Evolve solution to final time through repeated calls to Advance()
     void Evolve();
 
+    // Evolve solution in a multiblock framework by desired number of steps
+    void Evolve_MultiBlock(int multiblock_step, int max_block_step);
+
     // Tag cells for refinement
     void
     ErrorEst(int lev, amrex::TagBoxArray& tags, amrex::Real time, int ngrow)
@@ -151,6 +156,12 @@ public:
 
     void ReadCheckpointFile();
 
+    void SetMultiBlockPointer(MultiBlockContainer* mbc) { m_sim.set_mbc(mbc); }
+    void set_read_erf(ReadERFFunction fcn)
+    {
+        m_sim.set_read_erf(std::move(fcn));
+    }
+
 private:
     //
     // member variables

amr-wind/incflo.cpp

@@ -23,6 +23,10 @@ incflo::incflo()
     // constructor. No valid BoxArray and DistributionMapping have been defined.
     // But the arrays for them have been resized.
 
+    amrex::Print() << std::endl
+                   << "Initializing AMR-Wind." << std::endl
+                   << std::endl;
+
     // Check if dry run is requested and set up if so
     CheckAndSetUpDryRun();
 
@@ -140,7 +144,7 @@ void incflo::prepare_for_time_integration()
         return;
     }
 
-    if (m_do_initial_proj || m_initial_iterations > 0) {
+    if (m_initial_iterations > 0) {
         m_sim.pde_manager().prepare_boundaries();
     }
 

amr-wind/physics/VortexRing.cpp

@@ -192,7 +192,8 @@ void VortexRing::initialize_velocity(const VortexRingType& vorticity_theta)
 
     amrex::MLNodeLaplacian linop(
         mesh.Geom(0, mesh.finestLevel()), mesh.boxArray(0, mesh.finestLevel()),
-        mesh.DistributionMap(0, mesh.finestLevel()), info, {}, 1.0);
+        mesh.DistributionMap(0, mesh.finestLevel()), info,
+        amrex::Vector<amrex::FabFactory<amrex::FArrayBox> const*>{}, 1.0);
 
     amrex::Array<amrex::LinOpBCType, AMREX_SPACEDIM> bclo;
     amrex::Array<amrex::LinOpBCType, AMREX_SPACEDIM> bchi;

amr-wind/wind_energy/ABLBoundaryPlane.H

@@ -8,6 +8,8 @@
 #include "amr-wind/utilities/io_utils.H"
 #include <AMReX_BndryRegister.H>
 
+#include "amr-wind/wind_energy/ABLReadERFFunction.H"
+class MultiBlockContainer;
 namespace amr_wind {
 
 enum struct io_mode { output, input, undefined };
@@ -171,11 +173,28 @@ public:
 
     io_mode mode() const { return m_io_mode; }
 
+    MultiBlockContainer* mbc()
+    {
+        // Make sure that the double pointer m_mbc
+        // does not point to a nullptr
+        // before passing *m_mbc for use
+        if ((*m_mbc) == nullptr) {
+            amrex::Abort("Passing a null pointer for use!");
+        }
+        return *m_mbc;
+    }
+
 private:
     const amr_wind::SimTime& m_time;
     const FieldRepo& m_repo;
     const amrex::AmrCore& m_mesh;
 
+    // pointer to pointer : when ABL boundary plane gets initialized, amr-wind
+    // CFDsim does not yet have the up to date pointer to the mbc or the
+    // read_erf function, hence the use of double pointers
+    MultiBlockContainer** m_mbc;
+    ReadERFFunction* m_read_erf{nullptr};
+
 #ifdef AMR_WIND_USE_NETCDF
     void write_data(
         const ncutils::NCGroup& grp,

amr-wind/wind_energy/ABLBoundaryPlane.cpp

@@ -250,7 +250,11 @@ bool InletData::is_populated(amrex::Orientation ori) const
 }
 
 ABLBoundaryPlane::ABLBoundaryPlane(CFDSim& sim)
-    : m_time(sim.time()), m_repo(sim.repo()), m_mesh(sim.mesh())
+    : m_time(sim.time())
+    , m_repo(sim.repo())
+    , m_mesh(sim.mesh())
+    , m_mbc(sim.mbc())
+    , m_read_erf(sim.get_read_erf())
 {
     amrex::ParmParse pp("ABL");
     int pp_io_mode = -1;
@@ -287,7 +291,8 @@ ABLBoundaryPlane::ABLBoundaryPlane(CFDSim& sim)
     }
 #endif
 
-    if (!(m_out_fmt == "native" || m_out_fmt == "netcdf")) {
+    if (!((m_out_fmt == "native") || (m_out_fmt == "netcdf") ||
+          (m_out_fmt == "erf-multiblock"))) {
         amrex::Print() << "Warning: boundary output format not recognized, "
                           "changing to native format"
                        << std::endl;
@@ -350,6 +355,10 @@ void ABLBoundaryPlane::initialize_data()
                 "ABLBoundaryPlane: invalid variable requested: " + fname);
         }
     }
+    if ((m_io_mode == io_mode::output) && (m_out_fmt == "erf-multiblock")) {
+        amrex::Abort(
+            "ABLBoundaryPlane: can't output data in erf-multiblock mode");
+    }
 }
 
 void ABLBoundaryPlane::write_header()
@@ -856,6 +865,45 @@ void ABLBoundaryPlane::read_header()
                 m_in_data.define_level_data(ori, pbx, nc);
             }
         }
+    } else if (m_out_fmt == "erf-multiblock") {
+
+        m_in_times.push_back(-1.0e13); // create space for storing time at erf
+                                       // old and new timestep
+        m_in_times.push_back(-1.0e13);
+
+        int nc = 0;
+        for (auto* fld : m_fields) {
+            m_in_data.component(static_cast<int>(fld->id())) = nc;
+            nc += fld->num_comp();
+        }
+
+        for (amrex::OrientationIter oit; oit != nullptr; ++oit) {
+            auto ori = oit();
+
+            if (std::all_of(
+                    m_fields.begin(), m_fields.end(), [ori](const auto* fld) {
+                        return (
+                            (fld->bc_type()[ori] != BC::mass_inflow) &&
+                            (fld->bc_type()[ori] != BC::mass_inflow_outflow));
+                    })) {
+                continue;
+            }
+
+            m_in_data.define_plane(ori);
+
+            // restrict to level 0 for now for multiblock
+            const int lev = 0;
+
+            const amrex::Box& minBox = m_mesh.boxArray(lev).minimalBox();
+
+            amrex::IntVect plo(minBox.loVect());
+            amrex::IntVect phi(minBox.hiVect());
+            const int normal = ori.coordDir();
+            plo[normal] = ori.isHigh() ? minBox.hiVect()[normal] + 1 : -1;
+            phi[normal] = ori.isHigh() ? minBox.hiVect()[normal] + 1 : -1;
+            const amrex::Box pbx(plo, phi);
+            m_in_data.define_level_data(ori, pbx, nc);
+        }
     }
 }
 
@@ -1062,6 +1110,17 @@ void ABLBoundaryPlane::read_file(const bool nph_target_time)
     const amrex::Real time =
         nph_target_time ? m_time.current_time() + 0.5 * m_time.delta_t()
                         : m_time.new_time();
+
+    if (m_out_fmt == "erf-multiblock") {
+        ReadERFFunction read_erf = *m_read_erf;
+        if (read_erf != nullptr) {
+            read_erf(time, m_in_times, m_in_data, m_fields, mbc());
+        } else {
+            amrex::Abort("read_erf function is undefined.");
+        }
+        return;
+    }
+
     AMREX_ALWAYS_ASSERT(
         (m_in_times[0] <= time + constants::LOOSE_TOL) &&
         (time < m_in_times.back() + constants::LOOSE_TOL));

amr-wind/wind_energy/ABLReadERFFunction.H

@@ -0,0 +1,17 @@
+#ifndef ABLREADERFFUNCTION_H
+#define ABLREADERFFUNCTION_H
+#include <functional>
+
+class MultiBlockContainer;
+namespace amr_wind {
+class InletData;
+}
+
+using ReadERFFunction = std::function<void(
+    const amrex::Real time,
+    amrex::Vector<amrex::Real>&,
+    amr_wind::InletData&,
+    const amrex::Vector<amr_wind::Field*>&,
+    MultiBlockContainer*)>;
+
+#endif /* ABLREADERFFUNCTION_H */

cmake/amr-wind-utils.cmake

@@ -27,18 +27,57 @@ function(set_cuda_build_properties target)
 endfunction(set_cuda_build_properties)
 
 macro(init_amrex)
-  set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/submods/amrex")
-  include(${CMAKE_SOURCE_DIR}/cmake/set_amrex_options.cmake)
-  list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake")
-  add_subdirectory(${AMREX_SUBMOD_LOCATION})
-  set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/submods/amrex/Tools/Plotfile/amrex_fcompare
-    CACHE INTERNAL "Path to fcompare executable for regression tests")
+  if (${AMR_WIND_USE_INTERNAL_AMREX})
+    set(AMREX_SUBMOD_LOCATION "${PROJECT_SOURCE_DIR}/submods/amrex")
+    include(${PROJECT_SOURCE_DIR}/cmake/set_amrex_options.cmake)
+    list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake")
+    add_subdirectory(${AMREX_SUBMOD_LOCATION})
+    set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/submods/amrex/Tools/Plotfile/amrex_fcompare
+      CACHE INTERNAL "Path to fcompare executable for regression tests")
+  else()
+    set(CMAKE_PREFIX_PATH ${AMREX_DIR} ${CMAKE_PREFIX_PATH})
+    list(APPEND AMREX_COMPONENTS
+      "3D" "PIC" "PARTICLES" "PDOUBLE" "DOUBLE" "LSOLVERS")
+    if (AMR_WIND_ENABLE_MPI)
+      list(APPEND AMREX_COMPONENTS "MPI")
+    endif()
+    if (AMR_WIND_ENABLE_OPENMP)
+      list(APPEND AMREX_COMPONENTS "OMP")
+    endif()
+    if (AMR_WIND_ENABLE_CUDA)
+      list(APPEND AMREX_COMPONENTS "CUDA")
+    endif()
+    if (AMR_WIND_ENABLE_SYCL)
+      list(APPEND AMREX_COMPONENTS "SYCL")
+    endif()
+    if (AMR_WIND_ENABLE_ROCM)
+      list(APPEND AMREX_COMPONENTS "HIP")
+    endif()
+    if (AMR_WIND_ENABLE_HYPRE)
+      list(APPEND AMREX_COMPONENTS "HYPRE")
+    endif()
+    if (AMR_WIND_ENABLE_TINY_PROFILE)
+      list(APPEND AMREX_COMPONENTS "TINY_PROFILE")
+    endif()
+    separate_arguments(AMREX_COMPONENTS)
+    find_package(AMReX CONFIG REQUIRED
+      COMPONENTS ${AMREX_COMPONENTS})
+    message(STATUS "Found AMReX = ${AMReX_DIR}")
+    set(FCOMPARE_EXE ${AMReX_DIR}/../../../bin/amrex_fcompare
+      CACHE INTERNAL "Path to fcompare executable for regression tests")
+  endif()
 endmacro(init_amrex)
 
 macro(init_amrex_hydro)
-  set(AMREX_HYDRO_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/submods/AMReX-Hydro")
-  include(${CMAKE_SOURCE_DIR}/cmake/set_amrex_hydro_options.cmake)
-  add_subdirectory(${AMREX_HYDRO_SUBMOD_LOCATION})
+  if (${AMR_WIND_USE_INTERNAL_AMREX_HYDRO})
+    set(AMREX_HYDRO_SUBMOD_LOCATION "${PROJECT_SOURCE_DIR}/submods/AMReX-Hydro")
+    include(${PROJECT_SOURCE_DIR}/cmake/set_amrex_hydro_options.cmake)
+    add_subdirectory(${AMREX_HYDRO_SUBMOD_LOCATION})
+  else()
+    set(CMAKE_PREFIX_PATH ${AMReX-Hydro_DIR} ${CMAKE_PREFIX_PATH})
+    find_package(AMReX-Hydro CONFIG REQUIRED)
+    message(STATUS "Found AMReX-Hydro = ${AMReX-Hydro_DIR}")
+  endif()
 endmacro(init_amrex_hydro)
 
 macro(init_waves2amr)
@@ -79,7 +118,7 @@ macro(init_code_checks)
           COMMAND ${CMAKE_COMMAND} -E make_directory cppcheck
           # cppcheck ignores -isystem directories, so we change them to regular -I include directories (with no spaces either)
           COMMAND sed "s/isystem /I/g" ${CMAKE_BINARY_DIR}/compile_commands.json > cppcheck_compile_commands.json
-          COMMAND ${CPPCHECK_EXE} --template=gcc --inline-suppr --suppress=unusedFunction --suppress=useStlAlgorithm --suppress=missingIncludeSystem --std=c++17 --language=c++ --enable=all --project=cppcheck_compile_commands.json -i ${CMAKE_SOURCE_DIR}/submods/amrex/Src -i ${CMAKE_SOURCE_DIR}/submods/AMReX-Hydro -i ${CMAKE_SOURCE_DIR}/submods/Waves2AMR -i ${CMAKE_SOURCE_DIR}/submods/googletest --output-file=cppcheck-full-report.txt -j ${NP}
+          COMMAND ${CPPCHECK_EXE} --template=gcc --inline-suppr --suppress=unusedFunction --suppress=useStlAlgorithm --suppress=missingIncludeSystem --std=c++17 --language=c++ --enable=all --project=cppcheck_compile_commands.json -i ${PROJECT_SOURCE_DIR}/submods/amrex/Src -i ${PROJECT_SOURCE_DIR}/submods/AMReX-Hydro -i ${PROJECT_SOURCE_DIR}/submods/Waves2AMR -i ${PROJECT_SOURCE_DIR}/submods/googletest --output-file=cppcheck-full-report.txt -j ${NP}
           COMMENT "Run cppcheck on project compile_commands.json"
           BYPRODUCTS cppcheck-full-report.txt
           WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/cppcheck