From b3cf40a884f79c9127bb2d7fa0381995a2893f99 Mon Sep 17 00:00:00 2001 From: Mukul Dave Date: Mon, 10 Feb 2025 14:40:53 -0800 Subject: [PATCH] Changes for coupling with ERF using MultiBlock (#1478) --------- Co-authored-by: Jean M. Sexton Co-authored-by: Marc T. Henry de Frahan --- .clang-tidy | 2 +- CMakeLists.txt | 8 ++- amr-wind/CFDSim.H | 12 +++++ amr-wind/CMakeLists.txt | 2 +- amr-wind/incflo.H | 11 ++++ amr-wind/incflo.cpp | 6 ++- amr-wind/physics/VortexRing.cpp | 3 +- amr-wind/wind_energy/ABLBoundaryPlane.H | 19 +++++++ amr-wind/wind_energy/ABLBoundaryPlane.cpp | 63 ++++++++++++++++++++++- amr-wind/wind_energy/ABLReadERFFunction.H | 17 ++++++ cmake/amr-wind-utils.cmake | 59 +++++++++++++++++---- 11 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 amr-wind/wind_energy/ABLReadERFFunction.H diff --git a/.clang-tidy b/.clang-tidy index 0ed592f171..335fcbe7cd 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -73,7 +73,7 @@ CheckOptions: - { key: readability-identifier-naming.VariableCase, value: lower_case } - { key: readability-identifier-naming.GlobalConstantCase, value: UPPER_CASE } - { key: readability-identifier-naming.ClassIgnoredRegexp, value: "incflo"} - - { key: readability-identifier-naming.FunctionIgnoredRegexp, value: "(CheckAndSetUpDryRun|Godunov_|setVal|ComputeDt|ComputePrescribeDt|ApplyPredictor|ApplyCorrector|ApplyPrescribeStep|ApplyProjection|UpdateGradP|ReadCheckpointFile|Factory|ReadParameters|InitialProjection|InitialIterations|PrintMax|CheckForNans|InitData|Evolve|buildInfoGet|FAST_).*"} + - { key: readability-identifier-naming.FunctionIgnoredRegexp, value: "(CheckAndSetUpDryRun|Godunov_|setVal|ComputeDt|ComputePrescribeDt|ApplyPredictor|ApplyCorrector|ApplyPrescribeStep|ApplyProjection|UpdateGradP|ReadCheckpointFile|Factory|ReadParameters|InitialProjection|InitialIterations|PrintMax|CheckForNans|InitData|Evolve|buildInfoGet|FAST_|SetMultiBlockPointer).*"} - { key: readability-identifier-naming.GlobalConstantIgnoredRegexp, value: "^.*"} - { key: readability-identifier-naming.NamespaceIgnoredRegexp, value: "Hydro"} - { key: readability-identifier-naming.StructIgnoredRegexp, value: "(SC_DX|OpFM).*"} diff --git a/CMakeLists.txt b/CMakeLists.txt index 1594ddd4db..4b30635b3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required (VERSION 3.20 FATAL_ERROR) project(AMR-Wind CXX C) set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") include(CMakePackageConfigHelpers) include(GNUInstallDirs) include(amr-wind-utils) @@ -24,6 +24,10 @@ option(AMR_WIND_SAVE_GOLDS "Provide a directory in which to save golds during te option(AMR_WIND_ENABLE_FPE_TRAP_FOR_TESTS "Enable FPE trapping in tests" ON) option(AMR_WIND_TEST_WITH_PYTHON "Enable testing relying on python, like postprocessing" OFF) +#Use internal AMReX and AMReX-Hydro by default +option(AMR_WIND_USE_INTERNAL_AMREX "Use the AMReX submodule" ON) +option(AMR_WIND_USE_INTERNAL_AMREX_HYDRO "Use the AMReX-Hydro submodule" ON) + #Options for the executable option(AMR_WIND_ENABLE_MPI "Enable MPI" OFF) option(AMR_WIND_ENABLE_OPENMP "Enable OpenMP" OFF) @@ -200,7 +204,7 @@ if(AMR_WIND_ENABLE_ASCENT) target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_ASCENT) endif() -# Link with HELICS module +# Link with HELICS module if(AMR_WIND_ENABLE_HELICS) set(CMAKE_PREFIX_PATH ${HELICS_DIR} ${CMAKE_PREFIX_PATH}) find_package(HELICS 3 REQUIRED) diff --git a/amr-wind/CFDSim.H b/amr-wind/CFDSim.H index e343e48191..4ed8ce2411 100644 --- a/amr-wind/CFDSim.H +++ b/amr-wind/CFDSim.H @@ -9,6 +9,9 @@ #include "amr-wind/core/MeshMap.H" #include "amr-wind/helics.H" +#include "amr-wind/wind_energy/ABLReadERFFunction.H" +class MultiBlockContainer; + /** AMR-Wind * * All C++ code in AMR-Wind is organized within the amr_wind namespace. @@ -62,6 +65,12 @@ public: SimTime& time() { return m_time; } const SimTime& time() const { return m_time; } + void set_mbc(MultiBlockContainer* mbc) { m_mbc = mbc; } + MultiBlockContainer** mbc() { return &m_mbc; } + + void set_read_erf(ReadERFFunction fcn) { m_read_erf = std::move(fcn); } + ReadERFFunction* get_read_erf() { return &m_read_erf; } + //! Return the field repository FieldRepo& repo() { return m_repo; } FieldRepo& repo() const { return m_repo; } @@ -170,6 +179,9 @@ private: // State of solver - know if during an overset timestep or not bool m_during_overset_advance{false}; + + MultiBlockContainer* m_mbc{nullptr}; + ReadERFFunction m_read_erf{nullptr}; }; } // namespace amr_wind diff --git a/amr-wind/CMakeLists.txt b/amr-wind/CMakeLists.txt index 1757ceab0e..9336e53176 100644 --- a/amr-wind/CMakeLists.txt +++ b/amr-wind/CMakeLists.txt @@ -38,7 +38,7 @@ include(AMReXBuildInfo) generate_buildinfo(${amr_wind_lib_name} ${CMAKE_SOURCE_DIR}) # Generate AMR-Wind version header -configure_file("${CMAKE_SOURCE_DIR}/cmake/AMRWindVersion.H.in" +configure_file("${PROJECT_SOURCE_DIR}/cmake/AMRWindVersion.H.in" "${CMAKE_CURRENT_BINARY_DIR}/AMRWindVersion.H" @ONLY) target_link_libraries_system(${amr_wind_lib_name} PUBLIC AMReX::amrex AMReX-Hydro::amrex_hydro_api) diff --git a/amr-wind/incflo.H b/amr-wind/incflo.H index ae5d43978e..122a6dabc4 100644 --- a/amr-wind/incflo.H +++ b/amr-wind/incflo.H @@ -13,6 +13,8 @@ #include "amr-wind/core/FieldRepo.H" #include "amr-wind/overset/OversetOps.H" +#include "amr-wind/wind_energy/ABLReadERFFunction.H" +class MultiBlockContainer; namespace amr_wind { namespace pde { class PDEBase; @@ -58,6 +60,9 @@ public: // Evolve solution to final time through repeated calls to Advance() void Evolve(); + // Evolve solution in a multiblock framework by desired number of steps + void Evolve_MultiBlock(int multiblock_step, int max_block_step); + // Tag cells for refinement void ErrorEst(int lev, amrex::TagBoxArray& tags, amrex::Real time, int ngrow) @@ -151,6 +156,12 @@ public: void ReadCheckpointFile(); + void SetMultiBlockPointer(MultiBlockContainer* mbc) { m_sim.set_mbc(mbc); } + void set_read_erf(ReadERFFunction fcn) + { + m_sim.set_read_erf(std::move(fcn)); + } + private: // // member variables diff --git a/amr-wind/incflo.cpp b/amr-wind/incflo.cpp index ac8e32c505..2754f198b4 100644 --- a/amr-wind/incflo.cpp +++ b/amr-wind/incflo.cpp @@ -23,6 +23,10 @@ incflo::incflo() // constructor. No valid BoxArray and DistributionMapping have been defined. // But the arrays for them have been resized. + amrex::Print() << std::endl + << "Initializing AMR-Wind." << std::endl + << std::endl; + // Check if dry run is requested and set up if so CheckAndSetUpDryRun(); @@ -140,7 +144,7 @@ void incflo::prepare_for_time_integration() return; } - if (m_do_initial_proj || m_initial_iterations > 0) { + if (m_initial_iterations > 0) { m_sim.pde_manager().prepare_boundaries(); } diff --git a/amr-wind/physics/VortexRing.cpp b/amr-wind/physics/VortexRing.cpp index cc19537895..e6f075fa7c 100644 --- a/amr-wind/physics/VortexRing.cpp +++ b/amr-wind/physics/VortexRing.cpp @@ -192,7 +192,8 @@ void VortexRing::initialize_velocity(const VortexRingType& vorticity_theta) amrex::MLNodeLaplacian linop( mesh.Geom(0, mesh.finestLevel()), mesh.boxArray(0, mesh.finestLevel()), - mesh.DistributionMap(0, mesh.finestLevel()), info, {}, 1.0); + mesh.DistributionMap(0, mesh.finestLevel()), info, + amrex::Vector const*>{}, 1.0); amrex::Array bclo; amrex::Array bchi; diff --git a/amr-wind/wind_energy/ABLBoundaryPlane.H b/amr-wind/wind_energy/ABLBoundaryPlane.H index 5ae5082527..1e2eaff8ed 100644 --- a/amr-wind/wind_energy/ABLBoundaryPlane.H +++ b/amr-wind/wind_energy/ABLBoundaryPlane.H @@ -8,6 +8,8 @@ #include "amr-wind/utilities/io_utils.H" #include +#include "amr-wind/wind_energy/ABLReadERFFunction.H" +class MultiBlockContainer; namespace amr_wind { enum struct io_mode { output, input, undefined }; @@ -171,11 +173,28 @@ public: io_mode mode() const { return m_io_mode; } + MultiBlockContainer* mbc() + { + // Make sure that the double pointer m_mbc + // does not point to a nullptr + // before passing *m_mbc for use + if ((*m_mbc) == nullptr) { + amrex::Abort("Passing a null pointer for use!"); + } + return *m_mbc; + } + private: const amr_wind::SimTime& m_time; const FieldRepo& m_repo; const amrex::AmrCore& m_mesh; + // pointer to pointer : when ABL boundary plane gets initialized, amr-wind + // CFDsim does not yet have the up to date pointer to the mbc or the + // read_erf function, hence the use of double pointers + MultiBlockContainer** m_mbc; + ReadERFFunction* m_read_erf{nullptr}; + #ifdef AMR_WIND_USE_NETCDF void write_data( const ncutils::NCGroup& grp, diff --git a/amr-wind/wind_energy/ABLBoundaryPlane.cpp b/amr-wind/wind_energy/ABLBoundaryPlane.cpp index e98d4ba3f5..e0ed399106 100644 --- a/amr-wind/wind_energy/ABLBoundaryPlane.cpp +++ b/amr-wind/wind_energy/ABLBoundaryPlane.cpp @@ -250,7 +250,11 @@ bool InletData::is_populated(amrex::Orientation ori) const } ABLBoundaryPlane::ABLBoundaryPlane(CFDSim& sim) - : m_time(sim.time()), m_repo(sim.repo()), m_mesh(sim.mesh()) + : m_time(sim.time()) + , m_repo(sim.repo()) + , m_mesh(sim.mesh()) + , m_mbc(sim.mbc()) + , m_read_erf(sim.get_read_erf()) { amrex::ParmParse pp("ABL"); int pp_io_mode = -1; @@ -287,7 +291,8 @@ ABLBoundaryPlane::ABLBoundaryPlane(CFDSim& sim) } #endif - if (!(m_out_fmt == "native" || m_out_fmt == "netcdf")) { + if (!((m_out_fmt == "native") || (m_out_fmt == "netcdf") || + (m_out_fmt == "erf-multiblock"))) { amrex::Print() << "Warning: boundary output format not recognized, " "changing to native format" << std::endl; @@ -350,6 +355,10 @@ void ABLBoundaryPlane::initialize_data() "ABLBoundaryPlane: invalid variable requested: " + fname); } } + if ((m_io_mode == io_mode::output) && (m_out_fmt == "erf-multiblock")) { + amrex::Abort( + "ABLBoundaryPlane: can't output data in erf-multiblock mode"); + } } void ABLBoundaryPlane::write_header() @@ -856,6 +865,45 @@ void ABLBoundaryPlane::read_header() m_in_data.define_level_data(ori, pbx, nc); } } + } else if (m_out_fmt == "erf-multiblock") { + + m_in_times.push_back(-1.0e13); // create space for storing time at erf + // old and new timestep + m_in_times.push_back(-1.0e13); + + int nc = 0; + for (auto* fld : m_fields) { + m_in_data.component(static_cast(fld->id())) = nc; + nc += fld->num_comp(); + } + + for (amrex::OrientationIter oit; oit != nullptr; ++oit) { + auto ori = oit(); + + if (std::all_of( + m_fields.begin(), m_fields.end(), [ori](const auto* fld) { + return ( + (fld->bc_type()[ori] != BC::mass_inflow) && + (fld->bc_type()[ori] != BC::mass_inflow_outflow)); + })) { + continue; + } + + m_in_data.define_plane(ori); + + // restrict to level 0 for now for multiblock + const int lev = 0; + + const amrex::Box& minBox = m_mesh.boxArray(lev).minimalBox(); + + amrex::IntVect plo(minBox.loVect()); + amrex::IntVect phi(minBox.hiVect()); + const int normal = ori.coordDir(); + plo[normal] = ori.isHigh() ? minBox.hiVect()[normal] + 1 : -1; + phi[normal] = ori.isHigh() ? minBox.hiVect()[normal] + 1 : -1; + const amrex::Box pbx(plo, phi); + m_in_data.define_level_data(ori, pbx, nc); + } } } @@ -1062,6 +1110,17 @@ void ABLBoundaryPlane::read_file(const bool nph_target_time) const amrex::Real time = nph_target_time ? m_time.current_time() + 0.5 * m_time.delta_t() : m_time.new_time(); + + if (m_out_fmt == "erf-multiblock") { + ReadERFFunction read_erf = *m_read_erf; + if (read_erf != nullptr) { + read_erf(time, m_in_times, m_in_data, m_fields, mbc()); + } else { + amrex::Abort("read_erf function is undefined."); + } + return; + } + AMREX_ALWAYS_ASSERT( (m_in_times[0] <= time + constants::LOOSE_TOL) && (time < m_in_times.back() + constants::LOOSE_TOL)); diff --git a/amr-wind/wind_energy/ABLReadERFFunction.H b/amr-wind/wind_energy/ABLReadERFFunction.H new file mode 100644 index 0000000000..55436750bb --- /dev/null +++ b/amr-wind/wind_energy/ABLReadERFFunction.H @@ -0,0 +1,17 @@ +#ifndef ABLREADERFFUNCTION_H +#define ABLREADERFFUNCTION_H +#include + +class MultiBlockContainer; +namespace amr_wind { +class InletData; +} + +using ReadERFFunction = std::function&, + amr_wind::InletData&, + const amrex::Vector&, + MultiBlockContainer*)>; + +#endif /* ABLREADERFFUNCTION_H */ diff --git a/cmake/amr-wind-utils.cmake b/cmake/amr-wind-utils.cmake index a6a48b54fb..729d7fa7b0 100644 --- a/cmake/amr-wind-utils.cmake +++ b/cmake/amr-wind-utils.cmake @@ -27,18 +27,57 @@ function(set_cuda_build_properties target) endfunction(set_cuda_build_properties) macro(init_amrex) - set(AMREX_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/submods/amrex") - include(${CMAKE_SOURCE_DIR}/cmake/set_amrex_options.cmake) - list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake") - add_subdirectory(${AMREX_SUBMOD_LOCATION}) - set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/submods/amrex/Tools/Plotfile/amrex_fcompare - CACHE INTERNAL "Path to fcompare executable for regression tests") + if (${AMR_WIND_USE_INTERNAL_AMREX}) + set(AMREX_SUBMOD_LOCATION "${PROJECT_SOURCE_DIR}/submods/amrex") + include(${PROJECT_SOURCE_DIR}/cmake/set_amrex_options.cmake) + list(APPEND CMAKE_MODULE_PATH "${AMREX_SUBMOD_LOCATION}/Tools/CMake") + add_subdirectory(${AMREX_SUBMOD_LOCATION}) + set(FCOMPARE_EXE ${CMAKE_BINARY_DIR}/submods/amrex/Tools/Plotfile/amrex_fcompare + CACHE INTERNAL "Path to fcompare executable for regression tests") + else() + set(CMAKE_PREFIX_PATH ${AMREX_DIR} ${CMAKE_PREFIX_PATH}) + list(APPEND AMREX_COMPONENTS + "3D" "PIC" "PARTICLES" "PDOUBLE" "DOUBLE" "LSOLVERS") + if (AMR_WIND_ENABLE_MPI) + list(APPEND AMREX_COMPONENTS "MPI") + endif() + if (AMR_WIND_ENABLE_OPENMP) + list(APPEND AMREX_COMPONENTS "OMP") + endif() + if (AMR_WIND_ENABLE_CUDA) + list(APPEND AMREX_COMPONENTS "CUDA") + endif() + if (AMR_WIND_ENABLE_SYCL) + list(APPEND AMREX_COMPONENTS "SYCL") + endif() + if (AMR_WIND_ENABLE_ROCM) + list(APPEND AMREX_COMPONENTS "HIP") + endif() + if (AMR_WIND_ENABLE_HYPRE) + list(APPEND AMREX_COMPONENTS "HYPRE") + endif() + if (AMR_WIND_ENABLE_TINY_PROFILE) + list(APPEND AMREX_COMPONENTS "TINY_PROFILE") + endif() + separate_arguments(AMREX_COMPONENTS) + find_package(AMReX CONFIG REQUIRED + COMPONENTS ${AMREX_COMPONENTS}) + message(STATUS "Found AMReX = ${AMReX_DIR}") + set(FCOMPARE_EXE ${AMReX_DIR}/../../../bin/amrex_fcompare + CACHE INTERNAL "Path to fcompare executable for regression tests") + endif() endmacro(init_amrex) macro(init_amrex_hydro) - set(AMREX_HYDRO_SUBMOD_LOCATION "${CMAKE_SOURCE_DIR}/submods/AMReX-Hydro") - include(${CMAKE_SOURCE_DIR}/cmake/set_amrex_hydro_options.cmake) - add_subdirectory(${AMREX_HYDRO_SUBMOD_LOCATION}) + if (${AMR_WIND_USE_INTERNAL_AMREX_HYDRO}) + set(AMREX_HYDRO_SUBMOD_LOCATION "${PROJECT_SOURCE_DIR}/submods/AMReX-Hydro") + include(${PROJECT_SOURCE_DIR}/cmake/set_amrex_hydro_options.cmake) + add_subdirectory(${AMREX_HYDRO_SUBMOD_LOCATION}) + else() + set(CMAKE_PREFIX_PATH ${AMReX-Hydro_DIR} ${CMAKE_PREFIX_PATH}) + find_package(AMReX-Hydro CONFIG REQUIRED) + message(STATUS "Found AMReX-Hydro = ${AMReX-Hydro_DIR}") + endif() endmacro(init_amrex_hydro) macro(init_waves2amr) @@ -79,7 +118,7 @@ macro(init_code_checks) COMMAND ${CMAKE_COMMAND} -E make_directory cppcheck # cppcheck ignores -isystem directories, so we change them to regular -I include directories (with no spaces either) COMMAND sed "s/isystem /I/g" ${CMAKE_BINARY_DIR}/compile_commands.json > cppcheck_compile_commands.json - COMMAND ${CPPCHECK_EXE} --template=gcc --inline-suppr --suppress=unusedFunction --suppress=useStlAlgorithm --suppress=missingIncludeSystem --std=c++17 --language=c++ --enable=all --project=cppcheck_compile_commands.json -i ${CMAKE_SOURCE_DIR}/submods/amrex/Src -i ${CMAKE_SOURCE_DIR}/submods/AMReX-Hydro -i ${CMAKE_SOURCE_DIR}/submods/Waves2AMR -i ${CMAKE_SOURCE_DIR}/submods/googletest --output-file=cppcheck-full-report.txt -j ${NP} + COMMAND ${CPPCHECK_EXE} --template=gcc --inline-suppr --suppress=unusedFunction --suppress=useStlAlgorithm --suppress=missingIncludeSystem --std=c++17 --language=c++ --enable=all --project=cppcheck_compile_commands.json -i ${PROJECT_SOURCE_DIR}/submods/amrex/Src -i ${PROJECT_SOURCE_DIR}/submods/AMReX-Hydro -i ${PROJECT_SOURCE_DIR}/submods/Waves2AMR -i ${PROJECT_SOURCE_DIR}/submods/googletest --output-file=cppcheck-full-report.txt -j ${NP} COMMENT "Run cppcheck on project compile_commands.json" BYPRODUCTS cppcheck-full-report.txt WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/cppcheck