Merge pull request #107 from E3SM-Project/update-develop-20240229

Merge trilinos/develop into e3sm-project/develop

packages/framework/ini-files/config-specs.ini

@@ -2409,9 +2409,9 @@ use CUDA11-RUN-SERIAL-TESTS
 opt-set-cmake-var ROL_example_PinT_parabolic-control_AugmentedSystem_test_MPI_2_DISABLE BOOL FORCE : ON
 
 
-[rhel7_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.0.5_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all]
+[rhel7_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.0.5_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_pr]
 use rhel7_sems-cuda-11.4.2-sems-gnu-10.1.0-sems-openmpi-4.0.5_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables
-use PACKAGE-ENABLES|ALL
+use PACKAGE-ENABLES|PR
 opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : OFF
 
 

packages/krino/krino/krino_lib/Akri_AdaptivityHelpers.cpp

@@ -41,14 +41,14 @@ void filter_refinement_marker(const RefinementInterface & refinement, const stk:
     if (do_not_refine_or_unrefine_selector(*bucketPtr))
     {
       for (int i = 0; i < size; ++i)
-        if (markers[i] == Refinement_Marker::REFINE || markers[i] == Refinement_Marker::COARSEN)
-          markers[i] = Refinement_Marker::NOTHING;
+        if (markers[i] == static_cast<int>(Refinement_Marker::REFINE) || markers[i] == static_cast<int>(Refinement_Marker::COARSEN))
+          markers[i] = static_cast<int>(Refinement_Marker::NOTHING);
     }
     else if (bucketPtr->member(parentPart))
     {
       for (int i = 0; i < size; ++i)
-        if (markers[i] == Refinement_Marker::REFINE)
-          markers[i] = Refinement_Marker::NOTHING;
+        if (markers[i] == static_cast<int>(Refinement_Marker::REFINE))
+          markers[i] = static_cast<int>(Refinement_Marker::NOTHING);
     }
   }
 }
@@ -84,7 +84,7 @@ void perform_multilevel_adaptivity(RefinementInterface & refinement,
       int * markers = field_data<int>(elem_marker, *b_ptr);
       for (size_t i = 0; i < b_ptr->size(); ++i)
       {
-        if (markers[i] == Refinement_Marker::REFINE) ++num_marked_refine;
+        if (markers[i] == static_cast<int>(Refinement_Marker::REFINE)) ++num_marked_refine;
       }
     }
 

packages/krino/krino/krino_lib/Akri_AdaptivityHelpers.hpp

@@ -21,12 +21,12 @@ class RefinementInterface;
 
 stk::mesh::Selector cdfem_do_not_refine_or_unrefine_selector(const CDFEM_Support & cdfem_support);
 
-enum Refinement_Marker
-  {
-    COARSEN = -1,
-    NOTHING = 0,
-    REFINE = 1
-  };
+enum class Refinement_Marker
+{
+  COARSEN = -1,
+  NOTHING = 0,
+  REFINE = 1
+};
 
 void
 perform_multilevel_adaptivity(RefinementInterface & refinement,

packages/krino/krino/krino_lib/Akri_AnalyticSurfaceInterfaceGeometry.cpp

@@ -165,8 +165,6 @@ static void append_surface_edge_intersection_points(const stk::mesh::BulkData &
   }
 }
 
-
-
 FieldRef AnalyticSurfaceInterfaceGeometry::get_coordinates_field(const stk::mesh::BulkData & mesh) const
 {
   FieldRef coordsField = myCdfemSupport.get_coords_field();

packages/krino/krino/krino_lib/Akri_CDMesh.cpp

@@ -1033,56 +1033,14 @@ static void fill_nodes_of_elements_with_subelements_or_changed_phase(const stk::
   }
 }
 
-static
-void pack_shared_nodes_for_sharing_procs(const stk::mesh::BulkData & mesh,
-    const std::set<stk::mesh::Entity> & nodes,
-    stk::CommSparse &commSparse)
-{
-  std::vector<int> nodeSharedProcs;
-  stk::pack_and_communicate(commSparse,[&]()
-  {
-    for (auto node : nodes)
-    {
-      if (mesh.bucket(node).shared())
-      {
-        mesh.comm_shared_procs(node, nodeSharedProcs);
-        for (int procId : nodeSharedProcs)
-          commSparse.send_buffer(procId).pack(mesh.identifier(node));
-      }
-    }
-  });
-}
-
-static
-void unpack_shared_nodes(const stk::mesh::BulkData & mesh,
-    std::set<stk::mesh::Entity> & nodes,
-    stk::CommSparse &commSparse)
-{
-  stk::unpack_communications(commSparse, [&](int procId)
-  {
-    stk::CommBuffer & buffer = commSparse.recv_buffer(procId);
-
-    while ( buffer.remaining() )
-    {
-      stk::mesh::EntityId nodeId;
-      commSparse.recv_buffer(procId).unpack(nodeId);
-      stk::mesh::Entity node = mesh.get_entity(stk::topology::NODE_RANK, nodeId);
-      STK_ThrowRequire(mesh.is_valid(node));
-      nodes.insert(node);
-    }
-  });
-}
-
 static std::set<stk::mesh::Entity> get_nodes_of_elements_with_subelements_or_have_changed_phase(const stk::mesh::BulkData & mesh,
     const Phase_Support & phaseSupport,
     const std::vector<std::unique_ptr<Mesh_Element>> & meshElements)
 {
   std::set<stk::mesh::Entity> nodesOfElements;
   fill_nodes_of_elements_with_subelements_or_changed_phase(mesh, phaseSupport, meshElements, nodesOfElements);
 
-  stk::CommSparse commSparse(mesh.parallel());
-  pack_shared_nodes_for_sharing_procs(mesh, nodesOfElements, commSparse);
-  unpack_shared_nodes(mesh, nodesOfElements, commSparse);
+  communicate_shared_nodes_to_sharing_procs(mesh, nodesOfElements);
 
   return nodesOfElements;
 }
@@ -1426,7 +1384,7 @@ static void find_nearest_matching_prolong_facet(const stk::mesh::BulkData & mesh
     const std::vector<unsigned> & requiredFields,
     const SubElementNode & targetNode,
     const ProlongationFacet *& nearestProlongFacet,
-    FacetDistanceQuery & nearestFacetQuery,
+    FacetDistanceQuery<Facet> & nearestFacetQuery,
     bool & haveMissingRemoteProlongFacets)
 {
   const stk::math::Vector3d & targetCoordinates = targetNode.coordinates();
@@ -1450,7 +1408,7 @@ static void find_nearest_matching_prolong_facet(const stk::mesh::BulkData & mesh
 
         for (auto && prolong_facet : nearest_prolong_facets)
         {
-          FacetDistanceQuery facet_query(*prolong_facet->get_facet(), targetCoordinates);
+          FacetDistanceQuery<Facet> facet_query(*prolong_facet->get_facet(), targetCoordinates);
           if (nearestFacetQuery.empty() || facet_query.distance_squared() < nearestFacetQuery.distance_squared())
           {
             nearestProlongFacet = prolong_facet;
@@ -1526,7 +1484,7 @@ CDMesh::find_prolongation_node(const SubElementNode & targetNode) const
   STK_ThrowRequire(need_facets_for_prolongation());
 
   const ProlongationFacet * nearestProlongFacet = nullptr;
-  FacetDistanceQuery nearestFacetQuery;
+  FacetDistanceQuery<Facet> nearestFacetQuery;
 
   find_nearest_matching_prolong_facet(stk_bulk(), my_phase_prolong_tree_map, requiredFields, targetNode, nearestProlongFacet, nearestFacetQuery, my_missing_remote_prolong_facets);
 
@@ -1923,7 +1881,8 @@ CDMesh::snap_nearby_intersections_to_nodes(const InterfaceGeometry & interfaceGe
 { /* %TRACE[ON]% */ Trace trace__("krino::Mesh::snap_nearby_intersections_to_nodes(void)"); /* %TRACE% */
   stk::diag::TimeBlock timer__(my_timer_snap);
 
-  snap_to_node(stk_bulk(), interfaceGeometry, get_snapper(), domainsAtNodes);
+  const stk::mesh::Selector parentElementSelector = get_cdfem_parent_element_selector(get_active_part(), my_cdfem_support, my_phase_support);
+  snap_to_node(stk_bulk(), parentElementSelector, interfaceGeometry, get_snapper(), domainsAtNodes);
   for (auto && entry : domainsAtNodes)
   {
     const SubElementNode * node = get_mesh_node(stk_bulk().identifier(entry.first));

packages/krino/krino/krino_lib/Akri_CDMesh_Refinement.cpp

@@ -40,11 +40,11 @@ namespace {
 
     if (refine_level >= interface_max_refine_level )
     {
-      marker = Refinement_Marker::NOTHING;
+      marker = static_cast<int>(Refinement_Marker::NOTHING);
     }
     else
     {
-      marker = Refinement_Marker::REFINE;
+      marker = static_cast<int>(Refinement_Marker::REFINE);
     }
   }
 }
@@ -232,7 +232,7 @@ mark_nearest_node_on_cut_edges(const stk::mesh::BulkData& mesh,
 
 int determine_refinement_marker(const bool isElementIndicated, const int refinementIterCount, const int interfaceMinRefineLevel, const int elementRefineLevel)
 {
-  int marker = Refinement_Marker::NOTHING;
+  auto marker = Refinement_Marker::NOTHING;
   const int targetRefineLevel = isElementIndicated ? interfaceMinRefineLevel : 0;
   if (elementRefineLevel < targetRefineLevel)
   {
@@ -246,7 +246,7 @@ int determine_refinement_marker(const bool isElementIndicated, const int refinem
     // with refinement
     marker = Refinement_Marker::COARSEN;
   }
-  return marker;
+  return static_cast<int>(marker);
 }
 
 void

packages/krino/krino/krino_lib/Akri_Compute_Surface_Distance.cpp

@@ -33,7 +33,7 @@ Compute_Surface_Distance::calculate(
 }
 
 static void compute_distance_to_facets(const stk::mesh::BulkData & mesh,
-    const MeshSurface & facet_list,
+    const FacetedSurfaceBase & facets,
     const stk::mesh::Field<double>& coordinates,
     const stk::mesh::Field<double>& distance,
     const stk::mesh::Selector & volume_selector,
@@ -53,18 +53,18 @@ static void compute_distance_to_facets(const stk::mesh::BulkData & mesh,
       STK_ThrowAssert(&(dist[n]) != NULL);
 
       const stk::math::Vector3d xvec(coord+n*spatial_dimension, spatial_dimension);
-      dist[n] = facet_list.point_unsigned_distance(xvec, narrowBandSize, farFieldValue);
+      dist[n] = facets.point_unsigned_distance(xvec, narrowBandSize, farFieldValue);
     }
   }
 }
 
-void print_facet_info(const MeshSurface & facet_list, stk::ParallelMachine parallel)
+void print_facet_info(const FacetedSurfaceBase & facets, stk::ParallelMachine parallel)
 {
   constexpr int vec_size = 3;
   std::array <unsigned,vec_size> local_sizes, global_min, global_max;
-  local_sizes[0] = facet_list.storage_size();
-  local_sizes[1] = facet_list.size();
-  local_sizes[2] = facet_list.size()+facet_list.nonlocal_size();
+  local_sizes[0] = facets.storage_size();
+  local_sizes[1] = facets.size();
+  local_sizes[2] = facets.size()+facets.nonlocal_size();
 
   stk::all_reduce_min( parallel, local_sizes.data(), global_min.data(), vec_size );
   stk::all_reduce_max( parallel, local_sizes.data(), global_max.data(), vec_size );
@@ -75,30 +75,30 @@ void print_facet_info(const MeshSurface & facet_list, stk::ParallelMachine paral
   krinolog << "   Memory usage (mb):  min=" << global_min[0]/(1024.0*1024.0) << ", max=" << global_max[0]/(1024.0*1024.0) << stk::diag::dendl;
 }
 
-
 void
 Compute_Surface_Distance::calculate(
   const stk::mesh::BulkData & mesh,
   const stk::diag::Timer &parent_timer,
   const stk::mesh::Field<double>& coordinates,
   const stk::mesh::Field<double>& distance,
   const stk::mesh::Selector & volume_selector,
-  const stk::mesh::Selector & surface_selector,
+  const stk::mesh::Selector & surfaceSelector,
   const double narrowBandSize,
   const double farFieldValue)
 { /* %TRACE[ON]% */ Trace trace__("krino::Compute_Surface_Distance::compute_surface_distance(void)"); /* %TRACE% */
 
   stk::diag::Timer timer( "Compute Surface Distance", parent_timer );
   stk::diag::TimeBlock timer_(timer);
 
-  MeshSurface facet_list(mesh.mesh_meta_data(), coordinates, surface_selector, +1);
+  std::unique_ptr<FacetedSurfaceBase> facets = build_mesh_surface(mesh.mesh_meta_data(), coordinates, surfaceSelector, +1);
+
   FieldRef coordsField(coordinates);
   const BoundingBox nodeBbox = compute_nodal_bbox(mesh, volume_selector & stk::mesh::selectField(distance), coordsField);
-  facet_list.prepare_to_compute(0.0, nodeBbox, narrowBandSize); // Setup including communication of facets that are within this processors narrow band
+  facets->prepare_to_compute(0.0, nodeBbox, narrowBandSize); // Setup including communication of facets that are within this processors narrow band
 
-  print_facet_info(facet_list, mesh.parallel());
+  print_facet_info(*facets, mesh.parallel());
 
-  compute_distance_to_facets(mesh, facet_list, coordinates, distance, volume_selector, narrowBandSize, farFieldValue);
+  compute_distance_to_facets(mesh, *facets, coordinates, distance, volume_selector, narrowBandSize, farFieldValue);
 }
 
 } // namespace krino

packages/krino/krino/krino_lib/Akri_ContourElement.cpp

@@ -527,7 +527,7 @@ ContourElement::dump_subelement_details() const
   }
 
 void
-ContourElement::build_subelement_facets( Faceted_Surface & facets )
+ContourElement::build_subelement_facets( FacetedSurfaceBase & facets )
 {
   if (my_sign == 0)
   {

packages/krino/krino/krino_lib/Akri_ContourElement.hpp

@@ -74,7 +74,7 @@ class ContourElement {
   void dump_subelement_structure( void ) const;
   void dump_subelement_details( void ) const;
 
-  void build_subelement_facets( Faceted_Surface & facets );
+  void build_subelement_facets( FacetedSurfaceBase & facets );
 
   int gather_intg_pts( const int intg_pt_sign,
 		       sierra::ArrayContainer<double,DIM,NINT> & intg_pt_locations,

packages/krino/krino/krino_lib/Akri_ContourSubElement.cpp

@@ -43,7 +43,7 @@ ContourSubElement::ContourSubElement( const ContourElement *in_owner,
 }
 
 int
-ContourSubElement::build_facets( Faceted_Surface & facets )
+ContourSubElement::build_facets( FacetedSurfaceBase & facets )
 {
   int start_size = facets.size();
 
@@ -118,7 +118,7 @@ ContourSubElement::dump_details() const
 }
 
 int
-ContourSubElement::side_facets( Faceted_Surface & facets, int side ) const
+ContourSubElement::side_facets( FacetedSurfaceBase & facets, int side ) const
 {
   const std::string & owner_type = my_owner->dist_topology().name();
   const std::string & sub_type = topology().name();
@@ -1094,7 +1094,7 @@ ContourSubElement_Tri_3::is_degenerate( const std::array<int,6> & edge_node_ids,
 }
 
 int
-ContourSubElement_Tri_3::side_facets( Faceted_Surface & facets,
+ContourSubElement_Tri_3::side_facets( FacetedSurfaceBase & facets,
 			       int side ) const
 {
   STK_ThrowAssert( get_side_ids()[side] == -2 );
@@ -1105,15 +1105,9 @@ ContourSubElement_Tri_3::side_facets( Faceted_Surface & facets,
   const unsigned * const lnn = get_side_node_ordinals(topology(), side);
 
   if ( LevelSet::sign_change(0.0, (double) my_sign) )
-    {
-      std::unique_ptr<Facet> facet = std::make_unique<Facet2d>( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[1]]) );
-      facets.add( std::move(facet) );
-    }
+    facets.emplace_back_2d( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[1]]) );
   else
-    {
-      std::unique_ptr<Facet> facet = std::make_unique<Facet2d>( my_owner->coordinates(myCoords[lnn[1]]), my_owner->coordinates(myCoords[lnn[0]]) );
-      facets.add( std::move(facet) );
-    }
+    facets.emplace_back_2d( my_owner->coordinates(myCoords[lnn[1]]), my_owner->coordinates(myCoords[lnn[0]]) );
 
   return( num_facets );
 }
@@ -2086,7 +2080,7 @@ ContourSubElement_Tet_4::is_degenerate( const std::array<int,10> & edge_node_ids
 }
 
 int
-ContourSubElement_Tet_4::side_facets( Faceted_Surface & facets,
+ContourSubElement_Tet_4::side_facets( FacetedSurfaceBase & facets,
 			       int side ) const
 {
   STK_ThrowAssert( mySideIds[side] == -2 );
@@ -2097,15 +2091,9 @@ ContourSubElement_Tet_4::side_facets( Faceted_Surface & facets,
   const unsigned * const lnn = get_side_node_ordinals(topology(), side);
 
   if ( LevelSet::sign_change(0.0, (double) my_sign) )
-    {
-      std::unique_ptr<Facet> facet = std::make_unique<Facet3d>( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[1]]), my_owner->coordinates(myCoords[lnn[2]]) );
-      facets.add( std::move(facet) );
-    }
+    facets.emplace_back_3d( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[1]]), my_owner->coordinates(myCoords[lnn[2]]) );
   else
-    {
-      std::unique_ptr<Facet> facet = std::make_unique<Facet3d>( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[2]]), my_owner->coordinates(myCoords[lnn[1]]) );
-      facets.add( std::move(facet) );
-    }
+    facets.emplace_back_3d( my_owner->coordinates(myCoords[lnn[0]]), my_owner->coordinates(myCoords[lnn[2]]), my_owner->coordinates(myCoords[lnn[1]]) );
 
   return( num_facets );
 }

packages/krino/krino/krino_lib/Akri_ContourSubElement.hpp

@@ -64,10 +64,10 @@ class ContourSubElement {
 		       sierra::ArrayContainer<double,NINT> & determinants,
 		       int index = 0 );
 
-  int build_facets( Faceted_Surface & facets );
+  int build_facets( FacetedSurfaceBase & facets );
 
   // default implementation
-  virtual int side_facets( Faceted_Surface & facets, int side ) const;
+  virtual int side_facets( FacetedSurfaceBase & facets, int side ) const;
   virtual double side_area( int side ) const;
 
   stk::topology topology() const { return get_master_element().get_topology(); }
@@ -228,7 +228,7 @@ class ContourSubElement_Tri_3 : public ContourSubElementWithTopology<stk::topolo
                     const int subelement_sign = 0 );
   virtual ~ContourSubElement_Tri_3() {}
 
-  virtual int side_facets( Faceted_Surface & facets, int side ) const override;
+  virtual int side_facets( FacetedSurfaceBase & facets, int side ) const override;
   double side_area( int side ) const override;
 
 private:
@@ -295,7 +295,7 @@ class ContourSubElement_Tet_4 : public ContourSubElementWithTopology<stk::topolo
                     const int subelement_sign = 0 );
   virtual ~ContourSubElement_Tet_4() {}
 
-  virtual int side_facets( Faceted_Surface & facets, int side ) const override;
+  virtual int side_facets( FacetedSurfaceBase & facets, int side ) const override;
   virtual double side_area( int side ) const override;
 
 private: