Add AXOM_ENABLE_ASAN CMake option

src/cmake/AxomOptions.cmake

@@ -6,6 +6,13 @@
 # Defines CMake options for Axom's build system
 #------------------------------------------------------------------------------
 
+option(AXOM_ENABLE_ASAN "Enable AddressSanitizer for memory checking (Clang or GCC only)" OFF)
+if(AXOM_ENABLE_ASAN)
+    if(NOT (C_COMPILER_FAMILY_IS_CLANG OR C_COMPILER_FAMILY_IS_GNU))
+        message(FATAL_ERROR "AXOM_ENABLE_ASAN only supports Clang and GCC")
+    endif()
+endif()
+
 option(AXOM_ENABLE_SPARSEHASH "Enables Sparsehash." ON)
 option(AXOM_ENABLE_ALL_COMPONENTS "Enables all components by default" ON)
 option(AXOM_USE_64BIT_INDEXTYPE "Use 64-bit integers for axom::IndexType" OFF)

src/cmake/CMakeBasics.cmake

@@ -189,6 +189,13 @@ blt_append_custom_compiler_flag(FLAGS_VAR AXOM_NINJA_FLAGS
                   CLANG       "-fcolor-diagnostics"
                   )
 
+if(AXOM_ENABLE_ASAN)
+    message(STATUS "AddressSanitizer is ON (ENABLE_ASAN)")
+    foreach(_flagvar CMAKE_C_FLAGS CMAKE_CXX_FLAGS CMAKE_EXE_LINKER_FLAGS)
+        string(APPEND ${_flagvar} " -fsanitize=address -fno-omit-frame-pointer")
+    endforeach()
+endif()
+
 if(${AXOM_ENABLE_EXPORTS})
   set(CMAKE_ENABLE_EXPORTS ON)
 endif()

src/docs/sphinx/dev_guide/index.rst

@@ -51,6 +51,7 @@ changes are made, this guide should be updated accordingly.
    testing
    updating_tpls
    gpu_porting
+   memory_checking
    misc_tasks
 
 

src/docs/sphinx/dev_guide/memory_checking.rst

@@ -0,0 +1,90 @@
+.. ## Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and
+.. ## other Axom Project Developers. See the top-level LICENSE file for details.
+.. ##
+.. ## SPDX-License-Identifier: (BSD-3-Clause)
+
+.. _memorychecking-label:
+
+===============
+Memory Checking
+===============
+
+There are two commonly available memory checkers available to use with Axom on LC:
+`AddressSanitizer <https://github.com/google/sanitizers/wiki/AddressSanitizer>`_
+and `Valgrind <https://valgrind.org/>`_.
+
+AddressSanitizer
+----------------
+
+AddressSanitizer (aka Asan) is memory error detection tool that is a part of LLVM.  It
+very fast and easy to use but doesn't seem as robust as Valgrind.  It requires compile
+and link flags which are enabled via the CMake option ``AXOM_ENABLE_ASAN``.  Anything in our CMake
+system will get those flags after that is enabled but our third-party libraries (like MFEM)
+will not. After that just run your built executable and Asan will output a log to the screen
+after your program is done running.  Asan's behavior can be modified with a set of
+`environment variables <https://github.com/google/sanitizers/wiki/AddressSanitizerFlags>`_ .
+
+.. note::
+    Asan only works with the Clang and GCC compiler chains.  Our build system will throw
+    an error if you try to build with anything else while ``AXOM_ENABLE_ASAN`` is ``ON``.
+
+Here is a recommended workflow:
+
+.. code-block:: bash
+
+    ./config-build.py -hc host-configs/rzwhippet-toss_4_x86_64_ib-clang@14.0.6.cmake -DAXOM_ENABLE_ASAN=ON
+    cd build-rzwhippet-toss_4_x86_64_ib-clang@14.0.6-debug
+    srun -N1 --exclusive --mpi-bind=off make -j
+    LSAN_OPTIONS=suppressions=../suppressions.asan ASAN_OPTIONS=log_path=asan.out:log_exe_name=true srun -n2 <path to test>
+
+This will output files in the current directory for each process that follow the pattern:
+``asan.out.<exe name>.<pid>``.  It also sets your return code to a non-zero value if there
+were any non-suppressed memory errors.
+
+``LSAN_OPTIONS`` and ``ASAN_OPTIONS`` are delimited by ':'.
+
+Here is an explanation of the given options (all should be added to ``ASAN_OPTIONS`` unless noted):
+
+  * ``suppressions``: Location of memory leak suppression file (``LSAN_OPTIONS``)
+  * ``log_path``: Logs to the given file instead of to the screen. This is very helpful
+    to avoid intermingled lines on the screen from every process
+  * ``log_exe_name``: Adds executable name to log_path
+
+Helpful options:
+
+  * ``fast_unwind_on_malloc=0``: This improves Asan's stack tracing ability but also greatly slows
+    down the run
+  * ``exitcode=0``: This stops Asan from returning a a non-zero exit code from your executable
+    (defaults to 23) (``LSAN_OPTIONS``)
+
+
+Valgrind
+--------
+
+Valgrind is a very powerful set of tools that help with dynamic analysis tools.  We will
+focus on `memcheck <https://valgrind.org/docs/manual/mc-manual.html>`_ which is a memory
+error detection tool.
+
+Unlike Asan, valgrind does not need any special compiler flags.  Just build your executable
+and run your executable with ``valgrind``. Valgrind's suppression files are easily generated by
+valgrind with ``--gen-suppressions=all`` and are more customizable than Asan's.
+
+Here is a recommended workflow:
+
+.. code-block:: bash
+
+    ./config-build.py -hc host-configs/rzgenie-toss_3_x86_64_ib-gcc@8.1.0.cmake
+    cd build-rzgenie-toss_3_x86_64_ib-gcc@8.1.0-debug
+    srun -N1 --exclusive --mpi-bind=off make -j
+    srun -n2 valgrind --tool=memcheck --log-file=valgrind.out --leak-check=yes --show-leak-kinds=all --num-callers=20 --suppressions=../suppressions.valgrind <path to test>
+
+This will produce a file called ``valgrind.out`` in the current directory with a valgrind report.
+
+Here is an explanation of the given options:
+
+ * ``--tool=memcheck``: valgrind is a tool-suite so this runs the memcheck tool
+ * ``--log-file=valgrind.out``: Logs report to the given file
+ * ``--leak-check=yes``: Enables memory leak checks
+ * ``--show-leak-kinds=all```: Enables showing all memory leak kinds
+ * ``--num-callers=20``: Limits the size of the stack traces to 20
+ * ``--suppressions=../suppressions.valgrind``: Location of memory leak suppression file

suppressions.asan

@@ -0,0 +1,2 @@
+# Library that isn't built by us
+leak:libpsm2.so.2

suppressions.valgrind

@@ -0,0 +1,36 @@
+{
+   <mpi_init_leak>
+   Memcheck:Leak
+   ...
+   fun:PMPI_Init
+   fun:main
+}
+
+{
+   <mpi_init_leak2>
+   Memcheck:Leak
+   ...
+   fun:PMPI_Init
+}
+
+{
+   <mpi_init_param>
+   Memcheck:Param
+   rt_sigaction(act->sa_mask)
+   fun:__libc_sigaction
+   obj:/usr/lib64/libpsm2.so.2.1
+   obj:/usr/lib64/libpsm2.so.2.1
+   fun:psm2_ep_open
+   fun:psm_doinit
+   fun:MPID_Init
+   fun:MPIR_Init_thread
+   fun:PMPI_Init
+   fun:main
+}
+
+{
+   <psm2_lib>
+   Memcheck:Leak
+   ...
+   obj:/usr/lib64/libpsm2.so.2.1
+}