Enable fcompare tests with exawind driver

CMakeLists.txt

@@ -8,6 +8,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+option(EXAWIND_ENABLE_TESTS "Enable testing suite" OFF)
 option(EXAWIND_TEST_WITH_FCOMPARE "Check AMR-Wind test plots against gold files" OFF)
 option(EXAWIND_SAVE_GOLDS "Provide a directory in which to save golds during testing" OFF)
 option(EXAWIND_ENABLE_CUDA "Enable CUDA" OFF)
@@ -76,8 +77,10 @@ if(EXAWIND_ENABLE_CUDA)
   endforeach()
 endif()
 
-enable_testing()
-include(CTest)
-add_subdirectory(test)
+if(EXAWIND_ENABLE_TESTS)
+  enable_testing()
+  include(CTest)
+  add_subdirectory(test)
+endif()
 
 install(TARGETS ${EXAWIND_EXE_NAME})

test/CMakeLists.txt

@@ -14,10 +14,32 @@ endif()
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR}/CTestCustom.cmake)
 
+if(EXAWIND_TEST_WITH_FCOMPARE)
+  set(FCOMPARE_EXE "${FCOMPARE_EXE}" CACHE STRING "Path to fcompare executable for regression tests")
+  if("${EXAWIND_REFERENCE_GOLDS_DIRECTORY}" STREQUAL "")
+    message(FATAL_ERROR "To reference gold files, EXAWIND_REFERENCE_GOLDS_DIRECTORY must be set and exist")
+  else()
+    set(GOLD_FILES_DIRECTORY ${EXAWIND_REFERENCE_GOLDS_DIRECTORY}/${CMAKE_SYSTEM_NAME}/${CMAKE_CXX_COMPILER_ID}/${CMAKE_CXX_COMPILER_VERSION})
+    message(STATUS "Test golds directory for fcompare: ${GOLD_FILES_DIRECTORY}")
+  endif()
+endif()
+
+if(EXAWIND_SAVE_GOLDS)
+  if("${EXAWIND_SAVED_GOLDS_DIRECTORY}" STREQUAL "")
+    message(FATAL_ERROR "To save gold files, EXAWIND_SAVED_GOLDS_DIRECTORY must be set and the directory exist")
+  else()
+    if(EXISTS ${EXAWIND_SAVED_GOLDS_DIRECTORY})
+      set(SAVED_GOLDS_DIRECTORY ${EXAWIND_SAVED_GOLDS_DIRECTORY}/${CMAKE_SYSTEM_NAME}/${CMAKE_CXX_COMPILER_ID}/${CMAKE_CXX_COMPILER_VERSION})
+      message(STATUS "Gold files will be saved to: ${SAVED_GOLDS_DIRECTORY}")
+    else()
+      message(FATAL_ERROR "Specified directory for saving gold files does not exist: ${EXAWIND_SAVED_GOLDS_DIRECTORY}")
+    endif()
+  endif()
+endif()
+
 #=============================================================================
 # Functions for adding tests / Categories of tests
 #=============================================================================
-
 macro(setup_test)
     set(CURRENT_TEST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/test_files/${TEST_NAME})
     set(CURRENT_TEST_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/test_files/${TEST_NAME})
@@ -33,7 +55,7 @@ macro(setup_test)
       set(SAVE_GOLDS_COMMAND "&& cp -R ${PLOT_TEST} ${SAVED_GOLDS_DIRECTORY}/${TEST_NAME}/")
     endif()
     if(EXAWIND_TEST_WITH_FCOMPARE)
-      set(FCOMPARE_COMMAND "&& ${MPI_COMMANDS} ${FCOMPARE_EXE} ${FCOMPARE_TOLERANCE} ${PLOT_GOLD} ${PLOT_TEST}")
+      set(FCOMPARE_COMMAND "&& CUDA_LAUNCH_BLOCKING=1 ${FCOMPARE_EXE} ${FCOMPARE_TOLERANCE} ${PLOT_GOLD} ${PLOT_TEST}")
     endif()
     execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
                     ${CMAKE_CURRENT_SOURCE_DIR}/meshes

test/CTestCustom.cmake

@@ -1,5 +1,5 @@
 set(CTEST_CUSTOM_WARNING_EXCEPTION ".*: warning: could not create compact unwind*"
                                    ".*has no symbols*"
-                                   ".*resetting jobserver mode*")
-set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE "262144")
-set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "51200")
+                                   ".*submake: resetting jobserver mode*")
+set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE "131072")
+set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "131072")

test/test_files/amr-nalu-cylinder-motion/cylinder-amr.inp

@@ -6,7 +6,7 @@ tim.max_step = -1
 time.fixed_dt = 0.15
 time.cfl = 1.0
 time.plot_interval = 10
-time.checkpoint_interval = -10
+time.checkpoint_interval = -1
 io.plot_file = out/plt
 io.check_file = out/chk
 #               PHYSICS                 #

test/test_files/amr-nalu-cylinder/cylinder-amr.inp

@@ -6,7 +6,7 @@ tim.max_step = -1
 time.fixed_dt = 0.15
 time.cfl = 1.0
 time.plot_interval = 10
-time.checkpoint_interval = -10
+time.checkpoint_interval = -1
 io.plot_file = out/plt
 io.check_file = out/chk
 #               PHYSICS                 #

test/test_files/dam-break-block/dam-break-block-amr.inp

@@ -5,7 +5,7 @@ time.fixed_dt         =   0.001        # Use this constant dt if > 0
 time.cfl              =   0.95         # CFL factor
 
 time.plot_interval            =  10       # Steps between plot files
-time.checkpoint_interval      =  -100       # Steps between checkpoint files
+time.checkpoint_interval      =  -1       # Steps between checkpoint files
 
 io.int_outputs = iblank_cell
 

test/test_files/hybrid-multi-cylinder/cylinder-amr.inp

@@ -6,7 +6,7 @@ tim.max_step = -1
 time.fixed_dt = 0.15
 time.cfl = 1.0
 time.plot_interval = 10
-time.checkpoint_interval = -10
+time.checkpoint_interval = -1
 io.plot_file = out/plt
 io.check_file = out/chk
 #               PHYSICS                 #

test/test_files/sphere/sphere-amr.inp

@@ -15,8 +15,8 @@ time.cfl              =   1.00         # CFL factor
 #.......................................#
 io.KE_int = -1
 io.line_plot_int = -1
-time.plot_interval            =   5       # Steps between plot files
-time.checkpoint_interval      =  -5       # Steps between checkpoint files
+time.plot_interval            =  10       # Steps between plot files
+time.checkpoint_interval      =  -1       # Steps between checkpoint files
 
 #¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
 #               PHYSICS                 #

test/test_files/zalesak/zalesak-amr.inp

@@ -14,7 +14,7 @@ time.cfl              =   0.95         # CFL factor
 #            INPUT AND OUTPUT           #
 #.......................................#
 time.plot_interval            =  10       # Steps between plot files
-time.checkpoint_interval      =  -100       # Steps between checkpoint files
+time.checkpoint_interval      =  -1       # Steps between checkpoint files
 io.output_default_fields = 0
 io.outputs = density velocity vof