Merge branch 'development' into cppcheck_fixes

.codespell-ignore-words

@@ -12,3 +12,4 @@ nd
 ue
 bion
 aas
+checkin

.editorconfig

@@ -0,0 +1,49 @@
+# http://EditorConfig.org
+#
+# precedence of rules is bottom to top
+
+# this is the top-most EditorConfig file
+root = true
+
+
+[*.{c,h,cpp,hpp,H,py}]
+# 4 space indentation
+indent_style = space
+indent_size = 4
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+# unix-style newlines
+end_of_line = lf
+
+# newline ending in files
+insert_final_newline = true
+
+
+[*.md]
+# two end of line whitespaces are newlines without a paragraph
+trim_trailing_whitespace = false
+
+
+[*.rst]
+# Enforce UTF-8 encoding
+charset = utf-8
+
+# Unix-style newlines
+end_of_line = lf
+
+# Newline ending in files
+insert_final_newline = true
+
+# 3 space indentation
+indent_style = space
+indent_size = 3
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+[{Makefile,GNUmakefile,Make.*}]
+# TABs are part of its syntax
+indent_style = tab
+indent_size = unset

.github/workflows/check_params.py

@@ -48,7 +48,7 @@ def doit(castro_dir):
                 # remove comments
                 idx = line.find("#")
                 if idx > 0:
-                    line = line[idx:]
+                    line = line[:idx]
 
                 found_param = pattern.match(line)
                 if not found_param:

.zenodo.json

@@ -17,6 +17,11 @@
             "name": "Bell, John",
             "orcid": "0000-0002-5749-334X"
         },
+        {
+            "affiliation": "Department of Physics and Astronomy, Stony Brook University",
+            "name": "Chen, Zhi",
+            "orcid": "0000-0002-2839-107X"
+        },
         {
             "name": "Harpole, Alice",
             "orcid": "0000-0002-1530-781X"

CHANGES.md

@@ -1,3 +1,14 @@
+# 24.06
+
+  * Doc updates (#2839, #2842, #2846, #2851, #2854, #2860)
+
+  * Sync problem and Castro runtime parameters to recent Microphysics
+    changes (#2838, #2845, #2861)
+
+  * fix parsing in the `check_params.py` script (#2850)
+
+  * code cleaning (#2840, #2842, #2843)
+
 # 24.05
 
   * Changed how the shock flag is computed.  It is now computed once,
@@ -16,7 +27,7 @@
   * Fixed the Castro retry mechanism when amr.subcycling_mode = None
     (#2821, #2824, #2826)
 
-  * Added more amr parameters to the job_info file (#2828)
+  * Added more amr parameters to the `job_info` file (#2828)
 
   * Added OpenMP to the SDC burn loop (#2770)
 

Docs/source/Hydrodynamics.rst

@@ -609,7 +609,7 @@ behavior:
 Compute Primitive Variables
 ---------------------------
 
-We compute the primtive variables from the conserved variables.
+We compute the primitive variables from the conserved variables.
 
 -  :math:`\rho, \rho e`: directly copy these from the conserved state
    vector

Docs/source/Verification.rst

@@ -495,7 +495,7 @@ test problem. A hot sphere is centered at the origin in a spherical
 geometry. The spectrum from this sphere follows a Planck
 distribution. The ambient medium is at a much lower temperature. A
 frequency-dependent opacity makes the domain optically thin for high
-frequecies and optically thick for low frequency. At long times, the
+frequencies and optically thick for low frequency. At long times, the
 solution will be a combination of the blackbody radiation from the
 ambient medium plus the radiation that propagated from the hot sphere.
 An analytic solution exists :cite:`graziani:2008` which gives the

Docs/source/conf.py

@@ -82,7 +82,7 @@ def get_version():
 
 # General information about the project.
 project = 'Castro'
-copyright = '2018-2022, Castro development team'
+copyright = '2018-2024, Castro development team'
 author = 'Castro development team'
 
 html_logo = "castro_logo_hot_200.png"

Docs/source/gpu.rst

@@ -0,0 +1,58 @@
+*********************
+GPU Programming Model
+*********************
+
+CPUs and GPUs have separate memory, which means that working on both
+the host and device may involve managing the transfer of data between
+the memory on the host and that on the GPU.
+
+In Castro, the core design when running on GPUs is that all of the compute
+should be done on the GPU.
+
+When we compile with ``USE_CUDA=TRUE`` or ``USE_HIP=TRUE``, AMReX will allocate
+a pool of memory on the GPUs and all of the ``StateData`` will be stored there.
+As long as we then do all of the computation on the GPUs, then we don't need
+to manage any of the data movement manually.
+
+.. note::
+
+   We can tell AMReX to allocate the data using managed-memory by
+   setting:
+
+   ::
+
+      amrex.the_arena_is_managed = 1
+
+   This is generally not needed.
+
+The programming model used throughout Castro is C++-lambda-capturing
+by value.  We access the ``FArrayBox`` stored in the ``StateData``
+``MultiFab`` by creating an ``Array4`` object.  The ``Array4`` does
+not directly store a copy of the data, but instead has a pointer to
+the data in the ``FArrayBox``.  When we capture the ``Array4`` by
+value in the GPU kernel, the GPU gets access to the pointer to the
+underlying data.
+
+
+Most AMReX functions will work on the data directly on the GPU (like
+``.setVal()``).
+
+In rare instances where we might need to operate on the data on the
+host, we can force a copy to the host, do the work, and then copy
+back.  For an example, see the reduction done in  ``Gravity.cpp``.
+
+.. note::
+
+   For a thorough discussion of how the AMReX GPU offloading works
+   see :cite:`amrex-ecp`.
+
+
+Runtime parameters
+------------------
+
+The main exception for all data being on the GPUs all the time are the
+runtime parameters.  At the moment, these are allocated as managed
+memory and stored in global memory.  This is simply to make it easier
+to read them in and initialize them on the CPU at runtime.
+
+

Docs/source/index.rst

@@ -21,6 +21,7 @@ https://github.com/amrex-astro/Castro
    mpi_plus_x
    FlowChart
    software
+   gpu
    problem_setups
    timestepping
    creating_a_problem

Docs/source/refs.bib

@@ -1131,3 +1131,22 @@ @ARTICLE{doubledet2024
 title = {Sensitivity of Simulations of Double-detonation Type Ia Supernovae to Integration Methodology},
 journal = {The Astrophysical Journal},
 }
+
+
+@ARTICLE{amrex-ecp,
+       author = {{Myers}, Andrew and {Zhang}, Weiqun and {Almgren}, Ann and {Antoun}, Thierry and {Bell}, John and {Huebl}, Axel and {Sinn}, Alexander},
+        title = "{AMReX and pyAMReX: Looking Beyond ECP}",
+      journal = {arXiv e-prints},
+     keywords = {Computer Science - Distributed, Parallel, and Cluster Computing},
+         year = 2024,
+        month = mar,
+          eid = {arXiv:2403.12179},
+        pages = {arXiv:2403.12179},
+          doi = {10.48550/arXiv.2403.12179},
+archivePrefix = {arXiv},
+       eprint = {2403.12179},
+ primaryClass = {cs.DC},
+       adsurl = {https://ui.adsabs.harvard.edu/abs/2024arXiv240312179M},
+      adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+

Exec/Make.auto_source

@@ -12,19 +12,22 @@ INCLUDE_LOCATIONS += $(CASTRO_AUTO_SOURCE_DIR)
 
 CEXE_sources += extern_parameters.cpp
 CEXE_headers += extern_parameters.H
+CEXE_headers += extern_type.H
 
 # for dependency resolution
 AUTO_BUILD_SOURCES += $(CASTRO_AUTO_SOURCE_DIR)/extern_parameters.H
+AUTO_BUILD_SOURCES += $(CASTRO_AUTO_SOURCE_DIR)/extern_type.H
 
 # this is for finding runtime parameters
 EXTERN_PARAMETERS := $(foreach dir, $(INCLUDE_LOCATIONS),$(realpath $(wildcard $(dir)/_parameters)))
 
 $(CASTRO_AUTO_SOURCE_DIR)/extern_parameters.cpp: $(CASTRO_AUTO_SOURCE_DIR)/extern_parameters.H
+$(CASTRO_AUTO_SOURCE_DIR)/extern_type.H: $(CASTRO_AUTO_SOURCE_DIR)/extern_parameters.H
 
 $(CASTRO_AUTO_SOURCE_DIR)/extern_parameters.H: $(EXTERN_PARAMETERS)
 	@if [ ! -d $(CASTRO_AUTO_SOURCE_DIR) ]; then mkdir -p $(CASTRO_AUTO_SOURCE_DIR); fi
 	$(MICROPHYSICS_HOME)/util/build_scripts/write_probin.py \
-           --cxx_prefix $(CASTRO_AUTO_SOURCE_DIR)/extern --use_namespace \
+           --cxx_prefix $(CASTRO_AUTO_SOURCE_DIR)/extern \
            --pa "$(EXTERN_PARAMETERS)"
 
 

Exec/gravity_tests/DustCollapse/GNUmakefile

@@ -22,7 +22,7 @@ CASTRO_HOME  ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/EOS
 EOS_DIR     := gamma_law
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/Networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/Networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/gravity_tests/StarGrav/_prob_params

@@ -1,4 +1,4 @@
-model_name       character          ""          y
+model_name       string             ""          y
 
 m_0              real               1.e30_rt    y
 

Exec/gravity_tests/hse_convergence/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS := triple_alpha_plus_o.net
 

Exec/gravity_tests/hse_convergence_general/_prob_params

@@ -14,19 +14,19 @@ H_star                  real              500.e0_rt                 y
 
 atm_delta               real              25.e0_rt                  y
 
-fuel1_name              character         "helium-4"                y
+fuel1_name              string         "helium-4"                y
 
-fuel2_name              character         ""                        y
+fuel2_name              string         ""                        y
 
-fuel3_name              character         ""                        y
+fuel3_name              string         ""                        y
 
-fuel4_name              character         ""                        y
+fuel4_name              string         ""                        y
 
-ash1_name               character         "iron-56"                 y
+ash1_name               string         "iron-56"                 y
 
-ash2_name               character         ""                        y
+ash2_name               string         ""                        y
 
-ash3_name               character         ""                        y
+ash3_name               string         ""                        y
 
 fuel1_frac              real              1.0_rt                    y
 

Exec/gravity_tests/hydrostatic_adjust/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/gravity_tests/hydrostatic_adjust/_prob_params

@@ -1,6 +1,6 @@
 prob_type          integer          1                   y
 
-model_name         character        ""                  y
+model_name         string        ""                  y
 
 hse_rho_top        real             0.0_rt
 

Exec/hydro_tests/Sod_stellar/GNUmakefile

@@ -21,7 +21,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/rotating_torus/GNUmakefile

@@ -20,7 +20,7 @@ CASTRO_HOME  ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := polytrope
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/test_convect/GNUmakefile

@@ -18,7 +18,7 @@ USE_MODEL_PARSER = TRUE
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = ignition.net
 

Exec/hydro_tests/test_convect/_prob_params

@@ -1,4 +1,4 @@
-model_name           character          ""              y
+model_name           string          ""              y
 
 apply_vel_field      integer            0               y
 

Exec/hydro_tests/toy_convect/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := general_null
 NETWORK_INPUTS = hotcno.net
 

Exec/hydro_tests/toy_convect/_prob_params

@@ -2,7 +2,7 @@ H_min               real                1.e-4_rt      y
 
 cutoff_density      real                50.0_rt       y
 
-model_name          character           ""            y
+model_name          string              ""            y
 
 apply_vel_field     integer             0             y
 

Exec/radiation_tests/RadBreakout/_prob_params

@@ -7,7 +7,7 @@ rbasefac            real        0.99e0_rt      y
 filter_rhomax       real        -1.e20_rt      y
 filter_timemax      real        -1.e20_rt      y
 
-model_file          character   "model.input"  y
+model_file          string      "model.input"  y
 
 iye                 int         -1             n
 iinvmu              int         -1             n

Exec/reacting_tests/bubble_convergence/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := triple_alpha_plus_cago
 
 PROBLEM_DIR ?= ./

Exec/reacting_tests/nse_test/GNUmakefile

@@ -18,7 +18,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := aprox19
 USE_NSE_TABLE := TRUE
 

Exec/reacting_tests/reacting_bubble/GNUmakefile

@@ -17,7 +17,7 @@ CASTRO_HOME ?= ../../..
 # This sets the EOS directory in $(MICROPHYSICS_HOME)/eos
 EOS_DIR     := helmholtz
 
-# This sets the EOS directory in $(MICROPHYSICS_HOME)/networks
+# This sets the network directory in $(MICROPHYSICS_HOME)/networks
 NETWORK_DIR := ignition_simple
 
 PROBLEM_DIR ?= ./