From 243a5f2d268d25d53109d5790182f4059760d923 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 12:59:27 -0500 Subject: [PATCH 1/7] update hip action -- ubuntu 20.04 runner is going away (#1750) this syncs the dependency script up with amrex --- .../dependencies/dependencies_hip.sh | 20 ++++++++++--------- .github/workflows/hip.yml | 4 ++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dependencies/dependencies_hip.sh b/.github/workflows/dependencies/dependencies_hip.sh index 527379e7e8..76a2abd04c 100755 --- a/.github/workflows/dependencies/dependencies_hip.sh +++ b/.github/workflows/dependencies/dependencies_hip.sh @@ -33,7 +33,9 @@ sudo apt-key add rocm.gpg.key source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ... -echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \ +VERSION=${1-6.3.2} + +echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${VERSION} ${UBUNTU_CODENAME} main" \ | sudo tee /etc/apt/sources.list.d/rocm.list echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ | sudo tee -a /etc/profile.d/rocm.sh @@ -53,16 +55,16 @@ sudo apt-get install -y --no-install-recommends \ libnuma-dev \ libopenmpi-dev \ openmpi-bin \ - rocm-dev \ - roctracer-dev \ - rocprofiler-dev \ - rocrand-dev \ - rocfft-dev \ - rocprim-dev \ - rocsparse-dev + rocm-dev${VERSION} \ + roctracer-dev${VERSION} \ + rocprofiler-dev${VERSION} \ + rocrand-dev${VERSION} \ + rocfft-dev${VERSION} \ + rocprim-dev${VERSION} \ + rocsparse-dev${VERSION} # hiprand-dev is a new package that does not exist in old versions -sudo apt-get install -y --no-install-recommends hiprand-dev || true +sudo apt-get install -y --no-install-recommends hiprand-dev${VERSION} || true # activate # diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index cd3fac813e..fff66f72ec 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -8,7 +8,7 @@ concurrency: jobs: hip-compile: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: @@ -27,7 +27,7 @@ jobs: cd ../.. - name: Dependencies - run: .github/workflows/dependencies/dependencies_hip.sh + run: .github/workflows/dependencies/dependencies_hip.sh 6.3.2 - name: compile test_react with HIP (iso7) run: | From f9d54de7bdf27e928d93aaa59c2855c2012cc61d Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:08:05 -0500 Subject: [PATCH 2/7] Centralize some runtime parameters and their docs (#1749) in particular, this moves small_temp and small_dens to the top-level unit_test/_parameters, since every test needs them. We override it in a few tests, since they need much different defaults. This also moves the documentation about initializing composition up to a new centralized page, so we don't need to repeat it for every test. --- Docs/source/burn_cell.rst | 30 ++-------- Docs/source/comprehensive_tests.rst | 2 + Docs/source/eos_cell.rst | 10 +--- Docs/source/index.rst | 1 + Docs/source/jac_cell.rst | 13 +++-- Docs/source/one_zone_tests.rst | 2 + Docs/source/unit_test_runtime_parameters.rst | 58 +++++++++++++++++++ unit_test/_parameters | 7 +++ unit_test/burn_cell/_parameters | 3 - unit_test/burn_cell_metal_chem/_parameters | 4 +- .../burn_cell_primordial_chem/_parameters | 4 +- unit_test/burn_cell_sdc/_parameters | 3 - unit_test/eos_cell/_parameters | 3 - unit_test/jac_cell/_parameters | 3 - unit_test/nse_net_cell/_parameters | 3 - unit_test/nse_net_cell/make_table/_parameters | 3 - unit_test/nse_table_cell/_parameters | 3 - unit_test/part_func_cell/_parameters | 3 - unit_test/test_aprox_rates/_parameters | 3 - unit_test/test_conductivity/_parameters | 3 - unit_test/test_eos/_parameters | 3 - unit_test/test_jac/_parameters | 3 - unit_test/test_linear_algebra/_parameters | 3 - unit_test/test_neutrino_cooling/_parameters | 3 - unit_test/test_nse_interp/_parameters | 3 - unit_test/test_parameters/_parameters | 3 - unit_test/test_react/_parameters | 3 - unit_test/test_rhs/_parameters | 5 -- .../test_screening_templated/_parameters | 3 - unit_test/test_sdc/_parameters | 7 --- unit_test/test_sdc_vode_rhs/_parameters | 3 - unit_test/test_sdc_vode_rhs/vode_rhs_test.H | 2 +- 32 files changed, 88 insertions(+), 114 deletions(-) create mode 100644 Docs/source/unit_test_runtime_parameters.rst diff --git a/Docs/source/burn_cell.rst b/Docs/source/burn_cell.rst index 7ccfdde5e8..7541d3f261 100644 --- a/Docs/source/burn_cell.rst +++ b/Docs/source/burn_cell.rst @@ -52,39 +52,17 @@ The parameters that affect the thermodynamics are: * ``unit_test.temperature`` : the initial temperature -* ``unit_test.small_temp`` : the low temperature cutoff used in the equation of state +The composition can be set either by specifying individual mass fractions +or setting ``unit_test.uniform_xn`` as described in :ref:`sec:defining_unit_test_composition`. -* ``unit_test.small_dens`` : the low density cutoff used in the equation of state - -The composition can be set either by setting each mass fraction explicitly via the -parameters, ``unit_test.X1``, ``unit_test.X2``, ..., -e.g.: - -:: - - unit_test.X1 = 0.5 - unit_test.X2 = 0.2 - unit_test.X3 = 0.2 - unit_test.X4 = 0.1 - -where parameters up to ``X35`` are available. If the values don't sum to ``1`` -initially, then the test will do a normalization. This normalization can be -disabled by setting: +If the values don't sum to ``1`` initially, then the test will do a +normalization. This normalization can be disabled by setting: :: unit_test.skip_initial_normalization = 1 -Alternately, the composition can be set automatically by initializing all -of the mass fractions equally (to $1/N$, where $N$ is the number of species), -by setting: - -:: - - unit_test.uniform_xn = 1 - - Controlling time ---------------- diff --git a/Docs/source/comprehensive_tests.rst b/Docs/source/comprehensive_tests.rst index 5ea20ab663..32a4f507f9 100644 --- a/Docs/source/comprehensive_tests.rst +++ b/Docs/source/comprehensive_tests.rst @@ -1,3 +1,5 @@ +.. _sec:comprehensive_tests: + ************************ Comprehensive Unit Tests ************************ diff --git a/Docs/source/eos_cell.rst b/Docs/source/eos_cell.rst index 7399884eaa..c1a1a117ad 100644 --- a/Docs/source/eos_cell.rst +++ b/Docs/source/eos_cell.rst @@ -28,14 +28,8 @@ The parameters that affect the thermodynamics are: * ``unit_test.temperature`` : the initial temperature -* ``unit_test.small_temp`` : the low temperature cutoff used in the equation of state - -* ``unit_test.small_dens`` : the low density cutoff used in the equation of state - -The composition can be set in the same way as in ``burn_cell``, either -by setting each mass fraction explicitly via the parameters, -``unit_test.X1``, ``unit_test.X2``, ..., or forcing them to be all -equal via ``unit_test.uniform_xn=1``. +The composition can be set either by specifying individual mass fractions +or setting ``unit_test.uniform_xn`` as described in :ref:`sec:defining_unit_test_composition`. Building and Running the Code diff --git a/Docs/source/index.rst b/Docs/source/index.rst index a366bd78c9..37e13304a6 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -92,6 +92,7 @@ system. :hidden: unit_tests + unit_test_runtime_parameters comprehensive_tests one_zone_tests diff --git a/Docs/source/jac_cell.rst b/Docs/source/jac_cell.rst index 51db309555..cf7a73c2bc 100644 --- a/Docs/source/jac_cell.rst +++ b/Docs/source/jac_cell.rst @@ -38,14 +38,15 @@ The parameters that affect the thermodynamics are: * ``unit_test.temperature`` : the initial temperature -* ``unit_test.small_temp`` : the low temperature cutoff used in the equation of state +The composition can be set either by specifying individual mass fractions +or setting ``unit_test.uniform_xn`` as described in :ref:`sec:defining_unit_test_composition`. -* ``unit_test.small_dens`` : the low density cutoff used in the equation of state +If the values don't sum to ``1`` initially, then the test will do a +normalization. This normalization can be disabled by setting: -While the mass fractions can be set individually (using -``unit_test.X1``, ``unit_test.X2``, ...), it is recommended to use -``unit_test.uniform_xn=1`` to initialize all the mass fractions to be -equal. +:: + + unit_test.skip_initial_normalization = 1 Building and Running the Code diff --git a/Docs/source/one_zone_tests.rst b/Docs/source/one_zone_tests.rst index 8a34952ef4..2cf8ea57b8 100644 --- a/Docs/source/one_zone_tests.rst +++ b/Docs/source/one_zone_tests.rst @@ -1,3 +1,5 @@ +.. _sec:one_zone_tests: + ************** One Zone Tests ************** diff --git a/Docs/source/unit_test_runtime_parameters.rst b/Docs/source/unit_test_runtime_parameters.rst new file mode 100644 index 0000000000..fc6232c44a --- /dev/null +++ b/Docs/source/unit_test_runtime_parameters.rst @@ -0,0 +1,58 @@ +*********************************** +Unit Test Common Runtime Parameters +*********************************** + +There are a number of runtime parameters that are common to all (or most) of the unit tests. +These are defined in the top-level ``unit_test/_parameters`` file. + +Thermodynamics +============== + +The equation of state enforces minimum density and temperatures, which must be set +upon initialization. These are controlled by the following runtime parameters: + +* ``unit_test.small_temp`` : the low temperature cutoff used in the equation of state + +* ``unit_test.small_dens`` : the low density cutoff used in the equation of state + + +.. _sec:defining_unit_test_composition: + +Defining composition +==================== + +Most of the unit tests require a composition to be defined (for the +initial mass-fractions, $X_k$). There are a few ways this can be done +(depending on the test). + +* One-zone (``*_cell``) tests (see :ref:`sec:one_zone_tests`) usually do one of: + + * *Explicitly setting the individual mass fractions.* This is + controlled by the parameters ``unit_test.X1``, ``unit_test.X2``, ..., ``unit_test.X35``, + e.g.: + + :: + + unit_test.X1 = 0.5 + unit_test.X2 = 0.2 + unit_test.X3 = 0.2 + unit_test.X4 = 0.1 + + While many of the tests will renormalize the abundances, the user + should take care to ensure that the mass fractions sum to unity. + + * *Setting the composition to be uniform.* This is controlled by + ``unit_test.uniform_xn``. If this is set to ``1``, then each mass fraction + is initialized to ``1 / NumSpec``. + +* Comprehensive tests (see :ref:`sec:comprehensive_tests`) need many different compositions, since they are creating a cube + of varying thermodynamic properties, and thus require a prescription + to create the composition. This is done by setting ``unit_test.primary_species_1``, + ``unit_test.primary_species_2``, and ``unit_test.primary_species_3`` to one of the + *names* of the species in the network. + + The function ``setup_composition()`` is then used to set limits on + the species abundances (it takes a parameter which is the index into + the cube of data that is being initialized) which is then used by + ``get_xn()`` to create the individual mass fractions. Both of these + routines are contained in ``react_util.H``. diff --git a/unit_test/_parameters b/unit_test/_parameters index 2fd9224d2e..21be177521 100644 --- a/unit_test/_parameters +++ b/unit_test/_parameters @@ -40,5 +40,12 @@ X33 real 0.0e0 X34 real 0.0e0 X35 real 0.0e0 +# do we set all of the mass fractions equal (to 1/NumSpec)? uniform_xn bool 0 +# lowest temperature allowed in the EOS +small_temp real 1.e5 + +# lowest density allowed in the EOS +small_dens real 1.e5 + diff --git a/unit_test/burn_cell/_parameters b/unit_test/burn_cell/_parameters index 5a39427b8c..159e1d33ab 100644 --- a/unit_test/burn_cell/_parameters +++ b/unit_test/burn_cell/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - # the final time to integrate to tmax real 1.e-2 diff --git a/unit_test/burn_cell_metal_chem/_parameters b/unit_test/burn_cell_metal_chem/_parameters index 4d602783fc..627d677371 100644 --- a/unit_test/burn_cell_metal_chem/_parameters +++ b/unit_test/burn_cell_metal_chem/_parameters @@ -3,8 +3,8 @@ run_prefix string "burn_cell_metal_chem" # floor values of temperature and density -small_temp real 1.e1 -small_dens real 1.e-30 +small_temp real 1.e1 100 +small_dens real 1.e-30 100 # the final time to integrate to tmax real 1.e20 diff --git a/unit_test/burn_cell_primordial_chem/_parameters b/unit_test/burn_cell_primordial_chem/_parameters index 8aa77dfea5..a81e1ff435 100644 --- a/unit_test/burn_cell_primordial_chem/_parameters +++ b/unit_test/burn_cell_primordial_chem/_parameters @@ -3,8 +3,8 @@ run_prefix string "burn_cell_primordial_chem" # floor values of temperature and density -small_temp real 1.e1 -small_dens real 1.e-30 +small_temp real 1.e1 100 +small_dens real 1.e-30 100 # the final time to integrate to tmax real 1.e20 diff --git a/unit_test/burn_cell_sdc/_parameters b/unit_test/burn_cell_sdc/_parameters index d08fa3a2a4..c4b03562fd 100644 --- a/unit_test/burn_cell_sdc/_parameters +++ b/unit_test/burn_cell_sdc/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - # the final time to integrate to tmax real 1.e-2 diff --git a/unit_test/eos_cell/_parameters b/unit_test/eos_cell/_parameters index 8a48aeb859..e4186ca038 100644 --- a/unit_test/eos_cell/_parameters +++ b/unit_test/eos_cell/_parameters @@ -1,7 +1,4 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.e7 temperature real 3.e9 diff --git a/unit_test/jac_cell/_parameters b/unit_test/jac_cell/_parameters index 5894a90b97..26ec4ee758 100644 --- a/unit_test/jac_cell/_parameters +++ b/unit_test/jac_cell/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.e7 temperature real 3.e9 diff --git a/unit_test/nse_net_cell/_parameters b/unit_test/nse_net_cell/_parameters index 600c7cfca7..cfe765b8e8 100644 --- a/unit_test/nse_net_cell/_parameters +++ b/unit_test/nse_net_cell/_parameters @@ -2,9 +2,6 @@ run_prefix string "" -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.e7 temperature real 3.e9 diff --git a/unit_test/nse_net_cell/make_table/_parameters b/unit_test/nse_net_cell/make_table/_parameters index fb1905dabb..3d519057b8 100644 --- a/unit_test/nse_net_cell/make_table/_parameters +++ b/unit_test/nse_net_cell/make_table/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_dens real 1.e5 -small_temp real 1.e5 - rho_min real 1.e6 rho_max real 1.e10 diff --git a/unit_test/nse_table_cell/_parameters b/unit_test/nse_table_cell/_parameters index 4988b8ec76..70c4c6436e 100644 --- a/unit_test/nse_table_cell/_parameters +++ b/unit_test/nse_table_cell/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.23e9 temperature real 5.18e9 ye real 0.472 diff --git a/unit_test/part_func_cell/_parameters b/unit_test/part_func_cell/_parameters index 39a0b0c81f..9b1373362a 100644 --- a/unit_test/part_func_cell/_parameters +++ b/unit_test/part_func_cell/_parameters @@ -1,7 +1,4 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - temperature real 5.e9 diff --git a/unit_test/test_aprox_rates/_parameters b/unit_test/test_aprox_rates/_parameters index b546788335..b3d7d70ccd 100644 --- a/unit_test/test_aprox_rates/_parameters +++ b/unit_test/test_aprox_rates/_parameters @@ -6,6 +6,3 @@ temp_min real 1.e6 temp_max real 1.e12 metalicity_max real 0.1e0 - -small_temp real 1.e4 -small_dens real 1.e-4 diff --git a/unit_test/test_conductivity/_parameters b/unit_test/test_conductivity/_parameters index b546788335..b3d7d70ccd 100644 --- a/unit_test/test_conductivity/_parameters +++ b/unit_test/test_conductivity/_parameters @@ -6,6 +6,3 @@ temp_min real 1.e6 temp_max real 1.e12 metalicity_max real 0.1e0 - -small_temp real 1.e4 -small_dens real 1.e-4 diff --git a/unit_test/test_eos/_parameters b/unit_test/test_eos/_parameters index b546788335..b3d7d70ccd 100644 --- a/unit_test/test_eos/_parameters +++ b/unit_test/test_eos/_parameters @@ -6,6 +6,3 @@ temp_min real 1.e6 temp_max real 1.e12 metalicity_max real 0.1e0 - -small_temp real 1.e4 -small_dens real 1.e-4 diff --git a/unit_test/test_jac/_parameters b/unit_test/test_jac/_parameters index 918582dd7b..a708f46788 100644 --- a/unit_test/test_jac/_parameters +++ b/unit_test/test_jac/_parameters @@ -9,6 +9,3 @@ uniform_xn int 0 tmax real 0.1e0 -small_temp real 1.e5 -small_dens real 1.e5 - diff --git a/unit_test/test_linear_algebra/_parameters b/unit_test/test_linear_algebra/_parameters index 0a6cfaa175..7f058e92c1 100644 --- a/unit_test/test_linear_algebra/_parameters +++ b/unit_test/test_linear_algebra/_parameters @@ -1,4 +1 @@ @namespace: unit_test - -small_temp real 1.e5 -small_dens real 1.e5 diff --git a/unit_test/test_neutrino_cooling/_parameters b/unit_test/test_neutrino_cooling/_parameters index b546788335..b3d7d70ccd 100644 --- a/unit_test/test_neutrino_cooling/_parameters +++ b/unit_test/test_neutrino_cooling/_parameters @@ -6,6 +6,3 @@ temp_min real 1.e6 temp_max real 1.e12 metalicity_max real 0.1e0 - -small_temp real 1.e4 -small_dens real 1.e-4 diff --git a/unit_test/test_nse_interp/_parameters b/unit_test/test_nse_interp/_parameters index 4988b8ec76..70c4c6436e 100644 --- a/unit_test/test_nse_interp/_parameters +++ b/unit_test/test_nse_interp/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.23e9 temperature real 5.18e9 ye real 0.472 diff --git a/unit_test/test_parameters/_parameters b/unit_test/test_parameters/_parameters index ef2035f844..08e6c9e22b 100644 --- a/unit_test/test_parameters/_parameters +++ b/unit_test/test_parameters/_parameters @@ -5,7 +5,4 @@ dens_max real 1.e9 temp_min real 1.e6 temp_max real 1.e12 -small_temp real 1.e4 -small_dens real 1.e-4 - test_string string "test" diff --git a/unit_test/test_react/_parameters b/unit_test/test_react/_parameters index 6cfcdab221..43f495381e 100644 --- a/unit_test/test_react/_parameters +++ b/unit_test/test_react/_parameters @@ -7,6 +7,3 @@ temp_max real 1.e15 tmax real 0.1e0 -small_temp real 1.e5 -small_dens real 1.e5 - diff --git a/unit_test/test_rhs/_parameters b/unit_test/test_rhs/_parameters index 1c58257f74..fd85125029 100644 --- a/unit_test/test_rhs/_parameters +++ b/unit_test/test_rhs/_parameters @@ -4,8 +4,3 @@ dens_min real 1.e6 dens_max real 1.e9 temp_min real 1.e6 temp_max real 1.e15 - -uniform_xn int 0 - -small_temp real 1.e5 -small_dens real 1.e5 diff --git a/unit_test/test_screening_templated/_parameters b/unit_test/test_screening_templated/_parameters index 9bac5a5ccd..4fdce89a8b 100644 --- a/unit_test/test_screening_templated/_parameters +++ b/unit_test/test_screening_templated/_parameters @@ -7,7 +7,4 @@ temp_max real 1.e12 metalicity_max real 0.1e0 -small_temp real 1.e4 -small_dens real 1.e-4 - loops int 1 diff --git a/unit_test/test_sdc/_parameters b/unit_test/test_sdc/_parameters index 1b0bb5980f..1f023cde40 100644 --- a/unit_test/test_sdc/_parameters +++ b/unit_test/test_sdc/_parameters @@ -5,11 +5,4 @@ dens_max real 1.e9 temp_min real 1.e6 temp_max real 1.e15 -# uniform composition -uniform_xn int 0 - tmax real 0.1e0 - -small_temp real 1.e5 -small_dens real 1.e3 - diff --git a/unit_test/test_sdc_vode_rhs/_parameters b/unit_test/test_sdc_vode_rhs/_parameters index 276c539cad..c4918ad007 100644 --- a/unit_test/test_sdc_vode_rhs/_parameters +++ b/unit_test/test_sdc_vode_rhs/_parameters @@ -1,8 +1,5 @@ @namespace: unit_test -small_temp real 1.e5 -small_dens real 1.e5 - density real 1.e7 temperature real 3.e9 diff --git a/unit_test/test_sdc_vode_rhs/vode_rhs_test.H b/unit_test/test_sdc_vode_rhs/vode_rhs_test.H index 79234ee4f0..7a70eecdfe 100644 --- a/unit_test/test_sdc_vode_rhs/vode_rhs_test.H +++ b/unit_test/test_sdc_vode_rhs/vode_rhs_test.H @@ -23,7 +23,7 @@ void do_vode_rhs() for (int n = 1; n <= NumSpec; ++n) { - massfractions[n-1] = get_xn(n); + massfractions[n-1] = get_xn(n, unit_test_rp::uniform_xn); if (massfractions[n-1] < 0 || massfractions[n-1] > 1) { amrex::Error("mass fraction for " + short_spec_names_cxx[n-1] + " not initialized in the interval [0,1]!"); From af2a9441f04bf36dc75c2c6aa8931e6fe24fa9fc Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:17:39 -0500 Subject: [PATCH 3/7] replace AMREX_PARALLEL_FOR_3D -> amrex::ParallelFor (#1752) --- unit_test/test_aprox_rates/aprox_rates_util.cpp | 6 ++++-- unit_test/test_conductivity/conductivity_util.cpp | 3 ++- unit_test/test_eos/eos_util.cpp | 3 ++- unit_test/test_jac/main.cpp | 3 ++- unit_test/test_rhs/main.cpp | 3 ++- unit_test/test_sdc/main.cpp | 3 ++- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/unit_test/test_aprox_rates/aprox_rates_util.cpp b/unit_test/test_aprox_rates/aprox_rates_util.cpp index 65c0dfe2d8..6f2aef365e 100644 --- a/unit_test/test_aprox_rates/aprox_rates_util.cpp +++ b/unit_test/test_aprox_rates/aprox_rates_util.cpp @@ -24,7 +24,8 @@ void aprox_rates_test(const Box& bx, const int ini56 = network_spec_index("nickel-56"); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { eos_extra_t eos_state; @@ -444,7 +445,8 @@ void aprox_rates_extra_c12ag(const Box& bx, const int ini56 = network_spec_index("nickel-56"); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { eos_extra_t eos_state; diff --git a/unit_test/test_conductivity/conductivity_util.cpp b/unit_test/test_conductivity/conductivity_util.cpp index ca31bd202c..38163433eb 100644 --- a/unit_test/test_conductivity/conductivity_util.cpp +++ b/unit_test/test_conductivity/conductivity_util.cpp @@ -22,7 +22,8 @@ void cond_test_C(const amrex::Box& bx, const int ih1 = network_spec_index("hydrogen-1"); const int ihe4 = network_spec_index("helium-4"); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { // set the composition -- approximately solar diff --git a/unit_test/test_eos/eos_util.cpp b/unit_test/test_eos/eos_util.cpp index d778264446..ce20ab1d7a 100644 --- a/unit_test/test_eos/eos_util.cpp +++ b/unit_test/test_eos/eos_util.cpp @@ -26,7 +26,8 @@ void eos_test_C(const Box& bx, const int ihe4 = network_spec_index("helium-4"); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { // set the composition -- approximately solar diff --git a/unit_test/test_jac/main.cpp b/unit_test/test_jac/main.cpp index 2aa372b6f6..1150000fe3 100644 --- a/unit_test/test_jac/main.cpp +++ b/unit_test/test_jac/main.cpp @@ -172,7 +172,8 @@ void main_main () auto s = state.array(mfi); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { do_jac(i, j, k, s, vars); }); diff --git a/unit_test/test_rhs/main.cpp b/unit_test/test_rhs/main.cpp index e7ee5fcb20..7aba206a96 100644 --- a/unit_test/test_rhs/main.cpp +++ b/unit_test/test_rhs/main.cpp @@ -180,7 +180,8 @@ void main_main () auto s = state.array(mfi); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { do_rhs(i, j, k, s, vars); }); diff --git a/unit_test/test_sdc/main.cpp b/unit_test/test_sdc/main.cpp index 9b75a9ab57..1aabb86f65 100644 --- a/unit_test/test_sdc/main.cpp +++ b/unit_test/test_sdc/main.cpp @@ -197,7 +197,8 @@ void main_main () auto s = state.array(mfi); auto n_rhs = integrator_n_rhs.array(mfi); - AMREX_PARALLEL_FOR_3D(bx, i, j, k, + amrex::ParallelFor(bx, + [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k) { bool success = do_react(vars, i, j, k, s, n_rhs); From 5886c9c6ed9957f6741710c0c3a3232c1ecd4b7b Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:17:59 -0500 Subject: [PATCH 4/7] remove make stuff for sparse jacobian (#1753) this no longer seems to be available --- Make.Microphysics | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Make.Microphysics b/Make.Microphysics index 3cfab8b613..03748d2550 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -72,17 +72,6 @@ ifeq ($(USE_REACT), TRUE) DEFINES += -DREACTIONS endif -ifeq ($(USE_REACT_SPARSE_JACOBIAN), TRUE) - DEFINES += -DREACT_SPARSE_JACOBIAN - - # The following is sometimes useful to turn on for debugging sparse J indices - # (if a get/set/scale is called with (row, col) not in the sparse J, stop) - # Otherwise, set/scale do nothing, and get returns 0. - ifeq ($(USE_SPARSE_STOP_ON_OOB), TRUE) - DEFINES += -DSPARSE_STOP_ON_OOB - endif -endif - ifeq ($(USE_COMPILE_WITH_F2PY), TRUE) DEFINES += -DCOMPILE_WITH_F2PY endif From e9bb41c0e38f436c473222e2e37055a03284abc2 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:20:53 -0500 Subject: [PATCH 5/7] add a doc section about GPUs (#1751) --- Docs/source/design.rst | 37 +++++++++++++++++++++++++++++++++---- Docs/source/unit_tests.rst | 2 ++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Docs/source/design.rst b/Docs/source/design.rst index 018c61f267..cac9148656 100644 --- a/Docs/source/design.rst +++ b/Docs/source/design.rst @@ -41,7 +41,13 @@ and the generic solvers: * ``util/``: linear algebra solvers and other routines. -Design Philosophy + +.. note:: + + All quantities are assumed to be in CGS units, unless otherwise + specified. + +Design philosophy ================= Any application that uses Microphysics will at minimum need to @@ -67,7 +73,30 @@ solvers (ODE integration for the network and Newton-Raphson root finding for the EOS) are separated from the specific implementations of the microphysics. -.. note:: - All quantities are assumed to be in CGS units, unless otherwise - specified. + +GPU considerations +================== + +.. index:: GPUs + +All of the Microphysics routines are written to run on GPUs. This is +enabled in application codes by using the AMReX lambda-capturing +mechanism (see the [AMReX GPU +documentation](https://amrex-codes.github.io/amrex/docs_html/GPU.html) +for more information). + +This means leveraging the AMReX data-structures, macros, and +functions. The unit tests (see :ref:`sec:unit_tests`) provide a good +reference for how to interface the Microphysics solvers and physics +terms with an AMReX-based code. + +There are a few places where Microphysics behaves slightly differently +when running on a CPU vs. a GPU: + +* In the VODE integrator, we disable Jacobian-caching to save memory. + See :ref:`ch:networks:integrators`. + +* In general we disable printing from GPU kernels, due to register + pressure. Some output can be enabled by compiling with + ``USE_GPU_PRINTF=TRUE``. diff --git a/Docs/source/unit_tests.rst b/Docs/source/unit_tests.rst index 6c01f664c7..eb85893131 100644 --- a/Docs/source/unit_tests.rst +++ b/Docs/source/unit_tests.rst @@ -1,3 +1,5 @@ +.. _sec:unit_tests: + ********************** Overview of Unit Tests ********************** From b1a5bf79cfd47edaaa76916f622ed78aeda9e5e6 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:22:15 -0500 Subject: [PATCH 6/7] remove the CRSEGRNDOMP macro (#1755) this was removed from AMReX in 21.10 --- Make.Microphysics | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Make.Microphysics b/Make.Microphysics index 03748d2550..c84ca5cb77 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -59,10 +59,6 @@ ifeq ($(USE_COMPILE_WITH_F2PY), TRUE) sh dof2py.sh endif -# Use a coarse grained OMP approach -DEFINES += -DCRSEGRNDOMP - - # OpenACC support ifeq ($(USE_ACC), TRUE) DEFINES += -DACC From 3e1d2d752c8a8456009cbd80c251461dc98966b6 Mon Sep 17 00:00:00 2001 From: Michael Zingale Date: Thu, 13 Feb 2025 13:47:34 -0500 Subject: [PATCH 7/7] remove some old Fortran stuff from the build system (#1717) Co-authored-by: Eric T. Johnson --- Make.Microphysics | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/Make.Microphysics b/Make.Microphysics index c84ca5cb77..6f6d835e12 100644 --- a/Make.Microphysics +++ b/Make.Microphysics @@ -43,7 +43,7 @@ include $(AMREX_HOME)/Tools/GNUMake/Make.defs MICROPHYSICS_AUTO_SOURCE_DIR := $(TmpBuildDir)/microphysics_sources/$(optionsSuffix).EXE -all: build_status $(executable) starkiller_library +all: build_status $(executable) @echo SUCCESS build_status: @@ -51,14 +51,6 @@ build_status: --git_names "Microphysics AMReX" \ --git_dirs "$(TOP) $(AMREX_HOME)" -starkiller_library: $(executable) -ifeq ($(USE_COMPILE_WITH_F2PY), TRUE) - @echo Wrapping sources with f90wrap ... - sh dowrap.sh - @echo Linking objects with f2py ... - sh dof2py.sh -endif - # OpenACC support ifeq ($(USE_ACC), TRUE) DEFINES += -DACC @@ -68,10 +60,6 @@ ifeq ($(USE_REACT), TRUE) DEFINES += -DREACTIONS endif -ifeq ($(USE_COMPILE_WITH_F2PY), TRUE) - DEFINES += -DCOMPILE_WITH_F2PY -endif - #------------------------------------------------------------------------------ # AMReX @@ -198,20 +186,9 @@ $(objEXETempDir)/AMReX_buildInfo.o: .FORCE include $(AMREX_HOME)/Tools/GNUMake/Make.rules clean:: - $(SILENT) $(RM) extern.F90 - $(SILENT) $(RM) network_properties.F90 network_properties.H - $(SILENT) $(RM) $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.H $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.[fF]90 - $(SILENT) $(RM) extern_parameters.cpp extern_parameters_F.H extern_parameters.H - -# Older versions of CASTRO generated a "network.f90" file when using -# the general_null network, which conflicts with our generalized -# network.f90 file. So we should delete the local copy if we are -# trying to rebuild from an old directory. -# The same is true for "extern.f90". - -clean:: - $(RM) network.f90 - $(RM) extern.f90 + $(SILENT) $(RM) network_properties.H + $(SILENT) $(RM) $(MICROPHYSICS_AUTO_SOURCE_DIR)/*.H + $(SILENT) $(RM) extern_parameters.cpp extern_parameters.H # for debugging. To see the value of a Makefile variable,