diff --git a/.github/workflows/c-linter.yml b/.github/workflows/c-linter.yml index 295ac4ede4..435a7fa60a 100644 --- a/.github/workflows/c-linter.yml +++ b/.github/workflows/c-linter.yml @@ -6,7 +6,7 @@ jobs: cpp-linter: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -56,7 +56,7 @@ jobs: -run-linter - name: Archive clang tidy report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: clang-tidy-report path: clang-tidy-report.txt diff --git a/.github/workflows/check-ifdefs.yml b/.github/workflows/check-ifdefs.yml index 5558864238..3f4ab8daae 100644 --- a/.github/workflows/check-ifdefs.yml +++ b/.github/workflows/check-ifdefs.yml @@ -19,18 +19,9 @@ jobs: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' - - - name: Cache pip - uses: actions/cache@v3 - with: - # this path is specific to Ubuntu - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' - name: Run check-ifdefs run: | diff --git a/.github/workflows/check-makefiles.yml b/.github/workflows/check-makefiles.yml index 396393f383..33a21d604b 100644 --- a/.github/workflows/check-makefiles.yml +++ b/.github/workflows/check-makefiles.yml @@ -19,18 +19,9 @@ jobs: fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' - - - name: Cache pip - uses: actions/cache@v3 - with: - # this path is specific to Ubuntu - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' - name: Run check-ifdefs run: | diff --git a/.github/workflows/check-params.yml b/.github/workflows/check-params.yml index 324a0eb05f..992e6b9be8 100644 --- a/.github/workflows/check-params.yml +++ b/.github/workflows/check-params.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -28,19 +28,10 @@ jobs: cd ../.. - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Cache pip - uses: actions/cache@v3 - with: - # this path is specific to Ubuntu - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Run check-params run: | PYTHONPATH=external/Microphysics/util/build_scripts python .github/workflows/check_params.py . diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index f5191f1bd7..6964398e22 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -22,15 +22,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: '3.10' - - - name: Cache pip - uses: actions/cache@v3 - with: - # this path is specific to Ubuntu - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: "pip" - name: Install dependencies run: pip install -r ./requirements.txt diff --git a/.github/workflows/compiler-warnings.yml b/.github/workflows/compiler-warnings.yml index 1f95139323..0f6bd12fb1 100644 --- a/.github/workflows/compiler-warnings.yml +++ b/.github/workflows/compiler-warnings.yml @@ -5,7 +5,7 @@ jobs: compiler_warnings: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 3d021ab174..bd9d8795a8 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get the version id: get_version @@ -33,4 +33,4 @@ jobs: release_name: Release ${{ github.ref }} body: ${{ env.RELEASE_TXT }} draft: false - prerelease: false \ No newline at end of file + prerelease: false diff --git a/.github/workflows/detonation-sdc-compare.yml b/.github/workflows/detonation-sdc-compare.yml index b50dd3f91c..be850b35cf 100644 --- a/.github/workflows/detonation-sdc-compare.yml +++ b/.github/workflows/detonation-sdc-compare.yml @@ -5,7 +5,7 @@ jobs: detonation-sdc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/docs-test.yml b/.github/workflows/docs-test.yml index 054ca5a8f5..be80c197fc 100644 --- a/.github/workflows/docs-test.yml +++ b/.github/workflows/docs-test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install pandoc and doxygen @@ -22,18 +22,10 @@ jobs: sudo apt install pandoc doxygen - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' - - - name: Cache pip - uses: actions/cache@v3 - with: - # this path is specific to Ubuntu - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- + python-version: '3.11' + cache: "pip" - name: Install dependencies run: pip install -r ./requirements.txt diff --git a/.github/workflows/exact_riemann.yml b/.github/workflows/exact_riemann.yml index d369a87e78..996684ba78 100644 --- a/.github/workflows/exact_riemann.yml +++ b/.github/workflows/exact_riemann.yml @@ -5,7 +5,7 @@ jobs: exact_riemann: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/flame_wave-compare.yml b/.github/workflows/flame_wave-compare.yml index efb4abccc0..46ad5b4285 100644 --- a/.github/workflows/flame_wave-compare.yml +++ b/.github/workflows/flame_wave-compare.yml @@ -5,7 +5,7 @@ jobs: flame_wave-2d: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index d687274828..d5b3995fa3 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install pandoc and doxygen run: | @@ -18,27 +18,16 @@ jobs: sudo apt install pandoc doxygen - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' + cache: "pip" - name: Upgrade pip run: | # install pip=>20.1 to use "pip cache dir" python3 -m pip install --upgrade pip - - name: Get pip cache dir - id: pip-cache - run: echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache dependencies - uses: actions/cache@v3 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install dependencies run: python3 -m pip install -r ./requirements.txt @@ -48,7 +37,7 @@ jobs: GITHUB_BRANCH: 'main' run: ./deploy_docs_action.sh - - name: Build docs + - name: Build docs if: ${{ endsWith(github.ref, 'development') }} env: GITHUB_BRANCH: 'development' @@ -59,4 +48,4 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./out - keep_files: true \ No newline at end of file + keep_files: true diff --git a/.github/workflows/gpu_action.yml b/.github/workflows/gpu_action.yml index 10843ef256..6861161cbb 100644 --- a/.github/workflows/gpu_action.yml +++ b/.github/workflows/gpu_action.yml @@ -5,7 +5,7 @@ jobs: gpu-compilation: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index d00b3c9954..f74b1e7c7e 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -10,7 +10,7 @@ jobs: hip-compile: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/mhd-compare.yml b/.github/workflows/mhd-compare.yml index 8df15f2574..403ff79f5e 100644 --- a/.github/workflows/mhd-compare.yml +++ b/.github/workflows/mhd-compare.yml @@ -5,7 +5,7 @@ jobs: OrszagTang-3d: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/rad-compare.yml b/.github/workflows/rad-compare.yml index b123c972c8..775024000d 100644 --- a/.github/workflows/rad-compare.yml +++ b/.github/workflows/rad-compare.yml @@ -5,7 +5,7 @@ jobs: Rad2Tshock-1d: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/reacting-convergence-true-sdc.yml b/.github/workflows/reacting-convergence-true-sdc.yml index cc8dd1d471..bd810420f8 100644 --- a/.github/workflows/reacting-convergence-true-sdc.yml +++ b/.github/workflows/reacting-convergence-true-sdc.yml @@ -5,7 +5,7 @@ jobs: reacting-convergence-true-sdc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/sedov-compare.yml b/.github/workflows/sedov-compare.yml index 06cfca49f4..dd484b8525 100644 --- a/.github/workflows/sedov-compare.yml +++ b/.github/workflows/sedov-compare.yml @@ -5,7 +5,7 @@ jobs: Sedov-3d: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/uniform_cube.yml b/.github/workflows/uniform_cube.yml index 33d7e8b5df..edff5a8494 100644 --- a/.github/workflows/uniform_cube.yml +++ b/.github/workflows/uniform_cube.yml @@ -5,7 +5,7 @@ jobs: uniform_cube: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/uniform_sphere.yml b/.github/workflows/uniform_sphere.yml index 1dbd01f90b..e9ad3b9e4e 100644 --- a/.github/workflows/uniform_sphere.yml +++ b/.github/workflows/uniform_sphere.yml @@ -5,7 +5,7 @@ jobs: uniform_sphere: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/wdmerger_collision-compare.yml b/.github/workflows/wdmerger_collision-compare.yml index cd1e9c8b5f..99551f09dd 100644 --- a/.github/workflows/wdmerger_collision-compare.yml +++ b/.github/workflows/wdmerger_collision-compare.yml @@ -5,7 +5,7 @@ jobs: wdmerger_collision-2d: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/CHANGES.md b/CHANGES.md index 751300868c..0bb87dc598 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +# 24.01 + + * An option for unlimited PPM reconstruction was added (#2670) + + * An option allowing for optional passive sources to the conserved + state was added (#2678) + + * A script `diag_parser.py` was added to allow for easy parsing of + the global diagnostic files output at runtime (#2666, #2667) + # 23.12 * The radiation solver port to C++ has been completed (#2638, #2648) diff --git a/Docs/preprocess_files.py b/Docs/preprocess_files.py index 9467e9c946..d28b983fb2 100644 --- a/Docs/preprocess_files.py +++ b/Docs/preprocess_files.py @@ -1,5 +1,5 @@ """ -This script strips out preprocessor directives from C++ headers and Fortran files +This script strips out preprocessor directives from C++ headers and saves the results in source/preprocessed_files """ @@ -17,8 +17,7 @@ def strip_directives(filename, filepath, outpath): Read in file, remove all preprocessor directives and output. This is also going to switch square brackets initializing arrays to - parentheses and remove the new-line characters in these so sphinx - fortran is happy. + parentheses and remove the new-line characters """ with open(os.path.join(filepath, filename)) as infile: diff --git a/Docs/source/build_system.rst b/Docs/source/build_system.rst index ba27ac25b3..928f383871 100644 --- a/Docs/source/build_system.rst +++ b/Docs/source/build_system.rst @@ -42,17 +42,6 @@ Most of these are parameters from AMReX. ``HYPRE_DIR`` or ``HYPRE_OMP_DIR``. -Fortran Support -^^^^^^^^^^^^^^^ - -Radiation currently needs Fortran support. All of the other solvers -and problem set ups do not require Fortran. Fortran support in AMReX -is enabled / disabled via: - - * ``BL_NO_FORT``: if set to ``TRUE``, then no AMReX Fortran source will be built. - This cannot currently be used for the radiation solver. - - Parallelization and GPUs ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -273,15 +262,6 @@ This is the current build system process. * These headers are output into ``tmp_build_dir/castro_sources/``. -* (if Fortran support is enabled) The Fortran dependencies file is created - - * This creates the ``f90.depends`` file in the ``tmp_build_dir`` - - * The script ``amrex/Tools/F_scripts/dep.py`` is used - - * The hook for this is in ``amrex/Tools/GNUMake/Make.rules`` in the - ``$(depEXETempDir)/f90.depends`` target - * The C/C++ dependencies file is created * This creates the individual ``.d`` files in ``tmp_build_dir``, one for each source file diff --git a/Docs/source/faq.rst b/Docs/source/faq.rst index fb144d2a17..4e3d03daf3 100644 --- a/Docs/source/faq.rst +++ b/Docs/source/faq.rst @@ -54,12 +54,12 @@ Debugging I get more information?* The best thing to do is to recompile the code with ``TEST=TRUE`` - set in the ``GNUmakefile``. This will have AMReX catch the - signals raised in both C++ and Fortran functions. Behind the + set in the ``GNUmakefile``. This will have AMReX catch the + signals raised in C++ functions. Behind the scenes, this defines the ``AMREX_TESTING`` preprocessor flag, which will initialize memory allocated in fabs or multifabs to signaling NaNs (sNaN), and use the ``BLBackTrace::handler()`` - function to handle various signals raised in both C++ and Fortran + function to handle various signals raised in C++ functions. This is a Linux/UNIX capability. This gives us a chance to print out backtrace information. The signals include seg fault, floating point exceptions (NaNs, divided by zero and overflow), and diff --git a/Docs/source/rp_intro.rst b/Docs/source/rp_intro.rst index 59683e5853..09b4377d42 100644 --- a/Docs/source/rp_intro.rst +++ b/Docs/source/rp_intro.rst @@ -28,7 +28,7 @@ The Castro parameters that control the behavior of the code and physics modules are listed in ``_cpp_parameters`` and take the form of:: # comment describing the parameter - name type default need in Fortran? ifdef + name type default ifdef Here, @@ -45,13 +45,7 @@ Here, * `default` is the default value of the parameter. -The next columns are optional, but you need to fill in all of the -information up to and including any of the optional columns you need -(e.g., if you are going to provide "need in Fortran?" and "ifdef"). - - * `need in Fortran?` is ``y`` if the runtime parameter should be - made available in Fortran (through ``meth_params_module``). - Note: this option is deprecated. +The next column is optional: * `ifdef` provides the name of a preprocessor name that should wrap this parameter definition—it will only be compiled in if that diff --git a/Exec/Make.Castro b/Exec/Make.Castro index 79e747b5c8..3a8ffa20c8 100644 --- a/Exec/Make.Castro +++ b/Exec/Make.Castro @@ -35,13 +35,7 @@ EOS_HOME ?= $(MICROPHYSICS_HOME)/EOS NETWORK_HOME ?= $(MICROPHYSICS_HOME)/networks CONDUCTIVITY_HOME ?= $(MICROPHYSICS_HOME)/conductivity -# only radiation builds need Fortran -ifeq ($(USE_RAD), TRUE) - BL_NO_FORT := FALSE -else - BL_NO_FORT := TRUE -endif - +BL_NO_FORT := TRUE # AMReX is a git submodule of Castro. By default # we assume it is in the external/ directory. diff --git a/Exec/reacting_tests/nse_test/convergence_simplified_sdc_w_vel.sh b/Exec/reacting_tests/nse_test/convergence_simplified_sdc_w_vel.sh index 9d118c0718..0748f46b5b 100755 --- a/Exec/reacting_tests/nse_test/convergence_simplified_sdc_w_vel.sh +++ b/Exec/reacting_tests/nse_test/convergence_simplified_sdc_w_vel.sh @@ -10,14 +10,18 @@ problem.u0=1.e8 problem.v0=1.e8 " +mpiexec -n 8 ${EXEC} inputs.32 ${RUNPARAMS} >& /dev/null mpiexec -n 8 ${EXEC} inputs.64 ${RUNPARAMS} >& /dev/null mpiexec -n 8 ${EXEC} inputs.128 ${RUNPARAMS} >& /dev/null + +RichardsonConvergenceTest2d.gnu.ex coarFile=nse_test_32_plt00080 mediFile=nse_test_64_plt00160 fineFile=nse_test_128_plt00320 >& nse_convergence_simple_sdc_vlo.out + mpiexec -n 8 ${EXEC} inputs.256 ${RUNPARAMS} >& /dev/null -RichardsonConvergenceTest2d.gnu.ex coarFile=nse_test_64_plt00125 mediFile=nse_test_128_plt00250 fineFile=nse_test_256_plt00500 >& nse_convergence_simple_sdc_lo.out +RichardsonConvergenceTest2d.gnu.ex coarFile=nse_test_64_plt00160 mediFile=nse_test_128_plt00320 fineFile=nse_test_256_plt00640 >& nse_convergence_simple_sdc_lo.out mpiexec -n 8 ${EXEC} inputs.512 ${RUNPARAMS} >& /dev/null -RichardsonConvergenceTest2d.gnu.ex coarFile=nse_test_128_plt00250 mediFile=nse_test_256_plt00500 fineFile=nse_test_512_plt01000 >& nse_convergence_simple_sdc_hi.out +RichardsonConvergenceTest2d.gnu.ex coarFile=nse_test_128_plt00320 mediFile=nse_test_256_plt00640 fineFile=nse_test_512_plt01280 >& nse_convergence_simple_sdc_hi.out diff --git a/Exec/reacting_tests/nse_test/create_pretty_tables.py b/Exec/reacting_tests/nse_test/create_pretty_tables.py index ca8a3991f1..8c8d05ca3a 100644 --- a/Exec/reacting_tests/nse_test/create_pretty_tables.py +++ b/Exec/reacting_tests/nse_test/create_pretty_tables.py @@ -14,13 +14,21 @@ def sci_not(num): return r"${:5.3f} \times 10^{{{}}}$".format(round(mant, 3), exp) class Variable(): - def __init__(self, name, lo, o1, med, o2, hi): + def __init__(self, name, lo, o1, med, o2, hi, o3=None, vhi=None): self.name = name self.lo = float(lo) self.o1 = float(o1) self.med = float(med) self.o2 = float(o2) self.hi = float(hi) + if o3 is not None: + self.o3 = float(o3) + else: + self.o3 = None + if vhi is not None: + self.vhi = float(vhi) + else: + self.vhi = None def get_table_line(self, pretty_name=None, simple=False): if pretty_name is not None: @@ -29,69 +37,103 @@ def get_table_line(self, pretty_name=None, simple=False): name = self.name if simple: - _str = r" {:27} {:14.10g} {:5.3f} {:14.10g} {:5.3f} {:14.10g}" - return _str.format(name, self.lo, round(self.o1, 3), self.med, round(self.o2, 3), self.hi) + if self.o3 is None: + return rf" {name:27} {self.lo:14.10g} {round(self.o1, 3):5.3f} {self.med:14.10g} {round(self.o2, 3):5.3f} {self.hi:14.10g}" + else: + return rf" {name:27} {self.lo:14.10g} {round(self.o1, 3):5.3f} {self.med:14.10g} {round(self.o2, 3):5.3f} {self.hi:14.10g} {round(self.o3, 3):5.3f} {self.vhi:14.10g}" else: - _str = r" {:27} & {:23} & {:5.3f} & {:23} & {:5.3f} & {:23} \\" - return _str.format(name, sci_not(self.lo), round(self.o1, 3), sci_not(self.med), round(self.o2, 3), sci_not(self.hi)) + if self.o3 is None: + return rf" {name:27} & {sci_not(self.lo):23} & {round(self.o1, 3):5.3f} & {sci_not(self.med):23} & {round(self.o2, 3):5.3f} & {sci_not(self.hi):23} \\" + else: + return rf" {name:27} & {sci_not(self.lo):23} & {round(self.o1, 3):5.3f} & {sci_not(self.med):23} & {round(self.o2, 3):5.3f} & {sci_not(self.hi):23} & {round(self.o3, 3):5.3f} & {sci_not(self.vhi):23} \\" -class ConvergenceData(): +class ConvergenceData2(): def __init__(self): self.data = [] def add_variable(self, name, lo, order1, med, order2, hi): self.data.append(Variable(name, lo, order1, med, order2, hi)) -def read_convergence(file_lo, file_hi): +class ConvergenceData3(): + def __init__(self): + self.data = [] + + def add_variable(self, name, lo, order1, med, order2, hi, order3, vhi): + self.data.append(Variable(name, lo, order1, med, order2, hi, order3, vhi)) + +def read_convergence(file_lo, file_hi, file_vhi): # we'll wait until we find the L1 data lines_lo = [] - found_l1 = False - with open(file_lo, "r") as flo: - for line in flo: - if "L1 norm" in line: - found_l1 = True - continue - if not found_l1: - continue - # value data lines have 4 columns - if len(line.split()) == 4: - lines_lo.append(line.strip()) - lines_hi = [] - found_l1 = False - with open(file_hi, "r") as fhi: - for line in fhi: - if "L1 norm" in line: - found_l1 = True - continue - if not found_l1: + lines_vhi = [] + + fdata = [(lines_lo, file_lo), (lines_hi, file_hi)] + if file_vhi is not None: + fdata.append((lines_vhi, file_vhi)) + + for lines, filec in fdata: + found_l1 = False + with open(filec, "r") as fc: + for line in fc: + if "L1 norm" in line: + found_l1 = True + continue + if not found_l1: + continue + # value data lines have 4 columns + if len(line.split()) == 4: + lines.append(line.strip()) + + if file_vhi is None: + + cd = ConvergenceData2() + + for llo, lhi in zip(lines_lo, lines_hi): + + vlo, elo, o1, emed1 = llo.split() + vhi, emed2, o2, ehi = lhi.split() + + if "---" in o1 or "---" in o2: + print("skipping {}".format(vlo)) continue - # value data lines have 4 columns - if len(line.split()) == 4: - lines_hi.append(line.strip()) - cd = ConvergenceData() + if vlo != vhi: + sys.exit("error: variable mismatch") + + if emed1.strip() != emed2.strip(): + print(emed1, emed2) + sys.exit("error: error mismatch") + + cd.add_variable(vlo, elo, o1, emed1, o2, ehi) - for llo, lhi in zip(lines_lo, lines_hi): + else: - vlo, elo, o1, emed1 = llo.split() - vhi, emed2, o2, ehi = lhi.split() + cd = ConvergenceData3() - if "---" in o1 or "---" in o2: - print("skipping {}".format(vlo)) - continue + for llo, lhi, lvhi in zip(lines_lo, lines_hi, lines_vhi): + + vlo, elo, o1, emed1 = llo.split() + vhi, emed2, o2, ehi1 = lhi.split() + vvhi, ehi2, o3, evhi = lvhi.split() + + if "---" in o1 or "---" in o2 or "---" in o3: + print("skipping {}".format(vlo)) + continue - if vlo != vhi: - sys.exit("error: variable mismatch") + if vlo != vhi or vlo != vvhi: + sys.exit("error: variable mismatch") - if emed1.strip() != emed2.strip(): - print(emed1, emed2) - sys.exit("error: error mismatch") + if emed1.strip() != emed2.strip() or ehi1.strip() != ehi2.strip(): + print(emed1, emed2, ehi1, ehi2) + print(llo) + print(lhi) + print(lvhi) + sys.exit("error: error mismatch") - cd.add_variable(vlo, elo, o1, emed1, o2, ehi) + cd.add_variable(vlo, elo, o1, emed1, o2, ehi1, o3, evhi) return cd @@ -103,6 +145,8 @@ def read_convergence(file_lo, file_hi): help="name of the low resolution convergence output file") parser.add_argument("hifile", type=str, nargs=1, help="name of the high resolution convergence output file") + parser.add_argument("veryhifile", type=str, nargs="?", default=None, + help="(optional) name of the very high resolution convergence output file") args = parser.parse_args() @@ -129,8 +173,10 @@ def read_convergence(file_lo, file_hi): # sdc4 file_lo = args.lofile[0] file_hi = args.hifile[0] + file_vhi = args.veryhifile + print(file_vhi) - sdc4 = read_convergence(file_lo, file_hi) + sdc4 = read_convergence(file_lo, file_hi, file_vhi) for v in sdc4.data: if v.name in good_vars.keys(): diff --git a/Exec/reacting_tests/nse_test/inputs.128 b/Exec/reacting_tests/nse_test/inputs.128 index 47789635dc..3ce059a1a9 100644 --- a/Exec/reacting_tests/nse_test/inputs.128 +++ b/Exec/reacting_tests/nse_test/inputs.128 @@ -1,6 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- max_step = 15000 -stop_time = 0.025 +stop_time = 0.032 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 1 1 1 diff --git a/Exec/reacting_tests/nse_test/inputs.256 b/Exec/reacting_tests/nse_test/inputs.256 index 8e7234ba22..766365063f 100644 --- a/Exec/reacting_tests/nse_test/inputs.256 +++ b/Exec/reacting_tests/nse_test/inputs.256 @@ -1,6 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- max_step = 15000 -stop_time = 0.025 +stop_time = 0.032 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 1 1 1 diff --git a/Exec/reacting_tests/nse_test/inputs.32 b/Exec/reacting_tests/nse_test/inputs.32 new file mode 100644 index 0000000000..6b807f578d --- /dev/null +++ b/Exec/reacting_tests/nse_test/inputs.32 @@ -0,0 +1,74 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 15000 +stop_time = 0.032 + +# PROBLEM SIZE & GEOMETRY +geometry.is_periodic = 1 1 1 +geometry.coord_sys = 0 # 0 => cart, 1 => RZ 2=>spherical +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 2.e7 2.e7 2.e7 +amr.n_cell = 32 32 32 + + +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +# 0 = Interior 3 = Symmetry +# 1 = Inflow 4 = SlipWall +# 2 = Outflow 5 = NoSlipWall +# >>>>>>>>>>>>> BC FLAGS <<<<<<<<<<<<<<<< +castro.lo_bc = 0 0 0 +castro.hi_bc = 0 0 0 + +# WHICH PHYSICS +castro.do_hydro = 1 +castro.do_react = 1 + +castro.ppm_type = 1 +castro.ppm_temp_fix = 0 + +castro.use_flattening = 1 + +castro.riemann_solver = 0 + +castro.small_temp = 1.e7 + +# TIME STEP CONTROL +castro.cfl = 0.8 # cfl number for hyperbolic system +castro.init_shrink = 1.0 # scale back initial timestep +castro.change_max = 1.1 # scale back initial timestep +castro.fixed_dt = 4.e-4 + +# DIAGNOSTICS & VERBOSITY +castro.sum_interval = 1 # timesteps between computing mass +castro.v = 1 # verbosity in Castro.cpp +amr.v = 1 # verbosity in Amr.cpp +#amr.grid_log = grdlog # name of grid logging file + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed +amr.ref_ratio = 2 2 2 2 # refinement ratio +amr.regrid_int = 2 2 2 2 # how often to regrid +amr.blocking_factor = 8 # block factor in grid generation +amr.max_grid_size = 64 +amr.n_error_buf = 2 2 2 2 # number of buffer cells in error est + +# CHECKPOINT FILES +amr.checkpoint_files_output = 0 +amr.check_file = nse_test_32_chk # root name of checkpoint file +amr.check_int = 300 # number of timesteps between checkpoints + +# PLOTFILES +amr.plot_file = nse_test_32_plt # root name of plotfile +amr.plot_per = 0.24 +amr.derive_plot_vars = ALL + +# problem initialization + +problem.T0 = 5.e9 +problem.dT_fact = 0.2 +problem.rho0 = 1.e9 +problem.L_pert = 2.e7 +problem.nse_tol = 1.e2 + +# microphysics + +network.nse_table_interp_linear = 0 diff --git a/Exec/reacting_tests/nse_test/inputs.512 b/Exec/reacting_tests/nse_test/inputs.512 index 6426052e86..d1437273fe 100644 --- a/Exec/reacting_tests/nse_test/inputs.512 +++ b/Exec/reacting_tests/nse_test/inputs.512 @@ -1,6 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- max_step = 15000 -stop_time = 0.025 +stop_time = 0.032 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 1 1 1 diff --git a/Exec/reacting_tests/nse_test/inputs.64 b/Exec/reacting_tests/nse_test/inputs.64 index d066be3e1e..5e2867b41a 100644 --- a/Exec/reacting_tests/nse_test/inputs.64 +++ b/Exec/reacting_tests/nse_test/inputs.64 @@ -1,6 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- max_step = 15000 -stop_time = 0.025 +stop_time = 0.032 # PROBLEM SIZE & GEOMETRY geometry.is_periodic = 1 1 1 diff --git a/Exec/science/massive_star/README.md b/Exec/science/massive_star/README.md new file mode 100644 index 0000000000..161dee60ab --- /dev/null +++ b/Exec/science/massive_star/README.md @@ -0,0 +1,15 @@ +# massive star + +This setup models the convective burning shells in a massive star +leading up to core collapse. Some details: + +* The initial model is from MESA and is put onto a uniform grid using + the routines in the AMReX Astrophysics initial model repo + (https://github.com/amrex-astro/initial_models) in the + ``massive_star`` directory. + +* We use simplified-SDC together with aprox19 and an NSE table + (generated from pynucastro) + +* The Castro ``drive_initial_convection`` functionality is used to + establish the initial convective velocity field. diff --git a/Exec/science/massive_star/problem_initialize_state_data.H b/Exec/science/massive_star/problem_initialize_state_data.H index 0333d7da82..68fd2acfd8 100644 --- a/Exec/science/massive_star/problem_initialize_state_data.H +++ b/Exec/science/massive_star/problem_initialize_state_data.H @@ -7,6 +7,7 @@ #ifdef NSE_TABLE #include #include +#include #endif AMREX_GPU_HOST_DEVICE AMREX_INLINE @@ -128,8 +129,33 @@ void problem_initialize_state_data (int i, int j, int k, } if (problem::interpolate_pres == 1) { - eos(eos_input_rp, eos_state); - state(i,j,k,UTEMP) = eos_state.T; + + // we need to get T from P, rho, but consistent with NSE (if + // we are in NSE) + if (nse_check) { + // this will also change the composition, since the new T + // alters the NSE + nse_T_abar_from_p(eos_state.rho, eos_state.p, eos_state.aux[AuxZero::iye], + eos_state.T, eos_state.aux[AuxZero::iabar]); + state(i,j,k,UTEMP) = eos_state.T; + state(i,j,k,UFX+AuxZero::iabar) = eos_state.aux[AuxZero::iabar]; + + // now call the EOS with the new T and abar to get e + eos(eos_input_rt, eos_state); + + // finally, get the updated B/A + nse_table_t nse_state; + nse_state.T = state(i,j,k,UTEMP); + nse_state.rho = state(i,j,k,URHO); + nse_state.Ye = state(i,j,k,UFX+AuxZero::iye); + nse_interp(nse_state); + + state(i,j,k,UFX+AuxZero::ibea) = nse_state.bea; + + } else { + eos(eos_input_rp, eos_state); + state(i,j,k,UTEMP) = eos_state.T; + } } else { eos(eos_input_rt, eos_state); } diff --git a/Source/driver/Castro.H b/Source/driver/Castro.H index 755a540984..9cc45996c3 100644 --- a/Source/driver/Castro.H +++ b/Source/driver/Castro.H @@ -48,6 +48,8 @@ constexpr int PSTAR_BISECT_FACTOR = 5; #include #include +#include + using std::istream; using std::ostream; @@ -535,7 +537,6 @@ public: #include #endif - /// /// Estimate time step. /// @@ -1198,6 +1199,12 @@ public: static Vector > data_logs; static Vector > problem_data_logs; +/// +/// runtime parameters +// + static params_t params; + + protected: diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp index 9d0855d1d5..7d114996e8 100644 --- a/Source/driver/Castro.cpp +++ b/Source/driver/Castro.cpp @@ -76,6 +76,8 @@ int Castro::NUM_GROW_SRC = -1; int Castro::lastDtPlotLimited = 0; Real Castro::lastDtBeforePlotLimiting = 0.0; +params_t Castro::params; + Real Castro::num_zones_advanced = 0.0; Vector Castro::source_names; @@ -322,13 +324,6 @@ Castro::read_params () } #endif -#ifdef REACTIONS -#ifdef SIMPLIFIED_SDC - if (jacobian == 1) { - amrex::Abort("Simplified SDC requires the numerical Jacobian now (jacobian = 2)"); - } -#endif -#endif // sanity checks if (grown_factor < 1) { diff --git a/Source/driver/parse_castro_params.py b/Source/driver/parse_castro_params.py index cfa2ebb30c..aedeea6624 100755 --- a/Source/driver/parse_castro_params.py +++ b/Source/driver/parse_castro_params.py @@ -7,7 +7,7 @@ parameters have the format: - name type default need-in-fortran? ifdef + name type default ifdef the first three (name, type, default) are mandatory: @@ -22,8 +22,6 @@ the next are optional: - need-in-fortran: no longer used - ifdef: only define this parameter if the name provided is #ifdef-ed Any line beginning with a "#" is ignored @@ -131,7 +129,7 @@ def read_param_file(infile): return params -def write_headers(params, out_directory): +def write_headers(params, out_directory, struct_name): # output @@ -196,7 +194,7 @@ def write_headers(params, out_directory): cp.write("#endif\n") cp.close() - # write castro_queries.H + # write name_queries.H try: cq = open(f"{out_directory}/{nm}_queries.H", "w", encoding="UTF-8") except OSError: @@ -208,13 +206,15 @@ def write_headers(params, out_directory): if ifdef is None: for p in [q for q in params_nm if q.ifdef is None]: cq.write(p.get_default_string()) - cq.write(p.get_query_string("C++")) + cq.write(p.get_query_string()) + cq.write(p.get_query_struct_string(struct_name=struct_name, class_name="Castro")) cq.write("\n") else: cq.write(f"#ifdef {ifdef}\n") for p in [q for q in params_nm if q.ifdef == ifdef]: cq.write(p.get_default_string()) - cq.write(p.get_query_string("C++")) + cq.write(p.get_query_string()) + cq.write(p.get_query_struct_string(struct_name=struct_name, class_name="Castro")) cq.write("\n") cq.write("#endif\n") cq.write("\n") @@ -238,19 +238,64 @@ def write_headers(params, out_directory): jo.close() + # now write a single file that contains all of the parameter structs + try: + sf = open(f"{out_directory}/{struct_name}_type.H", "w", encoding="UTF-8") + except OSError: + sys.exit(f"unable to open {struct_name}_type.H for writing") + + sf.write(CWARNING) + sf.write(f"#ifndef {struct_name.upper()}_TYPE_H\n") + sf.write(f"#define {struct_name.upper()}_TYPE_H\n\n") + + sf.write("#include \n\n") + + for nm in namespaces: + + params_nm = [q for q in params if q.namespace == nm] + # sort by repr since None may be present + ifdefs = sorted({q.ifdef for q in params_nm}, key=repr) + + sf.write(f"struct {nm}_t {{\n") + print("namespace = ", nm) + for ifdef in ifdefs: + if ifdef is None: + for p in [q for q in params_nm if q.ifdef is None]: + sf.write(p.get_struct_entry()) + else: + sf.write(f"#ifdef {ifdef}\n") + for p in [q for q in params_nm if q.ifdef == ifdef]: + sf.write(p.get_struct_entry()) + sf.write("#endif\n") + + + sf.write("};\n\n") + + # now the parent struct + + sf.write(f"struct {struct_name}_t {{\n") + for nm in namespaces: + sf.write(f" {nm}_t {nm};\n") + sf.write("};\n\n") + + sf.write("#endif\n") + sf.close() + def main(): """the main driver""" parser = argparse.ArgumentParser() parser.add_argument("-o", type=str, default=None, help="output directory for the generated files") + parser.add_argument("-s", type=str, default="params", + help="name for the name struct that will hold the parameters") parser.add_argument("input_file", type=str, nargs=1, help="input file containing the list of parameters we will define") args = parser.parse_args() p = read_param_file(args.input_file[0]) - write_headers(p, args.o) + write_headers(p, args.o, args.s) if __name__ == "__main__": main() diff --git a/Source/hydro/Castro_mol_hydro.cpp b/Source/hydro/Castro_mol_hydro.cpp index 831260dbc2..05c3687eb9 100644 --- a/Source/hydro/Castro_mol_hydro.cpp +++ b/Source/hydro/Castro_mol_hydro.cpp @@ -57,28 +57,32 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) #endif { - // Declare local storage now. This should be done outside the MFIter loop, - // and then we will resize the Fabs in each MFIter loop iteration. Then, - // we apply an Elixir to ensure that their memory is saved until it is no - // longer needed (only relevant for the asynchronous case, usually on GPUs). - - FArrayBox flatn; - FArrayBox cond; - FArrayBox dq; - FArrayBox src_q; - FArrayBox shk; - FArrayBox qm, qp; - FArrayBox div; - FArrayBox q_int; - FArrayBox q_avg; - FArrayBox q_fc; - FArrayBox f_avg; - FArrayBox flux[AMREX_SPACEDIM]; - FArrayBox qe[AMREX_SPACEDIM]; + // Declare local storage now. This should be done outside the + // MFIter loop, and then we will resize the Fabs in each MFIter + // loop iteration. We use the async arenato ensure that their + // memory is saved until it is no longer needed (only relevant for + // the asynchronous case, usually on GPUs). + + FArrayBox flatn(The_Async_Arena()); + FArrayBox cond(The_Async_Arena()); + FArrayBox dq(The_Async_Arena()); + FArrayBox src_q(The_Async_Arena()); + FArrayBox shk(The_Async_Arena()); + FArrayBox qm(The_Async_Arena()), qp(The_Async_Arena()); + FArrayBox div(The_Async_Arena()); + FArrayBox q_int(The_Async_Arena()); + FArrayBox q_avg(The_Async_Arena()); + FArrayBox q_fc(The_Async_Arena()); + FArrayBox f_avg(The_Async_Arena()); + Vector flux, qe; + for (int n = 0; n < AMREX_SPACEDIM; ++n) { + flux.push_back(FArrayBox(The_Async_Arena())); + qe.push_back(FArrayBox(The_Async_Arena())); + } #if AMREX_SPACEDIM <= 2 - FArrayBox pradial; + FArrayBox pradial(The_Async_Arena()); #endif - FArrayBox avis; + FArrayBox avis(The_Async_Arena()); MultiFab& old_source = get_old_data(Source_Type); @@ -90,17 +94,12 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& obx = amrex::grow(bx, 1); const Box& obx2 = amrex::grow(bx, 2); - FArrayBox &statein = Sborder[mfi]; - Array4 const uin_arr = statein.array(); - - FArrayBox &stateout = S_new[mfi]; + Array4 const uin_arr = Sborder.array(mfi); - FArrayBox &source_in = old_source[mfi]; - auto source_in_arr = source_in.array(); + auto source_in_arr = old_source.array(mfi); // the output of this will be stored in the correct stage MF - FArrayBox &source_out = A_update[mfi]; - auto source_out_arr = source_out.array(); + auto source_out_arr = A_update.array(mfi); Real stage_weight = 1.0; @@ -110,7 +109,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get the flattening coefficient flatn.resize(obx, 1); - Elixir elix_flatn = flatn.elixir(); Array4 const q_arr = q.array(mfi); Array4 const flatn_arr = flatn.array(); @@ -134,8 +132,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get the interface states and shock variable shk.resize(obx, 1); - Elixir elix_shk = shk.elixir(); - Array4 const shk_arr = shk.array(); // Multidimensional shock detection @@ -172,30 +168,20 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) auto qaux_arr = qaux.array(mfi); flux[0].resize(xbx, NUM_STATE); - Elixir elix_flux_x = flux[0].elixir(); - qe[0].resize(gxbx, NGDNV); - Elixir elix_qe_x = qe[0].elixir(); #if AMREX_SPACEDIM >= 2 flux[1].resize(ybx, NUM_STATE); - Elixir elix_flux_y = flux[1].elixir(); - qe[1].resize(gybx, NGDNV); - Elixir elix_qe_y = qe[1].elixir(); #endif #if AMREX_SPACEDIM == 3 flux[2].resize(zbx, NUM_STATE); - Elixir elix_flux_z = flux[2].elixir(); - qe[2].resize(gzbx, NGDNV); - Elixir elix_qe_z = qe[2].elixir(); #endif avis.resize(obx, 1); auto avis_arr = avis.array(); - Elixir elix_avis = avis.elixir(); #ifndef AMREX_USE_GPU if (sdc_order == 4) { @@ -218,29 +204,23 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& nbx1 = amrex::grow(nbx, 1); qm.resize(obx2, NQ); - Elixir elix_qm = qm.elixir(); auto qm_arr = qm.array(); qp.resize(obx2, NQ); - Elixir elix_qp = qp.elixir(); auto qp_arr = qp.array(); q_int.resize(nbx1, 1); - Elixir elix_qint = q_int.elixir(); auto q_int_arr = q_int.array(); q_avg.resize(ibx[idir], NQ); - Elixir elix_qavg = q_avg.elixir(); auto q_avg_arr = q_avg.array(); #if AMREX_SPACEDIM >= 2 q_fc.resize(nbx, NQ); - Elixir elix_qfc = q_fc.elixir(); auto q_fc_arr = q_fc.array(); #endif f_avg.resize(ibx[idir], NUM_STATE); - Elixir elix_favg = f_avg.elixir(); auto f_avg_arr = f_avg.array(); for (int n = 0; n < NQ; n++) { @@ -449,8 +429,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) // get div{U} -- we'll use this for artificial viscosity div.resize(obx, 1); - Elixir elix_div = div.elixir(); - auto div_arr = div.array(); if (do_hydro) { @@ -460,11 +438,9 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) const Box& tbx = amrex::grow(bx, 2); qm.resize(tbx, NQ); - Elixir elix_qm = qm.elixir(); Array4 const qm_arr = qm.array(); qp.resize(tbx, NQ); - Elixir elix_qp = qp.elixir(); Array4 const qp_arr = qp.array(); // compute the fluxes and add artificial viscosity @@ -476,14 +452,12 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) if (ppm_type == 0) { dq.resize(obx, NQ); - Elixir elix_dq = dq.elixir(); auto dq_arr = dq.array(); // for well-balancing, we need to primitive variable // source terms const Box& qbx = amrex::grow(bx, NUM_GROW_SRC); src_q.resize(qbx, NQSRC); - Elixir elix_src_q = src_q.elixir(); Array4 const src_q_arr = src_q.array(); amrex::ParallelFor(qbx, @@ -566,7 +540,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) #ifdef DIFFUSION // add a diffusive flux cond.resize(obx, 1); - Elixir elix_cond = cond.elixir(); auto cond_arr = cond.array(); fill_temp_cond(obx, Sborder.array(mfi), cond_arr); @@ -667,7 +640,6 @@ Castro::construct_mol_hydro_source(Real time, Real dt, MultiFab& A_update) if (!Geom().IsCartesian()) { pradial.resize(xbx, 1); } - Elixir elix_pradial = pradial.elixir(); Array4 pradial_fab = pradial.array(); Array4 const qex_arr = qe[0].array(); diff --git a/Util/scripts/write_probdata.py b/Util/scripts/write_probdata.py index 2e44621cdc..68cb716aac 100755 --- a/Util/scripts/write_probdata.py +++ b/Util/scripts/write_probdata.py @@ -244,7 +244,7 @@ def write_probin(prob_param_files, cxx_prefix): fout.write(f" {p.get_default_string()}") if p.in_namelist: - fout.write(f" {p.get_query_string('C++')}") + fout.write(f" {p.get_query_string()}") fout.write("\n") fout.write(" }\n") diff --git a/external/Microphysics b/external/Microphysics index 49c97127a4..9d0655b75c 160000 --- a/external/Microphysics +++ b/external/Microphysics @@ -1 +1 @@ -Subproject commit 49c97127a43700ac5d7f9ac5c3aa296a0c9695a7 +Subproject commit 9d0655b75c2d7c0690fe637f8491fa572227a1dc diff --git a/external/amrex b/external/amrex index 2f47fa7361..a068330e6c 160000 --- a/external/amrex +++ b/external/amrex @@ -1 +1 @@ -Subproject commit 2f47fa7361bbf5793503cfb31b717bece889bde0 +Subproject commit a068330e6c66b5d9a7c6ca0e1c874f318e73f4cc