Merge pull request #801 from CliMA/ln/moist-hs

Add moist Held-Suarez run script

init

clean

clean params

clean params

Author: LenkaNovak

.buildkite/hierarchies/pipeline.yml

@@ -50,22 +50,18 @@ steps:
 
       - label: "Clima: GPU ClimaCoupler dry HS"
         command:
-          - "echo $$JULIA_DEPOT_PATH"
-
-          - echo "--- Instantiate AMIP env"
-          - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-          - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
-          - "julia --project=experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
-
-          - echo "--- Download artifacts"
-          - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
-          - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
-          - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
-          - "julia --project=artifacts artifacts/download_artifacts.jl"
-
-          - echo "--- Run simulation"
           - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_dry_held_suarez.jl"
         artifact_paths:  "dry_held_suarez/dry_held_suarez/clima_atmos/*"
+        agents:
+          queue: clima
+          slurm_mem: 20GB
+          slurm_gpus: 1
+          modules: common
+
+      - label: "Clima: GPU ClimaCoupler moist HS"
+        command:
+          - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_moist_held_suarez.jl"
+        artifact_paths:  "moist_held_suarez/moist_held_suarez/clima_atmos/*"
         agents:
           queue: clima
           slurm_mem: 20GB

.buildkite/pipeline.yml

@@ -378,15 +378,24 @@ steps:
 
   - group: "Hierarchy tests (1d)"
     steps:
-      - label: "Dry Held Suarez"
+      - label: ":construction: Dry Held Suarez"
         key: "dry_held_suarez"
         command:
-          - sed 's/t_end = "1000days"/t_end = "1days"/' experiments/ClimaEarth/run_dry_held_suarez.jl > experiments/ClimaEarth/run_dry_held_suarez_modified.jl
-          - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_dry_held_suarez_modified.jl"
+          - sed 's/t_end = "1000days"/t_end = "1days"/' experiments/ClimaEarth/run_dry_held_suarez.jl > experiments/ClimaEarth/run_dry_held_suarez_short.jl
+          - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_dry_held_suarez_short.jl"
         artifact_paths:  "dry_held_suarez/dry_held_suarez/clima_atmos/*"
         agents:
           slurm_mem: 20GB
 
+      - label: ":construction: Moist Held Suarez"
+        key: "moist_held_suarez"
+        command:
+          - sed 's/t_end = "1000days"/t_end = "1days"/' experiments/ClimaEarth/run_moist_held_suarez.jl > experiments/ClimaEarth/run_moist_held_suarez_short.jl
+          - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_moist_held_suarez_short.jl"
+        artifact_paths:  "moist_held_suarez/moist_held_suarez/clima_atmos/*"
+        agents:
+          slurm_mem: 20GB
+
   - group: "GPU integration tests"
     steps:
       # GPU RUNS: slabplanet

config/benchmark_configs/amip_diagedmf.yml

@@ -14,5 +14,5 @@ monthly_checkpoint: false
 run_name: "amip_diagedmf"
 start_date: "19790301"
 t_end: "12hours"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 use_coupler_diagnostics: false

config/benchmark_configs/gpu_amip_diagedmf.yml

@@ -14,5 +14,5 @@ monthly_checkpoint: false
 run_name: "gpu_amip_diagedmf"
 start_date: "19790301"
 t_end: "12hours"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 use_coupler_diagnostics: false

config/ci_configs/gpu_amip_target_topo_diagedmf_shortrun.yml

@@ -22,5 +22,5 @@ surface_setup: "PrescribedSurface"
 t_end: "200secs"
 topo_smoothing: true
 topography: "Earth"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 use_reference_state: false

config/ci_configs/interactive_debug.yml

@@ -18,5 +18,5 @@ run_name: "interactive_debug_run"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/amip_target.yml

@@ -19,5 +19,5 @@ run_name: "amip_target"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/amip_target_topo.yml

@@ -23,5 +23,5 @@ surface_setup: "PrescribedSurface"
 t_end: "100days"
 topo_smoothing: true
 topography: "Earth"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/amip_target_topo_diagedmf.yml

@@ -26,5 +26,5 @@ t_end: "30days"
 topo_smoothing: true
 topography: "Earth"
 coupler_toml_file: "toml/amip_target_topo_diagedmf.toml"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 use_reference_state: false

config/longrun_configs/dyamond_target.yml

@@ -12,4 +12,4 @@ monthly_checkpoint: false
 run_name: "dyamond_target"
 start_date: "19790301"
 t_end: "1days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"

config/longrun_configs/gpu_amip_target_topo.yml

@@ -23,6 +23,6 @@ surface_setup: "PrescribedSurface"
 t_end: "100days"
 topo_smoothing: true
 topography: "Earth"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"
 use_reference_state: false

config/longrun_configs/gpu_amip_target_topo_diagedmf.yml

@@ -26,5 +26,5 @@ t_end: "300days"
 topo_smoothing: true
 topography: "Earth"
 coupler_toml_file: "toml/amip_target_topo_diagedmf.toml"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 use_reference_state: false

config/longrun_configs/gpu_longrun_amip_dyamond.yml

@@ -12,4 +12,4 @@ monthly_checkpoint: false
 run_name: "gpu_longrun_amip_dyamond"
 start_date: "19790301"
 t_end: "1days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"

config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_atmos_sf_couple"
 start_date: "19790301"
 surface_setup: "DefaultMoninObukhov"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_atmos_sf_nocouple"
 start_date: "19790301"
 surface_setup: "DefaultMoninObukhov"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_coupler_sf.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_coupler_sf"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml

@@ -17,5 +17,5 @@ run_name: "slabplanet_aqua_coupler_sf_evolve_ocn"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_target.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_target"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_target_evolve_ocn.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_target_evolve_ocn"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_aqua_target_nocouple.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_aqua_target_nocouple"
 start_date: "19790301"
 surface_setup: "DefaultMoninObukhov"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml

@@ -17,5 +17,5 @@ run_name: "slabplanet_coupler_sf_evolve_ocn"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_target.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_target"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_target_evolve_ocn.yml

@@ -18,5 +18,5 @@ run_name: "slabplanet_target_evolve_ocn"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "100days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

config/longrun_configs/slabplanet_terra.yml

@@ -17,5 +17,5 @@ run_name: "slabplanet_terra"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
 t_end: "20days"
-turb_flux_partition: "CombinedStateFluxes"
+turb_flux_partition: "CombinedStateFluxesMOST"
 vert_diff: "true"

docs/src/fluxcalculator.md

@@ -3,7 +3,7 @@
 This modules contains abstract types and functions to calculate surface fluxes in the coupler, or to call flux calculating functions from the component models.
 
 Fluxes over a heterogeneous surface (e.g., from a gridpoint where atmospheric cell is overlying both land and ocean) can be handled in two different ways:
-1. **Combined fluxes** (called with `CombinedStateFluxes()`)
+1. **Combined fluxes** (called with `CombinedStateFluxesMOST()`)
   - these are calculated by averaging the surface properties for each gridpoint (e.g., land and ocean temperatures, albedos and roughness lengths are averaged, based on their respective area fractions), so the flux is calculated only once per gridpoint of the grid where we calculate fluxes. This is computationally faster, but it makes the fluxes on surface boundaries more diffuse. Currently, we use this method for calculating radiative fluxes in the atmosphere, and turbulent fluxes in the coupler (on the atmospheric grid). The fluxes are calculated in the atmospheric (host) model's cache, which can be setup to avoid allocating coupler fields.
 2. **Partitioned fluxes** (called with `PartitionedStateFluxes()`)
   - these are calculated separately for each surface type. It is then the fluxes (rather than the surface states) that are combined and passed to the atmospheric model as a boundary condition. This method ensures that each surface model receives fluxes that correspond to its state properties, resulting in a more accurate model evolution. However, it is more computationally expensive, and requires more communication between the component models.
@@ -13,9 +13,10 @@ Fluxes over a heterogeneous surface (e.g., from a gridpoint where atmospheric ce
 ```@docs
     ClimaCoupler.FluxCalculator.TurbulentFluxPartition
     ClimaCoupler.FluxCalculator.PartitionedStateFluxes
-    ClimaCoupler.FluxCalculator.CombinedStateFluxes
+    ClimaCoupler.FluxCalculator.CombinedStateFluxesMOST
     ClimaCoupler.FluxCalculator.combined_turbulent_fluxes!
-    ClimaCoupler.FluxCalculator.atmos_turbulent_fluxes!
+    ClimaCoupler.FluxCalculator.atmos_turbulent_fluxes_bulk!
+    ClimaCoupler.FluxCalculator.atmos_turbulent_fluxes_most!
     ClimaCoupler.FluxCalculator.surface_inputs
     ClimaCoupler.FluxCalculator.calculate_surface_air_density
     ClimaCoupler.FluxCalculator.get_surface_params

experiments/ClimaEarth/cli_options.jl

@@ -35,9 +35,9 @@ function argparse_settings()
         arg_type = Bool
         default = false
         "--turb_flux_partition"
-        help = "Method to partition turbulent fluxes. [`PartitionedStateFluxes`, `CombinedStateFluxes`]"
+        help = "Method to partition turbulent fluxes. [`PartitionedStateFluxes`, `CombinedStateFluxesMOST`]"
         arg_type = String
-        default = "CombinedStateFluxes"
+        default = "CombinedStateFluxesMOST"
         "--hourly_checkpoint"
         help = "Boolean flag indicating whether to checkpoint at intervals of 1 hour or multiple hours"
         arg_type = Bool

experiments/ClimaEarth/components/atmosphere/climaatmos.jl

@@ -29,8 +29,9 @@ function hasmoisture(integrator)
     return !(integrator.p.atmos.moisture_model isa CA.DryModel)
 end
 
-function atmos_init(::Type{FT}, atmos_config) where {FT}
+function atmos_init(atmos_config)
     # By passing `parsed_args` to `AtmosConfig`, `parsed_args` overwrites the default atmos config
+    FT = atmos_config.parsed_args["FLOAT_TYPE"] == "Float64" ? Float64 : Float32
     simulation = CA.get_simulation(atmos_config)
     (; integrator) = simulation
     Y = integrator.u
@@ -297,7 +298,7 @@ FluxCalculator.get_surface_params(sim::ClimaAtmosSimulation) = CAP.surface_fluxe
     get_atmos_config_dict(coupler_dict::Dict)
 
 Returns the specified atmospheric configuration (`atmos_config`) overwitten by arguments
-in the coupler dictionary (`config_dict`). The returned dictionary will then be passed to CA.AtmosConfig().
+in the coupler dictionary (`config_dict`). The returned `atmos_config` dictionary will then be passed to CA.AtmosConfig().
 The `atmos_config_repo` flag allows us to
 use a configuration specified within the ClimaCoupler repo, which is useful for direct
 coupled/atmos-only comparisons.
@@ -350,14 +351,18 @@ function get_atmos_config_dict(coupler_dict)
         joinpath(coupler_dict["mode_name"], coupler_dict["run_name"]),
         "clima_atmos",
     )
+
+    # merge configs
+    # (if there are common keys, the last dictionary in the `merge` arguments takes precedence)
     atmos_config = merge(atmos_config, Dict("output_dir" => atmos_output_dir))
+    coupler_config = merge(atmos_config, config_dict)
 
     # set restart file to the initial file saved in this location if it is not nothing
     # TODO this is hardcoded and should be fixed once we have a better restart system
     if !isnothing(atmos_config["restart_file"])
         atmos_config["restart_file"] = replace(atmos_config["restart_file"], "active" => "0000")
     end
-    return atmos_config
+    return atmos_config, coupler_config
 end
 
 
@@ -416,9 +421,10 @@ function get_new_cache(atmos_sim::ClimaAtmosSimulation, csf)
 end
 
 """
-    FluxCalculator.atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf)
+    FluxCalculator.atmos_turbulent_fluxes_most!(atmos_sim::ClimaAtmosSimulation, csf)
 
-Computes turbulent surface fluxes using ClimaAtmos's `update_surface_conditions!`. This
+Computes turbulent surface fluxes using ClimaAtmos's `update_surface_conditions!` and
+and the Monin Obukhov Similarity Theory. This
 requires that we define a new temporary parameter Tuple, `new_p`, and save the new surface state
 in it. We do not want `new_p` to live in the atmospheric model permanently, because that would also
 trigger flux calculation during Atmos `Interfacer.step!`. We only want to trigger this once per coupling
@@ -430,7 +436,7 @@ For debigging atmos, we can set the following atmos defaults:
  csf.beta .= 1
  csf = merge(csf, (;q_sfc = nothing))
 """
-function FluxCalculator.atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf)
+function FluxCalculator.atmos_turbulent_fluxes_most!(atmos_sim::ClimaAtmosSimulation, csf)
 
     if isnothing(atmos_sim.integrator.p.sfc_setup) # trigger flux calculation if not done in Atmos internally
         new_p = get_new_cache(atmos_sim, csf)

experiments/ClimaEarth/run_amip.jl

@@ -99,17 +99,13 @@ if isinteractive()
         parsed_args["config_file"]
 end
 
-## read in config dictionary from file, overriding the coupler defaults
+## read in config dictionary from file, overriding the coupler defaults in `parsed_args`
 config_dict = YAML.load_file(parsed_args["config_file"])
 config_dict = merge(parsed_args, config_dict)
 
 ## get component model dictionaries (if applicable)
-config_dict_atmos = get_atmos_config_dict(config_dict)
-atmos_config_object = CA.AtmosConfig(config_dict_atmos)
-
-## merge dictionaries of command line arguments, coupler dictionary and component model dictionaries
-## (if there are common keys, the last dictionary in the `merge` arguments takes precedence)
-config_dict = merge(config_dict_atmos, config_dict)
+atmos_config_dict, config_dict = get_atmos_config_dict(config_dict)
+atmos_config_object = CA.AtmosConfig(atmos_config_dict)
 
 ## read in some parsed command line arguments, required by this script
 mode_name = config_dict["mode_name"]
@@ -177,13 +173,13 @@ returns a `ComponentModelSimulation` object (see `Interfacer` docs for more deta
 
 #=
 ### Atmosphere
-This uses the `ClimaAtmos.jl` model, with parameterization options specified in the `config_dict_atmos` dictionary.
+This uses the `ClimaAtmos.jl` model, with parameterization options specified in the `atmos_config_object` dictionary.
 =#
 
 Utilities.show_memory_usage(comms_ctx)
 
 ## init atmos model component
-atmos_sim = atmos_init(FT, atmos_config_object);
+atmos_sim = atmos_init(atmos_config_object);
 Utilities.show_memory_usage(comms_ctx)
 
 thermo_params = get_thermo_params(atmos_sim) # TODO: this should be shared by all models #610
@@ -573,10 +569,10 @@ Decide on the type of turbulent flux partition (see `FluxCalculator` documentati
 turbulent_fluxes = nothing
 if config_dict["turb_flux_partition"] == "PartitionedStateFluxes"
     turbulent_fluxes = FluxCalculator.PartitionedStateFluxes()
-elseif config_dict["turb_flux_partition"] == "CombinedStateFluxes"
-    turbulent_fluxes = FluxCalculator.CombinedStateFluxes()
+elseif config_dict["turb_flux_partition"] == "CombinedStateFluxesMOST"
+    turbulent_fluxes = FluxCalculator.CombinedStateFluxesMOST()
 else
-    error("turb_flux_partition must be either PartitionedStateFluxes or CombinedStateFluxes")
+    error("turb_flux_partition must be either PartitionedStateFluxes or CombinedStateFluxesMOST")
 end
 
 #=
@@ -648,7 +644,7 @@ Interfacer.step!(ice_sim, Δt_cpl)
 
 # 4.turbulent fluxes: now we have all information needed for calculating the initial turbulent surface fluxes using the combined state
 # or the partitioned state method
-if cs.turbulent_fluxes isa FluxCalculator.CombinedStateFluxes
+if cs.turbulent_fluxes isa FluxCalculator.CombinedStateFluxesMOST
     ## import the new surface properties into the coupler (note the atmos state was also imported in step 3.)
     FieldExchanger.import_combined_surface_fields!(cs.fields, cs.model_sims, cs.turbulent_fluxes) # i.e. T_sfc, albedo, z0, beta, q_sfc
     ## calculate turbulent fluxes inside the atmos cache based on the combined surface state in each grid box
@@ -674,7 +670,7 @@ end
 FieldExchanger.reinit_model_sims!(cs.model_sims)
 
 # 6.update all fluxes: coupler re-imports updated atmos fluxes (radiative fluxes for both `turbulent_fluxes` types
-# and also turbulent fluxes if `turbulent_fluxes isa CombinedStateFluxes`,
+# and also turbulent fluxes if `turbulent_fluxes isa CombinedStateFluxesMOST`,
 # and sends them to the surface component models. If `turbulent_fluxes isa PartitionedStateFluxes`
 # atmos receives the turbulent fluxes from the coupler.
 FieldExchanger.import_atmos_fields!(cs.fields, cs.model_sims, cs.boundary_space, cs.turbulent_fluxes)
@@ -753,7 +749,7 @@ function solve_coupler!(cs)
 
         ## exchange combined fields and (if specified) calculate fluxes using combined states
         FieldExchanger.import_combined_surface_fields!(cs.fields, cs.model_sims, cs.turbulent_fluxes) # i.e. T_sfc, surface_albedo, z0, beta
-        if cs.turbulent_fluxes isa FluxCalculator.CombinedStateFluxes
+        if cs.turbulent_fluxes isa FluxCalculator.CombinedStateFluxesMOST
             FluxCalculator.combined_turbulent_fluxes!(cs.model_sims, cs.fields, cs.turbulent_fluxes) # this updates the surface thermo state, sfc_ts, in ClimaAtmos (but also unnecessarily calculates fluxes)
         elseif cs.turbulent_fluxes isa FluxCalculator.PartitionedStateFluxes
             ## calculate turbulent fluxes in surfaces and save the weighted average in coupler fields