Include ILAMB output in notebook and provide link to full diagnostic package

cupid/cupid_webpage.py

@@ -58,6 +58,17 @@ def build(config_path):
                         f"{run_dir}/ADF_output",
                         f"{run_dir}/computed_notebooks/_build/html/ADF",
                     )
+            if "external_tool" in control["compute_notebooks"][component][notebook]:
+                if (
+                    control["compute_notebooks"][component][notebook][
+                        "external_tool"
+                    ].get("tool_name")
+                    == "ILAMB"
+                ):
+                    shutil.copytree(
+                        f"{run_dir}/ILAMB_output",
+                        f"{run_dir}/computed_notebooks/_build/html/ILAMB",
+                    )
 
     # Originally used this code to copy jupyter book HTML to a location to host it online
 

examples/.gitignore

@@ -1 +1,3 @@
 */ADF_output
+*/ILAMB_output
+*/model_setup.txt

examples/external_diag_packages/config.yml

@@ -115,58 +115,24 @@ compute_notebooks:
         parameter_groups:
           none:
             adf_root: ../../examples/external_diag_packages/ADF_output/
+            key_plots: ["Surface_Wind_Stress_ANN_LatLon_Vector_Mean.png", "PRECT_ANN_LatLon_Mean.png", "PS_DJF_SHPolar_Mean.png", "TaylorDiag_ANN_Special_Mean.png"]
         external_tool:
           tool_name: 'ADF'
           vars: ['PRECT', 'SST']
           plotting_scripts: ["global_latlon_map", "global_latlon_vect_map"]
           analysis_scripts: ["amwg_table"]
           base_regridded_output: True
 
-#    glc:
-#      LIWG_SMB_diagnostic:
-#        parameter_groups:
-#          none:
-#            obs_path: '/glade/u/home/gunterl/obs_diagnostic_cesm/'
-#            obs_name: 'GrIS_MARv3.12_climo_1960_1999.nc'
-#            climo_nyears: 40
-
-#    ice:
-#      seaice:
-#        parameter_groups:
-#          none:
-#            cases:
-#              - g.e23_a16g.GJRAv4.TL319_t232_hycom1_N75.2024.005
-#              - g.e23_a16g.GJRAv4.TL319_t232_zstar_N65.2024.004
-#            begyr1: 245
-#            endyr1: 305
-#            begyr2: 245
-#            endyr2: 305
-#            nyears: 25
-
-#    lnd:
-#      land_comparison:
-#        parameter_groups:
-#          none:
-#            cases:
-#              - ctsm51d159_f45_GSWP3_bgccrop_1850pAD
-#              - ctsm51d159_f45_GSWP3_bgccrop_1850pSASU
-#            type:
-#              - 1850pAD
-#              - 1850pSASU
-
-#    ocn:
-#      ocean_surface:
-#        parameter_groups:
-#          none:
-#            Case: b.e23_alpha16b.BLT1850.ne30_t232.054
-#            savefigs: False
-#            mom6_tools_config:
-#              start_date: '0091-01-01'
-#              end_date: '0101-01-01'
-#              Fnames:
-#                native: 'mom6.h.native.????-??.nc'
-#                static: 'mom6.h.static.nc'
-#              oce_cat: /glade/u/home/gmarques/libs/oce-catalogs/reference-datasets.yml
+    lnd:
+      link_to_ILAMB:
+        parameter_groups:
+          none:
+            ilamb_root: ../../examples/external_diag_packages/ILAMB_output
+            key_plots: ["EcosystemandCarbonCycle/GrossPrimaryProductivity/FLUXCOM/CTSM51_global_bias.png", "EcosystemandCarbonCycle/LeafAreaIndex/AVHRR/CTSM51_global_bias.png", "EcosystemandCarbonCycle/GlobalNetEcosystemCarbonBalance/GCP/CTSM51_global_accumulate.png", "ILAMB_output_backup/HydrologyCycle/Permafrost/Brown2002/CTSM51_global_bias.png"]
+            print_table: False
+        external_tool:
+          tool_name: 'ILAMB'
+          ilamb_config_data_loc: '/glade/campaign/cesm/community/lmwg/diag/ILAMB/'
 
 ########### JUPYTER BOOK CONFIG ###########
 
@@ -192,23 +158,11 @@ book_toc:
 
     - caption: Atmosphere
       chapters:
-        - file: atm/link_to_ADF
-
-    # - caption: Ocean
-    #   chapters:
-    #       - file: ocn/ocean_surface
+       - file: atm/link_to_ADF
 
-    # - caption: Land
-    #   chapters:
-    #     - file: lnd/land_comparison
-
-    # - caption: Sea Ice
-    #   chapters:
-    #     - file: ice/seaice
-
-    # - caption: Land Ice
-    #   chapters:
-    #     - file: glc/LIWG_SMB_diagnostic
+    - caption: Land
+      chapters:
+        - file: lnd/link_to_ILAMB
 
 #####################################
 # Keys for Jupyter Book _config.yml #

examples/key_metrics/config.yml

@@ -131,6 +131,7 @@ compute_notebooks:
         parameter_groups:
           none:
             adf_root: ../../examples/key_metrics/ADF_output/
+            key_plots: ["Surface_Wind_Stress_ANN_LatLon_Vector_Mean.png", "PRECT_ANN_LatLon_Mean.png", "PS_DJF_SHPolar_Mean.png", "TaylorDiag_ANN_Special_Mean.png"]
         external_tool:
           tool_name: 'ADF'
           vars: ['SST', 'TS', 'SWCF', 'LWCF', 'PRECT', 'TAUX', 'TAUY',  'TGCLDLWP']

helper_scripts/.gitignore

@@ -1 +1,3 @@
+*.cfg
+*.txt
 *.yml

helper_scripts/generate_ilamb_config_files.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+import yaml
+
+
+def _parse_args():
+    """Parse command line arguments"""
+
+    parser = argparse.ArgumentParser(
+        description="Generate an ILAMB model_setup.txt file based on an existing CUPID YAML file",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    # Command line argument for location of CESM source code (required)
+    parser.add_argument(
+        "--cesm-root",
+        action="store",
+        dest="cesm_root",
+        required=True,
+        help="Location of CESM source code",
+    )
+    # Command line argument for CUPiD example from which to get config.yml
+    parser.add_argument(
+        "--cupid-config-loc",
+        action="store",
+        dest="cupid_config_loc",
+        default=None,
+        help="CUPiD config file to use information from for model_setup.txt",
+    )
+    parser.add_argument(
+        "--run-type",
+        action="store",
+        required=True,
+        help="either 'BGC' (biogeochemistry) or 'SP' (satellite phenology)",
+    )
+    return parser.parse_args()
+
+
+def generate_ilamb_cfg(cesm_root, cupid_config_loc, run_type):
+    """Create config file for use in ILAMB"""
+    sys.path.append(os.path.join(cesm_root, "cime"))
+
+    cupid_root = os.path.join(cesm_root, "tools", "CUPiD")
+    # Is cupid_config_loc a valid value?
+    if cupid_config_loc is None:
+        cupid_config_loc = os.path.join(cupid_root, "examples", "key_metrics")
+    if not os.path.exists(os.path.join(cupid_config_loc, "config.yml")):
+        raise KeyError(f"Can not find config.yml in {cupid_config_loc}")
+
+    with open(os.path.join(cupid_config_loc, "config.yml")) as c:
+        c_dict = yaml.safe_load(c)
+    ilamb_config_data_loc = c_dict["compute_notebooks"]["lnd"]["link_to_ILAMB"][
+        "external_tool"
+    ]["ilamb_config_data_loc"]
+
+    ilamb_config_loc = os.path.join(cesm_root, "tools", "CUPiD", "ilamb_aux")
+    print(run_type)
+    with open(
+        os.path.join(
+            ilamb_config_loc,
+            f"ilamb_nohoff_final_CLM_{run_type}_template.cfg",
+        ),
+    ) as cfg:
+        cfg_content = cfg.read()
+        cfg_content = cfg_content.replace("PATH/", ilamb_config_data_loc)
+    print(f"writing ilamb_nohoff_final_CLM_{run_type}.cfg")
+    with open(
+        os.path.join(ilamb_config_loc, f"ilamb_nohoff_final_CLM_{run_type}.cfg"),
+        "w",
+    ) as cfg:
+        cfg.write(cfg_content)
+    print(f"wrote {ilamb_config_loc}/ilamb_nohoff_final_CLM_{run_type}.cfg")
+
+
+def generate_ilamb_model_setup(cesm_root, cupid_config_loc, run_type):
+    """Create model_setup.txt file for use in ILAMB"""
+    sys.path.append(os.path.join(cesm_root, "cime"))
+
+    cupid_root = os.path.join(cesm_root, "tools", "CUPiD")
+    # Is cupid_config_loc a valid value?
+    if cupid_config_loc is None:
+        cupid_config_loc = os.path.join(cupid_root, "examples", "key_metrics")
+    if not os.path.exists(os.path.join(cupid_config_loc, "config.yml")):
+        raise KeyError(f"Can not find config.yml in {cupid_config_loc}")
+
+    with open(os.path.join(cupid_config_loc, "config.yml")) as c:
+        c_dict = yaml.safe_load(c)
+    base_case_output_dir = os.path.join(
+        c_dict["global_params"]["CESM_output_dir"],
+        c_dict["global_params"]["base_case_name"],
+    )
+    with open(cupid_config_loc + "model_setup.txt", "w") as ms:
+        ms.write(
+            "# Model Name    , Location of Files                                                                    ,  Shift From,  Shift To\n",  # noqa: E501
+        )
+        ms.write(
+            f"CTSM51          , {base_case_output_dir}/lnd/hist/regrid/\n",
+        )
+    print(f"wrote {cupid_config_loc}model_setup.txt")
+    print(
+        f"WARNING: ILAMB requires regridded output to be in {base_case_output_dir}/lnd/hist/regrid/ directory.",
+    )
+    print("You can now run ILAMB with the following commands:")
+    print("---")
+    print("qinteractive -l select=1:ncpus=16:mpiprocs=16:mem=100G -l walltime=06:00:00")
+    print("conda activate cupid-analysis")
+    print("export ILAMB_ROOT=../ilamb_aux")
+    if run_type == "SP":
+        print(
+            f"ilamb-run --config ../ilamb_aux/ilamb_nohoff_final_CLM_SP.cfg --build_dir {cupid_config_loc}ILAMB_output/ --df_errs ../ilamb_aux/quantiles_Whittaker_cmip5v6.parquet --define_regions ../ilamb_aux/DATA/regions/LandRegions.nc ../ilamb_aux/DATA/regions/Whittaker.nc --regions global --model_setup {cupid_config_loc}model_setup.txt --filter .clm2.h0.",  # noqa: E501
+        )
+    elif run_type == "BGC":
+        print(
+            f"ilamb-run --config ../ilamb_aux/ilamb_nohoff_final_CLM_BGC.cfg --build_dir {cupid_config_loc}ILAMB_output/ --df_errs ../ilamb_aux/quantiles_Whittaker_cmip5v6.parquet --define_regions ../ilamb_aux/DATA/regions/LandRegions.nc ../ilamb_aux/DATA/regions/Whittaker.nc --regions global --model_setup {cupid_config_loc}model_setup.txt --filter .clm2.h0.",  # noqa: E501
+        )
+    print("---------")
+
+
+if __name__ == "__main__":
+    args = vars(_parse_args())
+    print(args)
+    generate_ilamb_cfg(args["cesm_root"], args["cupid_config_loc"], args["run_type"])
+    generate_ilamb_model_setup(
+        args["cesm_root"],
+        args["cupid_config_loc"],
+        args["run_type"],
+    )

ilamb_aux/.gitignore

@@ -0,0 +1,2 @@
+ilamb_nohoff_final_CLM_BGC.cfg
+ilamb_nohoff_final_CLM_SP.cfg