Merge pull request NCAR#176 from TeaganKing/workflow

Updates in helper script for updated cupid workflow

cime_config/config_tool.xml

@@ -0,0 +1,192 @@
+<?xml version="1.0"?>
+
+<?xml-stylesheet type="text/xsl" href="definitions_variables.xsl" ?>
+
+<entry_id version="3.0">
+
+  <!-- Variables that configure which part(s) of CUPiD are run -->
+
+  <entry id="CUPID_GEN_TIMESERIES">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>TRUE</default_value>
+    <group>cupid_config</group>
+    <file>env_postprocessing.xml</file>
+    <desc>When TRUE, CUPiD will generate time series files from history output</desc>
+  </entry>
+
+  <entry id="CUPID_GEN_DIAGNOSTICS">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>TRUE</default_value>
+    <group>cupid_config</group>
+    <file>env_postprocessing.xml</file>
+    <desc>When TRUE, CUPiD will generate diagnostic plots</desc>
+  </entry>
+
+  <entry id="CUPID_GEN_HTML">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>TRUE</default_value>
+    <group>cupid_config</group>
+    <file>env_postprocessing.xml</file>
+    <desc>When TRUE, CUPiD will generate web pages from the diagnostic plots</desc>
+  </entry>
+
+  <!-- Variables that configure conda environments used by CUPiD -->
+
+  <entry id="CUPID_ANALYSIS_ENV">
+    <type>char</type>
+    <default_value>cupid-analysis</default_value>
+    <group>cupid_environments</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Conda environment to use when running CUPiD notebooks</desc>
+  </entry>
+
+  <entry id="CUPID_INFRASTRUCTURE_ENV">
+    <type>char</type>
+    <default_value>cupid-infrastructure</default_value>
+    <group>cupid_environments</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Conda environment to use when running CUPiD tools</desc>
+  </entry>
+
+  <!-- Variables that define what parts of the case(s) are analyzed -->
+
+  <entry id="CUPID_STARTDATE">
+    <type>char</type>
+    <default_value>0001-01-01</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>First year of output to include in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_NYEARS">
+    <type>integer</type>
+    <default_value>1</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Number of years of output to include in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_BASE_STARTDATE">
+    <type>char</type>
+    <default_value>0001-01-01</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>First year of baseline output to include in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_BASE_NYEARS">
+    <type>integer</type>
+    <default_value>100</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Number of years of baseline output to include in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_BASELINE_CASE">
+    <type>char</type>
+    <default_value>b.e23_alpha17f.BLT1850.ne30_t232.092</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Baseline case to use in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_BASELINE_ROOT">
+    <type>char</type>
+    <default_value>$DOUT_S_ROOT</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Baseline case root to use in CUPiD analysis</desc>
+  </entry>
+
+  <entry id="CUPID_EXAMPLE">
+    <type>char</type>
+    <default_value>key_metrics</default_value>
+    <group>cupid_analysis</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Example to use in CUPiD analysis</desc>
+  </entry>
+
+  <!-- Variables that define which components are analyzed -->
+
+  <entry id="CUPID_RUN_ALL">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>TRUE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not diagnostics for all components are run</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_ATM">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not ATM diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_OCN">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not OCN diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_LND">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not LND diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_ICE">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not ICE diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_ROF">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not ROF diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_GLC">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not GLC diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <entry id="CUPID_RUN_ADF">
+    <type>char</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>cupid_run_components</group>
+    <file>env_postprocessing.xml</file>
+    <desc>Boolean value that determines whether or not ADF diagnostics are run (if CUPID_RUN_ALL is FALSE)</desc>
+  </entry>
+
+  <help>
+    =========================================
+    Configuring CUPiD Postprocessing
+    =========================================
+  </help>
+
+</entry_id>

cupid/timeseries.py

@@ -65,9 +65,9 @@ def create_time_series(
          check if time series files already exist
      - overwrite_ts: list, boolean
          check if existing time series files will bew overwritten
-     - start_years: list, str or int
+     - start_years: list of ints
          first year for desired range of years
-     - end_years: list, str or int
+     - end_years: list of ints
          last year for desired range of years
      - height_dim: str
          name of height dimension for given component, eg 'lev'

helper_scripts/cesm_postprocessing.sh

@@ -0,0 +1,130 @@
+#!/bin/bash -e
+# This script is run by the CESM workflow when RUN_POSTPROCESSING=TRUE,
+# it is invoked from case.cupid and the expectation is that it is run
+# from CASEROOT.
+# One possible future development would be to make case.cupid a python script
+# and then update this to python as well (and take a CIME Case object as
+# an argument)
+
+# Function to add some number of years to a string that
+# is formatted as YYYY-MM-DD and print out the updated
+# string in the same format
+add_years() {
+  YEAR=`echo $1 | cut -d '-' -f 1`
+  MM=`echo $1 | cut -d '-' -f 2`
+  DD=`echo $1 | cut -d '-' -f 3`
+  NEW_YEAR=`printf '%04d' "$((YEAR + $2))"`-`printf '%02d' "${MM}"`-`printf '%02d' "${DD}"`
+  echo ${NEW_YEAR}
+}
+
+# Set variables that come from environment or CESM XML files
+CASEROOT=${PWD}
+SRCROOT=`./xmlquery --value SRCROOT`
+CUPID_EXAMPLE=`./xmlquery --value CUPID_EXAMPLE`
+CUPID_GEN_TIMESERIES=`./xmlquery --value CUPID_GEN_TIMESERIES`
+CUPID_GEN_DIAGNOSTICS=`./xmlquery --value CUPID_GEN_DIAGNOSTICS`
+CUPID_GEN_HTML=`./xmlquery --value CUPID_GEN_HTML`
+CUPID_BASELINE_CASE=`./xmlquery --value CUPID_BASELINE_CASE`
+CUPID_BASELINE_ROOT=`./xmlquery --value CUPID_BASELINE_ROOT`
+CUPID_STARTDATE=`./xmlquery --value CUPID_STARTDATE`
+CUPID_NYEARS=`./xmlquery --value CUPID_NYEARS`
+CUPID_ENDDATE=`add_years ${CUPID_STARTDATE} ${CUPID_NYEARS}`
+CUPID_BASE_STARTDATE=`./xmlquery --value CUPID_BASE_STARTDATE`
+CUPID_BASE_NYEARS=`./xmlquery --value CUPID_BASE_NYEARS`
+CUPID_BASE_ENDDATE=`add_years ${CUPID_BASE_STARTDATE} ${CUPID_BASE_NYEARS}`
+CUPID_RUN_ALL=`./xmlquery --value CUPID_RUN_ALL`
+CUPID_RUN_ATM=`./xmlquery --value CUPID_RUN_ATM`
+CUPID_RUN_OCN=`./xmlquery --value CUPID_RUN_OCN`
+CUPID_RUN_LND=`./xmlquery --value CUPID_RUN_LND`
+CUPID_RUN_ICE=`./xmlquery --value CUPID_RUN_ICE`
+CUPID_RUN_ROF=`./xmlquery --value CUPID_RUN_ROF`
+CUPID_RUN_GLC=`./xmlquery --value CUPID_RUN_GLC`
+CUPID_RUN_ADF=`./xmlquery --value CUPID_RUN_ADF`
+CUPID_INFRASTRUCTURE_ENV=`./xmlquery --value CUPID_INFRASTRUCTURE_ENV`
+CUPID_ANALYSIS_ENV=`./xmlquery --value CUPID_ANALYSIS_ENV`
+
+# Create directory for running CUPiD
+mkdir -p cupid-postprocessing
+cd cupid-postprocessing
+
+# If CUPID_RUN_ALL is TRUE, we don't add any component flags.
+# The lack of any component flags tells CUPiD to run all components.
+CUPID_FLAG_STRING=""
+if [ "${CUPID_RUN_ALL}" == "FALSE" ]; then
+  if [ "${CUPID_RUN_ATM}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --atm"
+  fi
+  if [ "${CUPID_RUN_OCN}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --ocn"
+  fi
+  if [ "${CUPID_RUN_LND}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --lnd"
+  fi
+  if [ "${CUPID_RUN_ICE}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --ice"
+  fi
+  if [ "${CUPID_RUN_ROF}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --rof"
+  fi
+  if [ "${CUPID_RUN_GLC}" == "TRUE" ]; then
+    CUPID_FLAG_STRING+=" --glc"
+  fi
+  if [ "${CUPID_FLAG_STRING}" == "" ]; then
+    echo "If CUPID_RUN_ALL is False, user must set at least one component"
+    exit 1
+  fi
+fi
+if [ "${CUPID_RUN_ALL}" == "TRUE" ]; then
+  echo "CUPID_RUN_ALL is True, running diagnostics for all components"
+fi
+
+# Use cupid-infrastructure environment for running these scripts
+# Note: on derecho, the cesmdev module creates a python conflict
+#       by setting $PYTHONPATH; since this is conda-based we
+#       want an empty PYTHONPATH environment variable
+unset PYTHONPATH
+conda activate ${CUPID_INFRASTRUCTURE_ENV}
+
+# 1. Generate CUPiD config file
+${SRCROOT}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \
+   --cesm-root ${SRCROOT} \
+   --case-root ${CASEROOT} \
+   --adf-output-root ${PWD} \
+   --cupid-example ${CUPID_EXAMPLE} \
+   --cupid-baseline-case ${CUPID_BASELINE_CASE} \
+   --cupid-baseline-root ${CUPID_BASELINE_ROOT} \
+   --cupid-startdate ${CUPID_STARTDATE} \
+   --cupid-enddate ${CUPID_ENDDATE} \
+   --cupid-base-startdate ${CUPID_BASE_STARTDATE} \
+   --cupid-base-enddate ${CUPID_BASE_ENDDATE} \
+
+# 2. Generate ADF config file
+if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then
+  ${SRCROOT}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \
+     --cesm-root ${SRCROOT} \
+     --cupid-config-loc . \
+     --adf-template ${SRCROOT}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \
+     --out-file adf_config.yml
+fi
+
+# 3. Generate timeseries files
+if [ "${CUPID_GEN_TIMESERIES}" == "TRUE" ]; then
+   ${SRCROOT}/tools/CUPiD/cupid/run_timeseries.py ${CUPID_FLAG_STRING}
+fi
+
+#4. Run ADF
+if [ "${CUPID_RUN_ADF}" == "TRUE" ]; then
+  conda deactivate
+  conda activate ${CUPID_ANALYSIS_ENV}
+  ${SRCROOT}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml
+fi
+
+# 5. Run CUPiD and build webpage
+conda deactivate
+conda activate ${CUPID_INFRASTRUCTURE_ENV}
+if [ "${CUPID_GEN_DIAGNOSTICS}" == "TRUE" ]; then
+  ${SRCROOT}/tools/CUPiD/cupid/run_diagnostics.py ${CUPID_FLAG_STRING}
+fi
+if [ "${CUPID_GEN_HTML}" == "TRUE" ]; then
+  ${SRCROOT}/tools/CUPiD/cupid/cupid_webpage.py
+fi