[develop] Integrate 'online-cmaq' into 'develop' branch: Part 2, build script and module files

.cicd/Jenkinsfile

@@ -15,7 +15,7 @@ pipeline {
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable hera
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Uncomment the following line to re-enable comprehensive tests

.gitignore

@@ -5,6 +5,7 @@ fix/
 include/
 lib/
 share/
+modulefiles/extrn_comp_build/
 sorc/*/
 tests/WE2E/WE2E_test_info.csv
 tests/WE2E/*.txt

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 84b28ec
+hash = e051e0e
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -44,5 +44,23 @@ hash = 8d21f25
 local_path = sorc/rrfs_utl
 required = True
 
+[arl_nexus]
+protocol = git
+repo_url = https://github.com/noaa-oar-arl/NEXUS
+# Specify either a branch name or a hash but not both.
+#branch = develop
+hash = 3842818
+local_path = sorc/arl_nexus
+required = True
+
+[aqm-utils]
+protocol = git
+repo_url = https://github.com/NOAA-EMC/AQM-utils
+# Specify either a branch name or a hash but not both.
+#branch = develop
+hash = 81a99ca
+local_path = sorc/AQM-utils
+required = True
+
 [externals_description]
 schema_version = 1.0.0

devbuild.sh

@@ -15,8 +15,8 @@ OPTIONS
       compiler to use; default depends on platform
       (e.g. intel | gnu | cray | gccgfortran)
   -a, --app=APPLICATION
-      weather model application to build
-      (e.g. ATM | ATMW | S2S | S2SW)
+      weather model application to build; for example, ATMAQ for Online-CMAQ
+      (e.g. ATM | ATMAQ | ATMW | S2S | S2SW)
   --ccpp="CCPP_SUITE1,CCPP_SUITE2..."
       CCPP suites (CCPP_SUITES) to include in build; delimited with ','
   --enable-options="OPTION1,OPTION2,..."
@@ -56,8 +56,7 @@ TARGETS
    all = builds all apps
    Or any combinations of (ufs, ufs_utils, upp, gsi, rrfs_utils)
 
-NOTE: This script is for internal developer use only;
-See User's Guide for detailed build instructions
+NOTE: See User's Guide for detailed build instructions
 
 EOF_USAGE
 }
@@ -87,6 +86,8 @@ Settings:
   BUILD_UPP=${BUILD_UPP}
   BUILD_GSI=${BUILD_GSI}
   BUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}
+  BUILD_NEXUS=${BUILD_NEXUS}
+  BUILD_AQM_UTILS=${BUILD_AQM_UTILS}
 
 EOF_SETTINGS
 }
@@ -123,6 +124,8 @@ BUILD_UFS_UTILS="off"
 BUILD_UPP="off"
 BUILD_GSI="off"
 BUILD_RRFS_UTILS="off"
+BUILD_NEXUS="off"
+BUILD_AQM_UTILS="off"
 
 # Make options
 CLEAN=false
@@ -182,36 +185,52 @@ while :; do
     upp) DEFAULT_BUILD=false; BUILD_UPP="on" ;;
     gsi) DEFAULT_BUILD=false; BUILD_GSI="on" ;;
     rrfs_utils) DEFAULT_BUILD=false; BUILD_RRFS_UTILS="on" ;;
+    nexus) DEFAULT_BUILD=false; BUILD_NEXUS="on" ;;
+    aqm_utils) DEFAULT_BUILD=false; BUILD_AQM_UTILS="on" ;;
     # unknown
     -?*|?*) usage_error "Unknown option $1" ;;
     *) break
   esac
   shift
 done
 
-# choose default apps to build
-if [ "${DEFAULT_BUILD}" = true ]; then
-  BUILD_UFS="on"
-  BUILD_UFS_UTILS="on"
-  BUILD_UPP="on"
-fi
-
 # Ensure uppercase / lowercase ============================================
 APPLICATION="${APPLICATION^^}"
 PLATFORM="${PLATFORM,,}"
 COMPILER="${COMPILER,,}"
+EXTERNALS="${EXTERNALS^^}"
 
 # check if PLATFORM is set
 if [ -z $PLATFORM ] ; then
   printf "\nERROR: Please set PLATFORM.\n\n"
   usage
   exit 0
 fi
-
 # set PLATFORM (MACHINE)
 MACHINE="${PLATFORM}"
 printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2
 
+# choose default apps to build
+if [ "${DEFAULT_BUILD}" = true ]; then
+  BUILD_UFS="on"
+  BUILD_UFS_UTILS="on"
+  BUILD_UPP="on"
+fi
+
+# Choose components to build for Online-CMAQ
+if [ "${APPLICATION}" = "ATMAQ" ]; then
+  if [ "${DEFAULT_BUILD}" = true ]; then
+    BUILD_NEXUS="on"
+    BUILD_AQM_UTILS="on"
+    BUILD_UPP="off"
+  fi
+  if [ "${PLATFORM}" = "wcoss2" ]; then
+    BUILD_POST_STAT="on"
+  else
+    BUILD_POST_STAT="off"
+  fi
+fi
+
 set -eu
 
 # automatically determine compiler
@@ -304,7 +323,8 @@ CMAKE_SETTINGS="\
  -DBUILD_UFS_UTILS=${BUILD_UFS_UTILS}\
  -DBUILD_UPP=${BUILD_UPP}\
  -DBUILD_GSI=${BUILD_GSI}\
- -DBUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}"
+ -DBUILD_NEXUS=${BUILD_NEXUS}\
+ -DBUILD_AQM_UTILS=${BUILD_AQM_UTILS}"
 
 if [ ! -z "${APPLICATION}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DAPP=${APPLICATION}"
@@ -318,6 +338,29 @@ fi
 if [ ! -z "${DISABLE_OPTIONS}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DDISABLE_OPTIONS=${DISABLE_OPTIONS}"
 fi
+if [ "${APPLICATION}" = "ATMAQ" ]; then
+  CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCPL_AQM=ON -DBUILD_POST_STAT=${BUILD_POST_STAT}"
+
+  # Copy module files to designated directory
+  EXTRN_BUILD_MOD_DIR="${SRW_DIR}/modulefiles/extrn_comp_build"
+  mkdir -p ${EXTRN_BUILD_MOD_DIR}
+  if [ "${BUILD_UFS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/ufs-weather-model/modulefiles/ufs_${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_ufs-weather-model.lua"
+    cp "${SRW_DIR}/sorc/ufs-weather-model/modulefiles/ufs_common.lua" ${EXTRN_BUILD_MOD_DIR}
+  fi
+  if [ "${BUILD_UFS_UTILS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/UFS_UTILS/modulefiles/build.${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_ufs-utils.lua"
+  fi
+  if [ "${BUILD_UPP}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/UPP/modulefiles/${PLATFORM}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_upp.lua" 
+  fi
+  if [ "${BUILD_NEXUS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/AQM-utils/parm/nexus_modulefiles/${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_nexus.lua"
+  fi
+  if [ "${BUILD_AQM_UTILS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/AQM-utils/modulefiles/build_${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_aqm-utils.lua"
+  fi
+fi
 
 # make settings
 MAKE_SETTINGS="-j ${BUILD_JOBS}"
@@ -393,6 +436,16 @@ if [ $USE_SUB_MODULES = true ]; then
         printf "... Loading RRFS_UTILS modules ...\n"
         load_module ""
     fi
+    if [ $BUILD_NEXUS = "on" ]; then
+        printf "... Loading NEXUS modules ...\n"
+        module use ${SRW_DIR}/sorc/arl_nexus/modulefiles
+        load_module ""
+    fi
+    if [ $BUILD_AQM_UTILS = "on" ]; then
+        printf "... Loading AQM-utils modules ...\n"
+        module use ${SRW_DIR}/sorc/AQM-utils/modulefiles
+        load_module ""
+    fi
 else
     module use ${SRW_DIR}/modulefiles
     module load ${MODULE_FILE}

jobs/JREGIONAL_POINT_SOURCE

@@ -8,7 +8,7 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_make_grid|task_run_fcst|cpl_aqm_parm" ${GLOBAL_VAR_DEFNS_FP}
+source_config_for_task "task_run_fcst|cpl_aqm_parm" ${GLOBAL_VAR_DEFNS_FP}
 . $USHdir/job_preamble.sh
 #
 #-----------------------------------------------------------------------
@@ -57,7 +57,7 @@ using NEXUS which will output for FV3 (in NetCDF format).
 if [ $RUN_ENVIR = "nco" ]; then
     export INPUT_DATA="${COMIN}"
 else
-    export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}"
+    export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT"
 fi
 mkdir_vrfy -p "${INPUT_DATA}"
 #
@@ -77,7 +77,7 @@ mkdir_vrfy -p "${DATA}"
 #
 #-----------------------------------------------------------------------
 #
-time $SCRIPTSdir/exregional_point_source.sh || \
+$SCRIPTSdir/exregional_point_source.sh || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #

modulefiles/build_cheyenne_gnu.lua

@@ -14,7 +14,7 @@ setenv("MKLROOT", "/glade/u/apps/opt/intel/2022.1/mkl/latest")
 load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0"))
 unload("netcdf")
 
-prepend_path("MODULEPATH","/glade/work/epicufsrt/GMTB/tools/gnu/11.2.0/hpc-stack-v1.2.0/modulefiles/stack")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/gnu11.2.0/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-gnu", os.getenv("hpc_gnu_ver") or "11.2.0"))
 load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25"))
@@ -25,8 +25,8 @@ load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5"))
 load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09"))
 load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4"))
 load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
-load(pathJoin("pio", os.getenv("pio_ver") or "2.5.3"))
-load(pathJoin("fms", os.getenv("fms_ver") or "2022.01"))
+load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7"))
+load(pathJoin("fms", os.getenv("fms_ver") or "2022.04"))
 
 setenv("CMAKE_C_COMPILER","mpicc")
 setenv("CMAKE_CXX_COMPILER","mpicxx")

modulefiles/build_cheyenne_intel.lua

@@ -14,7 +14,7 @@ load(pathJoin("python", os.getenv("python_ver") or "3.7.9"))
 load(pathJoin("ncarcompilers", os.getenv("ncarcompilers_ver") or "0.5.0"))
 unload("netcdf")
 
-prepend_path("MODULEPATH","/glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/modulefiles/stack")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/hpc-stack/intel2022.1/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1"))
 load(pathJoin("hpc-mpt", os.getenv("hpc_mpt_ver") or "2.25"))
@@ -25,8 +25,8 @@ load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5"))
 load(pathJoin("esmf", os.getenv("esmf_ver") or "8.3.0b09"))
 load(pathJoin("netcdf", os.getenv("netcdf_ver") or "4.7.4"))
 load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
-load(pathJoin("pio", os.getenv("pio_ver") or "2.5.3"))
-load(pathJoin("fms", os.getenv("fms_ver") or "2022.01"))
+load(pathJoin("pio", os.getenv("pio_ver") or "2.5.7"))
+load(pathJoin("fms", os.getenv("fms_ver") or "2022.04"))
 
 setenv("CMAKE_C_COMPILER","mpicc")
 setenv("CMAKE_CXX_COMPILER","mpicpc")

modulefiles/build_jet_intel.lua

@@ -10,7 +10,7 @@ load("sutils")
 
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
 
-prepend_path("MODULEPATH","/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack")
+prepend_path("MODULEPATH","/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
 load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))

modulefiles/build_orion_intel.lua

@@ -11,7 +11,7 @@ load("noaatools")
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.22.1"))
 load(pathJoin("python", os.getenv("python_ver") or "3.9.2"))
 
-prepend_path("MODULEPATH","/apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack")
+prepend_path("MODULEPATH","/work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
 load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
 load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))

modulefiles/build_wcoss2_intel.lua

@@ -11,37 +11,46 @@ load(pathJoin("PrgEnv-intel", os.getenv("PrgEnv_intel_ver")))
 load(pathJoin("intel", os.getenv("intel_ver")))
 load(pathJoin("craype", os.getenv("craype_ver")))
 load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver")))
-
 load(pathJoin("cmake", os.getenv("cmake_ver")))
-
-setenv("HPC_OPT","/apps/ops/para/libs")
-prepend_path("MODULEPATH", pathJoin("/apps/ops/para/libs/modulefiles/compiler/intel", os.getenv("intel_ver")))
-prepend_path("MODULEPATH", pathJoin("/apps/ops/para/libs/modulefiles/mpi/intel", os.getenv("intel_ver"), "cray-mpich", os.getenv("cray_mpich_ver")))
-
 load(pathJoin("jasper", os.getenv("jasper_ver")))
 load(pathJoin("zlib", os.getenv("zlib_ver")))
 load(pathJoin("libpng", os.getenv("libpng_ver")))
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
 load(pathJoin("netcdf", os.getenv("netcdf_ver")))
-load(pathJoin("pio", os.getenv("pio_ver")))
-load(pathJoin("esmf", os.getenv("esmf_ver")))
 load(pathJoin("fms", os.getenv("fms_ver")))
 load(pathJoin("bacio", os.getenv("bacio_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
 load(pathJoin("g2", os.getenv("g2_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
 load(pathJoin("ip", os.getenv("ip_ver")))
 load(pathJoin("sp", os.getenv("sp_ver")))
-load(pathJoin("w3nco", os.getenv("w3nco_ver")))
+load(pathJoin("w3emc", os.getenv("w3emc_ver")))
 
+load(pathJoin("w3nco", os.getenv("w3nco_ver")))
 load(pathJoin("libjpeg", os.getenv("libjpeg_ver")))
 load(pathJoin("cray-pals", os.getenv("cray_pals_ver")))
-
-load(pathJoin("w3emc", os.getenv("w3emc_ver")))
 load(pathJoin("nemsio", os.getenv("nemsio_ver")))
 load(pathJoin("sigio", os.getenv("sigio_ver")))
 load(pathJoin("sfcio", os.getenv("sfcio_ver")))
 load(pathJoin("wrf_io", os.getenv("wrf_io_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
+load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver")))
+
+
+setenv("HPC_OPT","/apps/ops/para/libs")
+prepend_path("MODULEPATH", pathJoin("/apps/ops/para/libs/modulefiles/compiler/intel", os.getenv("intel_para_ver")))
+prepend_path("MODULEPATH", pathJoin("/apps/ops/para/libs/modulefiles/mpi/intel", os.getenv("intel_para_ver"), "cray-mpich", os.getenv("cray_mpich_para_ver")))
+
+load(pathJoin("pio", os.getenv("pio_ver")))
+
+prepend_path("MODULEPATH", pathJoin("/apps/dev/lmodules/intel", os.getenv("intel_dev_ver")))
+prepend_path("MODULEPATH", pathJoin("/apps/dev/modulefiles/mpi/intel", os.getenv("intel_dev_ver"), "cray-mpich", os.getenv("cray_mpich_dev_ver")))
+
+load(pathJoin("esmf", os.getenv("esmf_ver")))
+load(pathJoin("gftl_shared", os.getenv("gftl_shared_ver")))
+load(pathJoin("mapl", os.getenv("mapl_ver")))
+
 
 setenv("CMAKE_C_COMPILER","cc")
 setenv("CMAKE_CXX_COMPILER","CC")

modulefiles/srw_common.lua

@@ -4,15 +4,15 @@ load_any("png/1.6.35", "libpng/1.6.37")
 
 load_any("netcdf/4.7.4", "netcdf-c/4.7.4")
 load_any("netcdf/4.7.4", "netcdf-fortran/4.5.4")
-load_any("pio/2.5.3", "parallelio/2.5.2")
+load_any("pio/2.5.7", "parallelio/2.5.2")
 load_any("esmf/8.3.0b09", "esmf/8.2.0")
-load("fms/2022.01")
+load("fms/2022.04")
 
 load("bufr/11.7.0")
 load("bacio/2.4.1")
-load("crtm/2.3.0")
+load("crtm/2.4.0")
 load("g2/3.4.5")
-load("g2tmpl/1.10.0")
+load("g2tmpl/1.10.2")
 load("ip/3.3.3")
 load("sp/2.3.3")
 load("w3emc/2.9.2")

modulefiles/tasks/hera/aqm_ics.local.lua

@@ -0,0 +1,2 @@
+load("miniconda_online-cmaq")
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/hera/aqm_lbcs.local.lua

@@ -0,0 +1 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/hera/fire_emission.local.lua

@@ -0,0 +1,2 @@
+load("hpss")
+load("miniconda_regional_workflow")

modulefiles/tasks/hera/miniconda_online-cmaq.lua

@@ -0,0 +1,5 @@
+prepend_path("MODULEPATH", "/contrib/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+setenv("AQM_ENV_FP", "/scratch2/NCEPDEV/naqfc/RRFS_CMAQ/PY_VENV")
+setenv("AQM_ENV", "online-cmaq")

modulefiles/tasks/hera/nexus_emission.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_online-cmaq")

modulefiles/tasks/hera/nexus_gfs_sfc.local.lua

@@ -0,0 +1,2 @@
+load("hpss")
+load("miniconda_regional_workflow")

modulefiles/tasks/hera/nexus_post_split.local.lua

@@ -0,0 +1,2 @@
+load("miniconda_online-cmaq")
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/hera/point_source.local.lua

@@ -0,0 +1 @@
+load("miniconda_online-cmaq")

modulefiles/tasks/hera/pre_post_stat.local.lua

@@ -0,0 +1 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/orion/aqm_ics.local.lua

@@ -0,0 +1,2 @@
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load("miniconda_online-cmaq")