From 8fee36f0307b0c08080e3a8fa8fa6703e7da5fce Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Tue, 6 Aug 2024 11:02:45 -0400
Subject: [PATCH] Clean-up temporary rundirs - take 2. (#2753)

This PR:
- is a follow-up to a previous PR that aggressively pruned run
directories.
- removes run directories for the current cycle in the clean-up if the
cycle is successful. If the cycle is not successful, cleanup is not
called and all run directories for the cycle are safe from being purged.
- also updates the PR template to list/query for any updates to
submodules.

---------

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
Co-authored-by: David Huber <david.huber@noaa.gov>
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 .github/pull_request_template.md  | 11 +++++++++++
 jobs/JGFS_ATMOS_FBWIND            |  2 +-
 jobs/JGFS_ATMOS_GEMPAK_META       |  2 +-
 jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC |  2 +-
 jobs/JGLOBAL_CLEANUP              |  6 +++---
 parm/config/gefs/config.base      |  1 -
 parm/config/gfs/config.base       |  1 -
 parm/config/gfs/config.fbwind     | 11 +++++++++++
 parm/config/gfs/config.resources  |  7 +++++++
 scripts/exglobal_cleanup.sh       | 33 +++++++++----------------------
 ush/jjob_header.sh                |  5 ++---
 workflow/rocoto/tasks.py          |  4 +++-
 12 files changed, 49 insertions(+), 36 deletions(-)
 create mode 100644 parm/config/gfs/config.fbwind

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index dbebfe8f6e..3f8fe65065 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
+  - [ ] GSI
+  - [ ] GSI-monitor
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
+  - [ ] wxflow
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.
diff --git a/jobs/JGFS_ATMOS_FBWIND b/jobs/JGFS_ATMOS_FBWIND
index 38e524afea..2071954b0d 100755
--- a/jobs/JGFS_ATMOS_FBWIND
+++ b/jobs/JGFS_ATMOS_FBWIND
@@ -6,7 +6,7 @@
 # GFS FBWIND PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base fbwind"
 
 ###################################
 # Specify NET and RUN Name and model
diff --git a/jobs/JGFS_ATMOS_GEMPAK_META b/jobs/JGFS_ATMOS_GEMPAK_META
index 3af463e241..480dc0f1d6 100755
--- a/jobs/JGFS_ATMOS_GEMPAK_META
+++ b/jobs/JGFS_ATMOS_GEMPAK_META
@@ -6,7 +6,7 @@
 # GFS GEMPAK META PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base gempak"
 
 
 ###############################################
diff --git a/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC b/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
index 0be52c0d19..a82d2805eb 100755
--- a/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
+++ b/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC
@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base gempak"
 
 ############################################
 # Set up model and cycle specific variables
diff --git a/jobs/JGLOBAL_CLEANUP b/jobs/JGLOBAL_CLEANUP
index 2f81003989..f3cfcae511 100755
--- a/jobs/JGLOBAL_CLEANUP
+++ b/jobs/JGLOBAL_CLEANUP
@@ -5,13 +5,13 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "cleanup" -c "base cleanup"
 
 "${SCRgfs}/exglobal_cleanup.sh"
 status=$?
-[[ ${status} -ne 0 ]] && exit "${status}"
+(( status != 0 )) && exit "${status}"
 
 ##########################################
 # Remove the Temporary working directory
 ##########################################
-cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
-[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+# DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# is removed in exglobal_cleanup.sh, nothing to do here.
 
 exit 0
 
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index d43ea09206..735743b568 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -97,7 +97,6 @@ export PSLOT="@PSLOT@"
 export EXPDIR="@EXPDIR@/${PSLOT}"
 export ROTDIR="@COMROOT@/${PSLOT}"
 
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
index 4ce3c8f45c..56005199aa 100644
--- a/parm/config/gfs/config.base
+++ b/parm/config/gfs/config.base
@@ -127,7 +127,6 @@ export DUMP_SUFFIX=""
 if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then
     export DUMP_SUFFIX="p"              # Use dumps from NCO GFS v15.3 parallel
 fi
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 
diff --git a/parm/config/gfs/config.fbwind b/parm/config/gfs/config.fbwind
new file mode 100644
index 0000000000..49fdb9e7b4
--- /dev/null
+++ b/parm/config/gfs/config.fbwind
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.gempak ##########
+# GFS fbwind step specific
+
+echo "BEGIN: config.fbwind"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" fbwind
+
+echo "END: config.fbwind"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 414b1ba16e..7b737d05f0 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -1124,6 +1124,13 @@ case ${step} in
     memory_gfs="2GB"
     ;;
 
+  "fbwind")
+    walltime="00:05:00"
+    ntasks=1
+    threads_per_task=1
+    memory="4GB"
+    ;;
+
   "mos_stn_prep")
     walltime="00:10:00"
     ntasks=3
diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index dcf1baef31..75b1f927bc 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -12,41 +12,26 @@ if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
-# Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days
-# purge_every_days should be a positive integer
-#purge_every_days=3
-
-# Find and delete files older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
-
-# Find and delete directories older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
+# In XML, DATAROOT is defined as:
+#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# cleanup is only executed after the entire cycle is successfully completed.
+# removing DATAROOT should be possible if that is the case.
+rm -rf "${DATAROOT}"
 
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################
 
-###############################################################
-# Clean up previous cycles; various depths
-# PRIOR CYCLE: Leave the prior cycle alone
-# shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${assim_freq} hours")
-# PREVIOUS to the PRIOR CYCLE
-GDATE=$(date --utc +%Y%m%d%H -d "${GDATE:0:8} ${GDATE:8:2} -${assim_freq} hours")
-
-# Remove the TMPDIR directory
-# TODO Only prepbufr is currently using this directory, and all jobs should be
-#   cleaning up after themselves anyway
-COMIN="${DATAROOT}/${GDATE}"
-[[ -d ${COMIN} ]] && rm -rf "${COMIN}"
-
 if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
 fi
 
+###############################################################
+# Clean up previous cycles; various depths
+
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours" )
+last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
 first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
 last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
 function remove_files() {
diff --git a/ush/jjob_header.sh b/ush/jjob_header.sh
index 45fa6402ae..dc75437f1a 100755
--- a/ush/jjob_header.sh
+++ b/ush/jjob_header.sh
@@ -39,7 +39,6 @@
 #                     [default: "YES"]
 #   - $pid          : Override the default process id
 #                     [default: $$]
-# 
 
 OPTIND=1
 while getopts "c:e:" option; do
@@ -99,7 +98,7 @@ for config in "${configs[@]:-''}"; do
     status=$?
     if (( status != 0 )); then
         echo "FATAL [${BASH_SOURCE[0]}]: Unable to load config config.${config}"
-    	exit "${status}"
+        exit "${status}"
     fi
 done
 
@@ -111,5 +110,5 @@ source "${HOMEgfs}/env/${machine}.env" "${env_job}"
 status=$?
 if (( status != 0 )); then
     echo "FATAL [${BASH_SOURCE[0]}]: Error while sourcing machine environment ${machine}.env for job ${env_job}"
-	exit "${status}"
+    exit "${status}"
 fi
diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 0c84eaba15..353d2aa943 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -61,6 +61,8 @@ def __init__(self, app_config: AppConfig, run: str) -> None:
 
         self.n_tiles = 6  # TODO - this needs to be elsewhere
 
+        # DATAROOT is set by prod_envir in ops.  Here, we use `STMP` to construct DATAROOT
+        dataroot_str = f"{self._base.get('STMP')}/RUNDIRS/{self._base.get('PSLOT')}/{self.run}.<cyclestr>@Y@m@d@H</cyclestr>"
         envar_dict = {'RUN_ENVIR': self._base.get('RUN_ENVIR', 'emc'),
                       'HOMEgfs': self.HOMEgfs,
                       'EXPDIR': self._base.get('EXPDIR'),
@@ -70,7 +72,7 @@ def __init__(self, app_config: AppConfig, run: str) -> None:
                       'PDY': '<cyclestr>@Y@m@d</cyclestr>',
                       'cyc': '<cyclestr>@H</cyclestr>',
                       'COMROOT': self._base.get('COMROOT'),
-                      'DATAROOT': self._base.get('DATAROOT')}
+                      'DATAROOT': dataroot_str}
 
         self.envars = self._set_envars(envar_dict)