ufs-community · zach1221 · May 31, 2024 · May 16, 2024 · May 16, 2024 · May 16, 2024
diff --git a/modulefiles/ufs_wcoss2.intel.lua b/modulefiles/ufs_wcoss2.intel.lua
@@ -18,27 +18,28 @@ load(pathJoin("cray-mpich", cray_mpich_ver))
 cmake_ver=os.getenv("cmake_ver") or "3.20.2"
 load(pathJoin("cmake", cmake_ver))
 
---prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/compiler/intel/19.1.3.304")
---prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.12")
+append_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.6.0_pnetcdf/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.12")
 
 local ufs_modules = {
-  {["jasper"]          = "2.0.25"},
-  {["zlib"]            = "1.2.11"},
-  {["libpng"]          = "1.6.37"},
-  {["hdf5-B"]          = "1.14.0"},
-  {["netcdf-B"]        = "4.9.2"},
-  {["pio-B"]           = "2.5.10"},
-  {["esmf-B"]          = "8.5.0"},
-  {["fms"]             = "2023.02.01"},
-  {["bacio"]           = "2.4.1"},
-  {["crtm"]            = "2.4.0"},
-  {["g2"]              = "3.4.5"},
-  {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
-  {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
-  {["gftl-shared"]     = "1.6.1"},
-  {["mapl-B"]          = "2.40.3"},
+  {["jasper"]      = "2.0.25"},
+  {["zlib"]        = "1.2.11"},
+  {["libpng"]      = "1.6.37"},
+  {["hdf5"]        = "1.14.0"},
+  {["netcdf"]      = "4.9.2"},
+  {["pio"]         = "2.5.10"},
+  {["esmf"]        = "8.6.0"},
+  {["fms"]         = "2023.04"},
+  {["bacio"]       = "2.4.1"},
+  {["crtm"]        = "2.4.0"},
+  {["g2"]          = "3.4.5"},
+  {["g2tmpl"]      = "1.10.2"},
+  {["ip"]          = "3.3.3"},
+  {["sp"]          = "2.3.3"},
+  {["w3emc"]       = "2.9.2"},
+  {["gftl-shared"] = "1.6.1"},
+  {["mapl"]        = "2.40.3-esmf-8.6.0"},
+  {["pnetcdf"]     = "1.12.2"},
+  {["scotch"]      = "7.0.4"},
 }
 
 for i = 1, #ufs_modules do
@@ -48,10 +49,6 @@ for i = 1, #ufs_modules do
   end
 end
 
-prepend_path("MODULEPATH", "/apps/prod/lmodules/INTEL_cray_mpich/19.1.3.304/cray-mpich/8.1.9")
-scotch_ver=os.getenv("scotch_ver") or "7.0.4"
-load(pathJoin("scotch",scotch_ver))
-
 setenv("CC", "cc")
 setenv("CXX", "CC")
 setenv("FC", "ftn")

diff --git a/tests/bl_date.conf b/tests/bl_date.conf
@@ -1 +1 @@
-export BL_DATE=20240524
+export BL_DATE=20240530
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -918,10 +918,6 @@ export_cice6() {
   export CICE_RESTART_DEFLATE=0
 
   export CICE_HISTORY_FORMAT='pnetcdf2'
-  if [[ ${MACHINE_ID} == wcoss2 ]]; then
-    export CICE_RESTART_FORMAT='hdf5'
-    export CICE_HISTORY_FORMAT='hdf5'
-  fi
   export CICE_HISTORY_IOTASKS=-99
   export CICE_HISTORY_REARR='box'
   export CICE_HISTORY_ROOT=-99

diff --git a/tests/logs/RegressionTests_acorn.log b/tests/logs/RegressionTests_acorn.log
diff --git a/tests/logs/RegressionTests_derecho.log b/tests/logs/RegressionTests_derecho.log
diff --git a/tests/logs/RegressionTests_gaea.log b/tests/logs/RegressionTests_gaea.log
diff --git a/tests/logs/RegressionTests_hera.log b/tests/logs/RegressionTests_hera.log
diff --git a/tests/logs/RegressionTests_hercules.log b/tests/logs/RegressionTests_hercules.log
diff --git a/tests/logs/RegressionTests_jet.log b/tests/logs/RegressionTests_jet.log
diff --git a/tests/logs/RegressionTests_orion.log b/tests/logs/RegressionTests_orion.log
diff --git a/tests/logs/RegressionTests_wcoss2.log b/tests/logs/RegressionTests_wcoss2.log
diff --git a/tests/rt.conf b/tests/rt.conf
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -28,7 +28,6 @@ usage() {
   echo "  -v  verbose output"
   echo "  -w  for weekly_test, skip comparing baseline results"
   echo
-  set -x
 }
 
 [[ $# -eq 0 ]] && usage
@@ -155,11 +154,11 @@ update_rtconf() {
 
 generate_log() {
   echo "rt.sh: Generating Regression Testing Log..."
-  set -x
   COMPILE_COUNTER=0
   FAILED_COMPILES=()
   TEST_COUNTER=0
   FAILED_TESTS=()
+  SKIPPED_TESTS=()
   FAILED_TEST_ID=()
   FAILED_COMPILE_LOGS=()
   FAILED_TEST_LOGS=()
@@ -255,14 +254,14 @@ EOF
           COMPILE_RESULT="FAILED: UNABLE TO START COMPILE"
           FAIL_LOG="N/A"
         elif [[ -f fail_compile_${COMPILE_ID} ]]; then
-          COMPILE_RESULT="FAILED: UNABLE TO COMPILE"
+          COMPILE_RESULT="FAILED: UNABLE TO FINISH COMPILE"
           FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
           if grep -q "quota" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
             COMPILE_RESULT="FAILED: DISK QUOTA ISSUE"
             FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
-          elif grep -q "timeout" "${LOG_DIR}/compile_${COMPILE_ID}.log"; then
-            COMPILE_RESULT="FAILED: TEST TIMED OUT"
-            FAIL_LOG="${LOG_DIR}/compile_${COMPILE_ID}.log"
+          elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"; then
+            COMPILE_RESULT="FAILED: COMPILE TIMED OUT"
+            FAIL_LOG="${RUNDIR_ROOT}/compile_${COMPILE_ID}/err"
           fi
         else
           COMPILE_RESULT="PASS"
@@ -330,21 +329,25 @@ EOF
         RT_TEST_TIME=""
         RT_TEST_MEM=""
         if [[ ${CREATE_BASELINE} == true && ${GEN_BASELINE} != "baseline" ]]; then
-          TEST_RESULT="SKIPPED (TEST DOES NOT GENERATE BASELINE)"
+          TEST_RESULT="SKIPPED: TEST DOES NOT GENERATE BASELINE"
+          SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
+        elif [[ ${COMPILE_RESULT} =~ FAILED ]]; then
+          TEST_RESULT="SKIPPED: ASSOCIATED COMPILE FAILED"
+          SKIPPED_TESTS+=("TEST ${TEST_NAME}_${COMPILER}: ${TEST_RESULT}")
         elif [[ ! -f "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log" ]]; then
-          TEST_RESULT="FAILED: UNABLE TO START RUN"
+          TEST_RESULT="FAILED: UNABLE TO START TEST"
           FAIL_LOG="N/A"
         elif [[ -f fail_test_${TEST_NAME}_${COMPILER} ]]; then
           if [[ -f "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log" ]]; then
             if grep -q "FAIL" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
-              TEST_RESULT="FAILED: UNABLE TO RUN COMPARISON"
+              TEST_RESULT="FAILED: UNABLE TO COMPLETE COMPARISON"
               FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
             # We need to catch a "PASS" in rt_*.log even if a fail_test_* files exists
             # I am not sure why this can happen.
             elif grep -q "PASS" "${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"; then
               TEST_RESULT="PASS"
             else
-              TEST_RESULT="FAILED: BASELINE COMPARISON"
+              TEST_RESULT="FAILED: UNSUCCESSFUL BASELINE COMPARISON"
               FAIL_LOG="${LOG_DIR}/rt_${TEST_NAME}_${COMPILER}.log"
             fi
           else
@@ -354,9 +357,9 @@ EOF
           if grep -q "quota" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
             TEST_RESULT="FAILED: DISK QUOTA ISSUE"
             FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
-          elif grep -q "timeout" "${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"; then
+          elif grep -q "TIME LIMIT" "${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"; then
             TEST_RESULT="FAILED: TEST TIMED OUT"
-            FAIL_LOG="${LOG_DIR}/run_${TEST_NAME}_${COMPILER}.log"
+            FAIL_LOG="${RUNDIR_ROOT}/${TEST_NAME}_${COMPILER}/err"
           fi
         else
           TEST_RESULT="PASS"
@@ -399,7 +402,7 @@ Starting Date/Time: ${TEST_START_TIME}
 Ending Date/Time: ${TEST_END_TIME}
 Total Time: ${elapsed_time}
 Compiles Completed: $((COMPILE_COUNTER-${#FAILED_COMPILES[@]}))/${COMPILE_COUNTER}
-Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}))/${TEST_COUNTER}
+Tests Completed: $((TEST_COUNTER-${#FAILED_TESTS[@]}-${#SKIPPED_TESTS[@]}))/${TEST_COUNTER}
 EOF
   # PRINT FAILED COMPILES
   if [[ "${#FAILED_COMPILES[@]}" -ne "0" ]]; then
@@ -674,14 +677,10 @@ done
 #B&N not run together
 [[ ${NEW_BASELINES_FILE} != '' && ${RUN_SINGLE_TEST} == true ]] && die "-b and -n options cannot be used at the same time"
 
-[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'
-
 if [[ ${RTVERBOSE} == true ]]; then
   set -x
 fi
 
-[[ -o xtrace ]] && set_x='set -x' || set_x='set +x'
-
 if [[ -z "${ACCNR}" ]]; then
   echo "Please use -a <account> to set group account to use on HPC"
   exit 1
@@ -694,12 +693,10 @@ echo "Account: ${ACCNR}"
 case ${MACHINE_ID} in
   wcoss2|acorn)
     echo "rt.sh: Setting up WCOSS2/Acorn"
-    set -x
     if [[ "${ECFLOW:-false}" == true ]] ; then
       module load ecflow/5.6.0.13
     fi
     module load intel/19.1.3.304 python/3.8.6
-    export colonifnco=":output"  # hack
 
     DISKNM="/lfs/h2/emc/nems/noscrub/emc.nems/RT"
     QUEUE="dev"
@@ -714,7 +711,6 @@ case ${MACHINE_ID} in
     ;;
   gaea)
     echo "rt.sh: Setting up gaea..."
-    set -x
     if [[ "${ROCOTO:-false}" == true ]] ; then
       module use /ncrc/proj/epic/rocoto/modulefiles
       module load rocoto
@@ -747,7 +743,6 @@ case ${MACHINE_ID} in
     ;;
   hera)
     echo "rt.sh: Setting up hera..."
-    set -x
     if [[ "${ROCOTO:-false}" == true ]] ; then
       module load rocoto
       ROCOTO_SCHEDULER=slurm
@@ -770,7 +765,6 @@ case ${MACHINE_ID} in
     ;;
   orion)
     echo "rt.sh: Setting up orion..."
-    set -x
     module load git/2.28.0
     module load gcc/10.2.0
     module load python/3.9.2
@@ -800,7 +794,6 @@ case ${MACHINE_ID} in
     ;;
   hercules)
     echo "rt.sh: Setting up hercules..."
-    set -x
     if [[ "${ROCOTO:-false}" == true ]] ; then
       module load contrib rocoto
       ROCOTO_SCHEDULER="slurm"
@@ -828,7 +821,6 @@ case ${MACHINE_ID} in
     ;;
   jet)
     echo "rt.sh: Setting up jet..."
-    set -x
     CurJetOS=$(lsb_release -is)
     echo "=======Running on ${CurJetOS}======="
     if [[ ${CurJetOS} == "CentOS" ]]; then
@@ -861,7 +853,6 @@ case ${MACHINE_ID} in
     ;;
   s4)
     echo "rt.sh: Setting up s4..."
-    set -x
     if [[ "${ROCOTO:-false}" == true ]] ; then
       module load rocoto/1.3.2
       ROCOTO_SCHEDULER=slurm
@@ -892,7 +883,6 @@ case ${MACHINE_ID} in
     ;;
   derecho)
     echo "rt.sh: Setting up derecho..."
-    set -x
     if [[ "${ROCOTO:-false}" == true ]] ; then
       module use /glade/work/epicufsrt/contrib/derecho/rocoto/modulefiles
       module load rocoto
@@ -930,7 +920,6 @@ case ${MACHINE_ID} in
     ;;
   stampede)
     echo "rt.sh: Setting up stampede..."
-    set -x
     export PYTHONPATH=
     if [[ "${ECFLOW:-false}" == true ]] ; then
       ECFLOW_START=
@@ -948,7 +937,6 @@ case ${MACHINE_ID} in
     ;;
   expanse)
     echo "rt.sh: Setting up expanse..."
-    set -x
     export PYTHONPATH=
 
     if [[ "${ECFLOW:-false}" == true ]] ; then
@@ -965,7 +953,6 @@ case ${MACHINE_ID} in
     ;;
   noaacloud)
     echo "rt.sh: Setting up noaacloud..."
-    set -x
     export PATH="/contrib/EPIC/bin:${PATH}"
     module use /apps/modules/modulefiles
 
@@ -987,7 +974,6 @@ case ${MACHINE_ID} in
     die "Unknown machine ID, please edit detect_machine.sh file"
     ;;
 esac
-eval "${set_x}"
 
 mkdir -p "${STMP}/${USER}"
 
@@ -1374,5 +1360,4 @@ fi
 
 ## Lets verify all tests were run and that they passed
 generate_log
-eval "${set_x}"
 echo "******Regression Testing Script Completed******"