parallel netcdf writes, inpost with no thread (#43)

* update parallel netcdf module files on dell * update fv3 to netcdf_parallel branch * update module files for hera, dell and conf file for cray * update post lib in cray module file * fix netlib on hera * Update FV3 submodule * Update baselines to develop-20200202 * Regtest logs on wcoss_dell_p3 * Regtest logs on wcoss_cray * Regtest logs on hera * update FV3 to point to NOAA-EMC develop branch Co-authored-by: Dusan Jovic <[email protected]>
ufs-community · Feb 4, 2020 · 31821b0 · 31821b0
1 parent 52795b8
commit 31821b0
Show file tree

Hide file tree

Showing 31 changed files with 11,609 additions and 13,229 deletions.
diff --git a/FV3 b/FV3
diff --git a/compsets/hera.input b/compsets/hera.input
@@ -31,8 +31,8 @@ platform hera.intel {
     # INPUTS is the input directory, which should contain fix and parm
     # files, plus any restarts or other inputs.
 
-    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
-    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
+    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
     INPUTS="@[BASELINE]"
 
     default_resources={

diff --git a/compsets/wcoss.input b/compsets/wcoss.input
@@ -28,8 +28,8 @@ platform wcoss.phase2 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
     MACHINE_ID='wcoss.phase2'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
 
     default_resources={
         TASKS=156
@@ -126,8 +126,8 @@ platform wcoss.phase1 {
     C768_THRD=1
 
     MACHINE_ID='wcoss.phase1'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=2700
@@ -213,8 +213,8 @@ platform wcoss_dell_p3 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
 
-    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
-    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
+    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202/"
+    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202/"
 
     default_resources={
         TASKS=156
@@ -314,8 +314,8 @@ platform wcoss.cray {
     BUILD_WALLTIME="3600"
     DEFAULT_TEST_WALLTIME=1800
 
-    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
-    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
+    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200202"
 
     execution_time_modules=[[[
       module load alps

diff --git a/conf/configure.fv3.hera.intel b/conf/configure.fv3.hera.intel
@@ -54,7 +54,7 @@ NETCDF_INC = -I$(NETCDF_ROOT)/include
 ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
   NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
 else
-  NETCDF_LIB += -L$(NETCDF)/lib -lnetcdff -lnetcdf
+  NETCDF_LIB = -L$(NETCDF)/lib -lnetcdff -lnetcdf
 endif
 
 FPPFLAGS := -fpp -Wp,-w $(INCLUDE)

diff --git a/conf/configure.fv3.wcoss_cray b/conf/configure.fv3.wcoss_cray
@@ -47,7 +47,7 @@ CFLAGS := $(INCLUDE)
 FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte
 
 CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO  -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
-CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML
+CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_PARALLEL_NETCDF
 
 ifeq ($(HYDRO),Y)
 CPPDEFS +=

diff --git a/log/report-hera.intel-log/build_fv3.exe.log b/log/report-hera.intel-log/build_fv3.exe.log
diff --git a/log/report-hera.intel-log/build_fv3_32bit.exe.log b/log/report-hera.intel-log/build_fv3_32bit.exe.log
diff --git a/log/report-hera.intel-log/build_fv3_32bit_debug.exe.log b/log/report-hera.intel-log/build_fv3_32bit_debug.exe.log
diff --git a/log/report-hera.intel-log/build_fv3_appbuilder.exe.log b/log/report-hera.intel-log/build_fv3_appbuilder.exe.log
diff --git a/log/report-hera.intel-log/build_fv3_ww3.exe.log b/log/report-hera.intel-log/build_fv3_ww3.exe.log
diff --git a/log/report-hera.intel-log/rtreport.txt b/log/report-hera.intel-log/rtreport.txt
diff --git a/log/report-wcoss.cray-log/build_fv3.exe.log b/log/report-wcoss.cray-log/build_fv3.exe.log
diff --git a/log/report-wcoss.cray-log/build_fv3_32bit.exe.log b/log/report-wcoss.cray-log/build_fv3_32bit.exe.log
diff --git a/log/report-wcoss.cray-log/build_fv3_appbuilder.exe.log b/log/report-wcoss.cray-log/build_fv3_appbuilder.exe.log
diff --git a/log/report-wcoss.cray-log/build_fv3_ww3.exe.log b/log/report-wcoss.cray-log/build_fv3_ww3.exe.log
diff --git a/log/report-wcoss.cray-log/rtreport.txt b/log/report-wcoss.cray-log/rtreport.txt
diff --git a/log/report-wcoss_dell_p3-log/build_fv3.exe.log b/log/report-wcoss_dell_p3-log/build_fv3.exe.log
diff --git a/log/report-wcoss_dell_p3-log/build_fv3_32bit.exe.log b/log/report-wcoss_dell_p3-log/build_fv3_32bit.exe.log
diff --git a/log/report-wcoss_dell_p3-log/build_fv3_appbuilder.exe.log b/log/report-wcoss_dell_p3-log/build_fv3_appbuilder.exe.log
diff --git a/log/report-wcoss_dell_p3-log/build_fv3_ww3.exe.log b/log/report-wcoss_dell_p3-log/build_fv3_ww3.exe.log
diff --git a/log/report-wcoss_dell_p3-log/rtreport.txt b/log/report-wcoss_dell_p3-log/rtreport.txt
diff --git a/modulefiles/hera.intel/fv3 b/modulefiles/hera.intel/fv3
@@ -23,7 +23,6 @@ module load contrib sutils
 ##
 module load intel/18.0.5.274
 module load impi/2018.0.4
-module load netcdf/4.7.0
 
 ##
 ## NCEP libraries (temporary version to match the CCPP requirements)
@@ -42,14 +41,16 @@ module load jasper/1.900.1
 module load png/1.2.44
 module load z/1.2.11
 ## load modules for nceppost grib
-module load post/8.0.1
+module load post/8.0.4
 
 ##
 ## load ESMF library for above compiler / MPI combination
 ## use pre-compiled EMSF library for above compiler / MPI combination
 ##
 module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0
+module load hdf5_parallel/1.10.6
+module load netcdf_parallel/4.7.4
+module load esmf/8.0.0_ParallelNetCDF
 
 ##
 ## load cmake

diff --git a/modulefiles/wcoss_cray/fv3 b/modulefiles/wcoss_cray/fv3
@@ -47,7 +47,7 @@ module load w3emc/2.3.1
 module load nemsio/2.2.4
 
 #post lib
-module load post/8.0.1
+module load post/8.0.4
 
 ## WCOSS Cray execution prereqs:
 module load rca

diff --git a/modulefiles/wcoss_dell_p3/fv3 b/modulefiles/wcoss_dell_p3/fv3
@@ -18,8 +18,6 @@ module-whatis "loads NEMS FV3 prerequisites on Venus and Mars"
 module load ips/18.0.1.163
 module load impi/18.0.1
 module load lsf/10.1
-module load NetCDF/4.5.0
-module load HDF5-serial/1.10.1
 module load cmake/3.10.0
 module load lsf/10.1
 
@@ -42,10 +40,12 @@ module load libpng/1.2.59
 module load zlib/1.2.11
 
 # load post lib
-module load post/8.0.1
+module load post/8.0.4
 
 module use /gpfs/dell2/emc/modeling/noscrub/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0
+module load hdf5_parallel/1.10.6
+module load netcdf_parallel/4.7.4  
+module load esmf/8.0.0_ParallelNetCDF
 
 ##
 ## load cmake