Single precision suite-4 from NEPTUNE -- branch jm-nrl-32bitfp-24cc09e -> main

CMakeLists.txt

@@ -82,6 +82,20 @@ foreach(typedef_module ${TYPEDEFS})
     list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module})
 endforeach()
 
+# This is a hack because I don't know how to get the pre_build to add these
+# modules to the install list --PAR
+if (PROJECT STREQUAL "CCPP-NEPTUNE")
+    # added for CCPP 5: ozne_def.mod, h2o_def.mod, module_radiation_surface.mod
+    list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/namelist_soilveg.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/set_soilveg_mod.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/physcons.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/funcphys.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/ozne_def.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/h2o_def.mod
+                            ${CMAKE_CURRENT_BINARY_DIR}/module_radiation_surface.mod
+        )
+endif (PROJECT STREQUAL "CCPP-NEPTUNE")
+
 #------------------------------------------------------------------------------
 # Set the sources: physics schemes
 set(SCHEMES $ENV{CCPP_SCHEMES})
@@ -291,30 +305,52 @@ elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel")
                  APPEND_STRING PROPERTY COMPILE_FLAGS " ${CMAKE_Fortran_FLAGS_DEFAULT_PREC} ${OpenMP_Fortran_FLAGS} ")
 
   else (PROJECT STREQUAL "CCPP-FV3")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_bfmicrophysics.f
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/sflx.f
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diff.f
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diag.f
-                                PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_model.f90
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/calpreciptype.f90
-                                PROPERTIES COMPILE_FLAGS "-r8 -free ${OpenMP_Fortran_FLAGS} ")
     SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/mersenne_twister.f
                                 PROPERTIES COMPILE_FLAGS "-r8 -ftz ${OpenMP_Fortran_FLAGS} ")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_water_prop.f90
-                                PROPERTIES COMPILE_FLAGS "-extend-source 132 -r8 -free ${OpenMP_Fortran_FLAGS} ")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/aer_cloud.F
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/wv_saturation.F
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/cldwat2m_micro.F
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/surface_perturbation.F90
-                                PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_mp_thompson_make_number_concentrations.F90
-                                PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
-    SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_SF_JSFC.F90
-                                ${LOCAL_CURRENT_SOURCE_DIR}/physics/module_BL_MYJPBL.F90
-                                PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
+    if ( $ENV{SINGLE} MATCHES "t" )
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_bfmicrophysics.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sflx.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diff.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diag.f
+                                  PROPERTIES COMPILE_FLAGS " ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_model.f90
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/calpreciptype.f90
+                                  PROPERTIES COMPILE_FLAGS " -free ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_water_prop.f90
+                                  PROPERTIES COMPILE_FLAGS "-extend-source 132 -free ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/aer_cloud.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/wv_saturation.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/cldwat2m_micro.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/surface_perturbation.F90
+                                  PROPERTIES COMPILE_FLAGS " ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_mp_thompson_make_number_concentrations.F90
+                                  PROPERTIES COMPILE_FLAGS "${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_SF_JSFC.F90
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/module_BL_MYJPBL.F90
+                                  PROPERTIES COMPILE_FLAGS " ${OpenMP_Fortran_FLAGS} ")
+    else ( $ENV{SINGLE} MATCHES "t" )
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_bfmicrophysics.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sflx.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diff.f
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/sfc_diag.f
+                                  PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_model.f90
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/calpreciptype.f90
+                                  PROPERTIES COMPILE_FLAGS "-r8 -free ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_nst_water_prop.f90
+                                  PROPERTIES COMPILE_FLAGS "-r8 -extend-source 132 -free ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/aer_cloud.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/wv_saturation.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/cldwat2m_micro.F
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/surface_perturbation.F90
+                                  PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_mp_thompson_make_number_concentrations.F90
+                                  PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
+      SET_SOURCE_FILES_PROPERTIES(${LOCAL_CURRENT_SOURCE_DIR}/physics/module_SF_JSFC.F90
+                                  ${LOCAL_CURRENT_SOURCE_DIR}/physics/module_BL_MYJPBL.F90
+                                  PROPERTIES COMPILE_FLAGS "-r8 ${OpenMP_Fortran_FLAGS} ")
+    endif ( $ENV{SINGLE} MATCHES "t" )
   endif (PROJECT STREQUAL "CCPP-FV3")
-
 else()
   message ("CMAKE_Fortran_COMPILER full path: " ${CMAKE_Fortran_COMPILER})
   message ("Fortran compiler: " ${CMAKE_Fortran_COMPILER_ID})
@@ -348,7 +384,7 @@ endforeach()
 set_target_properties(ccppphys PROPERTIES VERSION ${PROJECT_VERSION}
                                      SOVERSION ${PROJECT_VERSION_MAJOR})
 
-if (PROJECT STREQUAL "CCPP-FV3")
+if (PROJECT STREQUAL "CCPP-FV3" OR PROJECT STREQUAL "CCPP-NEPTUNE")
   # Define where to install the library
   install(TARGETS ccppphys
           EXPORT ccppphys-targets
@@ -364,4 +400,4 @@ if (PROJECT STREQUAL "CCPP-FV3")
   # Define where to install the C headers and Fortran modules
   #install(FILES ${HEADERS_C} DESTINATION include)
   install(FILES ${MODULES_F90} DESTINATION include)
-endif (PROJECT STREQUAL "CCPP-FV3")
+endif (PROJECT STREQUAL "CCPP-FV3" OR PROJECT STREQUAL "CCPP-NEPTUNE")

physics/GFS_MP_generic.F90

@@ -94,6 +94,7 @@ subroutine GFS_MP_generic_post_run(im, levs, kdt, nrcm, ncld, nncl, ntcw, ntrac,
         graupelprv, draincprv, drainncprv, diceprv, dsnowprv, dgraupelprv, dtp, errmsg, errflg)
 !
       use machine, only: kind_phys
+      use mod_calpreciptype
 
       implicit none
 

physics/GFS_rad_time_vary.fv3.F90

@@ -10,12 +10,21 @@ module GFS_rad_time_vary
 
       contains
 
+!>\defgroup GFS_rad_time_vary GFS RRTMG Update
+!!\ingroup RRTMG
+!! @{
+!! \section arg_table_GFS_rad_time_vary_init Argument Table
+!!
+      subroutine GFS_rad_time_vary_init
+      end subroutine GFS_rad_time_vary_init
+
 !>\defgroup mod_GFS_rad_time_vary GFS Radiation Time Update
 !> @{
 !> \section arg_table_GFS_rad_time_vary_timestep_init Argument Table
 !! \htmlinclude GFS_rad_time_vary_timestep_init.html
 !!
-      subroutine GFS_rad_time_vary_timestep_init (                                     &
+      subroutine GFS_rad_time_vary_timestep_init (nthrds, blksz, lrseeds, &
+               rseeds,      &
               lslwr, lsswr, isubc_lw, isubc_sw, icsdsw, icsdlw, cnx, cny, isc, jsc,    &
               imap, jmap, sec, kdt, imp_physics, imp_physics_zhao_carr, ps_2delt,      &
               ps_1delt, t_2delt, t_1delt, qv_2delt, qv_1delt, t, qv, ps, errmsg, errflg)
@@ -28,6 +37,10 @@ subroutine GFS_rad_time_vary_timestep_init (
          implicit none
 
          ! Interface variables
+         integer,                intent(in)    :: nthrds
+         integer,                intent(in)    :: blksz(:)
+         logical,                intent(in)    :: lrseeds
+         integer,                intent(in)    :: rseeds(:,:)
          integer,                intent(in)    :: isubc_lw, isubc_sw, cnx, cny, isc, jsc, kdt
          integer,                intent(in)    :: imp_physics, imp_physics_zhao_carr
          logical,                intent(in)    :: lslwr, lsswr
@@ -46,7 +59,7 @@ subroutine GFS_rad_time_vary_timestep_init (
 
          ! Local variables
          type (random_stat) :: stat
-         integer :: ix, j, i, nblks, ipseed
+         integer :: ii, ix, nb, j, i, nblks, ipseed
          integer :: numrdm(cnx*cny*2)
 
          ! Initialize CCPP error handling variables
@@ -55,24 +68,50 @@ subroutine GFS_rad_time_vary_timestep_init (
 
          if (lsswr .or. lslwr) then
 
-           !--- call to GFS_radupdate_timestep_init is now in GFS_rrtmg_setup_timestep_init
+           nblks = size(blksz)
+
+           !--- call to GFS_radupdate_run is now in GFS_rrtmg_setup_run
+
+!$OMP parallel num_threads(nthrds) default(none)        &
+!$OMP          private (nb,ix,ii, i,j)                      &
+!$OMP          shared (lrseeds,isubc_lw,isubc_sw,ipsdlim,ipsd0,ipseed) &
+!$OMP          shared (cnx,cny,sec,numrdm,stat,nblks,isc,jsc)  &
+!$OMP          shared (blksz,icsdsw,icsdlw,jmap,imap)
 
            !--- set up random seed index in a reproducible way for entire cubed-sphere face (lat-lon grid)
            if ((isubc_lw==2) .or. (isubc_sw==2)) then
-             ipseed = mod(nint(con_100*sqrt(sec)), ipsdlim) + 1 + ipsd0
-             call random_setseed (ipseed, stat)
-             call random_index (ipsdlim, numrdm, stat)
-
-             do ix=1,size(jmap)
-               j = jmap(ix)
-               i = imap(ix)
-               !--- for testing purposes, replace numrdm with '100'
-               icsdsw(ix) = numrdm(i+isc-1 + (j+jsc-2)*cnx)
-               icsdlw(ix) = numrdm(i+isc-1 + (j+jsc-2)*cnx + cnx*cny)
-             enddo
-
+!NRL If random seeds supplied by NEPTUNE
+             if(lrseeds) then
+               ii = 1
+               do nb=1,nblks
+                 do ix=1,blksz(nb)
+                   icsdsw(ix) = rseeds(ix,1)
+                   icsdlw(ix) = rseeds(ix,2)
+                 end do
+               enddo
+             else
+!$OMP single
+               ipseed = mod(nint(con_100*sqrt(sec)), ipsdlim) + 1 + ipsd0
+               call random_setseed (ipseed, stat)
+               call random_index (ipsdlim, numrdm, stat)
+!$OMP end single
+
+!$OMP do schedule (dynamic,1)
+               do nb=1,nblks
+                 do ix=1,blksz(nb)
+                   j = jmap(ix)
+                   i = imap(ix)
+                   !--- for testing purposes, replace numrdm with '100'
+                   icsdsw(ix) = numrdm(i+isc-1 + (j+jsc-2)*cnx)
+                   icsdlw(ix) = numrdm(i+isc-1 + (j+jsc-2)*cnx + cnx*cny)
+                 enddo
+               enddo
+!$OMP end do
+             end if
            endif  ! isubc_lw and isubc_sw
 
+!$OMP end parallel
+
            if (imp_physics == imp_physics_zhao_carr) then
              if (kdt == 1) then
                t_2delt  = t

physics/GFS_rad_time_vary.fv3.meta

@@ -7,6 +7,38 @@
 [ccpp-arg-table]
   name = GFS_rad_time_vary_timestep_init
   type = scheme
+[nthrds]
+  standard_name = omp_threads
+  long_name = number of OpenMP threads available for physics schemes
+  units = count
+  dimensions = ()
+  type = integer
+  intent = in
+  optional = F
+[blksz]
+  standard_name = ccpp_block_sizes
+  long_name = for explicit data blocking: block sizes of all blocks
+  units = count
+  dimensions = (ccpp_block_count)
+  type = integer
+  intent = in
+  optional = F
+[lrseeds]
+  standard_name = flag_for_model_specific_random_seeds
+  long_name = flag for model specific random seeds
+  units = flag
+  dimensions = ()
+  type = logical
+  intent = in
+  optional = F
+[rseeds]
+  standard_name = model_specific_random_seeds
+  long_name = model specific random seeds
+  units = none
+  dimensions = (horizontal_dimension,number_of_random_streams)
+  type = integer
+  intent = in
+  optional = F
 [lslwr]
   standard_name = flag_to_calc_lw
   long_name = logical flags for lw radiation calls

physics/GFS_rrtmg_post.meta

@@ -42,7 +42,7 @@
 [ltp]
   standard_name = extra_top_layer
   long_name = extra top layers
-  units = none
+  units = count
   dimensions = ()
   type = integer
   intent = in

physics/GFS_rrtmg_pre.F90

@@ -16,7 +16,7 @@ end subroutine GFS_rrtmg_pre_init
 !!
       ! Attention - the output arguments lm, im, lmk, lmp must not be set
       ! in the CCPP version - they are defined in the interstitial_create routine
-      subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
+      subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, lextop, ltp, n_var_lndp,        &
         imfdeepcnv, imfdeepcnv_gf, me, ncnd, ntrac, num_p3d, npdf3d, ncnvcld3d,&
         ntqv, ntcw,ntiw, ntlnc, ntinc, ncld, ntrw, ntsw, ntgl, ntwa, ntoz,     &
         ntclamt, nleffr, nieffr, nseffr, lndp_type, kdt, imp_physics,          &
@@ -37,11 +37,10 @@ subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
         clouds9, cldsa, cldfra, faersw1, faersw2, faersw3, faerlw1, faerlw2,   &
         faerlw3, alpha, errmsg, errflg)
 
-      use machine,                   only: kind_phys
+      use machine,                   only: kind_phys, r8=>kind_dbl_prec
 
       use physparam
-
-      use radcons,                   only: itsfc,ltp, lextop, qmin,  &
+      use radcons,                   only: itsfc, qmin,  &
                                            qme5, qme6, epsq, prsmin
       use funcphys,                  only: fpvs
 
@@ -79,7 +78,7 @@ subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
 
       implicit none
 
-      integer,              intent(in)  :: im, levs, lm, lmk, lmp, n_var_lndp, &
+      integer,              intent(in)  :: im, levs, lm, lmk, lmp, ltp, n_var_lndp, &
                                            imfdeepcnv,                         &
                                            imfdeepcnv_gf, me, ncnd, ntrac,     &
                                            num_p3d, npdf3d, ncnvcld3d, ntqv,   &
@@ -98,7 +97,7 @@ subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
 
       character(len=3), dimension(:), intent(in) :: lndp_var_list
 
-      logical,              intent(in) :: lsswr, lslwr, ltaerosol, lgfdlmprad, &
+      logical,              intent(in) :: lextop, lsswr, lslwr, ltaerosol, lgfdlmprad, &
                                           uni_cld, effr_in, do_mynnedmf,       &
                                           lmfshal, lmfdeep2, pert_clds
 
@@ -295,6 +294,8 @@ subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
           plyr(i,k1)    = prsl(i,k2)    * 0.01   ! pa to mb (hpa)
           tlyr(i,k1)    = tgrs(i,k2)
           prslk1(i,k1)  = prslk(i,k2)
+          rho(i,k1)     = prsl(i,k2)/(con_rd*tlyr(i,k1))
+          orho(i,k1)    = 1.0/rho(i,k1)
 
 !>  - Compute relative humidity.
           es  = min( prsl(i,k2),  fpvs( tgrs(i,k2) ) )  ! fpvs and prsl in pa
@@ -343,14 +344,15 @@ subroutine GFS_rrtmg_pre_run (im, levs, lm, lmk, lmp, n_var_lndp,        &
           enddo
         endif
       endif
-!
       if ( lextop ) then                 ! values for extra top layer
         do i = 1, IM
           plvl(i,llb) = prsmin
           if ( plvl(i,lla) <= prsmin ) plvl(i,lla) = 2.0*prsmin
           plyr(i,lyb)   = 0.5 * plvl(i,lla)
           tlyr(i,lyb)   = tlyr(i,lya)
           prslk1(i,lyb) = (plyr(i,lyb)*0.001) ** rocp ! plyr in hPa
+          rho(i,lyb)    = plyr(i,lyb) *100.0/(con_rd*tlyr(i,lyb))
+          orho(i,lyb)   = 1.0/rho(i,lyb)
           rhly(i,lyb)   = rhly(i,lya)
           qstl(i,lyb)   = qstl(i,lya)
         enddo

physics/GFS_rrtmg_pre.meta

@@ -49,6 +49,18 @@
   type = integer
   intent = in
   optional = F
+[lextop]
+  standard_name = flag_for_extra_top_layer_for_radiation_calculations
+  long_name = flag for extra top layer for radiation calculations
+  units = flag
+  dimensions = ()
+  type = logical
+[ltp]
+  standard_name = extra_top_layer
+  long_name = extra top layer
+  units = count
+  dimensions = ()
+  type = integer
 [n_var_lndp]
   standard_name = number_of_land_surface_variables_perturbed
   long_name = number of land surface variables perturbed