Merge pull request #156 from NCAR/fix_multi_restart

Fix multi-restart files in NUOPC cap

config_src/mct_driver/ocn_comp_mct.F90

@@ -119,7 +119,9 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
   integer                 :: year, month, day, hour, minute, seconds, seconds_n, seconds_d, rc
   character(len=240)      :: runid                !< Run ID
   character(len=32)       :: runtype              !< Run type
-  character(len=240)      :: restartfile          !< Path/Name of restart file
+  character(len=512)      :: restartfile          !< Path/Name of restart file
+  character(len=2048)     :: restartfiles         !< Path/Name of restart files.
+                                                  !! (same as restartfile if a single restart file is to be read in)
   integer                 :: nu                   !< i/o unit to read pointer file
   character(len=240)      :: restart_pointer_file !< File name for restart pointer file
   character(len=240)      :: restartpath          !< Path of the restart file
@@ -164,6 +166,7 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
   !logical :: lsend_precip_fact      !< If T,send precip_fact to cpl for use in fw balance
                                     !! (partially-coupled option)
   character(len=128) :: err_msg     !< Error message
+  integer :: iostat
 
   ! set the cdata pointers:
   call seq_cdata_setptrs(cdata_o, id=MOM_MCT_ID, mpicom=mpicom_ocn, &
@@ -296,15 +299,27 @@ subroutine ocn_init_mct( EClock, cdata_o, x2o_o, o2x_o, NLFilename )
     nu = shr_file_getUnit()
     restart_pointer_file = trim(glb%pointer_filename)
     if (is_root_pe()) write(glb%stdout,*) 'Reading ocn pointer file: ',restart_pointer_file
+    restartfile = ""; restartfiles = "";
     open(nu, file=restart_pointer_file, form='formatted', status='unknown')
-    read(nu,'(a)') restartfile
+    do
+      read(nu,'(a)', iostat=iostat) restartfile
+      if (len(trim(restartfiles))>1 .and. iostat<0) then
+        exit ! done reading restart files list.
+      else if (iostat/=0) then
+        call MOM_error(FATAL, 'Error reading rpointer.ocn')
+      endif
+      ! check if the length of restartfiles variable is sufficient:
+      if (len(restartfiles)-len(trim(restartfiles)) < len(trim(restartfile))) then
+        call MOM_error(FATAL, "Restart file name(s) too long.")
+      endif
+      restartfiles = trim(restartfiles) // " " // trim(restartfile)
+    enddo
     close(nu)
-    !restartfile = trim(restartpath) // trim(restartfile)
     if (is_root_pe()) then
-      write(glb%stdout,*) 'Reading restart file: ',trim(restartfile)
+      write(glb%stdout,*) 'Reading restart file(s): ',trim(restartfiles)
     end if
     call shr_file_freeUnit(nu)
-    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time_start, input_restart_file=trim(restartfile))
+    call ocean_model_init(glb%ocn_public, glb%ocn_state, time0, time_start, input_restart_file=trim(restartfiles))
   endif
   if (is_root_pe()) then
     write(glb%stdout,'(/12x,a/)') '======== COMPLETED MOM INITIALIZATION ========'
@@ -434,6 +449,9 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)
   integer                   :: ocn_cpl_dt   !< one ocn coupling interval in seconds. (to be received from cesm)
   real (kind=8)             :: mom_cpl_dt   !< one ocn coupling interval in seconds. (internal)
   integer                   :: ncouple_per_day !< number of ocean coupled call in one day (non-dim)
+  integer                   :: num_rest_files !< number of restart files written
+  integer                   :: i
+  character(len=8)          :: suffix
 
   ! reset shr logging to ocn log file:
   if (is_root_pe()) then
@@ -534,14 +552,28 @@ subroutine ocn_run_mct( EClock, cdata_o, x2o_o, o2x_o)
     write(restartname,'(A,".mom6.r.",I4.4,"-",I2.2,"-",I2.2,"-",I5.5)') trim(runid), year, month, day, seconds
 
     call save_restart(glb%ocn_state%dirs%restart_output_dir, glb%ocn_state%Time, glb%grid, &
-                      glb%ocn_state%restart_CSp, .false., filename=restartname, GV=glb%ocn_state%GV)
+                      glb%ocn_state%restart_CSp, .false., filename=restartname, GV=glb%ocn_state%GV, &
+                      num_rest_files=num_rest_files)
 
     ! write name of restart file in the rpointer file
     nu = shr_file_getUnit()
     if (is_root_pe()) then
       restart_pointer_file = trim(glb%pointer_filename)
       open(nu, file=restart_pointer_file, form='formatted', status='unknown')
       write(nu,'(a)') trim(restartname) //'.nc'
+
+      if (num_rest_files > 1) then
+        ! append i.th restart file name to rpointer
+        do i=1, num_rest_files-1
+          if (i < 10) then
+            write(suffix,'("_",I1)') i
+          else
+            write(suffix,'("_",I2)') i
+          endif
+          write(nu,'(a)') trim(restartname) // trim(suffix) // '.nc'
+        enddo
+      endif
+
       close(nu)
       write(glb%stdout,*) 'ocn restart pointer file written: ',trim(restartname)
     endif

config_src/nuopc_driver/mom_cap.F90

@@ -464,6 +464,8 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   integer                                :: iostat
   integer                                :: readunit
   character(len=512)                     :: restartfile          ! Path/Name of restart file
+  character(len=2048)                    :: restartfiles         ! Path/Name of restart files
+                                                                 ! (same as restartfile if single restart file)
   character(len=*), parameter            :: subname='(MOM_cap:InitializeAdvertise)'
   character(len=32)                      :: calendar
 !--------------------------------
@@ -653,10 +655,10 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
          return
   endif
 
-  restartfile = ""
+  restartfile = ""; restartfiles = ""
   if (runtype == "initial") then
 
-     restartfile = "n"
+     restartfiles = "n"
 
   else if (runtype == "continue") then ! hybrid or branch or continuos runs
 
@@ -675,16 +677,27 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
                      line=__LINE__, file=u_FILE_u, rcToReturn=rc)
                 return
             endif
-            read(readunit,'(a)', iostat=iostat) restartfile
-            if (iostat /= 0) then
-               call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading rpointer.ocn', &
-                    line=__LINE__, file=u_FILE_u, rcToReturn=rc)
-                return
-            endif
+            do
+              read(readunit,'(a)', iostat=iostat) restartfile
+              if (iostat /= 0) then
+                if (len(trim(restartfiles))>1 .and. iostat<0) then
+                  exit ! done reading restart files list.
+                else
+                   call ESMF_LogSetError(ESMF_RC_FILE_READ, msg=subname//' ERROR reading rpointer.ocn', &
+                     line=__LINE__, file=u_FILE_u, rcToReturn=rc)
+                   return
+                endif
+              endif
+              ! check if the length of restartfiles variable is sufficient:
+              if (len(restartfiles)-len(trim(restartfiles)) < len(trim(restartfile))) then
+                call MOM_error(FATAL, "Restart file name(s) too long.")
+              endif
+              restartfiles = trim(restartfiles) // " " // trim(restartfile)
+            enddo
             close(readunit)
          endif
          ! broadcast attribute set on master task to all tasks
-         call ESMF_VMBroadcast(vm, restartfile, count=ESMF_MAXSTR-1, rootPet=0, rc=rc)
+         call ESMF_VMBroadcast(vm, restartfiles, count=len(restartfiles), rootPet=0, rc=rc)
          if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=u_FILE_u)) return
       else
          call ESMF_LogWrite('MOM_cap: restart requested, use input.nml', ESMF_LOGMSG_WARNING)
@@ -693,7 +706,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
   endif
 
   ocean_public%is_ocean_pe = .true.
-  call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(restartfile))
+  call ocean_model_init(ocean_public, ocean_state, time0, time_start, input_restart_file=trim(restartfiles))
 
   call ocean_model_init_sfc(ocean_state, ocean_public)
 
@@ -1611,10 +1624,12 @@ subroutine ModelAdvance(gcomp, rc)
   integer                                :: writeunit
   integer                                :: localPet
   type(ESMF_VM)                          :: vm
-  integer                                :: n
+  integer                                :: n, i
   character(240)                         :: import_timestr, export_timestr
   character(len=128)                     :: fldname
   character(len=*),parameter             :: subname='(MOM_cap:ModelAdvance)'
+  character(len=8)                       :: suffix
+  integer                                :: num_rest_files
 
   rc = ESMF_SUCCESS
   if(profile_memory) call ESMF_VMLogMemInfo("Entering MOM Model_ADVANCE: ")
@@ -1832,6 +1847,12 @@ subroutine ModelAdvance(gcomp, rc)
 
         write(restartname,'(A,".mom6.r.",I4.4,"-",I2.2,"-",I2.2,"-",I5.5)') &
              trim(casename), year, month, day, seconds
+
+        call ESMF_LogWrite("MOM_cap: Writing restart :  "//trim(restartname), ESMF_LOGMSG_INFO, rc=rc)
+
+        ! write restart file(s)
+        call ocean_model_restart(ocean_state, restartname=restartname, num_rest_files=num_rest_files)
+
         if (localPet == 0) then
            ! Write name of restart file in the rpointer file - this is currently hard-coded for the ocean
            open(newunit=writeunit, file='rpointer.ocn', form='formatted', status='unknown', iostat=iostat)
@@ -1841,6 +1862,19 @@ subroutine ModelAdvance(gcomp, rc)
               return
            endif
            write(writeunit,'(a)') trim(restartname)//'.nc'
+
+           if (num_rest_files > 1) then
+              ! append i.th restart file name to rpointer
+              do i=1, num_rest_files-1
+                if (i < 10) then
+                  write(suffix,'("_",I1)') i
+                else
+                  write(suffix,'("_",I2)') i
+                endif
+                write(writeunit,'(a)') trim(restartname) // trim(suffix) // '.nc'
+              enddo
+           endif
+
            close(writeunit)
         endif
     else
@@ -1851,16 +1885,17 @@ subroutine ModelAdvance(gcomp, rc)
         write(restartname,'(A,I4.4,"-",I2.2,"-",I2.2,"-",I2.2,"-",I2.2,"-",I2.2)') &
              "MOM.res.", year, month, day, hour, minute, seconds
       endif
-    end if
-    call ESMF_LogWrite("MOM_cap: Writing restart :  "//trim(restartname), ESMF_LOGMSG_INFO, rc=rc)
 
-     ! write restart file(s)
-     call ocean_model_restart(ocean_state, restartname=restartname)
+      call ESMF_LogWrite("MOM_cap: Writing restart :  "//trim(restartname), ESMF_LOGMSG_INFO, rc=rc)
 
-     if (is_root_pe()) then
-       write(logunit,*) subname//' writing restart file ',trim(restartname)
-     endif
+      ! write restart file(s)
+      call ocean_model_restart(ocean_state, restartname=restartname)
+    end if
+
+    if (is_root_pe()) then
+      write(logunit,*) subname//' writing restart file ',trim(restartname)
     endif
+  endif
 
   !---------------
   ! Write diagnostics

config_src/nuopc_driver/mom_ocean_model_nuopc.F90

@@ -671,14 +671,15 @@ subroutine update_ocean_model(Ice_ocean_boundary, OS, Ocean_sfc, &
 end subroutine update_ocean_model
 
 !> This subroutine writes out the ocean model restart file.
-subroutine ocean_model_restart(OS, timestamp, restartname)
+subroutine ocean_model_restart(OS, timestamp, restartname, num_rest_files)
   type(ocean_state_type),     pointer    :: OS !< A pointer to the structure containing the
                                                !! internal ocean state being saved to a restart file
   character(len=*), optional, intent(in) :: timestamp !< An optional timestamp string that should be
                                                !! prepended to the file name. (Currently this is unused.)
   character(len=*), optional, intent(in) :: restartname !< Name of restart file to use
                                                !! This option distinguishes the cesm interface from the
                                                !! non-cesm interface
+  integer, optional, intent(out)         :: num_rest_files !< number of restart files written
 
   if (.not.MOM_state_is_synchronized(OS%MOM_CSp)) &
       call MOM_error(WARNING, "End of MOM_main reached with inconsistent "//&
@@ -689,8 +690,13 @@ subroutine ocean_model_restart(OS, timestamp, restartname)
       "restart files can only be created after the buoyancy forcing is applied.")
 
   if (present(restartname)) then
-     call save_restart(OS%dirs%restart_output_dir, OS%Time, OS%grid, &
-          OS%restart_CSp, GV=OS%GV, filename=restartname)
+     if (present(num_rest_files)) then
+       call save_restart(OS%dirs%restart_output_dir, OS%Time, OS%grid, &
+            OS%restart_CSp, GV=OS%GV, filename=restartname, num_rest_files=num_rest_files)
+     else
+       call save_restart(OS%dirs%restart_output_dir, OS%Time, OS%grid, &
+            OS%restart_CSp, GV=OS%GV, filename=restartname)
+     endif
      call forcing_save_restart(OS%forcing_CSp, OS%grid, OS%Time, &
           OS%dirs%restart_output_dir) ! Is this needed?
      if (OS%use_ice_shelf) then

src/framework/MOM_get_input.F90

@@ -21,7 +21,8 @@ module MOM_get_input
   character(len=240) :: &
     restart_input_dir = ' ',& !< The directory to read restart and input files.
     restart_output_dir = ' ',&!< The directory into which to write restart files.
-    output_directory = ' ', & !< The directory to use to write the model output.
+    output_directory = ' '    !< The directory to use to write the model output.
+  character(len=2048) :: &
     input_filename  = ' '     !< A string that indicates the input files or how
                               !! the run segment should be started.
 end type directories
@@ -46,7 +47,8 @@ subroutine get_MOM_input(param_file, dirs, check_params, default_input_filename,
     parameter_filename(npf), & ! List of files containing parameters.
     output_directory,        & ! Directory to use to write the model output.
     restart_input_dir,       & ! Directory for reading restart and input files.
-    restart_output_dir,      & ! Directory into which to write restart files.
+    restart_output_dir         ! Directory into which to write restart files.
+  character(len=2048) :: &
     input_filename             ! A string that indicates the input files or how
                                ! the run segment should be started.
   character(len=240) :: output_dir

src/framework/MOM_restart.F90

@@ -849,7 +849,7 @@ function query_initialized_4d_name(f_ptr, name, CS) result(query_initialized)
 end function query_initialized_4d_name
 
 !> save_restart saves all registered variables to restart files.
-subroutine save_restart(directory, time, G, CS, time_stamped, filename, GV)
+subroutine save_restart(directory, time, G, CS, time_stamped, filename, GV, num_rest_files)
   character(len=*),        intent(in)    :: directory !< The directory where the restart files
                                                   !! are to be written
   type(time_type),         intent(in)    :: time  !< The current model time
@@ -860,6 +860,7 @@ subroutine save_restart(directory, time, G, CS, time_stamped, filename, GV)
                                                   !! to the restart file names.
   character(len=*), optional, intent(in) :: filename !< A filename that overrides the name in CS%restartfile.
   type(verticalGrid_type), optional, intent(in) :: GV   !< The ocean's vertical grid structure
+  integer, optional, intent(out) :: num_rest_files      !< number of restart files written
 
   ! Local variables
   type(vardesc) :: vars(CS%max_fields)  ! Descriptions of the fields that
@@ -1056,6 +1057,9 @@ subroutine save_restart(directory, time, G, CS, time_stamped, filename, GV)
     num_files = num_files+1
 
   enddo
+
+  if (present(num_rest_files)) num_rest_files = num_files
+
 end subroutine save_restart
 
 !> restore_state reads the model state from previously generated files.  All