Merge pull request #24 from noaa-ocs-modeling/develop

rearrange README and rename scripts

README.md

@@ -12,27 +12,27 @@ ocean model ensembles.
 It utilizes [`nemspy`](https://github.com/noaa-ocs-modeling/NEMSpy) to generate NEMS configuration files, shares common
 configurations between runs, and organizes spinup and mesh partition into separate jobs for dependant submission.
 
-### Supported models and inputs:
+## Supported models and platforms
 
-- circulation models
-    - ADCIRC (uses [`adcircpy`](https://github.com/JaimeCalzadaNOAA/adcircpy))
-- forcing
-    - ATMESH
-    - WW3DATA
+- ### models
+    - #### circulation models
+        - ADCIRC (uses [`adcircpy`](https://github.com/JaimeCalzadaNOAA/adcircpy))
+    - #### forcings
+        - ATMESH
+        - WW3DATA
+- ### platforms
+    - #### local (no job manager)
+    - #### Hera (Slurm)
+    - #### Stampede2 (Slurm)
 
-### Supported platforms:
+## Usage
 
-- local (no job manager)
-- Hera (Slurm)
-- Stampede2 (Slurm)
+Example scripts can be found at `examples/<platform>`
 
-### Usage
+### 1. generate configuration directory
 
-Example scripts can be found at `examples/<platform>`.
-
-Here is an example for running a simple mesh over the Shinnecock Inlet with a coupled `(ATMESH + WW3DATA) -> ADCIRC` scenario,
-and prepping a configuration directory with run script for job submission on Hera. After running the following example on Hera,
-the user would then run `sh run.sh` to submit the jobs.
+The following code (`examples/hera/hera_shinnecock_ike.py`) creates a configuration for coupling `(ATMESH + WW3DATA) -> ADCIRC`
+on Hera, over a small Shinnecock Inlet mesh:
 
 ```python
 from datetime import datetime, timedelta
@@ -56,46 +56,126 @@ FORCINGS_DIRECTORY = Path('/scratch2/COASTAL/coastal/save/shared/models') / 'for
 # directory to which to write configuration
 OUTPUT_DIRECTORY = Path(__file__).parent.parent / 'data' / 'configuration' / 'hera_shinnecock_ike'
 
-if __name__ == '__main__':
-    # dictionary defining runs with ADCIRC value perturbations - in this case, a single run with no perturbation
-    runs = {f'test_case_1': (None, None)}
-
-    # initialize `nemspy` configuration object with forcing file locations, start and end times,  and processor assignment
-    nems = ModelingSystem(
-        start_time=datetime(2008, 8, 23),
-        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
-        interval=timedelta(hours=1),
-        atm=AtmosphericMeshEntry(filename=FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc', processors=1),
-        wav=WaveMeshEntry(filename=FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc', processors=1),
-        ocn=ADCIRCEntry(processors=11),
-    )
-
-    # describe connections between coupled components
-    nems.connect('ATM', 'OCN')
-    nems.connect('WAV', 'OCN')
-    nems.sequence = [
-        'ATM -> OCN',
-        'WAV -> OCN',
-        'ATM',
-        'WAV',
-        'OCN',
-    ]
-
-    # initialize `adcircpy` forcing objects
-    tidal_forcing = Tides()
-    tidal_forcing.use_all()
-    wind_forcing = AtmosphericMeshForcing(nws=17, interval_seconds=3600)
-    wave_forcing = WaveWatch3DataForcing(nrs=5, interval_seconds=3600)
-
-    # send run information to `adcircpy` and write the resulting configuration to output directory
-    write_adcirc_configurations(
-        nems,
-        runs,
-        MESH_DIRECTORY,
-        OUTPUT_DIRECTORY,
-        email_address='[email protected]',
-        platform=Platform.HERA,
-        spinup=timedelta(days=12.5),
-        forcings=[tidal_forcing, wind_forcing, wave_forcing],
-    )
+# dictionary defining runs with ADCIRC value perturbations - in this case, a single run with no perturbation
+runs = {f'test_case_1': (None, None)}
+
+# initialize `nemspy` configuration object with forcing file locations, start and end times,  and processor assignment
+nems = ModelingSystem(
+    start_time=datetime(2008, 8, 23),
+    end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
+    interval=timedelta(hours=1),
+    atm=AtmosphericMeshEntry(filename=FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc', processors=1),
+    wav=WaveMeshEntry(filename=FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc', processors=1),
+    ocn=ADCIRCEntry(processors=11),
+)
+
+# describe connections between coupled components
+nems.connect('ATM', 'OCN')
+nems.connect('WAV', 'OCN')
+nems.sequence = [
+    'ATM -> OCN',
+    'WAV -> OCN',
+    'ATM',
+    'WAV',
+    'OCN',
+]
+
+# initialize `adcircpy` forcing objects
+tidal_forcing = Tides()
+tidal_forcing.use_all()
+wind_forcing = AtmosphericMeshForcing(nws=17, interval_seconds=3600)
+wave_forcing = WaveWatch3DataForcing(nrs=5, interval_seconds=3600)
+
+# send run information to `adcircpy` and write the resulting configuration to output directory
+write_adcirc_configurations(
+    nems,
+    runs,
+    MESH_DIRECTORY,
+    OUTPUT_DIRECTORY,
+    email_address='[email protected]',
+    platform=Platform.HERA,
+    spinup=timedelta(days=12.5),
+    forcings=[tidal_forcing, wind_forcing, wave_forcing],
+)
+```
+
+This code will generate a directory `hera_shinnecock_ike/` with the following structure:
+
+```
+📦 hera_shinnecock_ike/
+┣ 📂 coldstart/
+┃  ┣ 📜 fort.13
+┃  ┣ 📜 fort.14
+┃  ┗ 📜 fort.15
+┣ 📂 runs/
+┃  ┗ 📂 test_case_1/
+┃    ┣ 📜 fort.13
+┃    ┣ 📜 fort.14
+┃    ┗ 📜 fort.15
+┣ 📜 config.rc.coldstart
+┣ 📜 config.rc.hotstart
+┣ 📜 model_configure.coldstart
+┣ 📜 model_configure.hotstart
+┣ 📜 nems.configure.coldstart
+┣ 📜 nems.configure.hotstart
+┣ 📜 job_adcprep_hera.job
+┣ 📜 job_nems_adcirc_hera.job.coldstart
+┣ 📜 job_nems_adcirc_hera.job.hotstart
+┗ 📜 run_hera.sh
+```
+
+_**Note:** the required NEMS configuration files (`nems.configure`, `model_configure`) do not yet exist in the run
+directories (`coldstart/`, `runs/test_case_1/`). These will be populated in the next step._
+
+### 2. run job submission script `run_<platform>.sh`
+
+Run `run_hera.sh`:
+
+```bash
+sh run_hera.sh
+``` 
+
+This will first create symbolic links to populate configuration directories,
+
+```
+📦 hera_shinnecock_ike/
+┣ 📂 coldstart/
+┃  ┣ 📜 fort.13
+┃  ┣ 📜 fort.14
+┃  ┣ 📜 fort.15
+┃  ┣ 🔗 config.rc -> ../config.rc.coldstart
+┃  ┣ 🔗 model_configure -> ../model_configure.hotstart
+┃  ┣ 🔗 nems.configure -> ../nems.configure.coldstart
+┃  ┣ 🔗 hera_adcprep.job -> ../job_adcprep_hera.job
+┃  ┗ 🔗 hera_nems_adcirc.job -> ../job_nems_adcirc_hera.job.coldstart
+┣ 📂 runs/
+┃  ┗ 📂 test_case_1/
+┃    ┣ 📜 fort.13
+┃    ┣ 📜 fort.14
+┃    ┣ 📜 fort.15
+┃    ┣ 🔗 config.rc -> ../../config.rc.hotstart
+┃    ┣ 🔗 model_configure -> ../../model_configure.hotstart
+┃    ┣ 🔗 nems.configure -> ../../nems.configure.hotstart
+┃    ┣ 🔗 hera_adcprep.job -> ../../job_adcprep_hera.job
+┃    ┗ 🔗 hera_nems_adcirc.job -> ../../job_nems_adcirc_hera.job.hotstart
+┣ 📜 config.rc.coldstart
+┣ 📜 config.rc.hotstart
+┣ 📜 model_configure.coldstart
+┣ 📜 model_configure.hotstart
+┣ 📜 nems.configure.coldstart
+┣ 📜 nems.configure.hotstart
+┣ 📜 job_adcprep_hera.job
+┣ 📜 job_nems_adcirc_hera.job.coldstart
+┣ 📜 job_nems_adcirc_hera.job.hotstart
+┗ 📜 run_hera.sh
+```
+
+and then submit the requested jobs to the queue:
+
+```bash
+   JOBID                  NAME CPUS NODE                      DEPENDENCY ACCOUNT PARTITION          SUBMIT_TIME           START_TIME             END_TIME
+16368044 ADCIRC_MESH_PARTITION    1    1                          (null) coastal      hera  2021-02-18T19:29:17                  N/A                  N/A
+16368045      ADCIRC_COLDSTART   11    1  afterany:16368044(unfulfilled) coastal      hera  2021-02-18T19:29:17                  N/A                  N/A
+16368046 ADCIRC_MESH_PARTITION    1    1  afterany:16368045(unfulfilled) coastal      hera  2021-02-18T19:29:17                  N/A                  N/A
+16368047       ADCIRC_HOTSTART   13    1  afterany:16368046(unfulfilled) coastal      hera  2021-02-18T19:29:17                  N/A                  N/A
 ```

coupledmodeldriver/adcirc.py

@@ -1,10 +1,8 @@
 import copy
 from datetime import timedelta
-from glob import glob
 import os
 from os import PathLike
 from pathlib import Path
-import re
 
 from adcircpy import AdcircMesh, AdcircRun
 from adcircpy.forcing.base import Forcing
@@ -147,7 +145,7 @@ def write_adcirc_configurations(
         slurm_log_filename=f'{adcprep_run_name}.out.log',
     )
     adcprep_script.write(
-        output_directory / f'{adcprep_script.platform.value}_adcprep.job', overwrite=True
+        output_directory / f'job_adcprep_{adcprep_script.platform.value}.job', overwrite=True
     )
 
     if spinup is not None:
@@ -190,7 +188,7 @@ def write_adcirc_configurations(
         )
 
     coldstart_run_script.write(
-        output_directory / f'{coldstart_run_script.platform.value}_nems_adcirc.job.coldstart',
+        output_directory / f'job_nems_adcirc_{coldstart_run_script.platform.value}.job.coldstart',
         overwrite=True,
     )
 
@@ -215,11 +213,8 @@ def write_adcirc_configurations(
             slurm_log_filename=f'{adcirc_hotstart_run_name}.out.log',
         )
 
-        hotstart_run_script.write(
-            output_directory
-            / f'{hotstart_run_script.platform.value}_nems_adcirc.job.hotstart',
-            overwrite=True,
-        )
+        hotstart_run_script.write(output_directory / f'job_nems_adcirc_{hotstart_run_script.platform.value}.job.hotstart',
+                                  overwrite=True)
 
     slurm = SlurmConfig(
         account=slurm_account,
@@ -274,24 +269,6 @@ def write_adcirc_configurations(
             driver.mesh.add_attribute(attribute_name)
         driver.mesh.set_attribute(attribute_name, value)
         driver.write(run_directory, overwrite=True, coldstart=None, hotstart='fort.15', driver=None)
-        for phase in ['hotstart']:
-            directory = run_directory / phase
-            if not directory.exists():
-                directory.mkdir()
-
-    pattern = re.compile(' p*adcirc')
-    replacement = ' NEMS.x'
-    for job_filename in glob(str(output_directory / '**' / 'slurm.job'), recursive=True):
-        with open(job_filename) as job_file:
-            text = job_file.read()
-        matched = pattern.search(text)
-        if matched:
-            LOGGER.debug(
-                f'replacing `{matched.group(0)}` with `{replacement}`' f' in "{job_filename}"'
-            )
-            text = re.sub(pattern, replacement, text)
-            with open(job_filename, 'w') as job_file:
-                job_file.write(text)
 
     run_script = RunScript(platform)
     run_script.write(output_directory / f'run_{platform.value}.sh', overwrite=True)

coupledmodeldriver/job_script.py

@@ -193,7 +193,7 @@ def __str__(self) -> str:
 
         if self.platform != Platform.LOCAL:
             lines.extend(
-                [self.slurm_header, '', 'set -e', '', ]
+                [self.slurm_header, '', 'set -e', '']
             )
 
         if self.modules is not None:
@@ -384,21 +384,24 @@ def __init__(self, platform: Platform):
 
     def __str__(self) -> str:
         lines = [
-            'DIRECTORY="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"',
+            'DIRECTORY="$(',
+            '    cd "$(dirname "$0")" >/dev/null 2>&1',
+            '    pwd -P',
+            ')"',
             '',
             '# prepare single coldstart directory',
             f'cd $DIRECTORY/coldstart',
-            f'ln -sf ../{self.platform.value}_adcprep.job adcprep.job',
-            f'ln -sf ../{self.platform.value}_nems_adcirc.job.coldstart nems_adcirc.job',
+            f'ln -sf ../job_adcprep_{self.platform.value}.job adcprep.job',
+            f'ln -sf ../job_nems_adcirc_{self.platform.value}.job.coldstart nems_adcirc.job',
             'cd $DIRECTORY',
             '',
             '# prepare every hotstart directory',
             bash_for_loop(
                 'for hotstart in $DIRECTORY//runs/*/',
                 [
                     'cd "$hotstart"',
-                    f'ln -sf ../../{self.platform.value}_adcprep.job adcprep.job',
-                    f'ln -sf ../../{self.platform.value}_nems_adcirc.job.hotstart nems_adcirc.job',
+                    f'ln -sf ../../job_adcprep_{self.platform.value}.job adcprep.job',
+                    f'ln -sf ../../job_nems_adcirc_{self.platform.value}.job.hotstart nems_adcirc.job',
                     'cd $DIRECTORY/',
                 ]
             ),
@@ -421,9 +424,11 @@ def __str__(self) -> str:
 
         if self.platform != Platform.LOCAL:
             # slurm queue output https://slurm.schedmd.com/squeue.html
-            squeue_command = 'squeue -u $USER -o "%.8F %.21j %.4C %.4D %.31E %.7a %.9P %.20V %.20S %.20e"'
+            squeue_command = 'squeue -u $USER -o "%.8i %.21j %.4C %.4D %.31E %.7a %.9P %.20V %.20S %.20e"'
             echo_squeue_command = squeue_command.replace('"', r'\"')
             lines.extend([
+                '',
+                '# display job queue with dependencies',
                 f'echo {echo_squeue_command}',
                 squeue_command,
             ])
@@ -473,7 +478,7 @@ def write(self, filename: PathLike, overwrite: bool = False):
             filename = Path(filename)
 
         if filename.is_dir():
-            filename = filename / f'run.sh'
+            filename = filename / f'run_{self.platform.value}.sh'
 
         output = f'{self}\n'
         if overwrite or not filename.exists():

tests/data/reference/hera_shinnecock_ike/coldstart/fort.15

@@ -1,4 +1,4 @@
-created on 2021-02-16 10:33                                     ! RUNDES
+created on 2021-02-18 15:59                                     ! RUNDES
  Shinacock Inlet Coarse Grid                                    ! RUNID
 1                                                               ! NFOVER
 1                                                               ! NABOUT

tests/data/reference/hera_shinnecock_ike/config.rc.coldstart

@@ -1,2 +1,2 @@
-# `config.rc` generated with NEMSpy 0.2.3.post16.dev0+f3fb2b8
+# `config.rc` generated with NEMSpy 0.3.1.post9.dev0+5c0cf20
 

tests/data/reference/hera_shinnecock_ike/config.rc.hotstart

@@ -1,4 +1,4 @@
-# `config.rc` generated with NEMSpy 0.2.3.post16.dev0+f3fb2b8
+# `config.rc` generated with NEMSpy 0.3.1.post9.dev0+5c0cf20
  atm_dir: input/shinnecock_ike/forcings
  atm_nam: wind_atm_fin_ch_time_vec.nc
  wav_dir: input/shinnecock_ike/forcings

tests/data/reference/hera_shinnecock_ike/job_adcprep_hera.job

@@ -0,0 +1,16 @@
+#!/bin/bash --login
+#SBATCH -J ADCIRC_MESH_PARTITION
+#SBATCH -A coastal
+#SBATCH --mail-type=ALL
+#SBATCH [email protected]
+#SBATCH --error=ADCIRC_MESH_PARTITION.err.log
+#SBATCH --output=ADCIRC_MESH_PARTITION.out.log
+#SBATCH -n 1
+#SBATCH --time=00:30:00
+
+set -e
+
+source /scratch2/COASTAL/coastal/save/shared/repositories/ADC-WW3-NWM-NEMS/modulefiles/hera/ESMF_NUOPC
+
+srun /scratch2/COASTAL/coastal/save/shared/repositories/ADC-WW3-NWM-NEMS/ADCIRC/work/adcprep --np 11 --partmesh
+srun /scratch2/COASTAL/coastal/save/shared/repositories/ADC-WW3-NWM-NEMS/ADCIRC/work/adcprep --np 11 --prepall