diff --git a/README.md b/README.md index dd9b04dd..b6fad041 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing import numpy from coupledmodeldriver import Platform -from coupledmodeldriver.nems import NEMSADCIRCRunConfiguration, NEMSADCIRCGenerationScript +from coupledmodeldriver.generate.nems import NEMSADCIRCRunConfiguration, NEMSADCIRCGenerationScript # paths to compiled `NEMS.x` and `adcprep` NEMS_EXECUTABLE = '/scratch2/COASTAL/coastal/save/shared/repositories/ADC-WW3-NWM-NEMS/NEMS/exe/NEMS.x' @@ -169,29 +169,6 @@ Running `generate_nems_adcirc.py` will read the JSON configuration and generate ``` 📦 hera_shinnecock_ike/ -┣ 📂 coldstart/ -┃ ┣ 📜 fort.13 -┃ ┣ 🔗 fort.14 -> ../fort.14 -┃ ┗ 📜 fort.15 -┣ 📂 runs/ -┃ ┗ 📂 test_case_1/ -┃ ┣ 📜 fort.13 -┃ ┣ 🔗 fort.14 -> ../../fort.14 -┃ ┗ 📜 fort.15 -┣ 📜 fort.14 -┣ 📜 nems.configure.coldstart -┣ 📜 nems.configure.hotstart -┣ 📜 config.rc.coldstart -┣ 📜 config.rc.hotstart -┣ 📜 model_configure.coldstart -┣ 📜 model_configure.hotstart -┣ 📜 job_adcprep_hera.job -┣ 📜 job_adcirc_hera.job.coldstart -┣ 📜 job_adcirc_hera.job.hotstart -┣ 📜 setup.sh.coldstart -┣ 📜 setup.sh.hotstart -┣ 📜 cleanup.sh -┣ 📜 setup_hera.sh ┣ ✎ configure_modeldriver.json ┣ ✎ configure_adcirc.json ┣ ✎ configure_nems.json @@ -200,24 +177,6 @@ Running `generate_nems_adcirc.py` will read the JSON configuration and generate ┣ ✎ configure_atmesh.json ┣ ✎ configure_ww3data.json ┣ ▶ generate_nems_adcirc.py -┗ ▶ run_hera.sh -``` - -_**Note:** the required NEMS configuration files (`nems.configure`, `model_configure`) do not yet exist in the run -directories (`coldstart/`, `runs/test_case_1/`). These will be populated in the next step._ - -### 3. run job submission script - -Running `run_hera.sh` will start the actual model run. - -```bash -sh run_hera.sh -``` - -This will first create symbolic links to populate configuration directories (by calling `setup_hera.sh`), - -``` -📦 hera_shinnecock_ike/ ┣ 📂 coldstart/ ┃ ┣ 📜 fort.13 ┃ ┣ 🔗 fort.14 -> ../fort.14 @@ -254,17 +213,18 @@ This will first create symbolic links to populate configuration directories (by ┣ 📜 setup.sh.hotstart ┣ 📜 cleanup.sh ┣ 📜 setup_hera.sh -┣ ✎ configure_modeldriver.json -┣ ✎ configure_adcirc.json -┣ ✎ configure_nems.json -┣ ✎ configure_slurm.json -┣ ✎ configure_tidal_forcing.json -┣ ✎ configure_atmesh.json -┣ ✎ configure_ww3data.json ┗ ▶ run_hera.sh ``` -and then submit the requested jobs to the queue (or run the commands directly if the platform is set to `LOCAL`): +### 3. run job submission script + +Running `run_hera.sh` will start the actual model run. + +```bash +sh run_hera.sh +``` + +This will submit the requested jobs to the queue (or run the commands directly if the platform is set to `LOCAL`): ```bash squeue -u $USER -o "%.8i %.21j %.4C %.4D %.31E %.20V %.20S %.20e" diff --git a/coupledmodeldriver/adcirc/__init__.py b/coupledmodeldriver/adcirc/__init__.py deleted file mode 100644 index 3cf0a98b..00000000 --- a/coupledmodeldriver/adcirc/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .adcirc import ADCIRCRunConfiguration, generate_adcirc_configuration -from .job_scripts import ADCIRCGenerationScript diff --git a/coupledmodeldriver/configurations.py b/coupledmodeldriver/configurations.py deleted file mode 100644 index 8b122ecc..00000000 --- a/coupledmodeldriver/configurations.py +++ /dev/null @@ -1,893 +0,0 @@ -from abc import ABC, abstractmethod -from datetime import datetime, timedelta -from enum import Enum -import json -from os import PathLike -from pathlib import Path -from typing import Any, Collection, Mapping, Union - -from adcircpy import AdcircMesh, AdcircRun -from adcircpy.forcing.base import Forcing -from adcircpy.forcing.tides import HAMTIDE, Tides -from adcircpy.forcing.tides.tides import TidalSource -from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing -from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing -from adcircpy.server import SlurmConfig -from nemspy import ModelingSystem -from nemspy.model import ADCIRCEntry, AtmosphericMeshEntry, ModelEntry, \ - WaveMeshEntry - -from .job_scripts import SlurmEmailType -from .platforms import Platform -from .utilities import LOGGER, convert_to_json, convert_value - - -class Model(Enum): - ADCIRC = 'ADCIRC' - TidalForcing = 'Tides' - ATMESH = 'ATMESH' - WW3DATA = 'WW3DATA' - - -class GWCESolutionScheme(Enum): - explicit = 'explicit' - semi_implicit = 'semi-implicit' - semi_implicit_legacy = 'semi-implicit-legacy' - - -class ConfigurationJSON(ABC): - name: PathLike = None - default_filename = f'configure.json' - field_types: {str: type} = {} - - def __init__(self, fields: {str: type} = None, configuration: {str: Any} = None): - if fields is None: - fields = {} - - fields.update(self.field_types) - - if configuration is None: - configuration = {field: None for field in fields} - - self.__fields = fields - self.__configuration = configuration - - @property - def fields(self) -> {str: type}: - return self.__fields - - @property - def configuration(self) -> {str: Any}: - return self.__configuration - - def update(self, configuration: {str: Any}): - for key, value in configuration.items(): - if key in self: - converted_value = convert_value(value, self.fields[key]) - if self[key] != converted_value: - value = converted_value - else: - return - self[key] = value - - def update_from_file(self, filename: PathLike): - with open(filename) as file: - configuration = json.load(file) - self.update(configuration) - - def __contains__(self, key: str) -> bool: - return key in self.configuration - - def __getitem__(self, key: str) -> Any: - return self.configuration[key] - - def __setitem__(self, key: str, value: Any): - if key in self.fields: - field_type = self.fields[key] - else: - field_type = type(value) - LOGGER.info( - f'adding new configuration entry "{key}: {field_type}"' f' to {self.name}"' - ) - self.__configuration[key] = convert_value(value, field_type) - if key not in self.fields: - self.__fields[key] = field_type - - def __eq__(self, other: 'ConfigurationJSON') -> bool: - return other.configuration == self.configuration - - def __repr__(self): - configuration_string = ', '.join( - [f'{key}={repr(value)}' for key, value in self.configuration.items()] - ) - return f'{self.__class__.__name__}({configuration_string})' - - @classmethod - def from_dict(cls, configuration: {str: Any}) -> 'ConfigurationJSON': - return cls(**configuration) - - def to_dict(self) -> {str: Any}: - return self.configuration - - @classmethod - def from_string(cls, string: str) -> 'ConfigurationJSON': - configuration = json.loads(string) - - configuration = { - key.lower(): convert_value(value, cls.field_types[key]) - if key in cls.field_types - else convert_to_json(value) - for key, value in configuration.items() - } - - return cls(**configuration) - - def __str__(self) -> str: - configuration = convert_to_json(self.configuration) - - if any(key != key.lower() for key in configuration): - configuration = {key.lower(): value for key, value in configuration.items()} - - return json.dumps(configuration, indent=2) - - @classmethod - def from_file(cls, filename: PathLike) -> 'ConfigurationJSON': - """ - create new object from an existing JSON file - - :param filename: path to JSON file - :return: configuration object - """ - - if not isinstance(filename, Path): - filename = Path(filename) - - if filename.is_dir(): - filename = filename / cls.default_filename - - with open(filename) as file: - LOGGER.debug(f'reading file "{filename}"') - configuration = json.load(file) - - configuration = { - key.lower(): convert_value(value, cls.field_types[key]) - if key in cls.field_types - else convert_to_json(value) - for key, value in configuration.items() - } - - return cls(**configuration) - - def to_file(self, filename: PathLike = None, overwrite: bool = False): - """ - Write script to file. - - :param filename: path to output file - :param overwrite: whether to overwrite existing file - """ - - if filename is None: - filename = self['output_directory'] - elif not isinstance(filename, Path): - filename = Path(filename) - - if filename.is_dir(): - filename = filename / self.default_filename - - if not filename.parent.exists(): - filename.mkdir(parents=True, exist_ok=True) - - configuration = convert_to_json(self.configuration) - - if any(key != key.lower() for key in configuration): - configuration = {key.lower(): value for key, value in configuration.items()} - - if overwrite or not filename.exists(): - with open(filename.absolute(), 'w') as file: - LOGGER.debug(f'writing to file "{filename}"') - json.dump(configuration, file, indent=2) - else: - LOGGER.debug(f'skipping existing file "{filename}"') - - -class NEMSJSON(ConfigurationJSON): - name = 'nems' - default_filename = f'configure_nems.json' - field_types = { - 'executable_path': Path, - 'modeled_start_time': datetime, - 'modeled_end_time': datetime, - 'interval': timedelta, - 'models': [ModelEntry], - 'connections': [[str]], - 'mediations': [str], - 'sequence': [str], - } - - def __init__( - self, - executable_path: PathLike, - modeled_start_time: datetime, - modeled_end_time: datetime, - interval: timedelta = None, - models: [ModelEntry] = None, - connections: [[str]] = None, - mediations: [[str]] = None, - sequence: [str] = None, - ): - super().__init__() - - self['executable_path'] = executable_path - self['modeled_start_time'] = modeled_start_time - self['modeled_end_time'] = modeled_end_time - self['interval'] = interval - self['models'] = models - self['connections'] = connections - self['mediations'] = mediations - self['sequence'] = sequence - - @property - def nemspy_modeling_system(self) -> ModelingSystem: - modeling_system = ModelingSystem( - start_time=self['modeled_start_time'], - end_time=self['modeled_end_time'], - interval=self['interval'], - **{model.model_type.value.lower(): model for model in self['models']}, - ) - for connection in self['connections']: - modeling_system.connect(*connection) - for mediation in self['mediations']: - modeling_system.mediate(*mediation) - - modeling_system.sequence = self['sequence'] - - return modeling_system - - def to_nemspy(self) -> ModelingSystem: - return self.nemspy_modeling_system - - @classmethod - def from_nemspy(cls, modeling_system: ModelingSystem, executable_path: PathLike = None): - if executable_path is None: - executable_path = 'NEMS.x' - return cls( - executable_path=executable_path, - modeled_start_time=modeling_system.start_time, - modeled_end_time=modeling_system.end_time, - interval=modeling_system.interval, - models=modeling_system.models, - connections=modeling_system.connections, - sequence=modeling_system.sequence, - ) - - -class SlurmJSON(ConfigurationJSON): - name = 'slurm' - default_filename = f'configure_slurm.json' - field_types = { - 'account': str, - 'tasks': int, - 'partition': str, - 'job_duration': timedelta, - 'run_directory': Path, - 'run_name': str, - 'email_type': SlurmEmailType, - 'email_address': str, - 'log_filename': Path, - 'modules': [str], - 'path_prefix': Path, - 'extra_commands': [str], - 'launcher': str, - 'nodes': int, - } - - def __init__( - self, - account: str, - tasks: int = None, - partition: str = None, - job_duration: timedelta = None, - run_directory: PathLike = None, - run_name: str = None, - email_type: SlurmEmailType = None, - email_address: str = None, - log_filename: PathLike = None, - modules: [str] = None, - path_prefix: Path = None, - extra_commands: [str] = None, - launcher: str = None, - nodes: int = None, - ): - super().__init__() - - self['account'] = account - self['tasks'] = tasks - self['partition'] = partition - self['job_duration'] = job_duration - self['run_directory'] = run_directory - self['run_name'] = run_name - self['email_type'] = email_type - self['email_address'] = email_address - self['log_filename'] = log_filename - self['modules'] = modules - self['path_prefix'] = path_prefix - self['extra_commands'] = extra_commands - self['launcher'] = launcher - self['nodes'] = nodes - - if self['email_type'] is None: - if self['email_address'] is not None: - self['email_type'] = SlurmEmailType.ALL - - def to_adcircpy(self) -> SlurmConfig: - return SlurmConfig( - account=self['account'], - ntasks=self['tasks'], - partition=self['partition'], - walltime=self['job_duration'], - filename=self['filename'] if 'filename' in self else None, - run_directory=self['run_directory'], - run_name=self['run_name'], - mail_type=self['email_type'], - mail_user=self['email_address'], - log_filename=self['log_filename'], - modules=self['modules'], - path_prefix=self['path_prefix'], - extra_commands=self['extra_commands'], - launcher=self['launcher'], - nodes=self['nodes'], - ) - - @classmethod - def from_adcircpy(cls, slurm_config: SlurmConfig): - instance = cls( - account=slurm_config._account, - tasks=slurm_config._slurm_ntasks, - partition=slurm_config._partition, - job_duration=slurm_config._walltime, - run_directory=slurm_config._run_directory, - run_name=slurm_config._run_name, - email_type=slurm_config._mail_type, - email_address=slurm_config._mail_user, - log_filename=slurm_config._log_filename, - modules=slurm_config._modules, - path_prefix=slurm_config._path_prefix, - extra_commands=slurm_config._extra_commands, - launcher=slurm_config._launcher, - nodes=slurm_config._nodes, - ) - - instance['filename'] = slurm_config._filename - - -class ModelJSON(ConfigurationJSON, ABC): - def __init__(self, model: Model, fields: {str: type} = None): - if not isinstance(model, Model): - model = Model[str(model).lower()] - if fields is None: - fields = {} - - fields.update(self.field_types) - - self.model = model - ConfigurationJSON.__init__(self, fields=fields) - - -class NEMSCapJSON(ConfigurationJSON, ABC): - field_types = { - 'processors': int, - 'nems_parameters': {str: str}, - } - - def __init__(self, processors: int, nems_parameters: {str: str} = None): - super().__init__(fields=self.fields, configuration=self.configuration) - self.fields.update(NEMSCapJSON.field_types) - - if nems_parameters is None: - nems_parameters = {} - - self['processors'] = processors - self['nems_parameters'] = nems_parameters - - @abstractmethod - def nemspy_entry(self) -> ModelEntry: - raise NotImplementedError() - - -class ADCIRCJSON(ModelJSON, NEMSCapJSON): - name = 'adcirc' - default_filename = f'configure_adcirc.json' - field_types = { - 'adcirc_executable_path': Path, - 'adcprep_executable_path': Path, - 'modeled_start_time': datetime, - 'modeled_end_time': datetime, - 'modeled_timestep': timedelta, - 'fort_13_path': Path, - 'fort_14_path': Path, - 'tidal_spinup_duration': timedelta, - 'tidal_spinup_timestep': timedelta, - 'gwce_solution_scheme': GWCESolutionScheme, - 'use_smagorinsky': bool, - 'source_filename': Path, - 'use_original_mesh': bool, - 'stations_file_path': Path, - 'write_surface_output': bool, - 'write_station_output': bool, - } - - def __init__( - self, - adcirc_executable_path: PathLike, - adcprep_executable_path: PathLike, - modeled_start_time: datetime, - modeled_end_time: datetime, - modeled_timestep: timedelta, - fort_13_path: PathLike, - fort_14_path: PathLike, - tidal_spinup_duration: timedelta = None, - tidal_spinup_timestep: timedelta = None, - forcings: [Forcing] = None, - gwce_solution_scheme: str = None, - use_smagorinsky: bool = None, - source_filename: PathLike = None, - slurm_configuration: SlurmJSON = None, - use_original_mesh: bool = False, - stations_file_path: PathLike = None, - write_surface_output: bool = True, - write_station_output: bool = False, - processors: int = 11, - nems_parameters: {str: str} = None, - ): - """ - - :param adcirc_executable_path: file path to `adcirc` or `NEMS.x` - :param adcprep_executable_path: file path to `adcprep` - :param modeled_start_time: start time in model run - :param modeled_end_time: edn time in model run - :param modeled_timestep: time interval between model steps - :param fort_13_path: file path to `fort.13` - :param fort_14_path: file path to `fort.14` - :param fort_14_path: file path to `fort.14` - :param tidal_spinup_duration: tidal spinup duration for ADCIRC coldstart - :param tidal_spinup_timestep: tidal spinup modeled time interval for ADCIRC coldstart - :param forcings: list of Forcing objects to apply to the mesh - :param gwce_solution_scheme: solution scheme (can be `explicit`, `semi-implicit`, or `semi-implicit-legacy`) - :param use_smagorinsky: whether to use Smagorinsky coefficient - :param source_filename: path to modulefile to `source` - :param slurm_configuration: Slurm configuration object - :param use_original_mesh: whether to symlink / copy original mesh instead of rewriting with `adcircpy` - :param stations_file_path: file path to stations file - :param write_surface_output: whether to write surface output to NetCDF - :param write_station_output: whether to write station output to NetCDF (only applicable if stations file exists) - :param processors: number of processors to use - :param nems_parameters: parameters to give to NEMS cap - """ - - if tidal_spinup_timestep is None: - tidal_spinup_timestep = modeled_timestep - - if forcings is None: - forcings = [] - - ModelJSON.__init__(self, model=Model.ADCIRC) - NEMSCapJSON.__init__(self, processors=processors, nems_parameters=nems_parameters) - - self['adcirc_executable_path'] = adcirc_executable_path - self['adcprep_executable_path'] = adcprep_executable_path - self['modeled_start_time'] = modeled_start_time - self['modeled_end_time'] = modeled_end_time - self['modeled_timestep'] = modeled_timestep - self['fort_13_path'] = fort_13_path - self['fort_14_path'] = fort_14_path - self['tidal_spinup_duration'] = tidal_spinup_duration - self['tidal_spinup_timestep'] = tidal_spinup_timestep - self['gwce_solution_scheme'] = gwce_solution_scheme - self['use_smagorinsky'] = use_smagorinsky - self['source_filename'] = source_filename - self['use_original_mesh'] = use_original_mesh - self['stations_file_path'] = stations_file_path - self['write_surface_output'] = write_surface_output - self['write_station_output'] = write_station_output - - self.forcings = forcings - self.slurm_configuration = slurm_configuration - - @property - def forcings(self) -> ['ForcingJSON']: - return self.__forcings - - @forcings.setter - def forcings(self, forcings: ['ForcingJSON']): - for index, forcing in enumerate(forcings): - if isinstance(forcing, Forcing): - forcings[index] = ForcingJSON.from_adcircpy(forcing) - self.__forcings = forcings - - @property - def slurm_configuration(self) -> [SlurmJSON]: - return self.__slurm_configuration - - @slurm_configuration.setter - def slurm_configuration(self, slurm_configuration: SlurmJSON): - if isinstance(slurm_configuration, SlurmConfig): - SlurmJSON.from_adcircpy(slurm_configuration) - self.__slurm_configuration = slurm_configuration - - @property - def adcircpy_mesh(self) -> AdcircMesh: - LOGGER.info(f'opening mesh "{self["fort_14_path"]}"') - mesh = AdcircMesh.open(self['fort_14_path'], crs=4326) - - LOGGER.debug(f'adding {len(self.forcings)} forcing(s) to mesh') - for forcing in self.forcings: - adcircpy_forcing = forcing.adcircpy_forcing - if ( - isinstance(adcircpy_forcing, Tides) - and self['tidal_spinup_duration'] is not None - ): - adcircpy_forcing.spinup_time = self['tidal_spinup_duration'] - adcircpy_forcing.start_date = self['modeled_start_time'] - adcircpy_forcing.end_date = self['modeled_end_time'] - if self['tidal_spinup_duration'] is not None: - adcircpy_forcing.start_date -= self['tidal_spinup_duration'] - mesh.add_forcing(adcircpy_forcing) - - if self['fort_13_path'] is not None: - LOGGER.info(f'reading attributes from "{self["fort_13_path"]}"') - if self['fort_13_path'].exists(): - mesh.import_nodal_attributes(self['fort_13_path']) - for attribute_name in mesh.get_nodal_attribute_names(): - mesh.set_nodal_attribute_state( - attribute_name, coldstart=True, hotstart=True - ) - else: - LOGGER.warning( - 'mesh values (nodal attributes) not found ' f'at "{self["fort_13_path"]}"' - ) - - if not mesh.has_nodal_attribute('primitive_weighting_in_continuity_equation'): - LOGGER.debug(f'generating tau0 in mesh') - mesh.generate_tau0() - - return mesh - - @property - def adcircpy_driver(self) -> AdcircRun: - # instantiate AdcircRun object. - driver = AdcircRun( - mesh=self.adcircpy_mesh, - start_date=self['modeled_start_time'], - end_date=self['modeled_end_time'], - spinup_time=self['tidal_spinup_duration'], - server_config=self.slurm_configuration.to_adcircpy() - if self.slurm_configuration is not None - else None, - ) - - if self['modeled_timestep'] is not None: - driver.timestep = self['modeled_timestep'] / timedelta(seconds=1) - - if self['gwce_solution_scheme'] is not None: - driver.gwce_solution_scheme = self['gwce_solution_scheme'].value - - if self['use_smagorinsky'] is not None: - driver.smagorinsky = self['use_smagorinsky'] - - if self['tidal_spinup_duration'] is not None: - spinup_start = self['modeled_start_time'] - self['tidal_spinup_duration'] - else: - spinup_start = None - - if self['write_station_output'] and self['stations_file_path'].exists(): - driver.import_stations(self['stations_file_path']) - driver.set_elevation_stations_output( - self['modeled_timestep'], - spinup=self['tidal_spinup_timestep'], - spinup_start=spinup_start, - ) - driver.set_velocity_stations_output( - self['modeled_timestep'], - spinup=self['tidal_spinup_timestep'], - spinup_start=spinup_start, - ) - - if self['write_surface_output']: - driver.set_elevation_surface_output( - self['modeled_timestep'], - spinup=self['tidal_spinup_timestep'], - spinup_start=spinup_start, - ) - driver.set_velocity_surface_output( - self['modeled_timestep'], - spinup=self['tidal_spinup_timestep'], - spinup_start=spinup_start, - ) - - return driver - - @property - def nemspy_entry(self) -> ADCIRCEntry: - return ADCIRCEntry(processors=self['processors'], **self['nems_parameters']) - - -class ForcingJSON(ConfigurationJSON, ABC): - field_types = {'resource': str} - - def __init__(self, resource: PathLike, fields: {str: type} = None): - if fields is None: - fields = {} - - fields.update(self.field_types) - fields.update(ForcingJSON.field_types) - - ConfigurationJSON.__init__(self, fields=fields) - - try: - resource = Path(resource).as_posix() - except: - pass - - self['resource'] = resource - - @property - @abstractmethod - def adcircpy_forcing(self) -> Forcing: - raise NotImplementedError - - def to_adcircpy(self) -> Forcing: - return self.adcircpy_forcing - - @classmethod - @abstractmethod - def from_adcircpy(cls, forcing: Forcing) -> 'ForcingJSON': - raise NotImplementedError() - - -class TidalForcingJSON(ForcingJSON): - name = 'tidal_forcing' - default_filename = f'configure_tidal_forcing.json' - field_types = {'tidal_source': TidalSource, 'constituents': [str]} - - def __init__( - self, - resource: PathLike = None, - tidal_source: TidalSource = TidalSource.TPXO, - constituents: [str] = None, - ): - if constituents is None: - constituents = 'ALL' - elif not isinstance(constituents, str): - constituents = list(constituents) - - super().__init__(resource=resource) - - self['tidal_source'] = tidal_source - self['constituents'] = constituents - - @property - def adcircpy_forcing(self) -> Forcing: - tides = Tides(tidal_source=self['tidal_source'], resource=self['resource']) - - constituents = [constituent.capitalize() for constituent in self['constituents']] - - if sorted(constituents) == sorted( - constituent.capitalize() for constituent in tides.constituents - ): - constituents = ['All'] - elif sorted(constituents) == sorted( - constituent.capitalize() for constituent in tides.major_constituents - ): - constituents = ['Major'] - - if 'All' in constituents: - tides.use_all() - else: - if 'Major' in constituents: - tides.use_major() - constituents.remove('Major') - for constituent in constituents: - if constituent not in tides.active_constituents: - tides.use_constituent(constituent) - - self['constituents'] = list(tides.active_constituents) - return tides - - @classmethod - def from_adcircpy(cls, forcing: Tides) -> 'TidalForcingJSON': - # TODO: workaround for this issue: https://github.com/JaimeCalzadaNOAA/adcircpy/pull/70#discussion_r607245713 - resource = forcing.tidal_dataset.path - if resource == HAMTIDE.OPENDAP_URL: - resource = None - - return cls( - resource=resource, - tidal_source=forcing.tidal_source, - constituents=forcing.active_constituents, - ) - - -class ATMESHForcingJSON(ForcingJSON, NEMSCapJSON): - name = 'atmesh' - default_filename = f'configure_atmesh.json' - field_types = { - 'nws': int, - 'modeled_timestep': timedelta, - } - - def __init__( - self, - resource: PathLike, - nws: int = 17, - modeled_timestep: timedelta = timedelta(hours=1), - processors: int = 1, - nems_parameters: {str: str} = None, - ): - ForcingJSON.__init__(self, resource=resource) - NEMSCapJSON.__init__(self, processors=processors, nems_parameters=nems_parameters) - - self['nws'] = nws - self['modeled_timestep'] = modeled_timestep - - @property - def adcircpy_forcing(self) -> Forcing: - return AtmosphericMeshForcing( - filename=self['resource'], - nws=self['nws'], - interval_seconds=self['modeled_timestep'] / timedelta(seconds=1), - ) - - @classmethod - def from_adcircpy(cls, forcing: AtmosphericMeshForcing) -> 'ATMESHForcingJSON': - return cls( - resource=forcing.filename, nws=forcing.NWS, modeled_timestep=forcing.interval, - ) - - @property - def nemspy_entry(self) -> AtmosphericMeshEntry: - return AtmosphericMeshEntry( - filename=self['resource'], processors=self['processors'], **self['nems_parameters'] - ) - - -class WW3DATAForcingJSON(ForcingJSON, NEMSCapJSON): - name = 'ww3data' - default_filename = f'configure_ww3data.json' - field_types = {'nrs': int, 'modeled_timestep': timedelta} - - def __init__( - self, - resource: PathLike, - nrs: int = 5, - modeled_timestep: timedelta = timedelta(hours=1), - processors: int = 1, - nems_parameters: {str: str} = None, - ): - ForcingJSON.__init__(self, resource=resource) - NEMSCapJSON.__init__(self, processors=processors, nems_parameters=nems_parameters) - - self['nrs'] = nrs - self['modeled_timestep'] = modeled_timestep - - @property - def adcircpy_forcing(self) -> Forcing: - return WaveWatch3DataForcing( - filename=self['resource'], - nrs=self['nrs'], - interval_seconds=self['modeled_timestep'], - ) - - @classmethod - def from_adcircpy(cls, forcing: WaveWatch3DataForcing) -> 'WW3DATAForcingJSON': - return cls( - resource=forcing.filename, nrs=forcing.NRS, modeled_timestep=forcing.interval, - ) - - @property - def nemspy_entry(self) -> WaveMeshEntry: - return WaveMeshEntry( - filename=self['resource'], processors=self['processors'], **self['nems_parameters'] - ) - - -class ModelDriverJSON(ConfigurationJSON): - name = 'modeldriver' - default_filename = f'configure_modeldriver.json' - field_types = { - 'platform': Platform, - 'runs': {str: {str: Any}}, - } - - def __init__( - self, platform: Platform, runs: {str: {str: Any}} = None, - ): - if runs is None: - runs = {'run_1': None} - - super().__init__() - - self['platform'] = platform - self['runs'] = runs - - -class RunConfiguration(ABC): - required: [ConfigurationJSON] = [] - forcings: [ForcingJSON] = [] - - def __init__(self, configurations: [ConfigurationJSON]): - self.__configurations = {} - self.configurations = configurations - - @property - def configurations(self) -> {str: ConfigurationJSON}: - return self.__configurations - - @configurations.setter - def configurations(self, configurations: {str: ConfigurationJSON}): - if isinstance(configurations, Collection) and not isinstance(configurations, Mapping): - configurations = {entry.name: entry for entry in configurations} - for name, configuration in configurations.items(): - self[name] = configuration - - @property - def nemspy_entries(self) -> [ModelEntry]: - return [ - configuration.nemspy_entry - for configuration in self.configurations.values() - if isinstance(configuration, NEMSCapJSON) - ] - - def __contains__(self, configuration: Union[str, ConfigurationJSON]) -> bool: - if isinstance(configuration, ConfigurationJSON): - configuration = configuration.name - return configuration in self.configurations - - def __getitem__(self, name: str) -> ConfigurationJSON: - return self.configurations[name] - - def __setitem__(self, name: str, value: ConfigurationJSON): - if isinstance(value, str): - if Path(value).exists(): - value = ConfigurationJSON.from_file(value) - else: - value = ConfigurationJSON.from_string(value) - elif isinstance(value, Forcing): - value = ForcingJSON.from_adcircpy(value) - self.configurations[name] = value - - @classmethod - def read_directory(cls, directory: PathLike) -> 'RunConfiguration': - if not isinstance(directory, Path): - directory = Path(directory) - if directory.is_file(): - directory = directory.parent - - configurations = [] - for configuration_class in cls.required: - filename = directory / configuration_class.default_filename - if filename.exists(): - configurations.append(configuration_class.from_file(filename)) - else: - raise FileNotFoundError(f'missing required configuration file "{filename}"') - - for configuration_class in cls.forcings: - filename = directory / configuration_class.default_filename - if filename.exists(): - configurations.append(configuration_class.from_file(filename)) - - return cls(configurations) - - def write_directory(self, directory: PathLike, overwrite: bool = False): - """ - :param directory: directory in which to write generated JSON configuration files - :param overwrite: whether to overwrite existing files - """ - - if not isinstance(directory, Path): - directory = Path(directory) - - if not directory.exists(): - directory.mkdir(parents=True, exist_ok=True) - - for configuration in self.__configurations.values(): - configuration.to_file(directory, overwrite=overwrite) diff --git a/coupledmodeldriver/configure/__init__.py b/coupledmodeldriver/configure/__init__.py new file mode 100644 index 00000000..92f8b8fc --- /dev/null +++ b/coupledmodeldriver/configure/__init__.py @@ -0,0 +1 @@ +from .base import ModelDriverJSON, SlurmJSON diff --git a/coupledmodeldriver/configure/base.py b/coupledmodeldriver/configure/base.py new file mode 100644 index 00000000..05baaef7 --- /dev/null +++ b/coupledmodeldriver/configure/base.py @@ -0,0 +1,309 @@ +from abc import ABC, abstractmethod +from datetime import timedelta +import json +from os import PathLike +from pathlib import Path +from typing import Any + +from adcircpy.server import SlurmConfig +from nemspy.model import ModelEntry + +from ..platforms import Platform +from ..script import SlurmEmailType +from ..utilities import LOGGER, convert_to_json, convert_value + + +class ConfigurationJSON(ABC): + name: PathLike + default_filename: str + field_types: {str: type} + + def __init__(self, fields: {str: type} = None, configuration: {str: Any} = None): + if not hasattr(self, 'fields'): + self.fields = {} + + if fields is not None: + self.fields.update(fields) + + if not hasattr(self, 'configuration'): + self.configuration = {field: None for field in self.fields} + + if configuration is not None: + self.configuration.update(configuration) + + def update(self, configuration: {str: Any}): + for key, value in configuration.items(): + if key in self: + converted_value = convert_value(value, self.fields[key]) + if self[key] != converted_value: + value = converted_value + else: + return + self[key] = value + + def update_from_file(self, filename: PathLike): + with open(filename) as file: + configuration = json.load(file) + self.update(configuration) + + def __contains__(self, key: str) -> bool: + return key in self.configuration + + def __getitem__(self, key: str) -> Any: + return self.configuration[key] + + def __setitem__(self, key: str, value: Any): + if key in self.fields: + field_type = self.fields[key] + else: + field_type = type(value) + LOGGER.info( + f'adding new configuration entry "{key}: {field_type}"' f' to {self.name}"' + ) + self.configuration[key] = convert_value(value, field_type) + if key not in self.fields: + self.fields[key] = field_type + + def __eq__(self, other: 'ConfigurationJSON') -> bool: + return other.configuration == self.configuration + + def __repr__(self): + configuration_string = ', '.join( + [f'{key}={repr(value)}' for key, value in self.configuration.items()] + ) + return f'{self.__class__.__name__}({configuration_string})' + + @classmethod + def from_dict(cls, configuration: {str: Any}) -> 'ConfigurationJSON': + return cls(**configuration) + + def to_dict(self) -> {str: Any}: + return self.configuration + + @classmethod + def from_string(cls, string: str) -> 'ConfigurationJSON': + configuration = json.loads(string) + + configuration = { + key.lower(): convert_value(value, cls.field_types[key]) + if key in cls.field_types + else convert_to_json(value) + for key, value in configuration.items() + } + + return cls(**configuration) + + def __str__(self) -> str: + configuration = convert_to_json(self.configuration) + + if any(key != key.lower() for key in configuration): + configuration = {key.lower(): value for key, value in configuration.items()} + + return json.dumps(configuration, indent=2) + + @classmethod + def from_file(cls, filename: PathLike) -> 'ConfigurationJSON': + """ + create new object from an existing JSON file + + :param filename: path to JSON file + :return: configuration object + """ + + if not isinstance(filename, Path): + filename = Path(filename) + + if filename.is_dir(): + filename = filename / cls.default_filename + + with open(filename) as file: + LOGGER.debug(f'reading file "{filename}"') + configuration = json.load(file) + + configuration = { + key.lower(): convert_value(value, cls.field_types[key]) + if key in cls.field_types + else convert_to_json(value) + for key, value in configuration.items() + } + + return cls(**configuration) + + def to_file(self, filename: PathLike = None, overwrite: bool = False): + """ + Write script to file. + + :param filename: path to output file + :param overwrite: whether to overwrite existing file + """ + + if filename is None: + filename = self['output_directory'] + elif not isinstance(filename, Path): + filename = Path(filename) + + if filename.is_dir(): + filename = filename / self.default_filename + + if not filename.parent.exists(): + filename.mkdir(parents=True, exist_ok=True) + + configuration = convert_to_json(self.configuration) + + if any(key != key.lower() for key in configuration): + configuration = {key.lower(): value for key, value in configuration.items()} + + if overwrite or not filename.exists(): + with open(filename.absolute(), 'w') as file: + LOGGER.debug(f'writing to file "{filename}"') + json.dump(configuration, file, indent=2) + else: + LOGGER.debug(f'skipping existing file "{filename}"') + + +class NEMSCapJSON(ConfigurationJSON, ABC): + field_types = { + 'processors': int, + 'nems_parameters': {str: str}, + } + + def __init__(self, processors: int, nems_parameters: {str: str} = None, **kwargs): + if nems_parameters is None: + nems_parameters = {} + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(NEMSCapJSON.field_types) + + ConfigurationJSON.__init__(self, **kwargs) + + self['processors'] = processors + self['nems_parameters'] = nems_parameters + + @abstractmethod + def nemspy_entry(self) -> ModelEntry: + raise NotImplementedError() + + +class SlurmJSON(ConfigurationJSON): + name = 'slurm' + default_filename = f'configure_slurm.json' + field_types = { + 'account': str, + 'tasks': int, + 'partition': str, + 'job_duration': timedelta, + 'run_directory': Path, + 'run_name': str, + 'email_type': SlurmEmailType, + 'email_address': str, + 'log_filename': Path, + 'modules': [str], + 'path_prefix': Path, + 'extra_commands': [str], + 'launcher': str, + 'nodes': int, + } + + def __init__( + self, + account: str, + tasks: int = None, + partition: str = None, + job_duration: timedelta = None, + run_directory: PathLike = None, + run_name: str = None, + email_type: SlurmEmailType = None, + email_address: str = None, + log_filename: PathLike = None, + modules: [str] = None, + path_prefix: Path = None, + extra_commands: [str] = None, + launcher: str = None, + nodes: int = None, + **kwargs, + ): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(SlurmJSON.field_types) + + ConfigurationJSON.__init__(self, **kwargs) + + self['account'] = account + self['tasks'] = tasks + self['partition'] = partition + self['job_duration'] = job_duration + self['run_directory'] = run_directory + self['run_name'] = run_name + self['email_type'] = email_type + self['email_address'] = email_address + self['log_filename'] = log_filename + self['modules'] = modules + self['path_prefix'] = path_prefix + self['extra_commands'] = extra_commands + self['launcher'] = launcher + self['nodes'] = nodes + + if self['email_type'] is None: + if self['email_address'] is not None: + self['email_type'] = SlurmEmailType.ALL + + def to_adcircpy(self) -> SlurmConfig: + return SlurmConfig( + account=self['account'], + ntasks=self['tasks'], + partition=self['partition'], + walltime=self['job_duration'], + filename=self['filename'] if 'filename' in self else None, + run_directory=self['run_directory'], + run_name=self['run_name'], + mail_type=self['email_type'], + mail_user=self['email_address'], + log_filename=self['log_filename'], + modules=self['modules'], + path_prefix=self['path_prefix'], + extra_commands=self['extra_commands'], + launcher=self['launcher'], + nodes=self['nodes'], + ) + + @classmethod + def from_adcircpy(cls, slurm_config: SlurmConfig): + instance = cls( + account=slurm_config._account, + tasks=slurm_config._slurm_ntasks, + partition=slurm_config._partition, + job_duration=slurm_config._walltime, + run_directory=slurm_config._run_directory, + run_name=slurm_config._run_name, + email_type=slurm_config._mail_type, + email_address=slurm_config._mail_user, + log_filename=slurm_config._log_filename, + modules=slurm_config._modules, + path_prefix=slurm_config._path_prefix, + extra_commands=slurm_config._extra_commands, + launcher=slurm_config._launcher, + nodes=slurm_config._nodes, + ) + + instance['filename'] = slurm_config._filename + + +class ModelDriverJSON(ConfigurationJSON): + name = 'modeldriver' + default_filename = f'configure_modeldriver.json' + field_types = { + 'platform': Platform, + 'runs': {str: {str: Any}}, + } + + def __init__(self, platform: Platform, runs: {str: {str: Any}} = None, **kwargs): + if runs is None: + runs = {'run_1': None} + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(ModelDriverJSON.field_types) + + ConfigurationJSON.__init__(self, **kwargs) + + self['platform'] = platform + self['runs'] = runs diff --git a/coupledmodeldriver/configure/forcings/__init__.py b/coupledmodeldriver/configure/forcings/__init__.py new file mode 100644 index 00000000..d0b0c902 --- /dev/null +++ b/coupledmodeldriver/configure/forcings/__init__.py @@ -0,0 +1,7 @@ +from .base import ( + ATMESHForcingJSON, + BestTrackForcingJSON, + OWIForcingJSON, + TidalForcingJSON, + WW3DATAForcingJSON, +) diff --git a/coupledmodeldriver/configure/forcings/base.py b/coupledmodeldriver/configure/forcings/base.py new file mode 100644 index 00000000..0def1dd4 --- /dev/null +++ b/coupledmodeldriver/configure/forcings/base.py @@ -0,0 +1,295 @@ +from abc import ABC, abstractmethod +from datetime import datetime, timedelta +from os import PathLike +from pathlib import Path + +from adcircpy import Tides +from adcircpy.forcing.base import Forcing +from adcircpy.forcing.tides import HAMTIDE +from adcircpy.forcing.tides.tides import TidalSource +from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing +from adcircpy.forcing.winds import BestTrackForcing +from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing +from adcircpy.forcing.winds.owi import OwiForcing +from nemspy.model import AtmosphericMeshEntry, WaveMeshEntry + +from coupledmodeldriver.configure.base import ConfigurationJSON, \ + NEMSCapJSON + + +class ForcingJSON(ConfigurationJSON, ABC): + @property + @abstractmethod + def adcircpy_forcing(self) -> Forcing: + raise NotImplementedError + + def to_adcircpy(self) -> Forcing: + return self.adcircpy_forcing + + @classmethod + @abstractmethod + def from_adcircpy(cls, forcing: Forcing) -> 'ForcingJSON': + raise NotImplementedError() + + +class TimestepForcingJSON(ForcingJSON, ABC): + field_types = {'modeled_timestep': timedelta} + + def __init__(self, modeled_timestep: timedelta, **kwargs): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(TimestepForcingJSON.field_types) + + ForcingJSON.__init__(self, **kwargs) + + self['modeled_timestep'] = modeled_timestep + + +class FileForcingJSON(ForcingJSON, ABC): + field_types = {'resource': str} + + def __init__(self, resource: PathLike, **kwargs): + try: + resource = Path(resource).as_posix() + except: + pass + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(FileForcingJSON.field_types) + + ForcingJSON.__init__(self, **kwargs) + + self['resource'] = resource + + +class TidalForcingJSON(FileForcingJSON): + name = 'tidal_forcing' + default_filename = f'configure_tidal_forcing.json' + field_types = {'tidal_source': TidalSource, 'constituents': [str]} + + def __init__( + self, + resource: PathLike = None, + tidal_source: TidalSource = TidalSource.TPXO, + constituents: [str] = None, + **kwargs, + ): + if constituents is None: + constituents = 'All' + elif not isinstance(constituents, str): + constituents = list(constituents) + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(TidalForcingJSON.field_types) + + FileForcingJSON.__init__(self, resource=resource, **kwargs) + + self['tidal_source'] = tidal_source + self['constituents'] = constituents + + @property + def adcircpy_forcing(self) -> Forcing: + tides = Tides(tidal_source=self['tidal_source'], resource=self['resource']) + + constituents = [constituent.capitalize() for constituent in self['constituents']] + + if sorted(constituents) == sorted( + constituent.capitalize() for constituent in tides.constituents + ): + constituents = ['All'] + elif sorted(constituents) == sorted( + constituent.capitalize() for constituent in tides.major_constituents + ): + constituents = ['Major'] + + if 'All' in constituents: + tides.use_all() + else: + if 'Major' in constituents: + tides.use_major() + constituents.remove('Major') + for constituent in constituents: + if constituent not in tides.active_constituents: + tides.use_constituent(constituent) + + self['constituents'] = list(tides.active_constituents) + return tides + + @classmethod + def from_adcircpy(cls, forcing: Tides) -> 'TidalForcingJSON': + # TODO: workaround for this issue: https://github.com/JaimeCalzadaNOAA/adcircpy/pull/70#discussion_r607245713 + resource = forcing.tidal_dataset.path + if resource == HAMTIDE.OPENDAP_URL: + resource = None + + return cls( + resource=resource, + tidal_source=forcing.tidal_source, + constituents=forcing.active_constituents, + ) + + +class WindForcingJSON(ForcingJSON, ABC): + field_types = {'nws': int} + + def __init__(self, nws: int, **kwargs): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(WindForcingJSON.field_types) + + ForcingJSON.__init__(self, **kwargs) + + self['nws'] = nws + + +class BestTrackForcingJSON(WindForcingJSON): + name = 'besttrack' + default_filename = f'configure_besttrack.json' + field_types = { + 'storm_id': str, + 'start_date': datetime, + 'end_date': datetime, + } + + def __init__( + self, storm_id: str, start_date: datetime, end_date: datetime, nws: int = 20, **kwargs + ): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(BestTrackForcingJSON.field_types) + + WindForcingJSON.__init__(self, nws=nws, **kwargs) + + self['storm_id'] = storm_id + self['start_date'] = start_date + self['end_date'] = end_date + + @property + def adcircpy_forcing(self) -> BestTrackForcing: + return BestTrackForcing( + storm_id=self['storm_id'], + nws=self['nws'], + start_date=self['start_date'], + end_date=self['end_date'], + ) + + @classmethod + def from_adcircpy(cls, forcing: BestTrackForcing) -> 'BestTrackForcingJSON': + return cls( + storm_id=forcing.storm_id, + nws=forcing.NWS, + start_date=forcing.start_date, + end_date=forcing.end_date, + ) + + +class OWIForcingJSON(WindForcingJSON, TimestepForcingJSON): + name = 'owi' + default_filename = f'configure_owi.json' + + def __init__(self, modeled_timestep: timedelta = timedelta(hours=1), **kwargs): + WindForcingJSON.__init__(self, nws=12, **kwargs) + TimestepForcingJSON.__init__(self, modeled_timestep=modeled_timestep, **kwargs) + + @property + def adcircpy_forcing(self) -> OwiForcing: + return OwiForcing(interval_seconds=self['modeled_timestep'] / timedelta(seconds=1), ) + + @classmethod + def from_adcircpy(cls, forcing: OwiForcing) -> 'OWIForcingJSON': + return cls(modeled_timestep=timedelta(seconds=forcing.interval)) + + +class ATMESHForcingJSON(WindForcingJSON, FileForcingJSON, TimestepForcingJSON, NEMSCapJSON): + name = 'atmesh' + default_filename = f'configure_atmesh.json' + + def __init__( + self, + resource: PathLike, + nws: int = 17, + modeled_timestep: timedelta = timedelta(hours=1), + processors: int = 1, + nems_parameters: {str: str} = None, + **kwargs, + ): + WindForcingJSON.__init__(self, nws=nws, **kwargs) + FileForcingJSON.__init__(self, resource=resource, **kwargs) + NEMSCapJSON.__init__( + self, processors=processors, nems_parameters=nems_parameters, **kwargs + ) + TimestepForcingJSON.__init__(self, modeled_timestep=modeled_timestep, **kwargs) + + @property + def adcircpy_forcing(self) -> Forcing: + return AtmosphericMeshForcing( + filename=self['resource'], + nws=self['nws'], + interval_seconds=self['modeled_timestep'] / timedelta(seconds=1), + ) + + @classmethod + def from_adcircpy(cls, forcing: AtmosphericMeshForcing) -> 'ATMESHForcingJSON': + return cls( + resource=forcing.filename, nws=forcing.NWS, modeled_timestep=forcing.interval, + ) + + @property + def nemspy_entry(self) -> AtmosphericMeshEntry: + return AtmosphericMeshEntry( + filename=self['resource'], processors=self['processors'], **self['nems_parameters'] + ) + + +class WaveForcingJSON(ForcingJSON, ABC): + field_types = {'nrs': int} + + def __init__(self, nrs: int, **kwargs): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(WaveForcingJSON.field_types) + + ForcingJSON.__init__(self, **kwargs) + + self['nrs'] = nrs + + +class WW3DATAForcingJSON(WaveForcingJSON, FileForcingJSON, TimestepForcingJSON, NEMSCapJSON): + name = 'ww3data' + default_filename = f'configure_ww3data.json' + + def __init__( + self, + resource: PathLike, + nrs: int = 5, + modeled_timestep: timedelta = timedelta(hours=1), + processors: int = 1, + nems_parameters: {str: str} = None, + **kwargs, + ): + WaveForcingJSON.__init__(self, nrs=nrs, **kwargs) + FileForcingJSON.__init__(self, resource=resource, **kwargs) + NEMSCapJSON.__init__( + self, processors=processors, nems_parameters=nems_parameters, **kwargs + ) + TimestepForcingJSON.__init__(self, modeled_timestep=modeled_timestep, **kwargs) + + @property + def adcircpy_forcing(self) -> Forcing: + return WaveWatch3DataForcing( + filename=self['resource'], + nrs=self['nrs'], + interval_seconds=self['modeled_timestep'], + ) + + @classmethod + def from_adcircpy(cls, forcing: WaveWatch3DataForcing) -> 'WW3DATAForcingJSON': + return cls( + resource=forcing.filename, nrs=forcing.NRS, modeled_timestep=forcing.interval, + ) + + @property + def nemspy_entry(self) -> WaveMeshEntry: + return WaveMeshEntry( + filename=self['resource'], processors=self['processors'], **self['nems_parameters'] + ) diff --git a/coupledmodeldriver/configure/models.py b/coupledmodeldriver/configure/models.py new file mode 100644 index 00000000..b52f7ce9 --- /dev/null +++ b/coupledmodeldriver/configure/models.py @@ -0,0 +1,259 @@ +from abc import ABC +from datetime import datetime, timedelta +from enum import Enum +from os import PathLike +from pathlib import Path + +from adcircpy import AdcircMesh, AdcircRun, Tides +from adcircpy.forcing.base import Forcing +from adcircpy.server import SlurmConfig +from nemspy.model import ADCIRCEntry + +from coupledmodeldriver.configure.forcings.base import ForcingJSON +from .base import ConfigurationJSON, NEMSCapJSON, SlurmJSON +from ..utilities import LOGGER + + +class Model(Enum): + ADCIRC = 'ADCIRC' + TidalForcing = 'Tides' + ATMESH = 'ATMESH' + WW3DATA = 'WW3DATA' + + +class ModelJSON(ConfigurationJSON, ABC): + def __init__(self, model: Model, **kwargs): + if not isinstance(model, Model): + model = Model[str(model).lower()] + + ConfigurationJSON.__init__(self, **kwargs) + + self.model = model + + +class GWCESolutionScheme(Enum): + explicit = 'explicit' + semi_implicit = 'semi-implicit' + semi_implicit_legacy = 'semi-implicit-legacy' + + +class ADCIRCJSON(ModelJSON, NEMSCapJSON): + name = 'adcirc' + default_filename = f'configure_adcirc.json' + field_types = { + 'adcirc_executable_path': Path, + 'adcprep_executable_path': Path, + 'modeled_start_time': datetime, + 'modeled_end_time': datetime, + 'modeled_timestep': timedelta, + 'fort_13_path': Path, + 'fort_14_path': Path, + 'tidal_spinup_duration': timedelta, + 'tidal_spinup_timestep': timedelta, + 'gwce_solution_scheme': GWCESolutionScheme, + 'use_smagorinsky': bool, + 'source_filename': Path, + 'use_original_mesh': bool, + 'stations_file_path': Path, + 'write_surface_output': bool, + 'write_station_output': bool, + } + + def __init__( + self, + adcirc_executable_path: PathLike, + adcprep_executable_path: PathLike, + modeled_start_time: datetime, + modeled_end_time: datetime, + modeled_timestep: timedelta, + fort_13_path: PathLike, + fort_14_path: PathLike, + tidal_spinup_duration: timedelta = None, + tidal_spinup_timestep: timedelta = None, + forcings: [Forcing] = None, + gwce_solution_scheme: str = None, + use_smagorinsky: bool = None, + source_filename: PathLike = None, + slurm_configuration: SlurmJSON = None, + use_original_mesh: bool = False, + stations_file_path: PathLike = None, + write_surface_output: bool = True, + write_station_output: bool = False, + processors: int = 11, + nems_parameters: {str: str} = None, + **kwargs, + ): + """ + + :param adcirc_executable_path: file path to `adcirc` or `NEMS.x` + :param adcprep_executable_path: file path to `adcprep` + :param modeled_start_time: start time in model run + :param modeled_end_time: edn time in model run + :param modeled_timestep: time interval between model steps + :param fort_13_path: file path to `fort.13` + :param fort_14_path: file path to `fort.14` + :param fort_14_path: file path to `fort.14` + :param tidal_spinup_duration: tidal spinup duration for ADCIRC coldstart + :param tidal_spinup_timestep: tidal spinup modeled time interval for ADCIRC coldstart + :param forcings: list of Forcing objects to apply to the mesh + :param gwce_solution_scheme: solution scheme (can be `explicit`, `semi-implicit`, or `semi-implicit-legacy`) + :param use_smagorinsky: whether to use Smagorinsky coefficient + :param source_filename: path to modulefile to `source` + :param slurm_configuration: Slurm configuration object + :param use_original_mesh: whether to symlink / copy original mesh instead of rewriting with `adcircpy` + :param stations_file_path: file path to stations file + :param write_surface_output: whether to write surface output to NetCDF + :param write_station_output: whether to write station output to NetCDF (only applicable if stations file exists) + :param processors: number of processors to use + :param nems_parameters: parameters to give to NEMS cap + """ + + if tidal_spinup_timestep is None: + tidal_spinup_timestep = modeled_timestep + if forcings is None: + forcings = [] + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(ADCIRCJSON.field_types) + + ModelJSON.__init__(self, model=Model.ADCIRC, **kwargs) + NEMSCapJSON.__init__( + self, processors=processors, nems_parameters=nems_parameters, **kwargs + ) + + self['adcirc_executable_path'] = adcirc_executable_path + self['adcprep_executable_path'] = adcprep_executable_path + self['modeled_start_time'] = modeled_start_time + self['modeled_end_time'] = modeled_end_time + self['modeled_timestep'] = modeled_timestep + self['fort_13_path'] = fort_13_path + self['fort_14_path'] = fort_14_path + self['tidal_spinup_duration'] = tidal_spinup_duration + self['tidal_spinup_timestep'] = tidal_spinup_timestep + self['gwce_solution_scheme'] = gwce_solution_scheme + self['use_smagorinsky'] = use_smagorinsky + self['source_filename'] = source_filename + self['use_original_mesh'] = use_original_mesh + self['stations_file_path'] = stations_file_path + self['write_surface_output'] = write_surface_output + self['write_station_output'] = write_station_output + + self.forcings = forcings + self.slurm_configuration = slurm_configuration + + @property + def forcings(self) -> ['ForcingJSON']: + return self.__forcings + + @forcings.setter + def forcings(self, forcings: ['ForcingJSON']): + for index, forcing in enumerate(forcings): + if isinstance(forcing, Forcing): + forcings[index] = ForcingJSON.from_adcircpy(forcing) + self.__forcings = forcings + + @property + def slurm_configuration(self) -> [SlurmJSON]: + return self.__slurm_configuration + + @slurm_configuration.setter + def slurm_configuration(self, slurm_configuration: SlurmJSON): + if isinstance(slurm_configuration, SlurmConfig): + SlurmJSON.from_adcircpy(slurm_configuration) + self.__slurm_configuration = slurm_configuration + + @property + def adcircpy_mesh(self) -> AdcircMesh: + LOGGER.info(f'opening mesh "{self["fort_14_path"]}"') + mesh = AdcircMesh.open(self['fort_14_path'], crs=4326) + + LOGGER.debug(f'adding {len(self.forcings)} forcing(s) to mesh') + for forcing in self.forcings: + adcircpy_forcing = forcing.adcircpy_forcing + if ( + isinstance(adcircpy_forcing, Tides) + and self['tidal_spinup_duration'] is not None + ): + adcircpy_forcing.spinup_time = self['tidal_spinup_duration'] + adcircpy_forcing.start_date = self['modeled_start_time'] + adcircpy_forcing.end_date = self['modeled_end_time'] + if self['tidal_spinup_duration'] is not None: + adcircpy_forcing.start_date -= self['tidal_spinup_duration'] + mesh.add_forcing(adcircpy_forcing) + + if self['fort_13_path'] is not None: + LOGGER.info(f'reading attributes from "{self["fort_13_path"]}"') + if self['fort_13_path'].exists(): + mesh.import_nodal_attributes(self['fort_13_path']) + for attribute_name in mesh.get_nodal_attribute_names(): + mesh.set_nodal_attribute_state( + attribute_name, coldstart=True, hotstart=True + ) + else: + LOGGER.warning( + 'mesh values (nodal attributes) not found ' f'at "{self["fort_13_path"]}"' + ) + + if not mesh.has_nodal_attribute('primitive_weighting_in_continuity_equation'): + LOGGER.debug(f'generating tau0 in mesh') + mesh.generate_tau0() + + return mesh + + @property + def adcircpy_driver(self) -> AdcircRun: + # instantiate AdcircRun object. + driver = AdcircRun( + mesh=self.adcircpy_mesh, + start_date=self['modeled_start_time'], + end_date=self['modeled_end_time'], + spinup_time=self['tidal_spinup_duration'], + server_config=self.slurm_configuration.to_adcircpy() + if self.slurm_configuration is not None + else None, + ) + + if self['modeled_timestep'] is not None: + driver.timestep = self['modeled_timestep'] / timedelta(seconds=1) + + if self['gwce_solution_scheme'] is not None: + driver.gwce_solution_scheme = self['gwce_solution_scheme'].value + + if self['use_smagorinsky'] is not None: + driver.smagorinsky = self['use_smagorinsky'] + + if self['tidal_spinup_duration'] is not None: + spinup_start = self['modeled_start_time'] - self['tidal_spinup_duration'] + else: + spinup_start = None + + if self['write_station_output'] and self['stations_file_path'].exists(): + driver.import_stations(self['stations_file_path']) + driver.set_elevation_stations_output( + self['modeled_timestep'], + spinup=self['tidal_spinup_timestep'], + spinup_start=spinup_start, + ) + driver.set_velocity_stations_output( + self['modeled_timestep'], + spinup=self['tidal_spinup_timestep'], + spinup_start=spinup_start, + ) + + if self['write_surface_output']: + driver.set_elevation_surface_output( + self['modeled_timestep'], + spinup=self['tidal_spinup_timestep'], + spinup_start=spinup_start, + ) + driver.set_velocity_surface_output( + self['modeled_timestep'], + spinup=self['tidal_spinup_timestep'], + spinup_start=spinup_start, + ) + + return driver + + @property + def nemspy_entry(self) -> ADCIRCEntry: + return ADCIRCEntry(processors=self['processors'], **self['nems_parameters']) diff --git a/coupledmodeldriver/configure/run.py b/coupledmodeldriver/configure/run.py new file mode 100644 index 00000000..b085de3b --- /dev/null +++ b/coupledmodeldriver/configure/run.py @@ -0,0 +1,93 @@ +from abc import ABC +from os import PathLike +from pathlib import Path +from typing import Collection, Mapping, Union + +from adcircpy.forcing.base import Forcing +from nemspy.model import ModelEntry + +from coupledmodeldriver.configure.forcings.base import ForcingJSON +from .base import ConfigurationJSON, NEMSCapJSON + + +class RunConfiguration(ABC): + required: [ConfigurationJSON] = [] + forcings: [ForcingJSON] = [] + + def __init__(self, configurations: [ConfigurationJSON]): + self.__configurations = {} + self.configurations = configurations + + @property + def configurations(self) -> {str: ConfigurationJSON}: + return self.__configurations + + @configurations.setter + def configurations(self, configurations: {str: ConfigurationJSON}): + if isinstance(configurations, Collection) and not isinstance(configurations, Mapping): + configurations = {entry.name: entry for entry in configurations} + for name, configuration in configurations.items(): + self[name] = configuration + + @property + def nemspy_entries(self) -> [ModelEntry]: + return [ + configuration.nemspy_entry + for configuration in self.configurations.values() + if isinstance(configuration, NEMSCapJSON) + ] + + def __contains__(self, configuration: Union[str, ConfigurationJSON]) -> bool: + if isinstance(configuration, ConfigurationJSON): + configuration = configuration.name + return configuration in self.configurations + + def __getitem__(self, name: str) -> ConfigurationJSON: + return self.configurations[name] + + def __setitem__(self, name: str, value: ConfigurationJSON): + if isinstance(value, str): + if Path(value).exists(): + value = ConfigurationJSON.from_file(value) + else: + value = ConfigurationJSON.from_string(value) + elif isinstance(value, Forcing): + value = ForcingJSON.from_adcircpy(value) + self.configurations[name] = value + + @classmethod + def read_directory(cls, directory: PathLike) -> 'RunConfiguration': + if not isinstance(directory, Path): + directory = Path(directory) + if directory.is_file(): + directory = directory.parent + + configurations = [] + for configuration_class in cls.required: + filename = directory / configuration_class.default_filename + if filename.exists(): + configurations.append(configuration_class.from_file(filename)) + else: + raise FileNotFoundError(f'missing required configuration file "{filename}"') + + for configuration_class in cls.forcings: + filename = directory / configuration_class.default_filename + if filename.exists(): + configurations.append(configuration_class.from_file(filename)) + + return cls(configurations) + + def write_directory(self, directory: PathLike, overwrite: bool = False): + """ + :param directory: directory in which to write generated JSON configuration files + :param overwrite: whether to overwrite existing files + """ + + if not isinstance(directory, Path): + directory = Path(directory) + + if not directory.exists(): + directory.mkdir(parents=True, exist_ok=True) + + for configuration in self.__configurations.values(): + configuration.to_file(directory, overwrite=overwrite) diff --git a/coupledmodeldriver/generate/__init__.py b/coupledmodeldriver/generate/__init__.py new file mode 100644 index 00000000..31b2ab4a --- /dev/null +++ b/coupledmodeldriver/generate/__init__.py @@ -0,0 +1,2 @@ +from .adcirc import * +from .nems import * diff --git a/coupledmodeldriver/generate/adcirc/__init__.py b/coupledmodeldriver/generate/adcirc/__init__.py new file mode 100644 index 00000000..c58f2e5a --- /dev/null +++ b/coupledmodeldriver/generate/adcirc/__init__.py @@ -0,0 +1,3 @@ +from .configure import ADCIRCRunConfiguration +from .generate import generate_adcirc_configuration +from .script import ADCIRCGenerationScript diff --git a/coupledmodeldriver/generate/adcirc/configure.py b/coupledmodeldriver/generate/adcirc/configure.py new file mode 100644 index 00000000..b9087d05 --- /dev/null +++ b/coupledmodeldriver/generate/adcirc/configure.py @@ -0,0 +1,182 @@ +from datetime import datetime, timedelta +from os import PathLike +from pathlib import Path + +from adcircpy import AdcircMesh, AdcircRun, Tides +from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing +from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing + +from coupledmodeldriver.configure.base import ModelDriverJSON, SlurmJSON +from coupledmodeldriver.configure.forcings import ( + ATMESHForcingJSON, + TidalForcingJSON, + WW3DATAForcingJSON, +) +from coupledmodeldriver.configure.forcings.base import ForcingJSON +from coupledmodeldriver.configure.models import ADCIRCJSON +from coupledmodeldriver.configure.run import RunConfiguration +from coupledmodeldriver.platforms import Platform + + +class ADCIRCRunConfiguration(RunConfiguration): + required = [ + ModelDriverJSON, + SlurmJSON, + ADCIRCJSON, + ] + forcings = [ + TidalForcingJSON, + ATMESHForcingJSON, + WW3DATAForcingJSON, + ] + + def __init__( + self, + fort13: PathLike, + fort14: PathLike, + modeled_start_time: datetime, + modeled_end_time: datetime, + modeled_timestep: timedelta, + tidal_spinup_duration: timedelta = None, + platform: Platform = None, + runs: {str: (float, str)} = None, + forcings: [ForcingJSON] = None, + adcirc_processors: int = None, + slurm_job_duration: timedelta = None, + slurm_partition: str = None, + slurm_email_address: str = None, + adcirc_executable: PathLike = None, + adcprep_executable: PathLike = None, + source_filename: PathLike = None, + ): + """ + Generate required configuration files for an ADCIRC run. + + :param fort13: path to input mesh values (`fort.13`) + :param fort14: path to input mesh nodes (`fort.14`) + :param modeled_start_time: start time within the modeled system + :param modeled_end_time: end time within the modeled system + :param modeled_timestep: time interval within the modeled system + :param adcirc_processors: numbers of processors to use for Slurm job + :param platform: HPC platform for which to configure + :param tidal_spinup_duration: spinup time for ADCIRC coldstart + :param runs: dictionary of run name to run value and mesh attribute name + :param slurm_job_duration: wall clock time of job + :param slurm_partition: Slurm partition + :param slurm_email_address: email address to send Slurm notifications + :param adcirc_executable: filename of compiled `adcirc` + :param adcprep_executable: filename of compiled `adcprep` + :param source_filename: path to module file to `source` + """ + + if platform is None: + platform = Platform.LOCAL + + if forcings is None: + forcings = [] + + if adcirc_processors is None: + adcirc_processors = 11 + + if adcprep_executable is None: + adcprep_executable = 'adcprep' + + slurm = SlurmJSON( + account=platform.value['slurm_account'], + tasks=adcirc_processors, + partition=slurm_partition, + job_duration=slurm_job_duration, + email_address=slurm_email_address, + ) + + adcirc = ADCIRCJSON( + adcirc_executable_path=adcirc_executable, + adcprep_executable_path=adcprep_executable, + modeled_start_time=modeled_start_time, + modeled_end_time=modeled_end_time, + modeled_timestep=modeled_timestep, + fort_13_path=fort13, + fort_14_path=fort14, + tidal_spinup_duration=tidal_spinup_duration, + source_filename=source_filename, + slurm_configuration=slurm, + processors=adcirc_processors, + ) + + driver = ModelDriverJSON(platform=platform, runs=runs) + super().__init__([driver, slurm, adcirc]) + + for forcing in forcings: + self.add_forcing(forcing) + + def add_forcing(self, forcing: ForcingJSON): + if not isinstance(forcing, ForcingJSON): + if isinstance(forcing, AtmosphericMeshForcing): + forcing = ATMESHForcingJSON.from_adcircpy(forcing) + elif isinstance(forcing, WaveWatch3DataForcing): + forcing = WW3DATAForcingJSON.from_adcircpy(forcing) + elif isinstance(forcing, Tides): + forcing = TidalForcingJSON.from_adcircpy(forcing) + else: + raise NotImplementedError(f'unable to parse object of type {type(forcing)}') + + if forcing not in self: + self.configurations[forcing.name] = forcing + self['adcirc'].forcings.append(forcing) + + @property + def adcircpy_mesh(self) -> AdcircMesh: + return self['adcirc'].adcircpy_mesh + + @property + def adcircpy_driver(self) -> AdcircRun: + return self['adcirc'].adcircpy_driver + + @classmethod + def from_configurations( + cls, + driver: ModelDriverJSON, + slurm: SlurmJSON, + adcirc: ADCIRCJSON, + forcings: [ForcingJSON] = None, + ) -> 'ADCIRCRunConfiguration': + instance = RunConfiguration([driver, slurm, adcirc]) + instance.__class__ = cls + + instance['modeldriver'] = driver + instance['slurm'] = slurm + instance['adcirc'] = adcirc + + if forcings is not None: + for forcing in forcings: + instance.add_forcing(forcing) + + return instance + + @classmethod + def read_directory(cls, directory: PathLike) -> 'ADCIRCRunConfiguration': + if not isinstance(directory, Path): + directory = Path(directory) + if directory.is_file(): + directory = directory.parent + + configurations = [] + for configuration_class in cls.required: + filename = directory / configuration_class.default_filename + if filename.exists(): + configurations.append(configuration_class.from_file(filename)) + else: + raise FileNotFoundError(f'missing required configuration file "{filename}"') + + forcings = [] + for configuration_class in cls.forcings: + filename = directory / configuration_class.default_filename + if filename.exists(): + forcings.append(configuration_class.from_file(filename)) + + return cls.from_configurations( + driver=configurations[0], + slurm=configurations[1], + adcirc=configurations[2], + forcings=forcings, + ) diff --git a/coupledmodeldriver/adcirc/adcirc.py b/coupledmodeldriver/generate/adcirc/generate.py similarity index 52% rename from coupledmodeldriver/adcirc/adcirc.py rename to coupledmodeldriver/generate/adcirc/generate.py index 20bc4783..6635eb24 100644 --- a/coupledmodeldriver/adcirc/adcirc.py +++ b/coupledmodeldriver/generate/adcirc/generate.py @@ -1,33 +1,18 @@ -from datetime import datetime, timedelta import logging import os from os import PathLike from pathlib import Path -from adcircpy import AdcircMesh, AdcircRun -from adcircpy.forcing.tides import Tides -from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing -from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing import numpy -from .job_scripts import AdcircMeshPartitionJob, AdcircRunJob -from ..configurations import ( - ADCIRCJSON, - ATMESHForcingJSON, - ForcingJSON, - ModelDriverJSON, - RunConfiguration, - SlurmJSON, - TidalForcingJSON, - WW3DATAForcingJSON, -) -from ..job_scripts import ( +from coupledmodeldriver.script import ( EnsembleCleanupScript, EnsembleRunScript, - EnsembleSetupScript, ) -from ..platforms import Platform -from ..utilities import LOGGER, create_symlink, get_logger +from coupledmodeldriver.utilities import LOGGER, create_symlink, \ + get_logger +from .configure import ADCIRCRunConfiguration +from .script import AdcircMeshPartitionJob, AdcircRunJob def generate_adcirc_configuration( @@ -106,7 +91,7 @@ def generate_adcirc_configuration( adcirc_coldstart_run_name = 'ADC_COLD_RUN' adcirc_hotstart_run_name = 'ADC_HOT_RUN' - adcprep_job_filename = output_directory / f'job_adcprep_{platform.name.lower()}.job' + adcprep_job_script_filename = output_directory / f'job_adcprep_{platform.name.lower()}.job' coldstart_run_script_filename = ( output_directory / f'job_adcirc_{platform.name.lower()}.job.coldstart' ) @@ -134,8 +119,10 @@ def generate_adcirc_configuration( source_filename=source_filename, ) - LOGGER.debug(f'writing mesh partitioning job script ' f'"{adcprep_job_filename.name}"') - adcprep_script.write(adcprep_job_filename, overwrite=overwrite) + LOGGER.debug( + f'writing mesh partitioning job script ' f'"{adcprep_job_script_filename.name}"' + ) + adcprep_script.write(adcprep_job_script_filename, overwrite=overwrite) LOGGER.debug(f'setting ADCIRC executable "{adcirc_executable_path}"') if tidal_spinup_duration is not None: @@ -179,9 +166,12 @@ def generate_adcirc_configuration( # instantiate AdcircRun object. driver = coupled_configuration.adcircpy_driver + local_fort13_filename = output_directory / 'fort.13' local_fort14_filename = output_directory / 'fort.14' if use_original_mesh: LOGGER.info(f'using original mesh from "{original_fort14_filename}"') + if original_fort13_filename.exists(): + create_symlink(original_fort13_filename, local_fort13_filename) create_symlink(original_fort14_filename, local_fort14_filename) else: LOGGER.info(f'rewriting original mesh to "{local_fort14_filename}"') @@ -208,13 +198,19 @@ def generate_adcirc_configuration( driver=None, ) if use_original_mesh: - if original_fort13_filename.exists(): - create_symlink(original_fort13_filename, coldstart_directory / 'fort.13') + if local_fort13_filename.exists(): + create_symlink('../fort.13', coldstart_directory / 'fort.13', relative=True) create_symlink('../fort.14', coldstart_directory / 'fort.14', relative=True) + create_symlink( + adcprep_job_script_filename, coldstart_directory / 'adcprep.job', relative=True + ) + create_symlink( + coldstart_run_script_filename, coldstart_directory / 'adcirc.job', relative=True + ) for run_name, (value, attribute_name) in runs.items(): - run_directory = runs_directory / run_name - LOGGER.debug(f'writing hotstart configuration to ' f'"{run_directory}"') + hotstart_directory = runs_directory / run_name + LOGGER.debug(f'writing hotstart configuration to ' f'"{hotstart_directory}"') if not isinstance(value, numpy.ndarray): value = numpy.full([len(driver.mesh.coords)], fill_value=value) if not driver.mesh.has_attribute(attribute_name): @@ -222,7 +218,7 @@ def generate_adcirc_configuration( driver.mesh.set_attribute(attribute_name, value) driver.write( - run_directory, + hotstart_directory, overwrite=overwrite, fort13=None if use_original_mesh else 'fort.13', fort14=None, @@ -231,18 +227,23 @@ def generate_adcirc_configuration( driver=None, ) if use_original_mesh: - if original_fort13_filename.exists(): - create_symlink(original_fort13_filename, run_directory / 'fort.13') - create_symlink('../../fort.14', run_directory / 'fort.14', relative=True) - - LOGGER.debug(f'writing ensemble setup script ' f'"{setup_script_filename.name}"') - setup_script = EnsembleSetupScript( - platform=platform, - adcprep_job_script=adcprep_job_filename.name, - coldstart_job_script=coldstart_run_script_filename.name, - hotstart_job_script=hotstart_run_script_filename.name, - ) - setup_script.write(setup_script_filename, overwrite=overwrite) + if local_fort13_filename.exists(): + create_symlink('../../fort.13', hotstart_directory / 'fort.13', relative=True) + create_symlink('../../fort.14', hotstart_directory / 'fort.14', relative=True) + create_symlink( + adcprep_job_script_filename, hotstart_directory / 'adcprep.job', relative=True + ) + create_symlink( + hotstart_run_script_filename, hotstart_directory / 'adcirc.job', relative=True + ) + try: + create_symlink( + '../../coldstart/fort.67.nc', hotstart_directory / 'fort.67.nc', relative=True + ) + except: + LOGGER.warning( + 'unable to link `fort.67.nc` from coldstart to hotstart; you must manually link or copy this file after coldstart completes' + ) LOGGER.info(f'writing ensemble run script "{run_script_filename.name}"') run_script = EnsembleRunScript(platform, setup_script_filename.name) @@ -251,167 +252,3 @@ def generate_adcirc_configuration( cleanup_script = EnsembleCleanupScript() LOGGER.debug(f'writing cleanup script "{cleanup_script_filename.name}"') cleanup_script.write(cleanup_script_filename, overwrite=overwrite) - - -class ADCIRCRunConfiguration(RunConfiguration): - required = [ - ModelDriverJSON, - SlurmJSON, - ADCIRCJSON, - ] - forcings = [ - TidalForcingJSON, - ATMESHForcingJSON, - WW3DATAForcingJSON, - ] - - def __init__( - self, - fort13: PathLike, - fort14: PathLike, - modeled_start_time: datetime, - modeled_end_time: datetime, - modeled_timestep: timedelta, - tidal_spinup_duration: timedelta = None, - platform: Platform = None, - runs: {str: (float, str)} = None, - forcings: [ForcingJSON] = None, - adcirc_processors: int = None, - slurm_job_duration: timedelta = None, - slurm_partition: str = None, - slurm_email_address: str = None, - adcirc_executable: PathLike = None, - adcprep_executable: PathLike = None, - source_filename: PathLike = None, - ): - """ - Generate required configuration files for an ADCIRC run. - - :param fort13: path to input mesh values (`fort.13`) - :param fort14: path to input mesh nodes (`fort.14`) - :param modeled_start_time: start time within the modeled system - :param modeled_end_time: end time within the modeled system - :param modeled_timestep: time interval within the modeled system - :param adcirc_processors: numbers of processors to use for Slurm job - :param platform: HPC platform for which to configure - :param tidal_spinup_duration: spinup time for ADCIRC coldstart - :param runs: dictionary of run name to run value and mesh attribute name - :param slurm_job_duration: wall clock time of job - :param slurm_partition: Slurm partition - :param slurm_email_address: email address to send Slurm notifications - :param adcirc_executable: filename of compiled `adcirc` - :param adcprep_executable: filename of compiled `adcprep` - :param source_filename: path to module file to `source` - """ - - if platform is None: - platform = Platform.LOCAL - - if forcings is None: - forcings = [] - - if adcirc_processors is None: - adcirc_processors = 11 - - if adcprep_executable is None: - adcprep_executable = 'adcprep' - - slurm = SlurmJSON( - account=platform.value['slurm_account'], - tasks=adcirc_processors, - partition=slurm_partition, - job_duration=slurm_job_duration, - email_address=slurm_email_address, - ) - - adcirc = ADCIRCJSON( - adcirc_executable_path=adcirc_executable, - adcprep_executable_path=adcprep_executable, - modeled_start_time=modeled_start_time, - modeled_end_time=modeled_end_time, - modeled_timestep=modeled_timestep, - fort_13_path=fort13, - fort_14_path=fort14, - tidal_spinup_duration=tidal_spinup_duration, - source_filename=source_filename, - slurm_configuration=slurm, - processors=adcirc_processors, - ) - - driver = ModelDriverJSON(platform=platform, runs=runs) - super().__init__([driver, slurm, adcirc]) - - for forcing in forcings: - self.add_forcing(forcing) - - def add_forcing(self, forcing: ForcingJSON): - if not isinstance(forcing, ForcingJSON): - if isinstance(forcing, AtmosphericMeshForcing): - forcing = ATMESHForcingJSON.from_adcircpy(forcing) - elif isinstance(forcing, WaveWatch3DataForcing): - forcing = WW3DATAForcingJSON.from_adcircpy(forcing) - elif isinstance(forcing, Tides): - forcing = TidalForcingJSON.from_adcircpy(forcing) - else: - raise NotImplementedError(f'unable to parse object of type {type(forcing)}') - - if forcing not in self: - self.configurations[forcing.name] = forcing - self['adcirc'].forcings.append(forcing) - - @property - def adcircpy_mesh(self) -> AdcircMesh: - return self['adcirc'].adcircpy_mesh - - @property - def adcircpy_driver(self) -> AdcircRun: - return self['adcirc'].adcircpy_driver - - @classmethod - def from_configurations( - cls, - driver: ModelDriverJSON, - slurm: SlurmJSON, - adcirc: ADCIRCJSON, - forcings: [ForcingJSON] = None, - ) -> 'ADCIRCRunConfiguration': - instance = RunConfiguration([driver, slurm, adcirc]) - instance.__class__ = cls - - instance['modeldriver'] = driver - instance['slurm'] = slurm - instance['adcirc'] = adcirc - - if forcings is not None: - for forcing in forcings: - instance.add_forcing(forcing) - - return instance - - @classmethod - def read_directory(cls, directory: PathLike) -> 'ADCIRCRunConfiguration': - if not isinstance(directory, Path): - directory = Path(directory) - if directory.is_file(): - directory = directory.parent - - configurations = [] - for configuration_class in cls.required: - filename = directory / configuration_class.default_filename - if filename.exists(): - configurations.append(configuration_class.from_file(filename)) - else: - raise FileNotFoundError(f'missing required configuration file "{filename}"') - - forcings = [] - for configuration_class in cls.forcings: - filename = directory / configuration_class.default_filename - if filename.exists(): - forcings.append(configuration_class.from_file(filename)) - - return cls.from_configurations( - driver=configurations[0], - slurm=configurations[1], - adcirc=configurations[2], - forcings=forcings, - ) diff --git a/coupledmodeldriver/adcirc/job_scripts.py b/coupledmodeldriver/generate/adcirc/script.py similarity index 97% rename from coupledmodeldriver/adcirc/job_scripts.py rename to coupledmodeldriver/generate/adcirc/script.py index 33c86178..844afab3 100644 --- a/coupledmodeldriver/adcirc/job_scripts.py +++ b/coupledmodeldriver/generate/adcirc/script.py @@ -2,8 +2,8 @@ from os import PathLike from pathlib import Path -from .. import Platform -from ..job_scripts import JobScript, Script +from coupledmodeldriver.platforms import Platform +from coupledmodeldriver.script import JobScript, Script class AdcircJob(JobScript): diff --git a/coupledmodeldriver/generate/nems/__init__.py b/coupledmodeldriver/generate/nems/__init__.py new file mode 100644 index 00000000..009504b6 --- /dev/null +++ b/coupledmodeldriver/generate/nems/__init__.py @@ -0,0 +1,4 @@ +from .base import NEMSJSON +from .configure import NEMSADCIRCRunConfiguration +from .generate import generate_nems_adcirc_configuration +from .script import NEMSADCIRCGenerationScript diff --git a/coupledmodeldriver/generate/nems/base.py b/coupledmodeldriver/generate/nems/base.py new file mode 100644 index 00000000..92b27988 --- /dev/null +++ b/coupledmodeldriver/generate/nems/base.py @@ -0,0 +1,84 @@ +from datetime import datetime, timedelta +from os import PathLike +from pathlib import Path + +from nemspy import ModelingSystem +from nemspy.model import ModelEntry + +from coupledmodeldriver.configure.base import ConfigurationJSON + + +class NEMSJSON(ConfigurationJSON): + name = 'nems' + default_filename = f'configure_nems.json' + field_types = { + 'executable_path': Path, + 'modeled_start_time': datetime, + 'modeled_end_time': datetime, + 'interval': timedelta, + 'models': [ModelEntry], + 'connections': [[str]], + 'mediations': [str], + 'sequence': [str], + } + + def __init__( + self, + executable_path: PathLike, + modeled_start_time: datetime, + modeled_end_time: datetime, + interval: timedelta = None, + models: [ModelEntry] = None, + connections: [[str]] = None, + mediations: [[str]] = None, + sequence: [str] = None, + **kwargs, + ): + if 'fields' not in kwargs: + kwargs['fields'] = {} + kwargs['fields'].update(NEMSJSON.field_types) + + ConfigurationJSON.__init__(self, **kwargs) + + self['executable_path'] = executable_path + self['modeled_start_time'] = modeled_start_time + self['modeled_end_time'] = modeled_end_time + self['interval'] = interval + self['models'] = models + self['connections'] = connections + self['mediations'] = mediations + self['sequence'] = sequence + + @property + def nemspy_modeling_system(self) -> ModelingSystem: + modeling_system = ModelingSystem( + start_time=self['modeled_start_time'], + end_time=self['modeled_end_time'], + interval=self['interval'], + **{model.model_type.value.lower(): model for model in self['models']}, + ) + for connection in self['connections']: + modeling_system.connect(*connection) + for mediation in self['mediations']: + modeling_system.mediate(*mediation) + + modeling_system.sequence = self['sequence'] + + return modeling_system + + def to_nemspy(self) -> ModelingSystem: + return self.nemspy_modeling_system + + @classmethod + def from_nemspy(cls, modeling_system: ModelingSystem, executable_path: PathLike = None): + if executable_path is None: + executable_path = 'NEMS.x' + return cls( + executable_path=executable_path, + modeled_start_time=modeling_system.start_time, + modeled_end_time=modeling_system.end_time, + interval=modeling_system.interval, + models=modeling_system.models, + connections=modeling_system.connections, + sequence=modeling_system.sequence, + ) diff --git a/coupledmodeldriver/generate/nems/configure.py b/coupledmodeldriver/generate/nems/configure.py new file mode 100644 index 00000000..f21646e1 --- /dev/null +++ b/coupledmodeldriver/generate/nems/configure.py @@ -0,0 +1,162 @@ +from datetime import datetime, timedelta +from os import PathLike +from pathlib import Path + +from adcircpy import Tides +from adcircpy.forcing.base import Forcing +from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing +from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing +from nemspy import ModelingSystem + +from coupledmodeldriver.configure.base import ModelDriverJSON, SlurmJSON +from coupledmodeldriver.configure.forcings import ( + ATMESHForcingJSON, + TidalForcingJSON, + WW3DATAForcingJSON, +) +from coupledmodeldriver.configure.forcings.base import ForcingJSON +from coupledmodeldriver.configure.models import ADCIRCJSON +from coupledmodeldriver.platforms import Platform +from .base import NEMSJSON +from ..adcirc import ADCIRCRunConfiguration + + +class NEMSADCIRCRunConfiguration(ADCIRCRunConfiguration): + required = [ + ModelDriverJSON, + NEMSJSON, + SlurmJSON, + ADCIRCJSON, + ] + + def __init__( + self, + fort13: PathLike, + fort14: PathLike, + modeled_start_time: datetime, + modeled_end_time: datetime, + modeled_timestep: timedelta, + nems_interval: timedelta, + nems_connections: [str], + nems_mediations: [str], + nems_sequence: [str], + tidal_spinup_duration: timedelta = None, + platform: Platform = None, + runs: {str: (float, str)} = None, + forcings: [ForcingJSON] = None, + adcirc_processors: int = None, + slurm_job_duration: timedelta = None, + slurm_partition: str = None, + slurm_email_address: str = None, + nems_executable: PathLike = None, + adcprep_executable: PathLike = None, + source_filename: PathLike = None, + ): + self.__nems = None + + super().__init__( + fort13=fort13, + fort14=fort14, + modeled_start_time=modeled_start_time, + modeled_end_time=modeled_end_time, + modeled_timestep=modeled_timestep, + tidal_spinup_duration=tidal_spinup_duration, + platform=platform, + runs=runs, + forcings=None, + adcirc_processors=adcirc_processors, + slurm_job_duration=slurm_job_duration, + slurm_partition=slurm_partition, + slurm_email_address=slurm_email_address, + adcprep_executable=adcprep_executable, + source_filename=source_filename, + ) + + nems = NEMSJSON( + executable_path=nems_executable, + modeled_start_time=modeled_start_time, + modeled_end_time=modeled_end_time, + interval=nems_interval, + models=self.nemspy_entries, + connections=nems_connections, + mediations=nems_mediations, + sequence=nems_sequence, + ) + + self.configurations[nems.name] = nems + + for forcing in forcings: + self.add_forcing(forcing) + + self['slurm']['tasks'] = self['nems'].nemspy_modeling_system.processors + + @property + def nemspy_modeling_system(self) -> ModelingSystem: + return self['nems'].nemspy_modeling_system + + def add_forcing(self, forcing: Forcing): + if not isinstance(forcing, ForcingJSON): + if isinstance(forcing, AtmosphericMeshForcing): + forcing = ATMESHForcingJSON.from_adcircpy(forcing) + self['nems']['models'].append(forcing.nemspy_entry) + elif isinstance(forcing, WaveWatch3DataForcing): + forcing = WW3DATAForcingJSON.from_adcircpy(forcing) + self['nems']['models'].append(forcing.nemspy_entry) + elif isinstance(forcing, Tides): + forcing = TidalForcingJSON.from_adcircpy(forcing) + else: + raise NotImplementedError(f'unable to parse object of type {type(forcing)}') + + if forcing not in self: + self[forcing.name] = forcing + self['adcirc'].forcings.append(forcing) + + @classmethod + def from_configurations( + cls, + driver: ModelDriverJSON, + nems: NEMSJSON, + slurm: SlurmJSON, + adcirc: ADCIRCJSON, + forcings: [ForcingJSON] = None, + ) -> 'NEMSADCIRCRunConfiguration': + instance = super().from_configurations( + driver=driver, slurm=slurm, adcirc=adcirc, forcings=None, + ) + instance.__class__ = cls + instance.configurations['nems'] = nems + + if forcings is not None: + for forcing in forcings: + instance.add_forcing(forcing) + + return instance + + @classmethod + def read_directory(cls, directory: PathLike) -> 'NEMSADCIRCRunConfiguration': + if not isinstance(directory, Path): + directory = Path(directory) + if directory.is_file(): + directory = directory.parent + + configurations = [] + for configuration_class in cls.required: + filename = directory / configuration_class.default_filename + if filename.exists(): + configurations.append(configuration_class.from_file(filename)) + else: + raise FileNotFoundError(f'missing required configuration file "{filename}"') + + forcings = [] + for configuration_class in cls.forcings: + filename = directory / configuration_class.default_filename + if filename.exists(): + forcings.append(configuration_class.from_file(filename)) + + return cls.from_configurations( + driver=configurations[0], + nems=configurations[1], + slurm=configurations[2], + adcirc=configurations[3], + forcings=forcings, + ) diff --git a/coupledmodeldriver/nems/nems_adcirc.py b/coupledmodeldriver/generate/nems/generate.py similarity index 60% rename from coupledmodeldriver/nems/nems_adcirc.py rename to coupledmodeldriver/generate/nems/generate.py index 8b17ed6e..6e7b0707 100644 --- a/coupledmodeldriver/nems/nems_adcirc.py +++ b/coupledmodeldriver/generate/nems/generate.py @@ -1,36 +1,19 @@ import copy -from datetime import datetime, timedelta import logging import os from os import PathLike from pathlib import Path -from adcircpy.forcing.base import Forcing -from adcircpy.forcing.tides import Tides -from adcircpy.forcing.waves.ww3 import WaveWatch3DataForcing -from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from nemspy import ModelingSystem -from .job_scripts import AdcircNEMSSetupScript -from ..adcirc import ADCIRCRunConfiguration -from ..adcirc.job_scripts import AdcircMeshPartitionJob, AdcircRunJob -from ..configurations import ( - ADCIRCJSON, - ATMESHForcingJSON, - ForcingJSON, - ModelDriverJSON, - NEMSJSON, - SlurmJSON, - TidalForcingJSON, - WW3DATAForcingJSON, -) -from ..job_scripts import ( +from coupledmodeldriver.script import ( EnsembleCleanupScript, EnsembleRunScript, - EnsembleSetupScript, ) -from ..platforms import Platform -from ..utilities import LOGGER, create_symlink, get_logger +from coupledmodeldriver.utilities import LOGGER, create_symlink, \ + get_logger +from .configure import NEMSADCIRCRunConfiguration +from ..adcirc.script import AdcircMeshPartitionJob, AdcircRunJob def generate_nems_adcirc_configuration( @@ -66,9 +49,7 @@ def generate_nems_adcirc_configuration( else: output_directory = output_directory.resolve().relative_to(Path().cwd()) - coupled_configuration = NEMSADCIRCRunConfiguration.read_directory( - configuration_directory - ) + coupled_configuration = NEMSADCIRCRunConfiguration.read_directory(configuration_directory) runs = coupled_configuration['modeldriver']['runs'] platform = coupled_configuration['modeldriver']['platform'] @@ -180,11 +161,9 @@ def generate_nems_adcirc_configuration( adcirc_hotstart_run_name = 'ADC_HOT_RUN' adcprep_job_script_filename = output_directory / f'job_adcprep_{platform.name.lower()}.job' - coldstart_setup_script_filename = output_directory / f'setup.sh.coldstart' coldstart_run_script_filename = ( output_directory / f'job_adcirc_{platform.name.lower()}.job.coldstart' ) - hotstart_setup_script_filename = output_directory / f'setup.sh.hotstart' hotstart_run_script_filename = ( output_directory / f'job_adcirc_{platform.name.lower()}.job.hotstart' ) @@ -213,17 +192,6 @@ def generate_nems_adcirc_configuration( ) adcprep_script.write(adcprep_job_script_filename, overwrite=overwrite) - coldstart_setup_script = AdcircNEMSSetupScript( - nems_configure_filename=Path('..') / 'nems.configure.coldstart', - model_configure_filename=Path('..') / 'model_configure.coldstart', - config_rc_filename=Path('..') / 'config.rc.coldstart', - ) - - LOGGER.debug( - f'writing coldstart setup script ' f'"{coldstart_setup_script_filename.name}"' - ) - coldstart_setup_script.write(coldstart_setup_script_filename, overwrite=overwrite) - LOGGER.debug(f'setting NEMS executable "{nems_executable}"') if tidal_spinup_nems is not None: coldstart_run_script = AdcircRunJob( @@ -260,12 +228,6 @@ def generate_nems_adcirc_configuration( coldstart_run_script.write(coldstart_run_script_filename, overwrite=overwrite) if tidal_spinup_nems is not None: - hotstart_setup_script = AdcircNEMSSetupScript( - nems_configure_filename=Path('../..') / 'nems.configure.hotstart', - model_configure_filename=Path('../..') / 'model_configure.hotstart', - config_rc_filename=Path('../..') / 'config.rc.hotstart', - fort67_filename=Path('../..') / 'coldstart/fort.67.nc', - ) hotstart_run_script = AdcircRunJob( platform=platform, slurm_tasks=nems.processors, @@ -281,20 +243,18 @@ def generate_nems_adcirc_configuration( source_filename=source_filename, ) - LOGGER.debug( - f'writing hotstart setup script ' f'"{hotstart_setup_script_filename.name}"' - ) - hotstart_setup_script.write(hotstart_setup_script_filename, overwrite=overwrite) - LOGGER.debug(f'writing hotstart run script ' f'"{hotstart_run_script_filename.name}"') hotstart_run_script.write(hotstart_run_script_filename, overwrite=overwrite) # instantiate AdcircRun object. driver = coupled_configuration.adcircpy_driver + local_fort13_filename = output_directory / 'fort.13' local_fort14_filename = output_directory / 'fort.14' if use_original_mesh: LOGGER.info(f'using original mesh from "{original_fort14_filename}"') + if original_fort13_filename.exists(): + create_symlink(original_fort13_filename, local_fort13_filename) create_symlink(original_fort14_filename, local_fort14_filename) else: LOGGER.info(f'rewriting original mesh to "{local_fort14_filename}"') @@ -321,13 +281,30 @@ def generate_nems_adcirc_configuration( driver=None, ) if use_original_mesh: - if original_fort13_filename.exists(): - create_symlink(original_fort13_filename, coldstart_directory / 'fort.13') + if local_fort13_filename.exists(): + create_symlink('../fort.13', coldstart_directory / 'fort.13', relative=True) create_symlink('../fort.14', coldstart_directory / 'fort.14', relative=True) + create_symlink( + f'../{adcprep_job_script_filename.name}', + coldstart_directory / 'adcprep.job', + relative=True, + ) + create_symlink( + f'../{coldstart_run_script_filename.name}', + coldstart_directory / 'adcirc.job', + relative=True, + ) + create_symlink( + '../nems.configure.coldstart', coldstart_directory / 'nems.configure', relative=True + ) + create_symlink( + '../model_configure.coldstart', coldstart_directory / 'model_configure', relative=True + ) + create_symlink('../config.rc.coldstart', coldstart_directory / 'config.rc', relative=True) for run_name, attributes in runs.items(): - run_directory = runs_directory / run_name - LOGGER.debug(f'writing hotstart configuration to ' f'"{run_directory}"') + hotstart_directory = runs_directory / run_name + LOGGER.debug(f'writing hotstart configuration to ' f'"{hotstart_directory}"') if attributes is not None: for name, value in attributes.items(): if name is not None: @@ -338,7 +315,7 @@ def generate_nems_adcirc_configuration( driver.mesh.set_attribute(name, value) driver.write( - run_directory, + hotstart_directory, overwrite=overwrite, fort13=None if use_original_mesh else 'fort.13', fort14=None, @@ -347,166 +324,45 @@ def generate_nems_adcirc_configuration( driver=None, ) if use_original_mesh: - if original_fort13_filename.exists(): - create_symlink(original_fort13_filename, run_directory / 'fort.13') - create_symlink('../../fort.14', run_directory / 'fort.14', relative=True) - - LOGGER.debug(f'writing ensemble setup script ' f'"{setup_script_filename.name}"') - setup_script = EnsembleSetupScript( - platform=platform, - adcprep_job_script=adcprep_job_script_filename.name, - coldstart_job_script=coldstart_run_script_filename.name, - hotstart_job_script=hotstart_run_script_filename.name, - coldstart_setup_script=coldstart_setup_script_filename.name, - hotstart_setup_script=hotstart_setup_script_filename.name, - ) - setup_script.write(setup_script_filename, overwrite=overwrite) + if local_fort13_filename.exists(): + create_symlink('../../fort.13', hotstart_directory / 'fort.13', relative=True) + create_symlink('../../fort.14', hotstart_directory / 'fort.14', relative=True) + create_symlink( + f'../../{adcprep_job_script_filename.name}', + hotstart_directory / 'adcprep.job', + relative=True, + ) + create_symlink( + f'../../{hotstart_run_script_filename.name}', + hotstart_directory / 'adcirc.job', + relative=True, + ) + create_symlink( + '../../nems.configure.hotstart', + hotstart_directory / 'nems.configure', + relative=True, + ) + create_symlink( + '../../model_configure.hotstart', + hotstart_directory / 'model_configure', + relative=True, + ) + create_symlink( + '../../config.rc.hotstart', hotstart_directory / 'config.rc', relative=True + ) + try: + create_symlink( + '../../coldstart/fort.67.nc', hotstart_directory / 'fort.67.nc', relative=True + ) + except: + LOGGER.warning( + 'unable to link `fort.67.nc` from coldstart to hotstart; you must manually link or copy this file after coldstart completes' + ) LOGGER.info(f'writing ensemble run script "{run_script_filename.name}"') - run_script = EnsembleRunScript(platform, setup_script_filename.name) + run_script = EnsembleRunScript(platform) run_script.write(run_script_filename, overwrite=overwrite) cleanup_script = EnsembleCleanupScript() LOGGER.debug(f'writing cleanup script "{cleanup_script_filename.name}"') cleanup_script.write(cleanup_script_filename, overwrite=overwrite) - - -class NEMSADCIRCRunConfiguration(ADCIRCRunConfiguration): - required = [ - ModelDriverJSON, - NEMSJSON, - SlurmJSON, - ADCIRCJSON, - ] - - def __init__( - self, - fort13: PathLike, - fort14: PathLike, - modeled_start_time: datetime, - modeled_end_time: datetime, - modeled_timestep: timedelta, - nems_interval: timedelta, - nems_connections: [str], - nems_mediations: [str], - nems_sequence: [str], - tidal_spinup_duration: timedelta = None, - platform: Platform = None, - runs: {str: (float, str)} = None, - forcings: [ForcingJSON] = None, - adcirc_processors: int = None, - slurm_job_duration: timedelta = None, - slurm_partition: str = None, - slurm_email_address: str = None, - nems_executable: PathLike = None, - adcprep_executable: PathLike = None, - source_filename: PathLike = None, - ): - self.__nems = None - - super().__init__( - fort13=fort13, - fort14=fort14, - modeled_start_time=modeled_start_time, - modeled_end_time=modeled_end_time, - modeled_timestep=modeled_timestep, - tidal_spinup_duration=tidal_spinup_duration, - platform=platform, - runs=runs, - forcings=None, - adcirc_processors=adcirc_processors, - slurm_job_duration=slurm_job_duration, - slurm_partition=slurm_partition, - slurm_email_address=slurm_email_address, - adcprep_executable=adcprep_executable, - source_filename=source_filename, - ) - - nems = NEMSJSON( - executable_path=nems_executable, - modeled_start_time=modeled_start_time, - modeled_end_time=modeled_end_time, - interval=nems_interval, - models=self.nemspy_entries, - connections=nems_connections, - mediations=nems_mediations, - sequence=nems_sequence, - ) - - self.configurations[nems.name] = nems - - for forcing in forcings: - self.add_forcing(forcing) - - self['slurm']['tasks'] = self['nems'].nemspy_modeling_system.processors - - @property - def nemspy_modeling_system(self) -> ModelingSystem: - return self['nems'].nemspy_modeling_system - - def add_forcing(self, forcing: Forcing): - if not isinstance(forcing, ForcingJSON): - if isinstance(forcing, AtmosphericMeshForcing): - forcing = ATMESHForcingJSON.from_adcircpy(forcing) - self['nems']['models'].append(forcing.nemspy_entry) - elif isinstance(forcing, WaveWatch3DataForcing): - forcing = WW3DATAForcingJSON.from_adcircpy(forcing) - self['nems']['models'].append(forcing.nemspy_entry) - elif isinstance(forcing, Tides): - forcing = TidalForcingJSON.from_adcircpy(forcing) - else: - raise NotImplementedError(f'unable to parse object of type {type(forcing)}') - - if forcing not in self: - self[forcing.name] = forcing - self['adcirc'].forcings.append(forcing) - - @classmethod - def from_configurations( - cls, - driver: ModelDriverJSON, - nems: NEMSJSON, - slurm: SlurmJSON, - adcirc: ADCIRCJSON, - forcings: [ForcingJSON] = None, - ) -> 'NEMSADCIRCRunConfiguration': - instance = super().from_configurations( - driver=driver, slurm=slurm, adcirc=adcirc, forcings=None, - ) - instance.__class__ = cls - instance.configurations['nems'] = nems - - if forcings is not None: - for forcing in forcings: - instance.add_forcing(forcing) - - return instance - - @classmethod - def read_directory(cls, directory: PathLike) -> 'NEMSADCIRCRunConfiguration': - if not isinstance(directory, Path): - directory = Path(directory) - if directory.is_file(): - directory = directory.parent - - configurations = [] - for configuration_class in cls.required: - filename = directory / configuration_class.default_filename - if filename.exists(): - configurations.append(configuration_class.from_file(filename)) - else: - raise FileNotFoundError(f'missing required configuration file "{filename}"') - - forcings = [] - for configuration_class in cls.forcings: - filename = directory / configuration_class.default_filename - if filename.exists(): - forcings.append(configuration_class.from_file(filename)) - - return cls.from_configurations( - driver=configurations[0], - nems=configurations[1], - slurm=configurations[2], - adcirc=configurations[3], - forcings=forcings, - ) diff --git a/coupledmodeldriver/generate/nems/script.py b/coupledmodeldriver/generate/nems/script.py new file mode 100644 index 00000000..fce6e810 --- /dev/null +++ b/coupledmodeldriver/generate/nems/script.py @@ -0,0 +1,28 @@ +from os import PathLike +from pathlib import Path + +from ..adcirc import ADCIRCGenerationScript + + +class NEMSADCIRCGenerationScript(ADCIRCGenerationScript): + def __str__(self): + lines = [ + 'from pathlib import Path', + '', + 'from coupledmodeldriver.adcirc.nems_adcirc import generate_nems_adcirc_configuration', + '', + '', + "if __name__ == '__main__':", + ' generate_nems_adcirc_configuration(output_directory=Path(__file__).parent, overwrite=True)', + ] + + return '\n'.join(lines) + + def write(self, filename: PathLike, overwrite: bool = False): + if not isinstance(filename, Path): + filename = Path(filename) + + if filename.is_dir(): + filename = filename / f'generate_nems_adcirc.py' + + super().write(filename, overwrite) diff --git a/coupledmodeldriver/nems/__init__.py b/coupledmodeldriver/nems/__init__.py deleted file mode 100644 index 34a8a006..00000000 --- a/coupledmodeldriver/nems/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .job_scripts import NEMSADCIRCGenerationScript -from .nems_adcirc import NEMSADCIRCRunConfiguration, generate_nems_adcirc_configuration diff --git a/coupledmodeldriver/nems/job_scripts.py b/coupledmodeldriver/nems/job_scripts.py deleted file mode 100644 index 21304ba2..00000000 --- a/coupledmodeldriver/nems/job_scripts.py +++ /dev/null @@ -1,59 +0,0 @@ -from os import PathLike -from pathlib import Path, PurePosixPath - -from ..adcirc import ADCIRCGenerationScript -from ..job_scripts import Script - - -class AdcircNEMSSetupScript(Script): - """ script for setting up ADCIRC NEMS configuration """ - - def __init__( - self, - nems_configure_filename: PathLike, - model_configure_filename: PathLike, - config_rc_filename: PathLike, - fort67_filename: PathLike = None, - ): - self.nems_configure_filename = PurePosixPath(nems_configure_filename) - self.model_configure_filename = PurePosixPath(model_configure_filename) - self.config_rc_filename = PurePosixPath(config_rc_filename) - self.fort67_filename = ( - PurePosixPath(fort67_filename) if fort67_filename is not None else None - ) - - commands = [ - f'ln -sf {self.nems_configure_filename} ./nems.configure', - f'ln -sf {self.model_configure_filename} ./model_configure', - f'ln -sf {self.config_rc_filename} ./config.rc', - f'ln -sf ./model_configure ./atm_namelist.rc', - ] - - if self.fort67_filename is not None: - commands.extend(['', f'ln -sf {self.fort67_filename} ./fort.67.nc']) - - super().__init__(commands) - - -class NEMSADCIRCGenerationScript(ADCIRCGenerationScript): - def __str__(self): - lines = [ - 'from pathlib import Path', - '', - 'from coupledmodeldriver.adcirc.nems_adcirc import generate_nems_adcirc_configuration', - '', - '', - "if __name__ == '__main__':", - ' generate_nems_adcirc_configuration(output_directory=Path(__file__).parent, overwrite=True)', - ] - - return '\n'.join(lines) - - def write(self, filename: PathLike, overwrite: bool = False): - if not isinstance(filename, Path): - filename = Path(filename) - - if filename.is_dir(): - filename = filename / f'generate_nems_adcirc.py' - - super().write(filename, overwrite) diff --git a/coupledmodeldriver/job_scripts.py b/coupledmodeldriver/script.py similarity index 78% rename from coupledmodeldriver/job_scripts.py rename to coupledmodeldriver/script.py index cbaff396..1fbed693 100644 --- a/coupledmodeldriver/job_scripts.py +++ b/coupledmodeldriver/script.py @@ -238,135 +238,27 @@ def write(self, filename: PathLike, overwrite: bool = False): self.__slurm_run_directory = None -class EnsembleSetupScript(Script): - def __init__( - self, - platform: Platform, - adcprep_job_script: PathLike = None, - coldstart_job_script: PathLike = None, - hotstart_job_script: PathLike = None, - coldstart_setup_script: PathLike = None, - hotstart_setup_script: PathLike = None, - commands: [str] = None, - ): - super().__init__(commands) - - if adcprep_job_script is None: - adcprep_job_script = f'job_adcprep_{self.platform.name.lower()}.job' - if coldstart_job_script is None: - coldstart_job_script = f'job_adcirc_{self.platform.name.lower()}.job.coldstart' - if hotstart_job_script is None: - hotstart_job_script = f'job_adcirc_{self.platform.name.lower()}.job.hotstart' - - self.platform = platform - self.adcprep_job_script = adcprep_job_script - self.coldstart_job_script = coldstart_job_script - self.hotstart_job_script = hotstart_job_script - self.coldstart_setup_script = coldstart_setup_script - self.hotstart_setup_script = hotstart_setup_script - - def __str__(self) -> str: - lines = [] - lines.extend( - [ - 'DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"', - '', - '# prepare single coldstart directory', - 'pushd ${DIRECTORY}/coldstart >/dev/null 2>&1', - ] - ) - - if self.coldstart_setup_script is not None: - lines.append(f'sh ../{self.coldstart_setup_script}') - - lines.extend( - [ - f'ln -sf ../{self.adcprep_job_script} adcprep.job', - f'ln -sf ../{self.coldstart_job_script} adcirc.job', - 'popd >/dev/null 2>&1', - '', - ] - ) - - hotstart_lines = [] - hotstart_lines.extend( - ['pushd ${hotstart} >/dev/null 2>&1', ] - ) - - if self.hotstart_setup_script is not None: - hotstart_lines.append(f'sh ../../{self.hotstart_setup_script}') - - hotstart_lines.extend( - [ - f'ln -sf ../../{self.adcprep_job_script} adcprep.job', - f'ln -sf ../../{self.hotstart_job_script} adcirc.job', - 'popd >/dev/null 2>&1', - ] - ) - - lines.extend( - [ - '# prepare every hotstart directory', - bash_for_loop('for hotstart in ${DIRECTORY}/runs/*/', hotstart_lines, ), - *(str(command) for command in self.commands), - ] - ) - - return '\n'.join(lines) - - def write(self, filename: PathLike, overwrite: bool = False): - if not isinstance(filename, Path): - filename = Path(filename) - - if filename.is_dir(): - filename = filename / f'setup_{self.platform.name.lower()}.sh' - - super().write(filename, overwrite) - - class EnsembleRunScript(Script): - def __init__( - self, platform: Platform, setup_script_name: PathLike = None, commands: [str] = None - ): + def __init__(self, platform: Platform, commands: [str] = None): self.platform = platform - self.setup_script_name = setup_script_name super().__init__(commands) def __str__(self) -> str: - lines = [] - - if self.setup_script_name is not None: - lines.extend( - [f'sh {self.setup_script_name}', '', ] - ) - - lines.extend( - [ - 'DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"', - '', - '# run single coldstart configuration', - 'pushd ${DIRECTORY}/coldstart >/dev/null 2>&1', - ] - ) - - lines.extend( - [self.coldstart, 'popd >/dev/null 2>&1', '', ] - ) - - lines.extend( - [ - '# run every hotstart configuration', - bash_for_loop( - 'for hotstart in ${DIRECTORY}/runs/*/', - [ - 'pushd ${hotstart} >/dev/null 2>&1', - self.hotstart, - 'popd >/dev/null 2>&1', - ], - ), - *(str(command) for command in self.commands), - ] - ) + lines = [ + 'DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"', + '', + '# run single coldstart configuration', + 'pushd ${DIRECTORY}/coldstart >/dev/null 2>&1', + self.coldstart, + 'popd >/dev/null 2>&1', + '', + '# run every hotstart configuration', + bash_for_loop( + 'for hotstart in ${DIRECTORY}/runs/*/', + ['pushd ${hotstart} >/dev/null 2>&1', self.hotstart, 'popd >/dev/null 2>&1', ], + ), + *(str(command) for command in self.commands), + ] if self.platform.value['uses_slurm']: # slurm queue output https://slurm.schedmd.com/squeue.html diff --git a/coupledmodeldriver/utilities.py b/coupledmodeldriver/utilities.py index 4b95528a..7a43889f 100644 --- a/coupledmodeldriver/utilities.py +++ b/coupledmodeldriver/utilities.py @@ -114,7 +114,11 @@ def create_symlink( symlink_filename.symlink_to(source_filename) except Exception as error: LOGGER.warning(f'could not create symbolic link: {error}') - shutil.copyfile(source_filename, symlink_filename) + try: + shutil.copyfile(source_filename, symlink_filename) + except: + if starting_directory is not None: + os.chdir(starting_directory) if starting_directory is not None: os.chdir(starting_directory) @@ -186,6 +190,8 @@ def convert_value(value: Any, to_type: type) -> Any: value = f'{hours}:{minutes}:{seconds}' else: value /= timedelta(seconds=1) + elif isinstance(value, CRS) and issubclass(to_type, str): + value = value.to_wkt() if issubclass(to_type, bool): value = eval(f'{value}') elif issubclass(to_type, datetime): diff --git a/examples/adcirc_only/hera_hsofs120m_sandy_spinup.py b/examples/adcirc_only/hera_hsofs120m_sandy_spinup.py index de3a37be..79771c2f 100644 --- a/examples/adcirc_only/hera_hsofs120m_sandy_spinup.py +++ b/examples/adcirc_only/hera_hsofs120m_sandy_spinup.py @@ -4,7 +4,7 @@ from pathlib import Path from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal.py b/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal.py index 38020958..c8d10e55 100644 --- a/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal.py +++ b/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal.py @@ -7,7 +7,7 @@ from adcircpy.forcing.tides.tides import TidalSource from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py b/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py index 4009a5b4..463d1773 100644 --- a/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py +++ b/examples/adcirc_only/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py @@ -9,7 +9,7 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_hsofs250m_sandy_spinup.py b/examples/adcirc_only/hera_hsofs250m_sandy_spinup.py index a0552559..9666a96e 100644 --- a/examples/adcirc_only/hera_hsofs250m_sandy_spinup.py +++ b/examples/adcirc_only/hera_hsofs250m_sandy_spinup.py @@ -4,7 +4,7 @@ from pathlib import Path from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal.py b/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal.py index a19154f2..97195a5c 100644 --- a/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal.py +++ b/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal.py @@ -7,7 +7,7 @@ from adcircpy.forcing.tides.tides import TidalSource from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py b/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py index 3b00a5ad..cd8888cb 100644 --- a/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py +++ b/examples/adcirc_only/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py @@ -9,7 +9,7 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/adcirc_only/hera_shinnecock_ike_spinup.py b/examples/adcirc_only/hera_shinnecock_ike_spinup.py index edc8b824..94e3e0e0 100644 --- a/examples/adcirc_only/hera_shinnecock_ike_spinup.py +++ b/examples/adcirc_only/hera_shinnecock_ike_spinup.py @@ -4,7 +4,7 @@ from pathlib import Path from coupledmodeldriver import Platform -from coupledmodeldriver.adcirc import ADCIRCGenerationScript, \ +from coupledmodeldriver.generate import ADCIRCGenerationScript, \ ADCIRCRunConfiguration # paths to compiled `NEMS.x` and `adcprep` diff --git a/examples/nems_adcirc/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py b/examples/nems_adcirc/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py index d6857f80..d6824c62 100644 --- a/examples/nems_adcirc/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py +++ b/examples/nems_adcirc/hera_hsofs120m_sandy_spinup_tidal_atmesh_ww3data.py @@ -9,8 +9,10 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from coupledmodeldriver import Platform -from coupledmodeldriver.nems import NEMSADCIRCGenerationScript, \ - NEMSADCIRCRunConfiguration +from coupledmodeldriver.generate import ( + NEMSADCIRCGenerationScript, + NEMSADCIRCRunConfiguration, +) # paths to compiled `NEMS.x` and `adcprep` NEMS_EXECUTABLE = ( diff --git a/examples/nems_adcirc/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py b/examples/nems_adcirc/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py index a0484559..b1e8524b 100644 --- a/examples/nems_adcirc/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py +++ b/examples/nems_adcirc/hera_hsofs250m_sandy_spinup_tidal_atmesh_ww3data.py @@ -9,8 +9,10 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from coupledmodeldriver import Platform -from coupledmodeldriver.nems import NEMSADCIRCGenerationScript, \ - NEMSADCIRCRunConfiguration +from coupledmodeldriver.generate import ( + NEMSADCIRCGenerationScript, + NEMSADCIRCRunConfiguration, +) # paths to compiled `NEMS.x` and `adcprep` NEMS_EXECUTABLE = ( diff --git a/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data.py b/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data.py index 084447e5..2af228c1 100644 --- a/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data.py +++ b/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data.py @@ -9,8 +9,10 @@ from adcircpy.forcing.winds.atmesh import AtmosphericMeshForcing from coupledmodeldriver import Platform -from coupledmodeldriver.nems import NEMSADCIRCGenerationScript, \ - NEMSADCIRCRunConfiguration +from coupledmodeldriver.generate import ( + NEMSADCIRCGenerationScript, + NEMSADCIRCRunConfiguration, +) # paths to compiled `NEMS.x` and `adcprep` NEMS_EXECUTABLE = ( diff --git a/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data_perturbed_mannings_n.py b/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data_perturbed_mannings_n.py index 4b8384ad..3461a38e 100644 --- a/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data_perturbed_mannings_n.py +++ b/examples/nems_adcirc/hera_shinnecock_ike_spinup_tidal_atmesh_ww3data_perturbed_mannings_n.py @@ -10,8 +10,10 @@ import numpy from coupledmodeldriver import Platform -from coupledmodeldriver.nems import NEMSADCIRCGenerationScript, \ - NEMSADCIRCRunConfiguration +from coupledmodeldriver.generate import ( + NEMSADCIRCGenerationScript, + NEMSADCIRCRunConfiguration, +) # paths to compiled `NEMS.x` and `adcprep` NEMS_EXECUTABLE = ( diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/adcirc.job b/tests/data/reference/hera_shinnecock_ike/coldstart/adcirc.job new file mode 100644 index 00000000..42a18eaa --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/adcirc.job @@ -0,0 +1,14 @@ +#!/bin/bash --login +#SBATCH -J ADC_COLD_RUN +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_COLD_RUN.err.log +#SBATCH --output=ADC_COLD_RUN.out.log +#SBATCH -n 600 +#SBATCH -N 15 +#SBATCH --time=06:00:00 + +set -e + +srun adcirc diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/adcprep.job b/tests/data/reference/hera_shinnecock_ike/coldstart/adcprep.job new file mode 100644 index 00000000..e8206091 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/adcprep.job @@ -0,0 +1,15 @@ +#!/bin/bash --login +#SBATCH -J ADC_MESH_DECOMP +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_MESH_DECOMP.err.log +#SBATCH --output=ADC_MESH_DECOMP.out.log +#SBATCH -n 1 +#SBATCH -N 1 +#SBATCH --time=06:00:00 + +set -e + +srun adcprep --np 600 --partmesh +srun adcprep --np 600 --prepall diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/config.rc b/tests/data/reference/hera_shinnecock_ike/coldstart/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/fort.15 b/tests/data/reference/hera_shinnecock_ike/coldstart/fort.15 index 0ba30426..ab94cecf 100644 --- a/tests/data/reference/hera_shinnecock_ike/coldstart/fort.15 +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:47 ! RUNDES +created on 2021-04-07 10:17 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/model_configure b/tests/data/reference/hera_shinnecock_ike/coldstart/model_configure new file mode 100644 index 00000000..4c112bc7 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 600 +start_year: 2008 +start_month: 8 +start_day: 10 +start_hour: 12 +start_minute: 0 +start_second: 0 +nhours_fcst: 300 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/hera_shinnecock_ike/coldstart/nems.configure b/tests/data/reference/hera_shinnecock_ike/coldstart/nems.configure new file mode 100644 index 00000000..0940c778 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/coldstart/nems.configure @@ -0,0 +1,20 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: OCN +EARTH_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 0 599 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + OCN + @ +:: diff --git a/tests/data/reference/hera_shinnecock_ike/config.rc.coldstart b/tests/data/reference/hera_shinnecock_ike/config.rc.coldstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/hera_shinnecock_ike/config.rc.coldstart +++ b/tests/data/reference/hera_shinnecock_ike/config.rc.coldstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/hera_shinnecock_ike/config.rc.hotstart b/tests/data/reference/hera_shinnecock_ike/config.rc.hotstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/hera_shinnecock_ike/config.rc.hotstart +++ b/tests/data/reference/hera_shinnecock_ike/config.rc.hotstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/hera_shinnecock_ike/configure_atmesh.json b/tests/data/reference/hera_shinnecock_ike/configure_atmesh.json index 07780a29..2af3fdd1 100644 --- a/tests/data/reference/hera_shinnecock_ike/configure_atmesh.json +++ b/tests/data/reference/hera_shinnecock_ike/configure_atmesh.json @@ -1,7 +1,7 @@ { "nws": 17, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/wind_atm_fin_ch_time_vec.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/hera_shinnecock_ike/configure_ww3data.json b/tests/data/reference/hera_shinnecock_ike/configure_ww3data.json index 394d5bb0..7d3691fe 100644 --- a/tests/data/reference/hera_shinnecock_ike/configure_ww3data.json +++ b/tests/data/reference/hera_shinnecock_ike/configure_ww3data.json @@ -1,7 +1,7 @@ { "nrs": 5, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/ww3.Constant.20151214_sxy_ike_date.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/hera_shinnecock_ike/model_configure.coldstart b/tests/data/reference/hera_shinnecock_ike/model_configure.coldstart index cad49d74..4c112bc7 100644 --- a/tests/data/reference/hera_shinnecock_ike/model_configure.coldstart +++ b/tests/data/reference/hera_shinnecock_ike/model_configure.coldstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/hera_shinnecock_ike/model_configure.hotstart b/tests/data/reference/hera_shinnecock_ike/model_configure.hotstart index c67f76cb..32b73a3f 100644 --- a/tests/data/reference/hera_shinnecock_ike/model_configure.hotstart +++ b/tests/data/reference/hera_shinnecock_ike/model_configure.hotstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/hera_shinnecock_ike/nems.configure.coldstart b/tests/data/reference/hera_shinnecock_ike/nems.configure.coldstart index a447b581..0940c778 100644 --- a/tests/data/reference/hera_shinnecock_ike/nems.configure.coldstart +++ b/tests/data/reference/hera_shinnecock_ike/nems.configure.coldstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: OCN EARTH_attributes:: diff --git a/tests/data/reference/hera_shinnecock_ike/nems.configure.hotstart b/tests/data/reference/hera_shinnecock_ike/nems.configure.hotstart index 1c9c1a28..c8363de3 100644 --- a/tests/data/reference/hera_shinnecock_ike/nems.configure.hotstart +++ b/tests/data/reference/hera_shinnecock_ike/nems.configure.hotstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: ATM WAV OCN EARTH_attributes:: diff --git a/tests/data/reference/hera_shinnecock_ike/run_hera.sh b/tests/data/reference/hera_shinnecock_ike/run_hera.sh index cfff659e..80c2aed5 100644 --- a/tests/data/reference/hera_shinnecock_ike/run_hera.sh +++ b/tests/data/reference/hera_shinnecock_ike/run_hera.sh @@ -1,5 +1,3 @@ -sh setup_hera.sh - DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # run single coldstart configuration diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcirc.job b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcirc.job new file mode 100644 index 00000000..2267f767 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcirc.job @@ -0,0 +1,14 @@ +#!/bin/bash --login +#SBATCH -J ADC_HOT_RUN +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_HOT_RUN.err.log +#SBATCH --output=ADC_HOT_RUN.out.log +#SBATCH -n 602 +#SBATCH -N 16 +#SBATCH --time=06:00:00 + +set -e + +srun adcirc diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcprep.job b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcprep.job new file mode 100644 index 00000000..e8206091 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/adcprep.job @@ -0,0 +1,15 @@ +#!/bin/bash --login +#SBATCH -J ADC_MESH_DECOMP +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_MESH_DECOMP.err.log +#SBATCH --output=ADC_MESH_DECOMP.out.log +#SBATCH -n 1 +#SBATCH -N 1 +#SBATCH --time=06:00:00 + +set -e + +srun adcprep --np 600 --partmesh +srun adcprep --np 600 --prepall diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/config.rc b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/fort.15 b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/fort.15 index 6ed2eaf4..cac1de0a 100644 --- a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/fort.15 +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:47 ! RUNDES +created on 2021-04-07 10:17 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/model_configure b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/model_configure new file mode 100644 index 00000000..32b73a3f --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 602 +start_year: 2008 +start_month: 8 +start_day: 23 +start_hour: 0 +start_minute: 0 +start_second: 0 +nhours_fcst: 348 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/nems.configure b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/nems.configure new file mode 100644 index 00000000..c8363de3 --- /dev/null +++ b/tests/data/reference/hera_shinnecock_ike/runs/test_case_1/nems.configure @@ -0,0 +1,38 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: ATM WAV OCN +EARTH_attributes:: + Verbosity = off +:: + +# ATM # +ATM_model: atmesh +ATM_petlist_bounds: 0 0 +ATM_attributes:: + Verbosity = off +:: + +# WAV # +WAV_model: ww3data +WAV_petlist_bounds: 1 1 +WAV_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 2 601 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + ATM -> OCN :remapMethod=redist + WAV -> OCN :remapMethod=redist + ATM + WAV + OCN + @ +:: diff --git a/tests/data/reference/hera_shinnecock_ike/setup.sh.coldstart b/tests/data/reference/hera_shinnecock_ike/setup.sh.coldstart deleted file mode 100644 index 7b759d6e..00000000 --- a/tests/data/reference/hera_shinnecock_ike/setup.sh.coldstart +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash --login -ln -sf ../nems.configure.coldstart ./nems.configure -ln -sf ../model_configure.coldstart ./model_configure -ln -sf ../config.rc.coldstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc diff --git a/tests/data/reference/hera_shinnecock_ike/setup.sh.hotstart b/tests/data/reference/hera_shinnecock_ike/setup.sh.hotstart deleted file mode 100644 index ba3d84f4..00000000 --- a/tests/data/reference/hera_shinnecock_ike/setup.sh.hotstart +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash --login -ln -sf ../../nems.configure.hotstart ./nems.configure -ln -sf ../../model_configure.hotstart ./model_configure -ln -sf ../../config.rc.hotstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc - -ln -sf ../../coldstart/fort.67.nc ./fort.67.nc diff --git a/tests/data/reference/hera_shinnecock_ike/setup_hera.sh b/tests/data/reference/hera_shinnecock_ike/setup_hera.sh deleted file mode 100644 index 34f05ea1..00000000 --- a/tests/data/reference/hera_shinnecock_ike/setup_hera.sh +++ /dev/null @@ -1,17 +0,0 @@ -DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# prepare single coldstart directory -pushd ${DIRECTORY}/coldstart >/dev/null 2>&1 -sh ../setup.sh.coldstart -ln -sf ../job_adcprep_hera.job adcprep.job -ln -sf ../job_adcirc_hera.job.coldstart adcirc.job -popd >/dev/null 2>&1 - -# prepare every hotstart directory -for hotstart in ${DIRECTORY}/runs/*/; do - pushd ${hotstart} >/dev/null 2>&1 - sh ../../setup.sh.hotstart - ln -sf ../../job_adcprep_hera.job adcprep.job - ln -sf ../../job_adcirc_hera.job.hotstart adcirc.job - popd >/dev/null 2>&1 -done diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/adcirc.job b/tests/data/reference/local_shinnecock_ike/coldstart/adcirc.job new file mode 100644 index 00000000..2f319dab --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/coldstart/adcirc.job @@ -0,0 +1,2 @@ +#!/bin/bash --login + adcirc diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/adcprep.job b/tests/data/reference/local_shinnecock_ike/coldstart/adcprep.job new file mode 100644 index 00000000..cabec8ab --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/coldstart/adcprep.job @@ -0,0 +1,3 @@ +#!/bin/bash --login + adcprep --np 11 --partmesh + adcprep --np 11 --prepall diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/config.rc b/tests/data/reference/local_shinnecock_ike/coldstart/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/coldstart/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/fort.15 b/tests/data/reference/local_shinnecock_ike/coldstart/fort.15 index 313f5490..ab94cecf 100644 --- a/tests/data/reference/local_shinnecock_ike/coldstart/fort.15 +++ b/tests/data/reference/local_shinnecock_ike/coldstart/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:46 ! RUNDES +created on 2021-04-07 10:17 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/model_configure b/tests/data/reference/local_shinnecock_ike/coldstart/model_configure new file mode 100644 index 00000000..bb4f6c64 --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/coldstart/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 11 +start_year: 2008 +start_month: 8 +start_day: 10 +start_hour: 12 +start_minute: 0 +start_second: 0 +nhours_fcst: 300 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/local_shinnecock_ike/coldstart/nems.configure b/tests/data/reference/local_shinnecock_ike/coldstart/nems.configure new file mode 100644 index 00000000..19d5cb10 --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/coldstart/nems.configure @@ -0,0 +1,20 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: OCN +EARTH_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 0 10 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + OCN + @ +:: diff --git a/tests/data/reference/local_shinnecock_ike/config.rc.coldstart b/tests/data/reference/local_shinnecock_ike/config.rc.coldstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/local_shinnecock_ike/config.rc.coldstart +++ b/tests/data/reference/local_shinnecock_ike/config.rc.coldstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/local_shinnecock_ike/config.rc.hotstart b/tests/data/reference/local_shinnecock_ike/config.rc.hotstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/local_shinnecock_ike/config.rc.hotstart +++ b/tests/data/reference/local_shinnecock_ike/config.rc.hotstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/local_shinnecock_ike/configure_atmesh.json b/tests/data/reference/local_shinnecock_ike/configure_atmesh.json index 07780a29..2af3fdd1 100644 --- a/tests/data/reference/local_shinnecock_ike/configure_atmesh.json +++ b/tests/data/reference/local_shinnecock_ike/configure_atmesh.json @@ -1,7 +1,7 @@ { "nws": 17, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/wind_atm_fin_ch_time_vec.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/local_shinnecock_ike/configure_ww3data.json b/tests/data/reference/local_shinnecock_ike/configure_ww3data.json index 394d5bb0..7d3691fe 100644 --- a/tests/data/reference/local_shinnecock_ike/configure_ww3data.json +++ b/tests/data/reference/local_shinnecock_ike/configure_ww3data.json @@ -1,7 +1,7 @@ { "nrs": 5, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/ww3.Constant.20151214_sxy_ike_date.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/local_shinnecock_ike/model_configure.coldstart b/tests/data/reference/local_shinnecock_ike/model_configure.coldstart index 9de7dc95..bb4f6c64 100644 --- a/tests/data/reference/local_shinnecock_ike/model_configure.coldstart +++ b/tests/data/reference/local_shinnecock_ike/model_configure.coldstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/local_shinnecock_ike/model_configure.hotstart b/tests/data/reference/local_shinnecock_ike/model_configure.hotstart index 5e37f826..96ecae64 100644 --- a/tests/data/reference/local_shinnecock_ike/model_configure.hotstart +++ b/tests/data/reference/local_shinnecock_ike/model_configure.hotstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/local_shinnecock_ike/nems.configure.coldstart b/tests/data/reference/local_shinnecock_ike/nems.configure.coldstart index 96c7e8ea..19d5cb10 100644 --- a/tests/data/reference/local_shinnecock_ike/nems.configure.coldstart +++ b/tests/data/reference/local_shinnecock_ike/nems.configure.coldstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: OCN EARTH_attributes:: diff --git a/tests/data/reference/local_shinnecock_ike/nems.configure.hotstart b/tests/data/reference/local_shinnecock_ike/nems.configure.hotstart index 899ca90e..c52b2a07 100644 --- a/tests/data/reference/local_shinnecock_ike/nems.configure.hotstart +++ b/tests/data/reference/local_shinnecock_ike/nems.configure.hotstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: ATM WAV OCN EARTH_attributes:: diff --git a/tests/data/reference/local_shinnecock_ike/run_local.sh b/tests/data/reference/local_shinnecock_ike/run_local.sh index 02bf95d0..a3bad3d6 100644 --- a/tests/data/reference/local_shinnecock_ike/run_local.sh +++ b/tests/data/reference/local_shinnecock_ike/run_local.sh @@ -1,5 +1,3 @@ -sh setup_local.sh - DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # run single coldstart configuration diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcirc.job b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcirc.job new file mode 100644 index 00000000..2f319dab --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcirc.job @@ -0,0 +1,2 @@ +#!/bin/bash --login + adcirc diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcprep.job b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcprep.job new file mode 100644 index 00000000..cabec8ab --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/adcprep.job @@ -0,0 +1,3 @@ +#!/bin/bash --login + adcprep --np 11 --partmesh + adcprep --np 11 --prepall diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/config.rc b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/fort.15 b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/fort.15 index af9575fd..cac1de0a 100644 --- a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/fort.15 +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:46 ! RUNDES +created on 2021-04-07 10:17 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/model_configure b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/model_configure new file mode 100644 index 00000000..96ecae64 --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 13 +start_year: 2008 +start_month: 8 +start_day: 23 +start_hour: 0 +start_minute: 0 +start_second: 0 +nhours_fcst: 348 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/local_shinnecock_ike/runs/test_case_1/nems.configure b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/nems.configure new file mode 100644 index 00000000..c52b2a07 --- /dev/null +++ b/tests/data/reference/local_shinnecock_ike/runs/test_case_1/nems.configure @@ -0,0 +1,38 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: ATM WAV OCN +EARTH_attributes:: + Verbosity = off +:: + +# ATM # +ATM_model: atmesh +ATM_petlist_bounds: 0 0 +ATM_attributes:: + Verbosity = off +:: + +# WAV # +WAV_model: ww3data +WAV_petlist_bounds: 1 1 +WAV_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 2 12 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + ATM -> OCN :remapMethod=redist + WAV -> OCN :remapMethod=redist + ATM + WAV + OCN + @ +:: diff --git a/tests/data/reference/local_shinnecock_ike/setup.sh.coldstart b/tests/data/reference/local_shinnecock_ike/setup.sh.coldstart deleted file mode 100644 index 7b759d6e..00000000 --- a/tests/data/reference/local_shinnecock_ike/setup.sh.coldstart +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash --login -ln -sf ../nems.configure.coldstart ./nems.configure -ln -sf ../model_configure.coldstart ./model_configure -ln -sf ../config.rc.coldstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc diff --git a/tests/data/reference/local_shinnecock_ike/setup.sh.hotstart b/tests/data/reference/local_shinnecock_ike/setup.sh.hotstart deleted file mode 100644 index ba3d84f4..00000000 --- a/tests/data/reference/local_shinnecock_ike/setup.sh.hotstart +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash --login -ln -sf ../../nems.configure.hotstart ./nems.configure -ln -sf ../../model_configure.hotstart ./model_configure -ln -sf ../../config.rc.hotstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc - -ln -sf ../../coldstart/fort.67.nc ./fort.67.nc diff --git a/tests/data/reference/local_shinnecock_ike/setup_local.sh b/tests/data/reference/local_shinnecock_ike/setup_local.sh deleted file mode 100644 index 79389109..00000000 --- a/tests/data/reference/local_shinnecock_ike/setup_local.sh +++ /dev/null @@ -1,17 +0,0 @@ -DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# prepare single coldstart directory -pushd ${DIRECTORY}/coldstart >/dev/null 2>&1 -sh ../setup.sh.coldstart -ln -sf ../job_adcprep_local.job adcprep.job -ln -sf ../job_adcirc_local.job.coldstart adcirc.job -popd >/dev/null 2>&1 - -# prepare every hotstart directory -for hotstart in ${DIRECTORY}/runs/*/; do - pushd ${hotstart} >/dev/null 2>&1 - sh ../../setup.sh.hotstart - ln -sf ../../job_adcprep_local.job adcprep.job - ln -sf ../../job_adcirc_local.job.hotstart adcirc.job - popd >/dev/null 2>&1 -done diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcirc.job b/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcirc.job new file mode 100644 index 00000000..eca5e2e3 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcirc.job @@ -0,0 +1,15 @@ +#!/bin/bash --login +#SBATCH -J ADC_COLD_RUN +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_COLD_RUN.err.log +#SBATCH --output=ADC_COLD_RUN.out.log +#SBATCH -n 1020 +#SBATCH -N 15 +#SBATCH --time=06:00:00 +#SBATCH --partition=development + +set -e + +ibrun adcirc diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcprep.job b/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcprep.job new file mode 100644 index 00000000..5876c780 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/adcprep.job @@ -0,0 +1,16 @@ +#!/bin/bash --login +#SBATCH -J ADC_MESH_DECOMP +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_MESH_DECOMP.err.log +#SBATCH --output=ADC_MESH_DECOMP.out.log +#SBATCH -n 1 +#SBATCH -N 1 +#SBATCH --time=06:00:00 +#SBATCH --partition=development + +set -e + +ibrun adcprep --np 1020 --partmesh +ibrun adcprep --np 1020 --prepall diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/config.rc b/tests/data/reference/stampede2_shinnecock_ike/coldstart/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/fort.15 b/tests/data/reference/stampede2_shinnecock_ike/coldstart/fort.15 index 0ba30426..b8032a51 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/coldstart/fort.15 +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:47 ! RUNDES +created on 2021-04-07 10:18 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/model_configure b/tests/data/reference/stampede2_shinnecock_ike/coldstart/model_configure new file mode 100644 index 00000000..365ac491 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 1020 +start_year: 2008 +start_month: 8 +start_day: 10 +start_hour: 12 +start_minute: 0 +start_second: 0 +nhours_fcst: 300 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/stampede2_shinnecock_ike/coldstart/nems.configure b/tests/data/reference/stampede2_shinnecock_ike/coldstart/nems.configure new file mode 100644 index 00000000..a812d08f --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/coldstart/nems.configure @@ -0,0 +1,20 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: OCN +EARTH_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 0 1019 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + OCN + @ +:: diff --git a/tests/data/reference/stampede2_shinnecock_ike/config.rc.coldstart b/tests/data/reference/stampede2_shinnecock_ike/config.rc.coldstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/config.rc.coldstart +++ b/tests/data/reference/stampede2_shinnecock_ike/config.rc.coldstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/stampede2_shinnecock_ike/config.rc.hotstart b/tests/data/reference/stampede2_shinnecock_ike/config.rc.hotstart index 715b7952..f9621b6b 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/config.rc.hotstart +++ b/tests/data/reference/stampede2_shinnecock_ike/config.rc.hotstart @@ -1,2 +1,2 @@ -# `config.rc` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 diff --git a/tests/data/reference/stampede2_shinnecock_ike/configure_atmesh.json b/tests/data/reference/stampede2_shinnecock_ike/configure_atmesh.json index 07780a29..2af3fdd1 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/configure_atmesh.json +++ b/tests/data/reference/stampede2_shinnecock_ike/configure_atmesh.json @@ -1,7 +1,7 @@ { "nws": 17, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/wind_atm_fin_ch_time_vec.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/stampede2_shinnecock_ike/configure_ww3data.json b/tests/data/reference/stampede2_shinnecock_ike/configure_ww3data.json index 394d5bb0..7d3691fe 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/configure_ww3data.json +++ b/tests/data/reference/stampede2_shinnecock_ike/configure_ww3data.json @@ -1,7 +1,7 @@ { "nrs": 5, - "modeled_timestep": 3600.0, "resource": "input/shinnecock_ike/forcings/ww3.Constant.20151214_sxy_ike_date.nc", "processors": 1, - "nems_parameters": {} + "nems_parameters": {}, + "modeled_timestep": 3600.0 } \ No newline at end of file diff --git a/tests/data/reference/stampede2_shinnecock_ike/model_configure.coldstart b/tests/data/reference/stampede2_shinnecock_ike/model_configure.coldstart index 1e730d0e..365ac491 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/model_configure.coldstart +++ b/tests/data/reference/stampede2_shinnecock_ike/model_configure.coldstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/stampede2_shinnecock_ike/model_configure.hotstart b/tests/data/reference/stampede2_shinnecock_ike/model_configure.hotstart index 9740b3f3..b3050087 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/model_configure.hotstart +++ b/tests/data/reference/stampede2_shinnecock_ike/model_configure.hotstart @@ -1,4 +1,4 @@ -# `model_configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 total_member: 1 print_esmf: .true. namelist: atm_namelist diff --git a/tests/data/reference/stampede2_shinnecock_ike/nems.configure.coldstart b/tests/data/reference/stampede2_shinnecock_ike/nems.configure.coldstart index 4f504b97..a812d08f 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/nems.configure.coldstart +++ b/tests/data/reference/stampede2_shinnecock_ike/nems.configure.coldstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: OCN EARTH_attributes:: diff --git a/tests/data/reference/stampede2_shinnecock_ike/nems.configure.hotstart b/tests/data/reference/stampede2_shinnecock_ike/nems.configure.hotstart index 67a71b9c..af969fad 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/nems.configure.hotstart +++ b/tests/data/reference/stampede2_shinnecock_ike/nems.configure.hotstart @@ -1,4 +1,4 @@ -# `nems.configure` generated with NEMSpy 1.1.3.post33.dev0+ff69a55 +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 # EARTH # EARTH_component_list: ATM WAV OCN EARTH_attributes:: diff --git a/tests/data/reference/stampede2_shinnecock_ike/run_stampede2.sh b/tests/data/reference/stampede2_shinnecock_ike/run_stampede2.sh index 610cd310..80c2aed5 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/run_stampede2.sh +++ b/tests/data/reference/stampede2_shinnecock_ike/run_stampede2.sh @@ -1,5 +1,3 @@ -sh setup_stampede2.sh - DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # run single coldstart configuration diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcirc.job b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcirc.job new file mode 100644 index 00000000..27b43558 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcirc.job @@ -0,0 +1,15 @@ +#!/bin/bash --login +#SBATCH -J ADC_HOT_RUN +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_HOT_RUN.err.log +#SBATCH --output=ADC_HOT_RUN.out.log +#SBATCH -n 1022 +#SBATCH -N 16 +#SBATCH --time=06:00:00 +#SBATCH --partition=development + +set -e + +ibrun adcirc diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcprep.job b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcprep.job new file mode 100644 index 00000000..5876c780 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/adcprep.job @@ -0,0 +1,16 @@ +#!/bin/bash --login +#SBATCH -J ADC_MESH_DECOMP +#SBATCH -A coastal +#SBATCH --mail-type=ALL +#SBATCH --mail-user=example@email.gov +#SBATCH --error=ADC_MESH_DECOMP.err.log +#SBATCH --output=ADC_MESH_DECOMP.out.log +#SBATCH -n 1 +#SBATCH -N 1 +#SBATCH --time=06:00:00 +#SBATCH --partition=development + +set -e + +ibrun adcprep --np 1020 --partmesh +ibrun adcprep --np 1020 --prepall diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/config.rc b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/config.rc new file mode 100644 index 00000000..f9621b6b --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/config.rc @@ -0,0 +1,2 @@ +# `config.rc` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 + diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/fort.15 b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/fort.15 index 6ed2eaf4..5d19d478 100644 --- a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/fort.15 +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/fort.15 @@ -1,4 +1,4 @@ -created on 2021-04-06 10:47 ! RUNDES +created on 2021-04-07 10:18 ! RUNDES Shinacock Inlet Coarse Grid ! RUNID 1 ! NFOVER 1 ! NABOUT diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/model_configure b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/model_configure new file mode 100644 index 00000000..b3050087 --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/model_configure @@ -0,0 +1,14 @@ +# `model_configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +total_member: 1 +print_esmf: .true. +namelist: atm_namelist +PE_MEMBER01: 1022 +start_year: 2008 +start_month: 8 +start_day: 23 +start_hour: 0 +start_minute: 0 +start_second: 0 +nhours_fcst: 348 +RUN_CONTINUE: .false. +ENS_SPS: .false. diff --git a/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/nems.configure b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/nems.configure new file mode 100644 index 00000000..af969fad --- /dev/null +++ b/tests/data/reference/stampede2_shinnecock_ike/runs/test_case_1/nems.configure @@ -0,0 +1,38 @@ +# `nems.configure` generated with NEMSpy 1.1.4.post3.dev0+b3c8a30 +# EARTH # +EARTH_component_list: ATM WAV OCN +EARTH_attributes:: + Verbosity = off +:: + +# ATM # +ATM_model: atmesh +ATM_petlist_bounds: 0 0 +ATM_attributes:: + Verbosity = off +:: + +# WAV # +WAV_model: ww3data +WAV_petlist_bounds: 1 1 +WAV_attributes:: + Verbosity = off +:: + +# OCN # +OCN_model: adcirc +OCN_petlist_bounds: 2 1021 +OCN_attributes:: + Verbosity = off +:: + +# Run Sequence # +runSeq:: + @3600 + ATM -> OCN :remapMethod=redist + WAV -> OCN :remapMethod=redist + ATM + WAV + OCN + @ +:: diff --git a/tests/data/reference/stampede2_shinnecock_ike/setup.sh.coldstart b/tests/data/reference/stampede2_shinnecock_ike/setup.sh.coldstart deleted file mode 100644 index 7b759d6e..00000000 --- a/tests/data/reference/stampede2_shinnecock_ike/setup.sh.coldstart +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash --login -ln -sf ../nems.configure.coldstart ./nems.configure -ln -sf ../model_configure.coldstart ./model_configure -ln -sf ../config.rc.coldstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc diff --git a/tests/data/reference/stampede2_shinnecock_ike/setup.sh.hotstart b/tests/data/reference/stampede2_shinnecock_ike/setup.sh.hotstart deleted file mode 100644 index ba3d84f4..00000000 --- a/tests/data/reference/stampede2_shinnecock_ike/setup.sh.hotstart +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash --login -ln -sf ../../nems.configure.hotstart ./nems.configure -ln -sf ../../model_configure.hotstart ./model_configure -ln -sf ../../config.rc.hotstart ./config.rc -ln -sf ./model_configure ./atm_namelist.rc - -ln -sf ../../coldstart/fort.67.nc ./fort.67.nc diff --git a/tests/data/reference/stampede2_shinnecock_ike/setup_stampede2.sh b/tests/data/reference/stampede2_shinnecock_ike/setup_stampede2.sh deleted file mode 100644 index 34f4dc24..00000000 --- a/tests/data/reference/stampede2_shinnecock_ike/setup_stampede2.sh +++ /dev/null @@ -1,17 +0,0 @@ -DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# prepare single coldstart directory -pushd ${DIRECTORY}/coldstart >/dev/null 2>&1 -sh ../setup.sh.coldstart -ln -sf ../job_adcprep_stampede2.job adcprep.job -ln -sf ../job_adcirc_stampede2.job.coldstart adcirc.job -popd >/dev/null 2>&1 - -# prepare every hotstart directory -for hotstart in ${DIRECTORY}/runs/*/; do - pushd ${hotstart} >/dev/null 2>&1 - sh ../../setup.sh.hotstart - ln -sf ../../job_adcprep_stampede2.job adcprep.job - ln -sf ../../job_adcirc_stampede2.job.hotstart adcirc.job - popd >/dev/null 2>&1 -done diff --git a/tests/test_configurations.py b/tests/test_configurations.py index 2e3d1722..047a8d94 100644 --- a/tests/test_configurations.py +++ b/tests/test_configurations.py @@ -5,15 +5,17 @@ import pytest from coupledmodeldriver import Platform -from coupledmodeldriver.configurations import ( - ADCIRCJSON, - ATMESHForcingJSON, +from coupledmodeldriver.configure.base import ( ModelDriverJSON, - NEMSJSON, SlurmJSON, +) +from coupledmodeldriver.configure.forcings import ( + ATMESHForcingJSON, TidalForcingJSON, WW3DATAForcingJSON, ) +from coupledmodeldriver.configure.models import ADCIRCJSON +from coupledmodeldriver.generate.nems import NEMSJSON def test_update(): @@ -151,7 +153,7 @@ def test_tidal(): ] configuration['tidal_source'] = 'TPXO' - configuration['resource'] = 'nonesistant/path/to/h_tpxo9.nc' + configuration['resource'] = 'nonexistent/path/to/h_tpxo9.nc' with pytest.raises((FileNotFoundError, OSError)): configuration.adcircpy_forcing diff --git a/tests/test_generation.py b/tests/test_generation.py index 3e08b7e0..c77769c0 100644 --- a/tests/test_generation.py +++ b/tests/test_generation.py @@ -13,7 +13,7 @@ import wget from coupledmodeldriver import Platform -from coupledmodeldriver.nems import ( +from coupledmodeldriver.generate.nems import ( NEMSADCIRCRunConfiguration, generate_nems_adcirc_configuration, )