Feature #1989: Add OMI use case (#2457)

Co-authored-by: George McCabe <[email protected]>

.github/parm/use_case_groups.json

@@ -124,6 +124,11 @@
     "index_list": "9",
     "run": false
   },
+  {
+    "category": "medium_range",
+    "index_list": "10",
+    "run": false
+  },
   {
     "category": "pbl",
     "index_list": "0",

docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py

@@ -0,0 +1,136 @@
+"""
+Grid-Stat: Using Python Embedding for Total Column Ozone
+================================================================================
+
+model_applications/medium_range/GridStat_fcstGFS_obsOMI
+_TotalColumnOzone.conf
+
+"""
+##############################################################################
+# Scientific Objective
+# --------------------
+# To provide useful statistical information on the relationship between observation
+# data in gridded format to a gridded forecast. The Ozone Monitoring Instrument (OMI)
+# data covers a 36 hour period and is compared to the average of the gridded forecast
+# files (all from the same initialization time). 
+
+##############################################################################
+# Datasets
+# --------
+#
+# | **Forecast:** GFS (1 degree Lat/Lon grid)
+# | **Observation:** NASA's Level-3 Aura/OMI Global Total Ozone Mapping Spectrometer-Like (TOMS-Like) Total Column Ozone gridded product OMTO3e (0.25deg Lat/Lon grid)
+# | **Location:** Click here for the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
+# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information.
+
+##############################################################################
+# METplus Components
+# ------------------
+#
+# This use case first runs PCPCombine on the forecast data to build a
+# 36-hour Total Colum Ozone mean/average file. Then the forecast data
+# are compared to the observation data using GridStat. This use case utilizes 
+# the METplus GridStat wrapper to read in the observation file using Python Embedding.
+
+##############################################################################
+# METplus Workflow
+# ----------------
+#
+# The following tools are used for each run time:
+#
+# PCPCombine (forecast) > GridStat
+#
+# It processes the following run times:
+#
+# | **Valid:** 2023-12-05 06Z (36 hour period covering 2023-12-03 18Z - 2023-12-05 06Z)
+# | **Init:** 2023-12-03 06Z
+# | **Forecast lead:** 48
+
+##############################################################################
+# METplus Configuration
+# ---------------------
+#
+# METplus first loads all of the configuration files found in parm/metplus_config,
+# then it loads any configuration files passed to METplus via the command line
+# parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf
+
+##############################################################################
+# MET Configuration
+# -----------------
+#
+# METplus sets environment variables based on user settings in the METplus configuration file. 
+# See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details. 
+#
+# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!**
+#
+# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to:
+# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>`
+#
+# .. note:: See the :ref:`GridStat MET Configuration<grid-stat-met-conf>` section of the User's Guide for more information on the environment variables used in the file below:
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped
+
+##############################################################################
+# Python Embedding
+# ----------------
+#
+# This use case uses one Python embedding script with GridStat to read the input observation data
+#
+# parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py
+#
+# .. highlight:: python
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py
+
+
+##############################################################################
+# Running METplus
+# ---------------
+#
+# Pass the use case configuration file to the run_metplus.py script
+# along with any user-specific system configuration files if desired::
+#
+#    run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf /path/to/user_system.conf
+#
+# See :ref:`running-metplus` for more information.
+#
+
+##############################################################################
+# Expected Output
+# ---------------
+#
+# A successful run will output the following both to the screen and to the logfile::
+#
+#   INFO: METplus has successfully finished running.
+#
+# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated.
+# Output for this use case will be found in /model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone
+# (relative to **OUTPUT_BASE**)
+# and will contain the following files:
+#
+# * PCPCombine/pcp_combine.tozone_l0.mean.f048.init2023120306.nc
+# * GridStat/grid_stat_GFS_vs_OMI_480000L_20231205_060000V.stat 
+
+##############################################################################
+# Keywords
+# --------
+#
+# .. note::
+#
+#   * PCPCombineToolUseCase
+#   * GridStatToolUseCase
+#   * MediumRangeAppUseCase
+#   * GRIBFileUseCase
+#   * PythonEmbeddingFileUseCase
+#   * RegriddingInToolUseCase
+#   * NOAAEMCOrgUseCase
+#
+#   Navigate to the :ref:`quick-search` page to discover other similar use cases.
+#
+#
+#
+# sphinx_gallery_thumbnail_path = '_static/medium_range-GridStat_fcstGFS_obsOMI_TotalColumnOzone.png'
+#

internal/tests/use_cases/all_use_cases.txt

@@ -115,6 +115,7 @@ Category: medium_range
 7::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf:: cfgrib_env,py_embed
 8::MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead::model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf:: netcdf4_env
 9::GridStat_fcstGEFS_obsCADB_BinaryObsPOE:: model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf:: netcdf4_env
+10::GridStat_fcstGFS_obsOMI_TotalColumnOzone:: model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf:: py_embed
 
 
 Category: pbl

parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf

@@ -0,0 +1,106 @@
+[config]
+
+# Documentation for this use case can be found at
+# https://metplus.readthedocs.io/en/latest/generated/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.html
+
+# For additional information, please see the METplus Users Guide.
+# https://metplus.readthedocs.io/en/latest/Users_Guide
+
+###
+# Processes to run
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list
+###
+
+PROCESS_LIST = PCPCombine, GridStat
+
+###
+# Time Info
+# LOOP_BY options are INIT, VALID, RETRO, and REALTIME
+# If set to INIT or RETRO:
+#   INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set
+# If set to VALID or REALTIME:
+#   VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set
+# LEAD_SEQ is the list of forecast leads to process
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control
+###
+
+LOOP_BY = VALID
+VALID_TIME_FMT = %Y%m%d%H
+VALID_BEG = 2023120506
+VALID_END = 2023120506
+VALID_INCREMENT = 21600
+
+LEAD_SEQ = 48
+
+###
+# File I/O
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info
+###
+
+# PCP_COMBINE
+FCST_PCP_COMBINE_RUN = True
+FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/fcst
+FCST_PCP_COMBINE_INPUT_TEMPLATE = tozne.pgbf{lead?fmt=%HH}.gfs.{init?fmt=%Y%m%d%H}
+FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/PCPCombine
+FCST_PCP_COMBINE_OUTPUT_TEMPLATE = pcp_combine.tozone_l0.mean.f{lead?fmt=%HHH}.init{init?fmt=%Y%m%d%H}.nc
+
+# GRID_STAT
+FCST_GRID_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR}
+FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE}
+OBS_GRID_STAT_INPUT_TEMPLATE = PYTHON_NUMPY
+GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/GridStat
+GRID_STAT_OUTPUT_TEMPLATE =
+
+###
+# Field Info
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info
+###
+
+CONFIG_DIR = {PARM_BASE}/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone
+
+# GRID_STAT
+GRID_STAT_ONCE_PER_FIELD = False
+FCST_VAR1_NAME = TOZNE_L0_mean
+FCST_VAR1_LEVELS = (*,*)
+OBS_VAR1_NAME = {CONFIG_DIR}/read_omi-aura_l3-omto3e.py {INPUT_BASE}/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/obs/OMI-Aura_L3-OMTO3e_{valid?fmt=%Y?shift=-86400}m{valid?fmt=%m%d?shift=-86400}_v883-{valid?fmt=%Y}m{valid?fmt=%m%d}t010905.he5 obs
+
+###
+# PCPCombine Settings
+# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pcpcombine
+###
+FCST_PCP_COMBINE_METHOD = DERIVE
+FCST_PCP_COMBINE_STAT_LIST = mean
+FCST_PCP_COMBINE_DERIVE_LOOKBACK = 42H
+FCST_PCP_COMBINE_MIN_FORECAST = 12
+FCST_PCP_COMBINE_MAX_FORECAST = 48
+FCST_IS_PROB = false
+FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB
+FCST_PCP_COMBINE_INPUT_ACCUMS = 6H
+FCST_PCP_COMBINE_INPUT_NAMES = TOZNE
+FCST_PCP_COMBINE_INPUT_LEVELS = L0
+FCST_PCP_COMBINE_INPUT_OPTIONS = GRIB_lvl_typ = 200;
+FCST_PCP_COMBINE_OUTPUT_ACCUM = 36H
+FCST_PCP_COMBINE_OUTPUT_NAME =
+
+###
+# GridStat Settings
+# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#gridstat
+###
+
+MODEL = GFS
+OBTYPE = OMI
+GRID_STAT_OUTPUT_PREFIX = {MODEL}_vs_{OBTYPE}
+GRID_STAT_DESC = NA
+GRID_STAT_REGRID_TO_GRID = G004
+GRID_STAT_REGRID_METHOD = BILIN
+GRID_STAT_REGRID_WIDTH = 2
+GRID_STAT_MASK_POLY =
+GRID_STAT_CLIMO_CDF_WRITE_BINS = False
+GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT
+GRID_STAT_OUTPUT_FLAG_CNT = STAT
+GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE
+GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE
+GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE
+GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE
+GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE
+GRID_STAT_GRID_WEIGHT_FLAG = COS_LAT

parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone/read_omi-aura_l3-omto3e.py

@@ -0,0 +1,92 @@
+"""
+Mallory Row - SAIC at NOAA/NWS/NCEP/EMC
+Designed to read in NASA OMI Total Column Ozone
+"""
+
+import sys
+import os
+import numpy as np
+import netCDF4 as netcdf
+import datetime
+
+print("Python Script:\t" + repr(sys.argv[0]))
+
+# Process script arguements
+if len(sys.argv) != 3:
+    print("Must specify the following elements: omi_file file_flag")
+    sys.exit(1)
+omi_file = os.path.expandvars(sys.argv[1]) 
+if not os.path.exists(omi_file):
+    print(f"OMI file {omi_file} does not exist, exit")
+    sys.exit(1)
+file_flag = sys.argv[2]
+if file_flag not in ['fcst', 'obs']:
+    print(f"File flag {file_flag} not valid (fcst, obs), exit")
+    sys.exit(1)
+
+print(f"Processing {omi_file} as {file_flag} data")
+
+# Read in OMI data
+omi_data = netcdf.Dataset(omi_file)
+omi_StartUTC = omi_data['/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES'].StartUTC
+omi_StartUTC_dt = datetime.datetime.strptime(omi_StartUTC.split(':')[0],
+                                             '%Y-%m-%dT%H')
+omi_EndUTC = omi_data['/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES'].EndUTC
+omi_EndUTC_dt = datetime.datetime.strptime(omi_EndUTC.split(':')[0],
+                                             '%Y-%m-%dT%H')
+omi_delta_lat = float(eval(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpacing
+)[0])
+omi_delta_lon = float(eval(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpacing
+)[1])
+omi_ColumnAmountO3 = omi_data[
+    '/HDFEOS/GRIDS/OMI Column Amount O3/Data Fields/ColumnAmountO3'
+]
+omi_lat_ll = float(eval(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpan
+)[2])
+omi_lon_ll = float(eval(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].GridSpan
+)[0])
+omi_nlat = int(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].NumberOfLatitudesInGrid
+)
+omi_nlon = int(
+    omi_data['/HDFEOS/GRIDS/OMI Column Amount O3'].NumberOfLongitudesInGrid
+)
+omi_ColumnAmountO3_vals = omi_ColumnAmountO3[:]
+omi_ColumnAmountO3_Units = omi_ColumnAmountO3.Units
+
+# There is no geolocation data, so construct it ourselves.
+latitude = np.arange(0., omi_nlat) * omi_delta_lat + omi_lat_ll + 0.125
+longitude = np.arange(0., omi_nlon) * omi_delta_lon + omi_lon_ll + 0.125
+
+# Set data up for MET
+met_data = omi_ColumnAmountO3_vals.copy()
+omi_MidPointUTC_dt = omi_StartUTC_dt + ((omi_EndUTC_dt - omi_StartUTC_dt)/2)
+print(f"Data runs from {omi_StartUTC_dt:%Y%m%d_%H%M%S} to "
+      +f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}...setting valid date as "
+      +f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}")
+met_data.attrs = {
+    'valid': f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}",
+    'init': f"{omi_EndUTC_dt:%Y%m%d_%H%M%S}",
+    'lead': '00',
+    'accum': '00',
+    'name': 'ColumnAmountO3',
+    'standard_name': 'total_column_ozone',
+    'long_name': 'total_column_ozone',
+    'level': 'TotalColumn',
+    'units': omi_ColumnAmountO3_Units,
+    'grid': {
+        'type': 'LatLon',
+        'name': 'OMI Grid',
+        'lat_ll': omi_lat_ll,
+        'lon_ll': omi_lon_ll,
+        'delta_lat': omi_delta_lat,
+        'delta_lon': omi_delta_lon,
+        'Nlat': omi_nlat,
+        'Nlon': omi_nlon,
+    }
+}
+attrs = met_data.attrs