CCD-1465 Removed all redundant tpn files for roman datamodel-represen…

…ted references. Certify parameter reference files (#1050)

Changes:
* allow subkinds to filekinds
  This allows generalizing the "all_" structure to parameter files.
  For example, now the following file is allowable:
    all_pars.tpn
  to generalize constraints that apply to all parameter reference files.
* Add specs and tpns that apply to all parameter reference files
* remove redundant tpns for rdm based references
* add pack the general rmap tpns
* remove asdf tag checking
  Too specific, just roman, and the roman check was too general,
  assuming that all tags are roman datamodels
* update certify tests
  expected outputs differ due to validation being done
  by the datamodel schema instead of tpns.
* script to recreate the test data
* fix reftype checking and make pars tpn stricter
* add tests for parameter references certification
* update tests to handle different testing environments
* update changelog
* fix rmap tests for different results

CHANGES.rst

@@ -1,3 +1,11 @@
+11.18.0 (unreleased)
+====================
+
+ROMAN
+-----
+
+- Removed all redundant tpn files for roman datamodel-represented references. Certify parameter reference files [#1050]
+
 11.17.24 (20204-06-10)
 =====================
 

crds/certify/certify.py

@@ -210,10 +210,6 @@ def certify(self):
         with self.error_on_exception(
                 "Checking ASDF Standard version for", repr(self.filename)):
                 self.check_asdf_standard_version()
-
-        with self.error_on_exception(
-                "Checking ASDF tag validity for", repr(self.filename)):
-                self.check_asdf_tag()
 
     def load(self):
         """Load and parse header from self.filename."""
@@ -498,23 +494,6 @@ def check_asdf_standard_version(self):
                     str(asdf_standard_requirement)
                 )
 
-    def check_asdf_tag(self):
-        """ Check that the tag for an asdf is valid of for a given file.
-        """
-        if self.observatory.lower() == 'roman':
-            from asdf.util import uri_match 
-            with asdf.open(self.filename, _force_raw_types=True) as f:
-                # This commented solution requires adding a new field to the rmap
-                # rmap = self.get_corresponding_rmap()
-                # assert f['roman']._tag == rmap['asdf_tag'] 
-                if not uri_match('asdf://stsci.edu/datamodels/roman/tags/reference_files/*{}-*'.format(f['roman']['meta']['reftype'].lower()), f['roman']._tag):
-                    log.error(
-                        "ASDF Tag Validation",
-                        f['roman']._tag,
-                        "does not match",
-                        'asdf://stsci.edu/datamodels/roman/tags/reference_files/{}-*'.format(f['roman']['meta']['reftype'].lower())
-                    )
-
 # ============================================================================
 
 def find_governing_rmap(context, reference):

crds/core/reftypes.py

@@ -260,7 +260,6 @@ def reference_props_to_validator_keys(self, instrument, filekind, field="tpn"):
         is nominally a .tpn filename and can vary by observatory, instrument, and type as well
         as by functions on the header of `filename`.
         """
-        results = []
         def append_tpn_level(results, instrument, filekind):
             """Append the validator key for one level of the `instrument`
             and `filekind` mutating list `results`.
@@ -273,9 +272,32 @@ def append_tpn_level(results, instrument, filekind):
                 log.verbose_warning("Can't find TPN key for",
                     (instrument, filekind, field), ":", str(exc),
                                     verbosity=75)
+
+        def subkinds(filekind, sep='-'):
+            """Generate list of subkinds
+
+            A subkind is defined as the parts of a filekind separated by a dash `-`.
+
+            This is initially implemented to support the parameter references files
+            where, conceptually, all "pars" files are a common filekind, yet practically
+            there is a separate filekind for each step. This allows applying common criteria
+            to all filekinds sharing the same prefix, such as "pars-".
+
+            The returned generator produces a list that goes from the specific to more general,
+            matching the overall philosophy of the calls to `append_tpn_levels`.
+
+            Examples
+            --------
+            >>> subkinds('pars-detector1')
+            ['pars', 'pars-detector1']
+            """
+            return reversed([filekind[:i] for i, ltr in enumerate(filekind) if ltr == sep] + [filekind])
+
+        results = []
         append_tpn_level(results, instrument, filekind)
         append_tpn_level(results, instrument, "all")
-        append_tpn_level(results, "all", filekind)
+        for subkind in subkinds(filekind):
+            append_tpn_level(results, "all", subkind)
         append_tpn_level(results, "all", "all")
         return results
 

crds/roman/locate.py

@@ -350,6 +350,9 @@ def ref_properties_from_header(filename):
             "Can't identify instrument of", repr(name), ":", str(exc)) from exc
     try:
         filekind = header.get('ROMAN.META.REFTYPE', 'UNDEFINED').lower()
+        # Not all headers are roman datamodels.
+        if filekind == 'undefined':
+            filekind = header.get('META.REFTYPE', 'UNDEFINED').lower()
         assert filekind in FILEKINDS, "Invalid file type " + repr(filekind)
     except Exception as exc:
         raise exceptions.CrdsNamingError("Can't identify ROMAN.META.REFTYPE of", repr(name))

crds/roman/specs/all_pars.spec

@@ -0,0 +1,9 @@
+{
+    'instrument' : 'all',
+    'filetype' : 'pars',
+    'suffix' : 'pars',
+    'file_ext': '.asdf',
+    'text_descr' : 'All parameters reference file types.',
+    'ld_tpn': 'all_pars_ld.tpn',
+    'tpn': 'all_pars.tpn',
+}

crds/roman/specs/combined_specs.json

@@ -10,6 +10,17 @@
             "text_descr":"All instruments and reference file types.",
             "tpn":"all_all.tpn",
             "unique_rowkeys":null
+        },
+        "pars":{
+            "extra_keys":null,
+            "file_ext":".asdf",
+            "filetype":"pars",
+            "instrument":"all",
+            "ld_tpn":"all_pars_ld.tpn",
+            "suffix":"pars",
+            "text_descr":"All parameters reference file types.",
+            "tpn":"all_pars.tpn",
+            "unique_rowkeys":null
         }
     },
     "wfi":{

crds/roman/tpns/all_all_ld.tpn

@@ -0,0 +1,20 @@
+#
+# Constraints defined in this file apply to every reference file
+#
+# Template file used by certify to check reference files
+# Some fields may be abbreviated to their first character:
+#
+# keytype = (Header|Group|Column)
+# datatype = (Integer|Real|Logical|Double|Character)
+# presence = (Optional|Required)
+#
+# NAME  KEYTYPE  DATATYPE   PRESENCE    VALUES
+#----------------------------------------------------------
+ROMAN.META.TELESCOPE          H   C   R       ROMAN
+ROMAN.META.INSTRUMENT.NAME    H   C   R
+ROMAN.META.REFTYPE            H   C   R
+ROMAN.META.USEAFTER           H   C   R       &JWSTDATE
+ROMAN.META.PEDIGREE           H   C   R       &JWSTPEDIGREE
+ROMAN.META.AUTHOR             H   C   R
+ROMAN.META.DESCRIPTION        H   C   R
+ROMAN.META.ORIGIN             H   C   R

crds/roman/tpns/all_pars.tpn

@@ -0,0 +1,29 @@
+#
+# Constraints defined in this file apply to every reference file
+#
+# Template file used by certify to check reference files
+# Some fields may be abbreviated to their first character:
+#
+# keytype = (Header|Group|Column)
+# datatype = (Integer|Real|Logical|Double|Character)
+# presence = (Optional|Required)
+#
+# NAME  KEYTYPE  DATATYPE   PRESENCE    VALUES
+#----------------------------------------------------------
+META.TELESCOPE                     H   C   R   ROMAN
+META.INSTRUMENT.NAME               H   C   R   WFI
+META.REFTYPE                       H   C   R
+META.USEAFTER                      H   C   R   &JWSTDATE
+META.PEDIGREE                      H   C   R   &JWSTPEDIGREE
+META.AUTHOR                        H   C   R
+META.DESCRIPTION                   H   C   R
+META.ORIGIN                        H   C   R
+META.INSTRUMENT.DETECTOR           H   C   O   WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\
+                                               WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\
+                                               ANY,N/A
+META.INSTRUMENT.OPTICAL_ELEMENT    H   C   O   F062,F087,F106,F129,F146,F158,F184,F213,GRISM,\
+                                               PRISM,CLEAR,DARK,N/A,ANY,UNKNOWN
+META.EXPOSURE.TYPE                 H   C   O   WFI_IMAGE,WFI_GRISM,WFI_PRISM,WFI_DARK,WFI_WSM_ACQ1,WFI_WSM_ACQ2,\
+                                               WFI_WSM_TRACK,WFI_WFSC,DEFOCUS_MODERATE,DEFOCUS_LARGE,\
+                                               WFI_WIM_ACQ,WFI_WIM_TRACK,WFI_PARALLEL,\
+                                               WFI_FLAT_EXTERNAL,WFI_FLAT_INTERNAL,WFI_RCS

crds/roman/tpns/wfi_all_ld.tpn

@@ -0,0 +1,24 @@
+# Contraints defined in this file apply to every REFTYPE of the WFI instrument
+
+# Template file used by certify to check reference files
+# Some fields may be abbreviated to their first character:
+#
+# keytype = (Header|Group|Column|Expression)
+# datatype = (Integer|Real|Logical|Double|Character|Expression)
+# presence = (Optional|Required)
+#
+# NAME  KEYTYPE  DATATYPE   PRESENCE    VALUES
+#----------------------------------------------------------
+ROMAN.META.INSTRUMENT.NAME        H   C   R   WFI
+
+ROMAN.META.INSTRUMENT.DETECTOR    H   C   O   WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\
+                                              WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\
+                                              ANY,N/A
+
+ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT      H   C   O   F062,F087,F106,F129,F146,F158,F184,F213,GRISM,\
+                                                       PRISM,CLEAR,DARK,N/A,ANY,UNKNOWN
+
+ROMAN.META.EXPOSURE.TYPE  H   C   O   WFI_IMAGE,WFI_GRISM,WFI_PRISM,WFI_DARK,WFI_WSM_ACQ1,WFI_WSM_ACQ2,\
+                                      WFI_WSM_TRACK,WFI_WFSC,DEFOCUS_MODERATE,DEFOCUS_LARGE,\
+                                      WFI_WIM_ACQ,WFI_WIM_TRACK,WFI_PARALLEL,\
+                                      WFI_FLAT_EXTERNAL,WFI_FLAT_INTERNAL,WFI_RCS