diff --git a/CHANGES.rst b/CHANGES.rst index d8dcb993b..f6e225945 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,11 @@ +11.18.0 (unreleased) +==================== + +ROMAN +----- + +- Removed all redundant tpn files for roman datamodel-represented references. Certify parameter reference files [#1050] + 11.17.24 (20204-06-10) ===================== diff --git a/crds/certify/certify.py b/crds/certify/certify.py index 4f5ad9cdc..8197c59ad 100644 --- a/crds/certify/certify.py +++ b/crds/certify/certify.py @@ -210,10 +210,6 @@ def certify(self): with self.error_on_exception( "Checking ASDF Standard version for", repr(self.filename)): self.check_asdf_standard_version() - - with self.error_on_exception( - "Checking ASDF tag validity for", repr(self.filename)): - self.check_asdf_tag() def load(self): """Load and parse header from self.filename.""" @@ -498,23 +494,6 @@ def check_asdf_standard_version(self): str(asdf_standard_requirement) ) - def check_asdf_tag(self): - """ Check that the tag for an asdf is valid of for a given file. - """ - if self.observatory.lower() == 'roman': - from asdf.util import uri_match - with asdf.open(self.filename, _force_raw_types=True) as f: - # This commented solution requires adding a new field to the rmap - # rmap = self.get_corresponding_rmap() - # assert f['roman']._tag == rmap['asdf_tag'] - if not uri_match('asdf://stsci.edu/datamodels/roman/tags/reference_files/*{}-*'.format(f['roman']['meta']['reftype'].lower()), f['roman']._tag): - log.error( - "ASDF Tag Validation", - f['roman']._tag, - "does not match", - 'asdf://stsci.edu/datamodels/roman/tags/reference_files/{}-*'.format(f['roman']['meta']['reftype'].lower()) - ) - # ============================================================================ def find_governing_rmap(context, reference): diff --git a/crds/core/reftypes.py b/crds/core/reftypes.py index 5bff81519..8b283a8c2 100644 --- a/crds/core/reftypes.py +++ b/crds/core/reftypes.py @@ -260,7 +260,6 @@ def reference_props_to_validator_keys(self, instrument, filekind, field="tpn"): is nominally a .tpn filename and can vary by observatory, instrument, and type as well as by functions on the header of `filename`. """ - results = [] def append_tpn_level(results, instrument, filekind): """Append the validator key for one level of the `instrument` and `filekind` mutating list `results`. @@ -273,9 +272,32 @@ def append_tpn_level(results, instrument, filekind): log.verbose_warning("Can't find TPN key for", (instrument, filekind, field), ":", str(exc), verbosity=75) + + def subkinds(filekind, sep='-'): + """Generate list of subkinds + + A subkind is defined as the parts of a filekind separated by a dash `-`. + + This is initially implemented to support the parameter references files + where, conceptually, all "pars" files are a common filekind, yet practically + there is a separate filekind for each step. This allows applying common criteria + to all filekinds sharing the same prefix, such as "pars-". + + The returned generator produces a list that goes from the specific to more general, + matching the overall philosophy of the calls to `append_tpn_levels`. + + Examples + -------- + >>> subkinds('pars-detector1') + ['pars', 'pars-detector1'] + """ + return reversed([filekind[:i] for i, ltr in enumerate(filekind) if ltr == sep] + [filekind]) + + results = [] append_tpn_level(results, instrument, filekind) append_tpn_level(results, instrument, "all") - append_tpn_level(results, "all", filekind) + for subkind in subkinds(filekind): + append_tpn_level(results, "all", subkind) append_tpn_level(results, "all", "all") return results diff --git a/crds/roman/locate.py b/crds/roman/locate.py index f03ec037f..f137cb341 100644 --- a/crds/roman/locate.py +++ b/crds/roman/locate.py @@ -350,6 +350,9 @@ def ref_properties_from_header(filename): "Can't identify instrument of", repr(name), ":", str(exc)) from exc try: filekind = header.get('ROMAN.META.REFTYPE', 'UNDEFINED').lower() + # Not all headers are roman datamodels. + if filekind == 'undefined': + filekind = header.get('META.REFTYPE', 'UNDEFINED').lower() assert filekind in FILEKINDS, "Invalid file type " + repr(filekind) except Exception as exc: raise exceptions.CrdsNamingError("Can't identify ROMAN.META.REFTYPE of", repr(name)) diff --git a/crds/roman/specs/all_pars.spec b/crds/roman/specs/all_pars.spec new file mode 100644 index 000000000..0df06ee52 --- /dev/null +++ b/crds/roman/specs/all_pars.spec @@ -0,0 +1,9 @@ +{ + 'instrument' : 'all', + 'filetype' : 'pars', + 'suffix' : 'pars', + 'file_ext': '.asdf', + 'text_descr' : 'All parameters reference file types.', + 'ld_tpn': 'all_pars_ld.tpn', + 'tpn': 'all_pars.tpn', +} diff --git a/crds/roman/specs/combined_specs.json b/crds/roman/specs/combined_specs.json index 89253e427..c0b0d8ad6 100644 --- a/crds/roman/specs/combined_specs.json +++ b/crds/roman/specs/combined_specs.json @@ -10,6 +10,17 @@ "text_descr":"All instruments and reference file types.", "tpn":"all_all.tpn", "unique_rowkeys":null + }, + "pars":{ + "extra_keys":null, + "file_ext":".asdf", + "filetype":"pars", + "instrument":"all", + "ld_tpn":"all_pars_ld.tpn", + "suffix":"pars", + "text_descr":"All parameters reference file types.", + "tpn":"all_pars.tpn", + "unique_rowkeys":null } }, "wfi":{ diff --git a/crds/roman/tpns/all_all.tpn b/crds/roman/tpns/all_all.tpn deleted file mode 100644 index 7da19b31a..000000000 --- a/crds/roman/tpns/all_all.tpn +++ /dev/null @@ -1,20 +0,0 @@ -# -# Constraints defined in this file apply to every reference file -# -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column) -# datatype = (Integer|Real|Logical|Double|Character) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.TELESCOPE H C R ROMAN -ROMAN.META.INSTRUMENT.NAME H C R -ROMAN.META.REFTYPE H C R -ROMAN.META.USEAFTER H C R &JWSTDATE -ROMAN.META.PEDIGREE H C R &JWSTPEDIGREE -ROMAN.META.AUTHOR H C R -ROMAN.META.DESCRIPTION H C R -ROMAN.META.ORIGIN H C R diff --git a/crds/roman/tpns/all_all_ld.tpn b/crds/roman/tpns/all_all_ld.tpn deleted file mode 120000 index 1c4cd658f..000000000 --- a/crds/roman/tpns/all_all_ld.tpn +++ /dev/null @@ -1 +0,0 @@ -all_all.tpn \ No newline at end of file diff --git a/crds/roman/tpns/all_all_ld.tpn b/crds/roman/tpns/all_all_ld.tpn new file mode 100644 index 000000000..7da19b31a --- /dev/null +++ b/crds/roman/tpns/all_all_ld.tpn @@ -0,0 +1,20 @@ +# +# Constraints defined in this file apply to every reference file +# +# Template file used by certify to check reference files +# Some fields may be abbreviated to their first character: +# +# keytype = (Header|Group|Column) +# datatype = (Integer|Real|Logical|Double|Character) +# presence = (Optional|Required) +# +# NAME KEYTYPE DATATYPE PRESENCE VALUES +#---------------------------------------------------------- +ROMAN.META.TELESCOPE H C R ROMAN +ROMAN.META.INSTRUMENT.NAME H C R +ROMAN.META.REFTYPE H C R +ROMAN.META.USEAFTER H C R &JWSTDATE +ROMAN.META.PEDIGREE H C R &JWSTPEDIGREE +ROMAN.META.AUTHOR H C R +ROMAN.META.DESCRIPTION H C R +ROMAN.META.ORIGIN H C R diff --git a/crds/roman/tpns/all_pars.tpn b/crds/roman/tpns/all_pars.tpn new file mode 100644 index 000000000..785d50dd1 --- /dev/null +++ b/crds/roman/tpns/all_pars.tpn @@ -0,0 +1,29 @@ +# +# Constraints defined in this file apply to every reference file +# +# Template file used by certify to check reference files +# Some fields may be abbreviated to their first character: +# +# keytype = (Header|Group|Column) +# datatype = (Integer|Real|Logical|Double|Character) +# presence = (Optional|Required) +# +# NAME KEYTYPE DATATYPE PRESENCE VALUES +#---------------------------------------------------------- +META.TELESCOPE H C R ROMAN +META.INSTRUMENT.NAME H C R WFI +META.REFTYPE H C R +META.USEAFTER H C R &JWSTDATE +META.PEDIGREE H C R &JWSTPEDIGREE +META.AUTHOR H C R +META.DESCRIPTION H C R +META.ORIGIN H C R +META.INSTRUMENT.DETECTOR H C O WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\ + WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\ + ANY,N/A +META.INSTRUMENT.OPTICAL_ELEMENT H C O F062,F087,F106,F129,F146,F158,F184,F213,GRISM,\ + PRISM,CLEAR,DARK,N/A,ANY,UNKNOWN +META.EXPOSURE.TYPE H C O WFI_IMAGE,WFI_GRISM,WFI_PRISM,WFI_DARK,WFI_WSM_ACQ1,WFI_WSM_ACQ2,\ + WFI_WSM_TRACK,WFI_WFSC,DEFOCUS_MODERATE,DEFOCUS_LARGE,\ + WFI_WIM_ACQ,WFI_WIM_TRACK,WFI_PARALLEL,\ + WFI_FLAT_EXTERNAL,WFI_FLAT_INTERNAL,WFI_RCS diff --git a/crds/roman/tpns/wfi_all.tpn b/crds/roman/tpns/wfi_all.tpn deleted file mode 100644 index 03a2371c2..000000000 --- a/crds/roman/tpns/wfi_all.tpn +++ /dev/null @@ -1,24 +0,0 @@ -# Contraints defined in this file apply to every REFTYPE of the WFI instrument - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.NAME H C R WFI - -ROMAN.META.INSTRUMENT.DETECTOR H C O WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\ - WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\ - ANY,N/A - -ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT H C O F062,F087,F106,F129,F146,F158,F184,F213,GRISM,\ - PRISM,CLEAR,DARK,N/A,ANY,UNKNOWN - -ROMAN.META.EXPOSURE.TYPE H C O WFI_IMAGE,WFI_GRISM,WFI_PRISM,WFI_DARK,WFI_WSM_ACQ1,WFI_WSM_ACQ2,\ - WFI_WSM_TRACK,WFI_WFSC,DEFOCUS_MODERATE,DEFOCUS_LARGE,\ - WFI_WIM_ACQ,WFI_WIM_TRACK,WFI_PARALLEL,\ - WFI_FLAT_EXTERNAL,WFI_FLAT_INTERNAL,WFI_RCS diff --git a/crds/roman/tpns/wfi_all_ld.tpn b/crds/roman/tpns/wfi_all_ld.tpn deleted file mode 120000 index 0c2b206e0..000000000 --- a/crds/roman/tpns/wfi_all_ld.tpn +++ /dev/null @@ -1 +0,0 @@ -wfi_all.tpn \ No newline at end of file diff --git a/crds/roman/tpns/wfi_all_ld.tpn b/crds/roman/tpns/wfi_all_ld.tpn new file mode 100644 index 000000000..03a2371c2 --- /dev/null +++ b/crds/roman/tpns/wfi_all_ld.tpn @@ -0,0 +1,24 @@ +# Contraints defined in this file apply to every REFTYPE of the WFI instrument + +# Template file used by certify to check reference files +# Some fields may be abbreviated to their first character: +# +# keytype = (Header|Group|Column|Expression) +# datatype = (Integer|Real|Logical|Double|Character|Expression) +# presence = (Optional|Required) +# +# NAME KEYTYPE DATATYPE PRESENCE VALUES +#---------------------------------------------------------- +ROMAN.META.INSTRUMENT.NAME H C R WFI + +ROMAN.META.INSTRUMENT.DETECTOR H C O WFI01,WFI02,WFI03,WFI04,WFI05,WFI06,WFI07,WFI08,WFI09,\ + WFI10,WFI11,WFI12,WFI13,WFI14,WFI15,WFI16,WFI17,WFI18,\ + ANY,N/A + +ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT H C O F062,F087,F106,F129,F146,F158,F184,F213,GRISM,\ + PRISM,CLEAR,DARK,N/A,ANY,UNKNOWN + +ROMAN.META.EXPOSURE.TYPE H C O WFI_IMAGE,WFI_GRISM,WFI_PRISM,WFI_DARK,WFI_WSM_ACQ1,WFI_WSM_ACQ2,\ + WFI_WSM_TRACK,WFI_WFSC,DEFOCUS_MODERATE,DEFOCUS_LARGE,\ + WFI_WIM_ACQ,WFI_WIM_TRACK,WFI_PARALLEL,\ + WFI_FLAT_EXTERNAL,WFI_FLAT_INTERNAL,WFI_RCS diff --git a/crds/roman/tpns/wfi_area.tpn b/crds/roman/tpns/wfi_area.tpn deleted file mode 100644 index f297e86a9..000000000 --- a/crds/roman/tpns/wfi_area.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's AREA reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_dark.tpn b/crds/roman/tpns/wfi_dark.tpn deleted file mode 100644 index 38683f014..000000000 --- a/crds/roman/tpns/wfi_dark.tpn +++ /dev/null @@ -1,16 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's DARK reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R - -ROMAN.META.EXPOSURE.TYPE H C R - -ROMAN.META.EXPOSURE.MA_TABLE_NUMBER H I R diff --git a/crds/roman/tpns/wfi_distortion.tpn b/crds/roman/tpns/wfi_distortion.tpn deleted file mode 100644 index 593f98670..000000000 --- a/crds/roman/tpns/wfi_distortion.tpn +++ /dev/null @@ -1,13 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's DISTORTION reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R -ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT H C R diff --git a/crds/roman/tpns/wfi_flat.tpn b/crds/roman/tpns/wfi_flat.tpn deleted file mode 100644 index 14e971279..000000000 --- a/crds/roman/tpns/wfi_flat.tpn +++ /dev/null @@ -1,14 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's FLAT reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R - -ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT H C R diff --git a/crds/roman/tpns/wfi_gain.tpn b/crds/roman/tpns/wfi_gain.tpn deleted file mode 100644 index a9a06e814..000000000 --- a/crds/roman/tpns/wfi_gain.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's GAIN reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_inverselinearity.tpn b/crds/roman/tpns/wfi_inverselinearity.tpn deleted file mode 100644 index c46e00163..000000000 --- a/crds/roman/tpns/wfi_inverselinearity.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Constraints defined in this file apply the WFI instrument's Inverse Linearity reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_ipc.tpn b/crds/roman/tpns/wfi_ipc.tpn deleted file mode 100644 index 2124a3387..000000000 --- a/crds/roman/tpns/wfi_ipc.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Constraints defined in this file apply the WFI instrument's IPC reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_linearity.tpn b/crds/roman/tpns/wfi_linearity.tpn deleted file mode 100644 index 680dea4af..000000000 --- a/crds/roman/tpns/wfi_linearity.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's LINEARITY reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_mask.tpn b/crds/roman/tpns/wfi_mask.tpn deleted file mode 100644 index 684465815..000000000 --- a/crds/roman/tpns/wfi_mask.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's MASK reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_photom.tpn b/crds/roman/tpns/wfi_photom.tpn deleted file mode 100644 index 9b42931e3..000000000 --- a/crds/roman/tpns/wfi_photom.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's PHOTOM reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_readnoise.tpn b/crds/roman/tpns/wfi_readnoise.tpn deleted file mode 100644 index 389b1489a..000000000 --- a/crds/roman/tpns/wfi_readnoise.tpn +++ /dev/null @@ -1,14 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's READNOISE reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R - -ROMAN.META.EXPOSURE.TYPE H C R diff --git a/crds/roman/tpns/wfi_refpix.tpn b/crds/roman/tpns/wfi_refpix.tpn deleted file mode 100644 index 36cfd77d5..000000000 --- a/crds/roman/tpns/wfi_refpix.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's REFPIX reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/crds/roman/tpns/wfi_saturation.tpn b/crds/roman/tpns/wfi_saturation.tpn deleted file mode 100644 index a21c05c63..000000000 --- a/crds/roman/tpns/wfi_saturation.tpn +++ /dev/null @@ -1,12 +0,0 @@ -# Contraints defined in this file apply the WFI instrument's SATURATION reference type. - -# Template file used by certify to check reference files -# Some fields may be abbreviated to their first character: -# -# keytype = (Header|Group|Column|Expression) -# datatype = (Integer|Real|Logical|Double|Character|Expression) -# presence = (Optional|Required) -# -# NAME KEYTYPE DATATYPE PRESENCE VALUES -#---------------------------------------------------------- -ROMAN.META.INSTRUMENT.DETECTOR H C R diff --git a/test/certify/test_certify.py b/test/certify/test_certify.py index 9d42d010f..e34fca3e8 100644 --- a/test/certify/test_certify.py +++ b/test/certify/test_certify.py @@ -789,7 +789,7 @@ def test_certify_roman_invalid_asdf_schema(roman_test_cache_state, roman_data, c certify.certify_file(fpath, "roman_0003.pmap", observatory="roman") out = caplog.text assert "Validation error" in out - assert "This ain't no valid time" in out + assert "yesterday" in out @mark.roman @@ -803,22 +803,7 @@ def test_certify_roman_invalid_asdf_tpn(roman_test_cache_state, roman_data, capl with caplog.at_level(logging.INFO, logger="CRDS"): certify.certify_file(f"{roman_data}/roman_wfi16_f158_flat_invalid_tpn.asdf", "roman_0003.pmap", observatory="roman") out = caplog.text - expected_out = f"""Certifying '{roman_data}/roman_wfi16_f158_flat_invalid_tpn.asdf' as 'ASDF' relative to context 'roman_0003.pmap' -In 'roman_wfi16_f158_flat_invalid_tpn.asdf' : Checking 'ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT [FITS unknown]' : Value 'BAD' is not one of ['ANY', -'CLEAR', -'DARK', -'F062', -'F087', -'F106', -'F129', -'F146', -'F158', -'F184', -'F213', -'GRISM', -'N/A', -'PRISM', -'UNKNOWN']""" + expected_out = """Validation error : 'BAD' is not one of ['F062', 'F087', 'F106', 'F129', 'F146', 'F158', 'F184', 'F213', 'GRISM', 'PRISM', 'DARK']""" for msg in expected_out.splitlines(): assert msg.strip() in out @@ -860,10 +845,7 @@ def test_certify_roman_invalid_spec_asdf_tpn(roman_test_cache_state, roman_data, with caplog.at_level(logging.INFO, logger="CRDS"): certify.certify_file(f"{roman_data}/roman_wfi16_grism_flat_invalid_tpn.asdf", "roman_0003.pmap", observatory="roman") out = caplog.text - expected_out = f"""Certifying '{roman_data}/roman_wfi16_grism_flat_invalid_tpn.asdf' as 'ASDF' relative to context 'roman_0003.pmap' -In 'roman_wfi16_grism_flat_invalid_tpn.asdf' : Error mapping reference names and values to dataset names and values : Bad USEAFTER time format = 'yesterday' -In 'roman_wfi16_grism_flat_invalid_tpn.asdf' : Checking 'ROMAN.META.USEAFTER [USEAFTER]' : Invalid 'Jwstdate' format 'yesterday' should be '2018-12-22T00:00:00' -In 'roman_wfi16_grism_flat_invalid_tpn.asdf' : Checking ASDF tag validity for '{roman_data}/roman_wfi16_grism_flat_invalid_tpn.asdf' : 'dict' object has no attribute '_tag'""" + expected_out = """'BAD' is not one of ['F062', 'F087', 'F106', 'F129', 'F146', 'F158', 'F184', 'F213', 'GRISM', 'PRISM', 'DARK']""" for msg in expected_out.splitlines(): assert msg.strip() in out @@ -2991,3 +2973,40 @@ def test_or_bars_certify_bad_value(jwst_serverless_state, jwst_data, caplog): 1 errors""".splitlines() for line in expected: assert line in out + + +@mark.roman +@mark.certify +def test_certify_pars(roman_test_cache_state, roman_data, caplog): + """Test that parameter references certify positive. + """ + with caplog.at_level(logging.INFO, logger="CRDS"): + certify.certify_file(f"{roman_data}/roman_wfi_pars-exposurepipeline.asdf", "roman_0006.pmap", observatory="roman") + out = caplog.text + assert len(out.splitlines()) <= 2 + + +@mark.roman +@mark.certify +def test_certify_pars_badtype(roman_test_cache_state, roman_data, caplog): + """Test parameter references check reftype. + """ + with caplog.at_level(logging.INFO, logger="CRDS"): + certify.certify_file(f"{roman_data}/roman_wfi_pars-exposurepipeline_badtype.asdf", "roman_0006.pmap", observatory="roman") + out = caplog.text + expected = """Can't identify ROMAN.META.REFTYPE""".splitlines() + for line in expected: + assert line in out + + +@mark.roman +@mark.certify +def test_certify_pars_badschema(roman_test_cache_state, roman_data, caplog): + """Test parameter references check general schema validation failures. + """ + with caplog.at_level(logging.INFO, logger="CRDS"): + certify.certify_file(f"{roman_data}/roman_wfi_pars-exposurepipeline_invalid_schema.asdf", "roman_0006.pmap", observatory="roman") + out = caplog.text + expected = """Invalid instrument 'bad'""".splitlines() + for line in expected: + assert line in out diff --git a/test/core/test_rmap.py b/test/core/test_rmap.py index 8249fb12d..60ec92ba4 100644 --- a/test/core/test_rmap.py +++ b/test/core/test_rmap.py @@ -449,7 +449,7 @@ def test_validate_mapping_ambiguous_roman(roman_serverless_state, roman_data, ca with caplog.at_level(logging.INFO, logger="CRDS"): r.validate_mapping() out = caplog.text - expected = """ Match('ROMAN.META.INSTRUMENT.DETECTOR [DETECTOR]', 'ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT [FITS unknown]') : ('WFI01', 'F158') : + expected = """Match('ROMAN.META.INSTRUMENT.DETECTOR', 'ROMAN.META.INSTRUMENT.OPTICAL_ELEMENT') : ('WFI01', 'F158') : ---------------------------------------- Match case ('WFI01', 'F158') : UseAfter({ diff --git a/test/data/roman/mkdata.py b/test/data/roman/mkdata.py new file mode 100644 index 000000000..aefbd8ac4 --- /dev/null +++ b/test/data/roman/mkdata.py @@ -0,0 +1,52 @@ +"""Make test data + +To keep up with Roman datamodel changes, the test data need to be updated regularly. + +Execute with + + $ cd + $ python -m mkdata +""" +import os + +import roman_datamodels as rdm +import roman_datamodels.maker_utils as mu + + +def mkdata(): + """Create FlatRefModels with various bad settings""" + + # Good model + flatref = mu.mk_datamodel(rdm.datamodels.FlatRefModel, shape=(2, 2), + meta={'instrument': {'name': 'WFI', 'detector': 'WFI16', 'optical_element': 'F158'}}) + flatref.save('roman_wfi16_f158_flat_small.asdf') + flatrefgrism = mu.mk_datamodel(rdm.datamodels.FlatRefModel, shape=(2, 2), + meta={'instrument': {'name': 'WFI', 'detector': 'WFI16', 'optical_element': 'GRISM'}}) + flatrefgrism.save('roman_wfi16_grism_flat_small.asdf') + + # Turn off validations so we can make bad decisions + os.environ['ROMAN_VALIDATE'] = 'false' + + # Bad reftype + bad = flatref.copy() + bad.meta.reftype = 'badtype' + bad.save('roman_wfi16_f158_flat_badtype.asdf') + + # Invalid schema + bad = flatref.copy() + bad.meta.useafter = "yesterday" + bad.save('roman_wfi16_f158_flat_invalid_schema.asdf') + bad = flatrefgrism.copy() + bad.meta.useafter = "yesterday" + bad.save('roman_wfi16_grism_flat_invalid_schema.asdf') + + # Invalid TPN. Really just another bad schema choice. + bad = flatref.copy() + bad.meta.instrument.optical_element = 'BAD' + bad.save('roman_wfi16_f158_flat_invalid_tpn.asdf') + bad = flatrefgrism.copy() + bad.meta.instrument.optical_element = 'BAD' + bad.save('roman_wfi16_grism_flat_invalid_tpn.asdf') + +if __name__ == '__main__': + mkdata() diff --git a/test/data/roman/roman_wfi16_f158_badtype_small.asdf b/test/data/roman/roman_wfi16_f158_badtype_small.asdf deleted file mode 100644 index 3a7d15cf6..000000000 Binary files a/test/data/roman/roman_wfi16_f158_badtype_small.asdf and /dev/null differ diff --git a/test/data/roman/roman_wfi16_f158_flat_badtype.asdf b/test/data/roman/roman_wfi16_f158_flat_badtype.asdf new file mode 100644 index 000000000..3631ecd88 Binary files /dev/null and b/test/data/roman/roman_wfi16_f158_flat_badtype.asdf differ diff --git a/test/data/roman/roman_wfi16_f158_flat_invalid_schema.asdf b/test/data/roman/roman_wfi16_f158_flat_invalid_schema.asdf index e3cf87306..10e2f5c32 100644 Binary files a/test/data/roman/roman_wfi16_f158_flat_invalid_schema.asdf and b/test/data/roman/roman_wfi16_f158_flat_invalid_schema.asdf differ diff --git a/test/data/roman/roman_wfi16_f158_flat_invalid_tpn.asdf b/test/data/roman/roman_wfi16_f158_flat_invalid_tpn.asdf index 2f3086862..b7572eb90 100644 Binary files a/test/data/roman/roman_wfi16_f158_flat_invalid_tpn.asdf and b/test/data/roman/roman_wfi16_f158_flat_invalid_tpn.asdf differ diff --git a/test/data/roman/roman_wfi16_f158_flat_small.asdf b/test/data/roman/roman_wfi16_f158_flat_small.asdf index 0ac5ea21e..583ceddf5 100644 Binary files a/test/data/roman/roman_wfi16_f158_flat_small.asdf and b/test/data/roman/roman_wfi16_f158_flat_small.asdf differ diff --git a/test/data/roman/roman_wfi16_grism_flat_invalid_schema.asdf b/test/data/roman/roman_wfi16_grism_flat_invalid_schema.asdf index 8955d24f2..6406b7d7c 100644 Binary files a/test/data/roman/roman_wfi16_grism_flat_invalid_schema.asdf and b/test/data/roman/roman_wfi16_grism_flat_invalid_schema.asdf differ diff --git a/test/data/roman/roman_wfi16_grism_flat_invalid_tpn.asdf b/test/data/roman/roman_wfi16_grism_flat_invalid_tpn.asdf index 7a5e069bd..b7572eb90 100644 Binary files a/test/data/roman/roman_wfi16_grism_flat_invalid_tpn.asdf and b/test/data/roman/roman_wfi16_grism_flat_invalid_tpn.asdf differ diff --git a/test/data/roman/roman_wfi16_grism_flat_small.asdf b/test/data/roman/roman_wfi16_grism_flat_small.asdf index 9479aa5b8..0f8525eff 100644 Binary files a/test/data/roman/roman_wfi16_grism_flat_small.asdf and b/test/data/roman/roman_wfi16_grism_flat_small.asdf differ diff --git a/test/data/roman/roman_wfi_pars-exposurepipeline.asdf b/test/data/roman/roman_wfi_pars-exposurepipeline.asdf new file mode 100644 index 000000000..7186ce791 --- /dev/null +++ b/test/data/roman/roman_wfi_pars-exposurepipeline.asdf @@ -0,0 +1,260 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.5.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 3.2.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.5.0 + software: !core/software-1.0.0 {name: asdf, version: 3.2.0} +class: romancal.pipeline.exposure_pipeline.ExposurePipeline +meta: + author: rcal developer + date: '2024-06-07T15:46:47' + description: Parameters for calibration step romancal.pipeline.exposure_pipeline.ExposurePipeline + instrument: {name: WFI} + origin: + pedigree: GROUND + reftype: PARS-EXPOSUREPIPELINE + telescope: ROMAN + useafter: 2022-02-02T00:00:00 +name: ExposurePipeline +parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_calibrated_ramp: false + save_results: false + search_output_file: true + skip: false + suffix: null +steps: +- class: romancal.dq_init.dq_init_step.DQInitStep + name: dq_init + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.saturation.saturation_step.SaturationStep + name: saturation + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.refpix.refpix_step.RefPixStep + name: refpix + parameters: + cosine_interpolate: true + fft_interpolate: true + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + remove_offset: true + remove_trends: true + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.linearity.linearity_step.LinearityStep + name: linearity + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.dark_current.dark_current_step.DarkCurrentStep + name: dark_current + parameters: + dark_output: null + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.ramp_fitting.ramp_fit_step.RampFitStep + name: rampfit + parameters: + algorithm: ols_cas22 + input_dir: '' + maximum_cores: none + opt_name: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_opt: false + save_results: false + search_output_file: true + skip: false + suffix: rampfit + threshold_constant: null + threshold_intercept: null + use_ramp_jump_detection: true +- class: romancal.assign_wcs.assign_wcs_step.AssignWcsStep + name: assign_wcs + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.flatfield.flat_field_step.FlatFieldStep + name: flatfield + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.photom.photom_step.PhotomStep + name: photom + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.source_detection.source_detection_step.SourceDetectionStep + name: source_detection + parameters: + bkg_boxsize: 9 + bkg_estimator: median + bkg_filter_size: 3 + bkg_sigma: 2.0 + calc_threshold: true + fit_psf: true + input_dir: '' + kernel_fwhm: null + max_sources: null + min_separation: 11 + output_cat_filetype: asdf + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + peakmax: 100000.0 + post_hooks: [] + pre_hooks: [] + roundhi: 1.0 + roundlo: -1.0 + save_catalogs: false + save_results: false + scalar_threshold: null + search_output_file: true + sharphi: 1.0 + sharplo: 0.0 + skip: false + snr_threshold: 3.0 + suffix: null +- class: romancal.tweakreg.tweakreg_step.TweakRegStep + name: tweakreg + parameters: + abs_fitgeometry: rshift + abs_minobj: 15 + abs_nclip: 3 + abs_refcat: GAIADR3 + abs_searchrad: 6.0 + abs_separation: 0.1 + abs_sigma: 3.0 + abs_tolerance: 0.7 + abs_use2dhist: true + catalog_format: ascii.ecsv + catalog_path: '' + catfile: '' + enforce_user_order: false + expand_refcat: false + fitgeometry: rshift + input_dir: '' + minobj: 15 + nclip: 3 + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: true + post_hooks: [] + pre_hooks: [] + save_abs_catalog: false + save_results: false + search_output_file: true + searchrad: 2.0 + separation: 1.0 + sigma: 3.0 + skip: false + suffix: null + tolerance: 0.7 + use2dhist: true + use_custom_catalogs: false +... diff --git a/test/data/roman/roman_wfi_pars-exposurepipeline_badtype.asdf b/test/data/roman/roman_wfi_pars-exposurepipeline_badtype.asdf new file mode 100644 index 000000000..724794858 --- /dev/null +++ b/test/data/roman/roman_wfi_pars-exposurepipeline_badtype.asdf @@ -0,0 +1,260 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.5.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 3.2.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.5.0 + software: !core/software-1.0.0 {name: asdf, version: 3.2.0} +class: romancal.pipeline.exposure_pipeline.ExposurePipeline +meta: + author: rcal developer + date: '2024-06-07T15:46:47' + description: Parameters for calibration step romancal.pipeline.exposure_pipeline.ExposurePipeline + instrument: {name: WFI} + origin: + pedigree: GROUND + reftype: bad reftype + telescope: ROMAN + useafter: 2022-02-02T00:00:00 +name: ExposurePipeline +parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_calibrated_ramp: false + save_results: false + search_output_file: true + skip: false + suffix: null +steps: +- class: romancal.dq_init.dq_init_step.DQInitStep + name: dq_init + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.saturation.saturation_step.SaturationStep + name: saturation + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.refpix.refpix_step.RefPixStep + name: refpix + parameters: + cosine_interpolate: true + fft_interpolate: true + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + remove_offset: true + remove_trends: true + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.linearity.linearity_step.LinearityStep + name: linearity + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.dark_current.dark_current_step.DarkCurrentStep + name: dark_current + parameters: + dark_output: null + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.ramp_fitting.ramp_fit_step.RampFitStep + name: rampfit + parameters: + algorithm: ols_cas22 + input_dir: '' + maximum_cores: none + opt_name: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_opt: false + save_results: false + search_output_file: true + skip: false + suffix: rampfit + threshold_constant: null + threshold_intercept: null + use_ramp_jump_detection: true +- class: romancal.assign_wcs.assign_wcs_step.AssignWcsStep + name: assign_wcs + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.flatfield.flat_field_step.FlatFieldStep + name: flatfield + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.photom.photom_step.PhotomStep + name: photom + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.source_detection.source_detection_step.SourceDetectionStep + name: source_detection + parameters: + bkg_boxsize: 9 + bkg_estimator: median + bkg_filter_size: 3 + bkg_sigma: 2.0 + calc_threshold: true + fit_psf: true + input_dir: '' + kernel_fwhm: null + max_sources: null + min_separation: 11 + output_cat_filetype: asdf + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + peakmax: 100000.0 + post_hooks: [] + pre_hooks: [] + roundhi: 1.0 + roundlo: -1.0 + save_catalogs: false + save_results: false + scalar_threshold: null + search_output_file: true + sharphi: 1.0 + sharplo: 0.0 + skip: false + snr_threshold: 3.0 + suffix: null +- class: romancal.tweakreg.tweakreg_step.TweakRegStep + name: tweakreg + parameters: + abs_fitgeometry: rshift + abs_minobj: 15 + abs_nclip: 3 + abs_refcat: GAIADR3 + abs_searchrad: 6.0 + abs_separation: 0.1 + abs_sigma: 3.0 + abs_tolerance: 0.7 + abs_use2dhist: true + catalog_format: ascii.ecsv + catalog_path: '' + catfile: '' + enforce_user_order: false + expand_refcat: false + fitgeometry: rshift + input_dir: '' + minobj: 15 + nclip: 3 + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: true + post_hooks: [] + pre_hooks: [] + save_abs_catalog: false + save_results: false + search_output_file: true + searchrad: 2.0 + separation: 1.0 + sigma: 3.0 + skip: false + suffix: null + tolerance: 0.7 + use2dhist: true + use_custom_catalogs: false +... diff --git a/test/data/roman/roman_wfi_pars-exposurepipeline_invalid_schema.asdf b/test/data/roman/roman_wfi_pars-exposurepipeline_invalid_schema.asdf new file mode 100644 index 000000000..41982f616 --- /dev/null +++ b/test/data/roman/roman_wfi_pars-exposurepipeline_invalid_schema.asdf @@ -0,0 +1,260 @@ +#ASDF 1.0.0 +#ASDF_STANDARD 1.5.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.1.0 +asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', + name: asdf, version: 3.2.0} +history: + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.5.0 + software: !core/software-1.0.0 {name: asdf, version: 3.2.0} +class: romancal.pipeline.exposure_pipeline.ExposurePipeline +meta: + author: rcal developer + date: '2024-06-07T15:46:47' + description: Parameters for calibration step romancal.pipeline.exposure_pipeline.ExposurePipeline + instrument: {name: BAD} + origin: + pedigree: GROUND + reftype: PARS-EXPOSUREPIPELINE + telescope: ROMAN + useafter: 2022-02-02T00:00:00 +name: ExposurePipeline +parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_calibrated_ramp: false + save_results: false + search_output_file: true + skip: false + suffix: null +steps: +- class: romancal.dq_init.dq_init_step.DQInitStep + name: dq_init + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.saturation.saturation_step.SaturationStep + name: saturation + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.refpix.refpix_step.RefPixStep + name: refpix + parameters: + cosine_interpolate: true + fft_interpolate: true + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + remove_offset: true + remove_trends: true + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.linearity.linearity_step.LinearityStep + name: linearity + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.dark_current.dark_current_step.DarkCurrentStep + name: dark_current + parameters: + dark_output: null + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.ramp_fitting.ramp_fit_step.RampFitStep + name: rampfit + parameters: + algorithm: ols_cas22 + input_dir: '' + maximum_cores: none + opt_name: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_opt: false + save_results: false + search_output_file: true + skip: false + suffix: rampfit + threshold_constant: null + threshold_intercept: null + use_ramp_jump_detection: true +- class: romancal.assign_wcs.assign_wcs_step.AssignWcsStep + name: assign_wcs + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.flatfield.flat_field_step.FlatFieldStep + name: flatfield + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.photom.photom_step.PhotomStep + name: photom + parameters: + input_dir: '' + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + suffix: null +- class: romancal.source_detection.source_detection_step.SourceDetectionStep + name: source_detection + parameters: + bkg_boxsize: 9 + bkg_estimator: median + bkg_filter_size: 3 + bkg_sigma: 2.0 + calc_threshold: true + fit_psf: true + input_dir: '' + kernel_fwhm: null + max_sources: null + min_separation: 11 + output_cat_filetype: asdf + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + peakmax: 100000.0 + post_hooks: [] + pre_hooks: [] + roundhi: 1.0 + roundlo: -1.0 + save_catalogs: false + save_results: false + scalar_threshold: null + search_output_file: true + sharphi: 1.0 + sharplo: 0.0 + skip: false + snr_threshold: 3.0 + suffix: null +- class: romancal.tweakreg.tweakreg_step.TweakRegStep + name: tweakreg + parameters: + abs_fitgeometry: rshift + abs_minobj: 15 + abs_nclip: 3 + abs_refcat: GAIADR3 + abs_searchrad: 6.0 + abs_separation: 0.1 + abs_sigma: 3.0 + abs_tolerance: 0.7 + abs_use2dhist: true + catalog_format: ascii.ecsv + catalog_path: '' + catfile: '' + enforce_user_order: false + expand_refcat: false + fitgeometry: rshift + input_dir: '' + minobj: 15 + nclip: 3 + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: true + post_hooks: [] + pre_hooks: [] + save_abs_catalog: false + save_results: false + search_output_file: true + searchrad: 2.0 + separation: 1.0 + sigma: 3.0 + skip: false + suffix: null + tolerance: 0.7 + use2dhist: true + use_custom_catalogs: false +...