diff --git a/CHANGES.rst b/CHANGES.rst index 45ad79728..92e9bdc6d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -46,7 +46,10 @@ - When possible, display name of ASDF file that caused version mismatch warning. [#306] -- Issue a warning when an unrecognized tag is encountered. [#295] +- Issue a warning when an unrecognized tag is encountered. [#295] This warning + is silenced by default, but can be enabled with a parameter to the + ``AsdfFile`` constructor, or to ``AsdfFile.open``. Also added an option for + ignoring warnings from unrecognized schema tags. [#319] - Fix bug with loading JSON schemas in Python 3.5. [#317] diff --git a/asdf/asdf.py b/asdf/asdf.py index 36c3fd28a..47f62fb5d 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -42,7 +42,8 @@ class AsdfFile(versioning.VersionedMixin): """ The main class that represents a ASDF file. """ - def __init__(self, tree=None, uri=None, extensions=None, version=None): + def __init__(self, tree=None, uri=None, extensions=None, version=None, + ignore_version_mismatch=True, ignore_unrecognized_tag=False): """ Parameters ---------- @@ -65,7 +66,16 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None): The ASDF version to use when writing out. If not provided, it will write out in the latest version supported by asdf. + + ignore_version_mismatch : bool, optional + When `True`, do not raise warnings for mismatched schema versions. + Set to `True` by default. + + ignore_unrecognized_tag : bool, optional + When `True`, do not raise warnings for unrecognized tags. Set to + `False` by default. """ + if extensions is None or extensions == []: self._extensions = extension._builtin_extension_list else: @@ -77,6 +87,9 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None): extensions.insert(0, extension.BuiltinExtension()) self._extensions = extension.AsdfExtensionList(extensions) + self._ignore_version_mismatch = ignore_version_mismatch + self._ignore_unrecognized_tag = ignore_unrecognized_tag + self._fd = None self._external_asdf_by_uri = {} self._blocks = block.BlockManager(self) @@ -428,7 +441,6 @@ def _find_asdf_version_in_comments(cls, comments): def _open_asdf(cls, self, fd, uri=None, mode='r', validate_checksums=False, do_not_fill_defaults=False, - ignore_version_mismatch=False, _get_yaml_content=False, _force_raw_types=False): """Attempt to populate AsdfFile data from file-like object""" @@ -466,7 +478,7 @@ def _open_asdf(cls, self, fd, uri=None, mode='r', # We parse the YAML content into basic data structures # now, but we don't do anything special with it until # after the blocks have been read - tree = yamlutil.load_tree(reader, self, ignore_version_mismatch) + tree = yamlutil.load_tree(reader, self, self._ignore_version_mismatch) has_blocks = fd.seek_until(constants.BLOCK_MAGIC, 4, include=True) elif yaml_token == constants.BLOCK_MAGIC: has_blocks = True @@ -493,7 +505,6 @@ def _open_asdf(cls, self, fd, uri=None, mode='r', def _open_impl(cls, self, fd, uri=None, mode='r', validate_checksums=False, do_not_fill_defaults=False, - ignore_version_mismatch=False, _get_yaml_content=False, _force_raw_types=False): """Attempt to open file-like object as either AsdfFile or AsdfInFits""" @@ -505,6 +516,7 @@ def _open_impl(cls, self, fd, uri=None, mode='r', from . import fits_embed return fits_embed.AsdfInFits.open(fd, uri=uri, validate_checksums=validate_checksums, + ignore_version_mismatch=self._ignore_version_mismatch, extensions=self._extensions) except ValueError: pass @@ -514,7 +526,6 @@ def _open_impl(cls, self, fd, uri=None, mode='r', return cls._open_asdf(self, fd, uri=uri, mode=mode, validate_checksums=validate_checksums, do_not_fill_defaults=do_not_fill_defaults, - ignore_version_mismatch=ignore_version_mismatch, _get_yaml_content=_get_yaml_content, _force_raw_types=_force_raw_types) @@ -523,7 +534,8 @@ def open(cls, fd, uri=None, mode='r', validate_checksums=False, extensions=None, do_not_fill_defaults=False, - ignore_version_mismatch=False, + ignore_version_mismatch=True, + ignore_unrecognized_tag=False, _force_raw_types=False): """ Open an existing ASDF file. @@ -556,19 +568,25 @@ def open(cls, fd, uri=None, mode='r', ignore_version_mismatch : bool, optional When `True`, do not raise warnings for mismatched schema versions. + Set to `True` by default. + + ignore_unrecognized_tag : bool, optional + When `True`, do not raise warnings for unrecognized tags. Set to + `False` by default. Returns ------- asdffile : AsdfFile The new AsdfFile object. """ - self = cls(extensions=extensions) + self = cls(extensions=extensions, + ignore_version_mismatch=ignore_version_mismatch, + ignore_unrecognized_tag=ignore_unrecognized_tag) return cls._open_impl( self, fd, uri=uri, mode=mode, validate_checksums=validate_checksums, do_not_fill_defaults=do_not_fill_defaults, - ignore_version_mismatch=ignore_version_mismatch, _force_raw_types=_force_raw_types) def _write_tree(self, tree, fd, pad_blocks): diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py index 396776ec8..d58c174dc 100644 --- a/asdf/asdftypes.py +++ b/asdf/asdftypes.py @@ -165,6 +165,7 @@ def __init__(self): self._unnamed_types = set() self._hooks_by_type = {} self._all_types = set() + self._has_warned = {} def add_type(self, asdftype): """ @@ -215,7 +216,28 @@ def from_custom_type(self, custom_type, version='latest'): return write_type_index.from_custom_type(custom_type) - def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=False): + def _get_version_mismatch(self, name, version, latest_version): + warning_string = None + + if (latest_version.major, latest_version.minor) != \ + (version.major, version.minor): + warning_string = \ + "'{}' with version {} found in file{{}}, but latest " \ + "supported version is {}".format( + name, version, latest_version) + + return warning_string + + def _warn_version_mismatch(self, ctx, tag, warning_string, fname): + if warning_string is not None: + # Ensure that only a single warning occurs per tag per AsdfFile + # TODO: If it is useful to only have a single warning per file on + # disk, then use `fname` in the key instead of `ctx`. + if not (ctx, tag) in self._has_warned: + warnings.warn(warning_string.format(fname)) + self._has_warned[(ctx, tag)] = True + + def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=True): """ Given a YAML tag, adjust it to the best supported version. @@ -224,24 +246,37 @@ def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=False): the earliest understood version if none are less than the version in the file. - Raises a warning if it could not find a match where the major - and minor numbers are the same. + If ``ignore_version_mismatch==False``, this function raises a warning + if it could not find a match where the major and minor numbers are the + same. """ warning_string = None + name, version = split_tag_version(tag) + + fname = " '{}'".format(ctx._fname) if ctx._fname else '' + if tag in self._type_by_tag: + asdftype = self._type_by_tag[tag] + # Issue warnings for the case where there exists a class for the + # given tag due to the 'supported_versions' attribute being + # defined, but this tag is not the latest version of the type. + # This prevents 'supported_versions' from affecting the behavior of + # warnings that are purely related to YAML validation. + if not ignore_version_mismatch and hasattr(asdftype, '_latest_version'): + warning_string = self._get_version_mismatch( + name, version, asdftype._latest_version) + self._warn_version_mismatch(ctx, tag, warning_string, fname) return tag - fname = " '{}'".format(ctx._fname) if ctx._fname else '' if tag in self._best_matches: best_tag, warning_string = self._best_matches[tag] - if warning_string: - warnings.warn(warning_string.format(fname)) + if not ignore_version_mismatch: + self._warn_version_mismatch(ctx, tag, warning_string, fname) return best_tag - name, version = split_tag_version(tag) versions = self._versions_by_type_name.get(name) if versions is None: return tag @@ -251,16 +286,12 @@ def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=False): i = bisect.bisect_left(versions, version) i = max(0, i - 1) - best_version = versions[i] if not ignore_version_mismatch: - if (best_version.major, best_version.minor) != \ - (version.major, version.minor): - warning_string = \ - "'{}' with version {} found in file{{}}, but latest " \ - "supported version is {}".format( - name, version, best_version) - warnings.warn(warning_string.format(fname)) + warning_string = self._get_version_mismatch( + name, version, versions[-1]) + self._warn_version_mismatch(ctx, tag, warning_string, fname) + best_version = versions[i] best_tag = join_tag_version(name, best_version) self._best_matches[tag] = best_tag, warning_string if tag != best_tag: @@ -418,6 +449,7 @@ def __new__(mcls, name, bases, attrs): new_attrs = copy(attrs) new_attrs['version'] = version new_attrs['supported_versions'] = set() + new_attrs['_latest_version'] = cls.version siblings.append( ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs)) setattr(cls, '__versioned_siblings', siblings) diff --git a/asdf/fits_embed.py b/asdf/fits_embed.py index a80e48bba..c20a5bf03 100644 --- a/asdf/fits_embed.py +++ b/asdf/fits_embed.py @@ -140,11 +140,10 @@ class AsdfInFits(asdf.AsdfFile): ff.write_to('test.fits') # doctest: +SKIP """ - def __init__(self, hdulist=None, tree=None, uri=None, extensions=None): + def __init__(self, hdulist=None, tree=None, **kwargs): if hdulist is None: hdulist = fits.HDUList() - super(AsdfInFits, self).__init__( - tree=tree, uri=uri, extensions=extensions) + super(AsdfInFits, self).__init__(tree=tree, **kwargs) self._blocks = _EmbeddedBlockManager(hdulist, self) self._hdulist = hdulist self._close_hdulist = False @@ -162,7 +161,8 @@ def close(self): self._tree = {} @classmethod - def open(cls, fd, uri=None, validate_checksums=False, extensions=None): + def open(cls, fd, uri=None, validate_checksums=False, extensions=None, + ignore_version_mismatch=True, ignore_unrecognized_tag=False): """Creates a new AsdfInFits object based on given input data Parameters @@ -185,6 +185,9 @@ def open(cls, fd, uri=None, validate_checksums=False, extensions=None): A list of extensions to the ASDF to support when reading and writing ASDF files. See `asdftypes.AsdfExtension` for more information. + + ignore_version_mismatch : bool, optional + When `True`, do not raise warnings for mismatched schema versions. """ close_hdulist = False if isinstance(fd, fits.hdu.hdulist.HDUList): @@ -202,7 +205,9 @@ def open(cls, fd, uri=None, validate_checksums=False, extensions=None): msg = "Failed to parse given file '{}'. Is it FITS?" raise ValueError(msg.format(file_obj.uri)) - self = cls(hdulist, uri=uri, extensions=extensions) + self = cls(hdulist, uri=uri, extensions=extensions, + ignore_version_mismatch=ignore_version_mismatch, + ignore_unrecognized_tag=ignore_unrecognized_tag) self._close_hdulist = close_hdulist try: diff --git a/asdf/tests/helpers.py b/asdf/tests/helpers.py index 5dfd4e8b0..a05cf4692 100644 --- a/asdf/tests/helpers.py +++ b/asdf/tests/helpers.py @@ -267,6 +267,9 @@ def display_warnings(_warnings): msg : str String containing the warning messages to be displayed """ + if len(_warnings) == 0: + return "No warnings occurred (was one expected?)" + msg = "Unexpected warning(s) occurred:\n" for warning in _warnings: msg += "{}:{}: {}: {}\n".format( diff --git a/asdf/tests/test_asdftypes.py b/asdf/tests/test_asdftypes.py index 34da38158..383968ab6 100644 --- a/asdf/tests/test_asdftypes.py +++ b/asdf/tests/test_asdftypes.py @@ -14,7 +14,7 @@ from .. import util from .. import versioning -from . import helpers +from . import helpers, CustomTestType from astropy.tests.helper import catch_warnings @@ -97,7 +97,7 @@ def test_version_mismatch(): buff = helpers.yaml_to_asdf(yaml) with catch_warnings() as warning: - with asdf.AsdfFile.open(buff) as ff: + with asdf.AsdfFile.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 1 @@ -108,7 +108,7 @@ def test_version_mismatch(): # Make sure warning is repeatable buff.seek(0) with catch_warnings() as warning: - with asdf.AsdfFile.open(buff) as ff: + with asdf.AsdfFile.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 1 @@ -116,6 +116,15 @@ def test_version_mismatch(): "'tag:stsci.edu:asdf/core/complex' with version 42.0.0 found in file, " "but latest supported version is 1.0.0") + # Make sure the warning does not occur if it is being ignored (default) + buff.seek(0) + with catch_warnings() as warning: + with asdf.AsdfFile.open(buff) as ff: + assert isinstance(ff.tree['a'], complex) + + assert len(warning) == 0, helpers.display_warnings(warning) + + # If the major and minor match, there should be no warning. yaml = """ a: !core/complex-1.0.1 @@ -124,7 +133,7 @@ def test_version_mismatch(): buff = helpers.yaml_to_asdf(yaml) with catch_warnings() as warning: - with asdf.AsdfFile.open(buff) as ff: + with asdf.AsdfFile.open(buff, ignore_version_mismatch=False) as ff: assert isinstance(ff.tree['a'], complex) assert len(warning) == 0 @@ -144,7 +153,7 @@ def test_version_mismatch_file(tmpdir): handle.write(buff.read()) with catch_warnings() as w: - with asdf.AsdfFile.open(testfile) as ff: + with asdf.AsdfFile.open(testfile, ignore_version_mismatch=False) as ff: assert ff._fname == "file://{}".format(testfile) assert isinstance(ff.tree['a'], complex) @@ -154,6 +163,54 @@ def test_version_mismatch_file(tmpdir): "'file://{}', but latest supported version is 1.0.0".format(testfile)) +def test_version_mismatch_with_supported_versions(): + """Make sure that defining the supported_versions field does not affect + whether or not schema mismatch warnings are triggered.""" + + class CustomFlow(object): + pass + + class CustomFlowType(CustomTestType): + version = '1.1.0' + supported_versions = ['1.0.0', '1.1.0'] + name = 'custom_flow' + organization = 'nowhere.org' + standard = 'custom' + types = [CustomFlow] + + class CustomFlowExtension(object): + @property + def types(self): + return [CustomFlowType] + + @property + def tag_mapping(self): + return [('tag:nowhere.org:custom', + 'http://nowhere.org/schemas/custom{tag_suffix}')] + + @property + def url_mapping(self): + return [('http://nowhere.org/schemas/custom/', + util.filepath_to_url(TEST_DATA_PATH) + + '/{url_suffix}.yaml')] + + yaml = """ +flow_thing: + ! + c: 100 + d: 3.14 +""" + buff = helpers.yaml_to_asdf(yaml) + with catch_warnings() as w: + data = asdf.AsdfFile.open( + buff, ignore_version_mismatch=False, + extensions=CustomFlowExtension()) + assert len(w) == 1, helpers.display_warnings(w) + assert str(w[0].message) == ( + "'tag:nowhere.org:custom/custom_flow' with version 1.0.0 found in " + "file, but latest supported version is 1.1.0") + + def test_versioned_writing(): from ..tags.core.complex import ComplexType @@ -297,6 +354,12 @@ def test_undefined_tag(): "tag:nowhere.org:custom/{} is not recognized, converting to raw " "Python data structure".format(tag)) + # Make sure no warning occurs if explicitly ignored + buff.seek(0) + with catch_warnings() as warning: + afile = asdf.AsdfFile.open(buff, ignore_unrecognized_tag=True) + assert len(warning) == 0 + def test_newer_tag(): # This test simulates a scenario where newer versions of CustomFlow @@ -365,13 +428,10 @@ def url_mapping(self): with catch_warnings() as warning: asdf.AsdfFile.open(old_buff, extensions=CustomFlowExtension()) - assert len(warning) == 2, helpers.display_warnings(warning) - assert str(warning[0].message) == ( - "'tag:nowhere.org:custom/custom_flow' with version 1.0.0 found " - "in file, but latest supported version is 1.1.0") + assert len(warning) == 1, helpers.display_warnings(warning) # We expect this warning since it will not be possible to convert version # 1.0.0 of CustomFlow to a CustomType (by design, for testing purposes). - assert str(warning[1].message).startswith( + assert str(warning[0].message).startswith( "Failed to convert " "tag:nowhere.org:custom/custom_flow-1.0.0 to custom type") @@ -535,7 +595,7 @@ def url_mapping(self): with catch_warnings() as _warnings: data = asdf.AsdfFile.open(buff, extensions=CustomFlowExtension()) - assert len(_warnings) == 2 - assert str(_warnings[1].message) == ( + assert len(_warnings) == 1 + assert str(_warnings[0].message) == ( "Version 1.1.0 of tag:nowhere.org:custom/custom_flow is not compatible " "with any existing tag implementations") diff --git a/asdf/tests/test_fits_embed.py b/asdf/tests/test_fits_embed.py index e3fd1eb48..6ddd2c0bd 100644 --- a/asdf/tests/test_fits_embed.py +++ b/asdf/tests/test_fits_embed.py @@ -18,7 +18,7 @@ from .. import asdf from .. import fits_embed from .. import open as asdf_open -from .helpers import assert_tree_match, yaml_to_asdf +from .helpers import assert_tree_match, yaml_to_asdf, display_warnings TEST_DATA_PATH = os.path.join(os.path.dirname(__file__), 'data') @@ -230,7 +230,8 @@ def test_version_mismatch_file(): testfile = os.path.join(TEST_DATA_PATH, 'version_mismatch.fits') with catch_warnings() as w: - with asdf.AsdfFile.open(testfile) as fits_handle: + with asdf.AsdfFile.open(testfile, + ignore_version_mismatch=False) as fits_handle: assert fits_handle.tree['a'] == complex(0j) # This is the warning that we expect from opening the FITS file assert len(w) == 1 @@ -238,10 +239,23 @@ def test_version_mismatch_file(): "'tag:stsci.edu:asdf/core/complex' with version 7.0.0 found in file " "'file://{}', but latest supported version is 1.0.0".format(testfile)) + # Make sure warning does not occur when warning is ignored (default) with catch_warnings() as w: - with fits_embed.AsdfInFits.open(testfile) as fits_handle: + with asdf.AsdfFile.open(testfile) as fits_handle: + assert fits_handle.tree['a'] == complex(0j) + assert len(w) == 0, display_warnings(w) + + with catch_warnings() as w: + with fits_embed.AsdfInFits.open(testfile, + ignore_version_mismatch=False) as fits_handle: assert fits_handle.tree['a'] == complex(0j) assert len(w) == 1 assert str(w[0].message) == ( "'tag:stsci.edu:asdf/core/complex' with version 7.0.0 found in file " "'file://{}', but latest supported version is 1.0.0".format(testfile)) + + # Make sure warning does not occur when warning is ignored (default) + with catch_warnings() as w: + with fits_embed.AsdfInFits.open(testfile) as fits_handle: + assert fits_handle.tree['a'] == complex(0j) + assert len(w) == 0, display_warnings(w) diff --git a/asdf/tests/test_schema.py b/asdf/tests/test_schema.py index 313416483..d4772ab23 100644 --- a/asdf/tests/test_schema.py +++ b/asdf/tests/test_schema.py @@ -204,7 +204,7 @@ def test_schema_example(filename, example): try: with catch_warnings() as w: - ff._open_impl(ff, buff, ignore_version_mismatch=True) + ff._open_impl(ff, buff) # Do not tolerate any warnings that occur during schema validation, # other than a few that we expect to occur under certain circumstances _assert_warnings(w) diff --git a/asdf/yamlutil.py b/asdf/yamlutil.py index de72e3c75..8481190b2 100644 --- a/asdf/yamlutil.py +++ b/asdf/yamlutil.py @@ -258,10 +258,10 @@ def walker(node): tag_type = ctx.type_index.from_yaml_tag(ctx, tag_name) # This means the tag did not correspond to any type in our type index. - # TODO: maybe this warning should be possible to suppress? if tag_type is None: - warnings.warn("{} is not recognized, converting to raw Python data " - "structure".format(tag_name)) + if not ctx._ignore_unrecognized_tag: + warnings.warn("{} is not recognized, converting to raw Python " + "data structure".format(tag_name)) return node real_tag = ctx.type_index.get_real_tag(tag_name)