Implement schema backwards compatibility (#272)

* Add unit test for older schema

* Minor refactor of debug code for _get_yaml_content

* Add new wcs unit test for backwards compatible schema conversion

* Fix handling of obsgeovel and obsgeoloc in tag v1.0.0

* Add unit test for GCRS frame backwards compatibility...

This was necessary since the obsgeovel and obsgeoloc fields were
affected by the schema version change but are not used in Galactocentric
frames, so were not previously being tested.

* Add unit test for using tags that are not defined...

ASDF should not fail to read a file, even if the file makes reference to
schema tags that are not defined. In these cases, ASDF should simply
provide a raw Python object that represents the deserialized object.

* Add unit test for tag implementation newer than schema read...

This is an important case that is was not being handled properly.
If an exact version match was not found for a schema version,
the "best match" was used. However, the tag implementation for this "best
match" might not be compatible with the given schema version, which would
result in an unhandled error.

* Modify extension type hierarchy to diffentiate custom classes...

This update allows extension types to be differentiated between those
that ASDF provides as part of the library, and those that are
user-defined. The ASDF types are automatically added to the list of
built-in extensions used by the library, whereas user-defined types are
not.

* Extension classes now have `supported_version` field...

Custom extension classes can now declare explicit support for various
schema versions using the `supported_version` field. If this field is
provided by an extension class, it will be used to determine whether that
class can be used to convert a particular version of a schema to/from a
custom class. If the field is not provided, or if a particular schema
version is not listed, the extension class will simply return raw Python
data structures representing the schema.

* Add unit test for supported_versions for CustomType

asdf/__init__.py

@@ -15,7 +15,7 @@
 # ----------------------------------------------------------------------------
 
 if _ASDF_SETUP_ is False:
-    __all__ = ['AsdfFile', 'AsdfType', 'AsdfExtension',
+    __all__ = ['AsdfFile', 'CustomType', 'AsdfExtension',
                'Stream', 'open', 'test', 'commands',
                'ValidationError']
 
@@ -35,7 +35,7 @@
         raise ImportError("asdf requires numpy")
 
     from .asdf import AsdfFile
-    from .asdftypes import AsdfType
+    from .asdftypes import CustomType
     from .extension import AsdfExtension
     from .stream import Stream
     from . import commands

asdf/asdf.py

@@ -448,16 +448,18 @@ def _open_asdf(cls, self, fd, uri=None, mode='r',
             self.version = version
 
         yaml_token = fd.read(4)
-        yaml_content = b''
         tree = {}
         has_blocks = False
         if yaml_token == b'%YAM':
             reader = fd.reader_until(
                 constants.YAML_END_MARKER_REGEX, 7, 'End of YAML marker',
                 include=True, initial_content=yaml_token)
 
+            # For testing: just return the raw YAML content
             if _get_yaml_content:
                 yaml_content = reader.read()
+                fd.close()
+                return yaml_content
             else:
                 # We parse the YAML content into basic data structures
                 # now, but we don't do anything special with it until
@@ -469,11 +471,6 @@ def _open_asdf(cls, self, fd, uri=None, mode='r',
         elif yaml_token != b'':
             raise IOError("ASDF file appears to contain garbage after header.")
 
-        # For testing: just return the raw YAML content
-        if _get_yaml_content:
-            fd.close()
-            return yaml_content
-
         if has_blocks:
             self._blocks.read_internal_blocks(
                 fd, past_magic=True, validate_checksums=validate_checksums)

asdf/asdftypes.py

@@ -9,14 +9,15 @@
 import re
 
 import six
+from copy import copy
 
 
 from .compat import lru_cache
 from .extern import semver
 
 from . import tagged
 from . import util
-from . import versioning
+from .versioning import get_version_map, version_to_string
 
 
 __all__ = ['format_tag', 'AsdfTypeIndex', 'AsdfType']
@@ -51,7 +52,7 @@ def join_tag_version(name, version):
     """
     Join the root and version of a tag back together.
     """
-    return '{0}-{1}'.format(name, versioning.version_to_string(version))
+    return '{0}-{1}'.format(name, version_to_string(version))
 
 
 class _AsdfWriteTypeIndex(object):
@@ -106,7 +107,7 @@ def add_by_tag(name, version):
                 add_by_tag(name, version)
         else:
             try:
-                version_map = versioning.get_version_map(version)
+                version_map = get_version_map(version)
             except ValueError:
                 raise ValueError(
                     "Don't know how to write out ASDF version {0}".format(
@@ -160,6 +161,7 @@ def __init__(self):
         self._type_by_tag = {}
         self._versions_by_type_name = {}
         self._best_matches = {}
+        self._real_tag = {}
         self._unnamed_types = set()
         self._hooks_by_type = {}
         self._all_types = set()
@@ -225,6 +227,8 @@ def fix_yaml_tag(self, tag):
         Raises a warning if it could not find a match where the major
         and minor numbers are the same.
         """
+        warning_string = None
+
         if tag in self._type_by_tag:
             return tag
 
@@ -243,30 +247,32 @@ def fix_yaml_tag(self, tag):
 
         # The versions list is kept sorted, so bisect can be used to
         # quickly find the best option.
-
         i = bisect.bisect_left(versions, version)
         i = max(0, i - 1)
 
         best_version = versions[i]
-        if best_version[:2] == version[:2]:
-            # Major and minor match, so only patch and devel differs
-            # -- no need for alarm
-            warning_string = None
-        else:
-            warning_string = (
-                "'{0}' with version {1} found in file, but asdf only "
-                "understands version {2}.".format(
+        if best_version[:2] != version[:2]:
+            warning_string = \
+                "'{}' with version {} found in file, but asdf only supports " \
+                "version {}".format(
                     name,
                     semver.format_version(*version),
-                    semver.format_version(*best_version)))
-
-        if warning_string:
+                    semver.format_version(*best_version))
             warnings.warn(warning_string)
 
         best_tag = join_tag_version(name, best_version)
         self._best_matches[tag] = best_tag, warning_string
+        if tag != best_tag:
+            self._real_tag[best_tag] = tag
         return best_tag
 
+    def get_real_tag(self, tag):
+        if tag in self._real_tag:
+            return self._real_tag[tag]
+        elif tag in self._type_by_tag:
+            return tag
+        return None
+
     def from_yaml_tag(self, tag):
         """
         From a given YAML tag string, return the corresponding
@@ -320,10 +326,9 @@ def _from_tree_tagged_missing_requirements(cls, tree, ctx):
     return tree
 
 
-class AsdfTypeMeta(type):
+class ExtensionTypeMeta(type):
     """
-    Keeps track of `AsdfType` subclasses that are created, and stores
-    them in `AsdfTypeIndex`.
+    Custom class constructor for extension types.
     """
     _import_cache = {}
 
@@ -358,6 +363,10 @@ def _find_in_bases(cls, attrs, bases, name, default=None):
                 return getattr(base, name)
         return default
 
+    @property
+    def versioned_siblings(mcls):
+        return getattr(mcls, '__versioned_siblings') or []
+
     def __new__(mcls, name, bases, attrs):
         requires = mcls._find_in_bases(attrs, bases, 'requires', [])
         if not mcls._has_required_modules(requires):
@@ -375,7 +384,7 @@ def __new__(mcls, name, bases, attrs):
                 new_types.append(typ)
             attrs['types'] = new_types
 
-        cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs)
+        cls = super(ExtensionTypeMeta, mcls).__new__(mcls, name, bases, attrs)
 
         if hasattr(cls, 'name'):
             if isinstance(cls.name, six.string_types):
@@ -386,14 +395,42 @@ def __new__(mcls, name, bases, attrs):
             elif cls.name is not None:
                 raise TypeError("name must be string or list")
 
+        if hasattr(cls, 'supported_versions'):
+            if not isinstance(cls.supported_versions, (list, set)):
+                raise TypeError(
+                    "supported_versions attribute must be list or set")
+            supported_versions = set()
+            for version in cls.supported_versions:
+                supported_versions.add(version_to_string(version))
+            cls.supported_versions = supported_versions
+            siblings = list()
+            for version in cls.supported_versions:
+                if version != version_to_string(cls.version):
+                    new_attrs = copy(attrs)
+                    new_attrs['version'] = version
+                    new_attrs['supported_versions'] = set()
+                    siblings.append(
+                       ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs))
+            setattr(cls, '__versioned_siblings', siblings)
+
+        return cls
+
+
+class AsdfTypeMeta(ExtensionTypeMeta):
+    """
+    Keeps track of `AsdfType` subclasses that are created, and stores them in
+    `AsdfTypeIndex`.
+    """
+    def __new__(mcls, name, bases, attrs):
+        cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs)
+        # Classes using this metaclass get added to the list of built-in
+        # extensions
         _all_asdftypes.add(cls)
 
         return cls
 
 
-@six.add_metaclass(AsdfTypeMeta)
-@six.add_metaclass(util.InheritDocstrings)
-class AsdfType(object):
+class ExtensionType(object):
     """
     The base class of all custom types in the tree.
 
@@ -417,6 +454,11 @@ class AsdfType(object):
     version : 3-tuple of int
         The version of the standard the type is defined in.
 
+    supported_versions : set
+        If provided, indicates explicit compatibility with the given set of
+        versions. Other versions of the same schema that are not included in
+        this set will not be converted to custom types with this class.
+
     yaml_tag : str
         The YAML tag to use for the type.  If not provided, it will be
         automatically generated from name, organization, standard and
@@ -439,6 +481,7 @@ class AsdfType(object):
     organization = 'stsci.edu'
     standard = 'asdf'
     version = (1, 0, 0)
+    supported_versions = set()
     types = []
     handle_dynamic_subclasses = False
     validators = {}
@@ -450,7 +493,7 @@ def make_yaml_tag(cls, name):
         return format_tag(
             cls.organization,
             cls.standard,
-            versioning.version_to_string(cls.version),
+            version_to_string(cls.version),
             name)
 
     @classmethod
@@ -488,3 +531,37 @@ def from_tree_tagged(cls, tree, ctx):
         with the tag directly.
         """
         return cls.from_tree(tree.data, ctx)
+
+    @classmethod
+    def incompatible_version(cls, version):
+        """
+        If this tag class explicitly identifies compatible versions then this
+        checks whether a given version is compatible or not. Otherwise, all
+        versions are assumed to be compatible.
+
+        Child classes can override this method to affect how version
+        compatiblity for this type is determined.
+        """
+        if cls.supported_versions:
+            if version_to_string(version) not in cls.supported_versions:
+                return True
+        return False
+
+
+@six.add_metaclass(AsdfTypeMeta)
+@six.add_metaclass(util.InheritDocstrings)
+class AsdfType(ExtensionType):
+    """
+    Base class for all built-in ASDF types. Types that inherit this class will
+    be automatically added to the list of built-ins. This should *not* be used
+    for user-defined extensions.
+    """
+
+@six.add_metaclass(ExtensionTypeMeta)
+@six.add_metaclass(util.InheritDocstrings)
+class CustomType(ExtensionType):
+    """
+    Base class for all user-defined types. Unlike classes that inherit
+    AsdfType, classes that inherit this class will *not* automatically be added
+    to the list of built-ins. This should be used for user-defined extensions.
+    """

asdf/extension.py

@@ -120,6 +120,9 @@ def __init__(self, extensions):
             for typ in extension.types:
                 self._type_index.add_type(typ)
                 validators.update(typ.validators)
+                for sibling in typ.versioned_siblings:
+                    self._type_index.add_type(sibling)
+                    validators.update(sibling.validators)
         self._tag_mapping = resolver.Resolver(tag_mapping, 'tag')
         self._url_mapping = resolver.Resolver(url_mapping, 'url')
         self._validators = validators