From 65bce60fa2545e488389b465f8faef4158e02520 Mon Sep 17 00:00:00 2001 From: Ben Andersen Date: Thu, 7 Mar 2024 00:53:42 +1100 Subject: [PATCH] Added custom package orderer and fixed package orderer sort logic Signed-off-by: Ben Andersen --- src/rez/cli/config.py | 4 +- src/rez/config.py | 20 +- .../solver/packages/pyvariants/2/package.py | 7 +- .../data/tests/solver/packages/reorderable.py | 31 + src/rez/package_order.py | 678 +++++++++++++----- src/rez/resolved_context.py | 17 +- src/rez/resolver.py | 4 +- src/rez/solver.py | 35 +- src/rez/tests/test_completion.py | 2 +- src/rez/tests/test_context.py | 76 +- src/rez/tests/test_packages.py | 4 + src/rez/tests/test_packages_order.py | 141 ++-- src/rez/tests/test_solver.py | 511 ++++++++++++- src/rez/tests/test_version.py | 25 + src/rez/utils/yaml.py | 13 + src/rez/version/_version.py | 11 +- 16 files changed, 1310 insertions(+), 269 deletions(-) create mode 100644 src/rez/data/tests/solver/packages/reorderable.py diff --git a/src/rez/cli/config.py b/src/rez/cli/config.py index c04ce1008e..498fd666fd 100644 --- a/src/rez/cli/config.py +++ b/src/rez/cli/config.py @@ -30,7 +30,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): from rez.config import config - from rez.utils.yaml import dump_yaml + from rez.utils.yaml import dump_yaml, YamlDumpable from rez.utils.data_utils import convert_json_safe if opts.search_list: @@ -54,7 +54,7 @@ def command(opts, parser, extra_arg_groups=None): except KeyError: raise ValueError("no such setting: %r" % opts.FIELD) - if isinstance(data, (dict, list)): + if isinstance(data, (dict, list, YamlDumpable)): if opts.json: txt = json.dumps(convert_json_safe(data)) else: diff --git a/src/rez/config.py b/src/rez/config.py index e8542d1ede..4068f83cec 100644 --- a/src/rez/config.py +++ b/src/rez/config.py @@ -83,9 +83,8 @@ def _validate(self, data): if self.key in self.config.overrides: return data - if not self.config.locked: - - # next, env-var + # next, env-var + if self._env_var_name and not self.config.locked: value = os.getenv(self._env_var_name) if value is not None: if self.key in _deprecated_settings: @@ -361,6 +360,19 @@ def _parse_env_var(self, value): return value +class PackageOrderers(Setting): + @cached_class_property + def schema(cls): + from rez.package_order import from_pod, OrdererDict + return Or( + Schema(And( + [Use(from_pod)], + Use(OrdererDict) + )), None) + + _env_var_name = None + + config_schema = Schema({ "packages_path": PathList, "plugin_path": PathList, @@ -496,7 +508,7 @@ def _parse_env_var(self, value): "env_var_separators": Dict, "variant_select_mode": VariantSelectMode_, "package_filter": OptionalDictOrDictList, - "package_orderers": OptionalDictOrDictList, + "package_orderers": PackageOrderers, "new_session_popen_args": OptionalDict, "context_tracking_amqp": OptionalDict, "context_tracking_extra_fields": OptionalDict, diff --git a/src/rez/data/tests/solver/packages/pyvariants/2/package.py b/src/rez/data/tests/solver/packages/pyvariants/2/package.py index ed06df9da4..0374627d38 100644 --- a/src/rez/data/tests/solver/packages/pyvariants/2/package.py +++ b/src/rez/data/tests/solver/packages/pyvariants/2/package.py @@ -1,4 +1,9 @@ name = "pyvariants" version = "2" -variants = [["python-2.7.0"], ["python-2.6.8", "nada"]] +variants = [ + ["python-2.7.0"], + ["python-2.6.8", "nada"], + ["python-2.6.8"], + ["nada"], +] diff --git a/src/rez/data/tests/solver/packages/reorderable.py b/src/rez/data/tests/solver/packages/reorderable.py new file mode 100644 index 0000000000..952da665d6 --- /dev/null +++ b/src/rez/data/tests/solver/packages/reorderable.py @@ -0,0 +1,31 @@ +name = 'reorderable' + +versions = [ + "3.1.1", + "3.0.0", + "2.2.1", + "2.2.0", + "2.1.5", + "2.1.1", + "2.1.0", + "2.0.6", + "2.0.5", + "2.0.0", + "1.9.1", + "1.9.0", +] + +# Note - we've intentionally left out timestamps for 2.2.0 and 1.9.1 to +# make sure the system still works +version_overrides = { + "3.1.1": {"timestamp": 1470728488}, + "3.0.0": {"timestamp": 1470728486}, + "2.1.5": {"timestamp": 1470728484}, + "2.2.1": {"timestamp": 1470728482}, + "2.1.1": {"timestamp": 1470728480}, + "2.1.0": {"timestamp": 1470728478}, + "2.0.6": {"timestamp": 1470728476}, + "2.0.5": {"timestamp": 1470728474}, + "2.0.0": {"timestamp": 1470728472}, + "1.9.0": {"timestamp": 1470728470}, +} diff --git a/src/rez/package_order.py b/src/rez/package_order.py index e00839feaa..cc1376faf9 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -2,15 +2,48 @@ # Copyright Contributors to the Rez Project +from abc import ABCMeta, abstractmethod from inspect import isclass from hashlib import sha1 +from typing import Mapping from rez.config import config -from rez.utils.data_utils import cached_class_property +from rez.exceptions import ConfigurationError +from rez.utils.data_utils import cached_class_property, cached_property +from rez.utils.yaml import YamlDumpable from rez.version import Version +from rez.version._version import _Comparable, _ReversedComparable +DEFAULT_TOKEN = "" -class PackageOrder(object): + +class FallbackComparable(_Comparable): + """First tries to compare objects using the main_comparable, but if that + fails, compares using the fallback_comparable object. + """ + + def __init__(self, main_comparable, fallback_comparable): + self.main_comparable = main_comparable + self.fallback_comparable = fallback_comparable + + def __eq__(self, other): + try: + return self.main_comparable == other.main_comparable + except Exception: + return self.fallback_comparable == other.fallback_comparable + + def __lt__(self, other): + try: + return self.main_comparable < other.main_comparable + except Exception: + return self.fallback_comparable < other.fallback_comparable + + def __repr__(self): + return '%s(%r, %r)' % (type(self).__name__, self.main_comparable, + self.fallback_comparable) + + +class PackageOrder(YamlDumpable, metaclass=ABCMeta): """Package reorderer base class.""" #: Orderer name @@ -19,50 +52,142 @@ class PackageOrder(object): def __init__(self): pass - def reorder(self, iterable, key=None): - """Put packages into some order for consumption. - - You can safely assume that the packages referred to by `iterable` are - all versions of the same package family. + def sort_key(self, package_name, version_like): + """Returns a sort key usable for sorting these packages within the + same family - Note: - Returning None, and an unchanged `iterable` list, are not the same - thing. Returning None may cause rez to pass the package list to the - next orderer; whereas a package list that has been reordered (even - if the unchanged list is returned) is not passed onto another orderer. + Args: + package_name: (str) The family name of the package we are sorting + version_like: (Version|_LowerBound|_UpperBound|_Bound|VersionRange) + the version-like object you wish to generate a key for + Returns: + Comparable object + """ + from rez.version._version import Version, _LowerBound, _UpperBound, _Bound, VersionRange + if isinstance(version_like, VersionRange): + return tuple(self.sort_key(package_name, bound) + for bound in version_like.bounds) + elif isinstance(version_like, _Bound): + return (self.sort_key(package_name, version_like.lower), + self.sort_key(package_name, version_like.upper)) + elif isinstance(version_like, _LowerBound): + inclusion_key = -2 if version_like.inclusive else -1 + return (self.sort_key(package_name, version_like.version), + inclusion_key) + elif isinstance(version_like, _UpperBound): + inclusion_key = 2 if version_like.inclusive else 1 + return (self.sort_key(package_name, version_like.version), + inclusion_key) + elif isinstance(version_like, Version): + # finally, the bit that we actually use the sort_key_implementation + # for... + # Need to use a FallbackComparable because we can compare versions + # of different packages... + return FallbackComparable( + self.sort_key_implementation(package_name, version_like), + version_like) + else: + raise TypeError(version_like) + + @abstractmethod + def sort_key_implementation(self, package_name, version): + """Returns a sort key usable for sorting these packages within the + same family Args: - iterable: Iterable list of packages, or objects that contain packages. - key (typing.Callable[typing.Any, Package]): Callable, where key(iterable) - gives a :class:`~rez.packages.Package`. If None, iterable is assumed - to be a list of :class:`~rez.packages.Package` objects. + package_name: (str) The family name of the package we are sorting + version: (Version) the version object you wish to generate a key for Returns: - list: Reordered ``iterable`` + Comparable object """ raise NotImplementedError + # override this if you have an orderer that doesn't store packages + # in the standard way + @property + def packages(self): + """Returns an iterable over the list of package family names that this + order applies to + Returns: + (iterable(str|None)) Package families that this orderer is used for + """ + return self._packages + + @packages.setter + def packages(self, packages): + if packages is None: + # Apply to all packages + self._packages = None + elif isinstance(packages, str): + self._packages = [packages] + else: + self._packages = sorted(packages) + + def to_yaml_pod(self): + data = self.to_pod() + data['type'] = self.name + return data + + @abstractmethod def to_pod(self): raise NotImplementedError + @classmethod + def from_pod(cls, data): + raise NotImplementedError + @property def sha1(self): - return sha1(repr(self).encode('utf-8')).hexdigest() + return sha1(repr(self).encode("utf-8")).hexdigest() - def __str__(self): - raise NotImplementedError + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, str(self)) def __eq__(self, other): + return type(self) == type(other) and str(self) == str(other) + + # need to implement this to avoid infinite recursion! + @abstractmethod + def __str__(self): raise NotImplementedError def __ne__(self, other): return not self == other - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, str(self)) + +class PackageBasedPackageOrder(PackageOrder, metaclass=ABCMeta): + """ + A package orderer that can directly apply to packages + """ + + def __init__(self, packages=None): + """ + packages (Optional[Union[str, Iterable[str]]): packages that this orderer should sort + If not provided, will apply to all packages. + """ + super().__init__() + if packages is None: + packages = [DEFAULT_TOKEN] + elif isinstance(packages, str): + packages = [packages] + self.packages = packages + + @cached_property + def applies_to_all_packages(self): + return self.packages == [DEFAULT_TOKEN] + + @cached_property + def get_packages_str(self): + if self.applies_to_all_packages: + return "*" + return str(self.packages) + + def __str__(self): + return self.get_packages_str -class NullPackageOrder(PackageOrder): +class NullPackageOrder(PackageBasedPackageOrder): """An orderer that does not change the order - a no op. This orderer is useful in cases where you want to apply some default orderer @@ -71,11 +196,10 @@ class NullPackageOrder(PackageOrder): """ name = "no_order" - def reorder(self, iterable, key=None): - return list(iterable) - - def __str__(self): - return "{}" + def sort_key_implementation(self, package_name, version): + # python's sort will preserve the order of items that compare equal, so + # to not change anything, we just return the same object for all... + return 0 def __eq__(self, other): return type(self) == type(other) @@ -87,29 +211,46 @@ def to_pod(self): .. code-block:: yaml type: no_order + packages: ["foo"] """ - return {} + return { + "packages": self.packages, + } @classmethod def from_pod(cls, data): - return cls() + return cls(packages=data.get("packages")) -class SortedOrder(PackageOrder): +class SortedOrder(PackageBasedPackageOrder): """An orderer that sorts based on :attr:`Package.version `. """ name = "sorted" - def __init__(self, descending): + def __init__(self, descending, packages=None): + """ + descending (bool) + packages (Optional[Union[str, Iterable[str]]): packages that this orderer should sort + If not provided, will apply to all packages. + """ + super().__init__(packages) self.descending = descending - def reorder(self, iterable, key=None): - key = key or (lambda x: x) - return sorted(iterable, key=lambda x: key(x).version, - reverse=self.descending) + def sort_key_implementation(self, package_name, version): + # Note that the name "descending" can be slightly confusing - it + # indicates that the final ordering this Order gives should be + # version descending (ie, the default) - however, the sort_key itself + # returns its results in "normal" ascending order (because it needs to + # be used "alongside" normally-sorted objects like versions). + # when the key is passed to sort(), though, it is always invoked with + # reverse=True... + if self.descending: + return version + else: + return _ReversedComparable(version) def __str__(self): - return str(self.descending) + return f"{self.get_packages_str}, descending={self.descending}" def __eq__(self, other): return ( @@ -124,13 +265,20 @@ def to_pod(self): .. code-block:: yaml type: sorted + packages: ["foo", "bar"] descending: true """ - return {"descending": self.descending} + return { + "packages": self.packages, + "descending": self.descending, + } @classmethod def from_pod(cls, data): - return cls(descending=data["descending"]) + return cls( + descending=data["descending"], + packages=data.get("packages"), + ) class PerFamilyOrder(PackageOrder): @@ -139,37 +287,37 @@ class PerFamilyOrder(PackageOrder): name = "per_family" def __init__(self, order_dict, default_order=None): - """Create a reorderer. - + """ Args: - order_dict (dict[str, PackageOrder]): Orderers to apply to + order_dict (Dict[str, PackageOrder]): Orderers to apply to each package family. - default_order (PackageOrder): Orderer to apply to any packages + default_order (Optional[PackageOrder]): Orderer to apply to any packages not specified in ``order_dict``. """ self.order_dict = order_dict.copy() + if default_order is None: + default_order = NullPackageOrder() self.default_order = default_order - def reorder(self, iterable, key=None): - try: - item = next(iter(iterable)) - except: - return None - - key = key or (lambda x: x) - package = key(item) - - orderer = self.order_dict.get(package.name) - if orderer is None: - orderer = self.default_order + def sort_key_implementation(self, package_name, version): + orderer = self.order_dict.get(package_name) if orderer is None: - return None + if self.default_order is not None: + orderer = self.default_order + else: + # shouldn't get here, because applies_to should protect us... + raise RuntimeError( + "package family orderer %r does not apply to package family %r", + (self, package_name)) + return orderer.sort_key_implementation(package_name, version) - return orderer.reorder(iterable, key) + @property + def packages(self): + return iter(self.order_dict) def __str__(self): - items = sorted((x[0], str(x[1])) for x in self.order_dict.items()) - return str((items, str(self.default_order))) + items = sorted((x[0], repr(x[1])) for x in self.order_dict.items()) + return f"{items}, default_order={repr(self.default_order)}" def __eq__(self, other): return ( @@ -226,10 +374,9 @@ def from_pod(cls, data): for d in data["orderers"]: d = d.copy() - fams = d.pop("packages") orderer = from_pod(d) - for fam in fams: + for fam in orderer.packages: order_dict[fam] = orderer d = data.get("default_order") @@ -239,7 +386,7 @@ def from_pod(cls, data): return cls(order_dict, default_order) -class VersionSplitPackageOrder(PackageOrder): +class VersionSplitPackageOrder(PackageBasedPackageOrder): """Orders package versions <= a given version first. For example, given the versions [5, 4, 3, 2, 1], an orderer initialized @@ -247,38 +394,23 @@ class VersionSplitPackageOrder(PackageOrder): """ name = "version_split" - def __init__(self, first_version): + def __init__(self, first_version, packages=None): """Create a reorderer. Args: first_version (Version): Start with versions <= this value. + packages (Union[str, Iterable[str]): packages that this orderer should sort + If not provided, will apply to all packages. """ self.first_version = first_version + super().__init__(packages) - def reorder(self, iterable, key=None): - key = key or (lambda x: x) - - # sort by version descending - descending = sorted(iterable, key=lambda x: key(x).version, reverse=True) - - above = [] - below = [] - is_above = True - - for item in descending: - if is_above: - package = key(item) - is_above = (package.version > self.first_version) - - if is_above: - above.append(item) - else: - below.append(item) - - return below + above + def sort_key_implementation(self, package_name, version): + priority_key = 1 if version <= self.first_version else 0 + return priority_key, version def __str__(self): - return str(self.first_version) + return f"{self.get_packages_str}, first_version='{self.first_version}'" def __eq__(self, other): return ( @@ -293,16 +425,21 @@ def to_pod(self): .. code-block:: yaml type: version_split + packages: ["foo", "bar"] first_version: "3.0.0" """ - return dict(first_version=str(self.first_version)) + return dict(packages=self.packages, + first_version=str(self.first_version)) @classmethod def from_pod(cls, data): - return cls(Version(data["first_version"])) + return cls( + first_version=Version(data["first_version"]), + packages=data.get("packages"), + ) -class TimestampPackageOrder(PackageOrder): +class TimestampPackageOrder(PackageBasedPackageOrder): """A timestamp order function. Given a time ``T``, this orderer returns packages released before ``T``, in descending @@ -345,7 +482,7 @@ class TimestampPackageOrder(PackageOrder): """ name = "soft_timestamp" - def __init__(self, timestamp, rank=0): + def __init__(self, timestamp, rank=0, packages=None): """Create a reorderer. Args: @@ -353,81 +490,93 @@ def __init__(self, timestamp, rank=0): are preferred. rank (int): If non-zero, allow version changes at this rank or above past the timestamp. + packages (Union[str, Iterable[str]): packages that this orderer should sort + If not provided, will apply to all packages. """ self.timestamp = timestamp self.rank = rank - - def reorder(self, iterable, key=None): + super().__init__(packages) + + # dictionary mapping from package family to the first-version-after + # the given timestamp + self._cached_first_after = {} + self._cached_sort_key = {} + + def get_first_after(self, package_family): + first_after = self._cached_first_after.get(package_family, KeyError) + if first_after is KeyError: + first_after = self._calc_first_after(package_family) + self._cached_first_after[package_family] = first_after + return first_after + + def _calc_first_after(self, package_family): + from rez.packages import iter_packages + descending = sorted(iter_packages(package_family), + key=lambda p: p.version, + reverse=True) first_after = None - key = key or (lambda x: x) - - # sort by version descending - descending = sorted(iterable, key=lambda x: key(x).version, reverse=True) - - for i, o in enumerate(descending): - package = key(o) - if package.timestamp: - if package.timestamp > self.timestamp: - first_after = i - else: - break - - if first_after is None: # all packages are before T - return None - - before = descending[first_after + 1:] - after = list(reversed(descending[:first_after + 1])) - - if not self.rank: # simple case - return before + after - - # include packages after timestamp but within rank - if before and after: - package = key(before[0]) - first_prerank = package.version.trim(self.rank - 1) - found = False - - for i, o in enumerate(after): - package = key(o) - prerank = package.version.trim(self.rank - 1) - if prerank != first_prerank: - found = True - break - - if not found: - # highest version is also within rank, so result is just - # simple descending list - return descending - - if i: - before = list(reversed(after[:i])) + before - after = after[i:] - - # ascend below rank, but descend within - after_ = [] - postrank = [] - prerank = None - - for o in after: - package = key(o) - prerank_ = package.version.trim(self.rank - 1) - - if prerank_ == prerank: - postrank.append(o) + for i, package in enumerate(descending): + if not package.timestamp: + continue + if package.timestamp > self.timestamp: + first_after = package.version + else: + break + + if not self.rank: + return first_after + + # if we have rank, then we need to then go back UP the + # versions, until we find one whose trimmed version doesn't + # match. + # Note that we COULD do this by simply iterating through + # an ascending sequence, in which case we wouldn't have to + # "switch direction" after finding the first result after + # by timestamp... but we're making the assumption that the + # timestamp break will be closer to the higher end of the + # version, and that we'll therefore have to check fewer + # timestamps this way... + trimmed_version = package.version.trim(self.rank - 1) + first_after = None + for after_package in reversed(descending[:i]): + if after_package.version.trim(self.rank - 1) != trimmed_version: + return after_package.version + + return first_after + + def _calc_sort_key(self, package_name, version): + first_after = self.get_first_after(package_name) + if first_after is None: + # all packages are before T + is_before = True + else: + is_before = int(version < first_after) + if is_before: + return is_before, version + else: + if self.rank: + return (is_before, + _ReversedComparable(version.trim(self.rank - 1)), + version.tokens[self.rank - 1:]) else: - after_.extend(reversed(postrank)) - postrank = [o] - prerank = prerank_ + return is_before, _ReversedComparable(version) - after_.extend(reversed(postrank)) - return before + after_ + def sort_key_implementation(self, package_name, version): + cache_key = (package_name, str(version)) + result = self._cached_sort_key.get(cache_key) + if result is None: + result = self._calc_sort_key(package_name, version) + self._cached_sort_key[cache_key] = result + + return result def __str__(self): - return str((self.timestamp, self.rank)) + return f"{self.get_packages_str}, timestamp={self.timestamp}, rank={self.rank}" def __eq__(self, other): return ( type(other) == type(self) + and self.packages == other.packages and self.timestamp == other.timestamp and self.rank == other.rank ) @@ -439,38 +588,218 @@ def to_pod(self): .. code-block:: yaml type: soft_timestamp + packages: ["foo", "bar"] timestamp: 1234567 rank: 3 """ - return dict(timestamp=self.timestamp, + return dict(packages=self.packages, + timestamp=self.timestamp, rank=self.rank) @classmethod def from_pod(cls, data): - return cls(timestamp=data["timestamp"], rank=data.get("rank", 0)) + return cls( + timestamp=data["timestamp"], + rank=data.get("rank", 0), + packages=data.get("packages"), + ) + + +class CustomPackageOrder(PackageOrder): + """A package order that allows explicit specification of version ordering. + Specified through the "packages" attributes, which should be a dict which + maps from a package family name to a list of version ranges to prioritize, + in decreasing priority order. + As an example, consider a package splunge which has versions: + [1.0, 1.1, 1.2, 1.4, 2.0, 2.1, 3.0, 3.2] + By default, version priority is given to the higest version, so version + priority, from most to least preferred, is: + [3.2, 3.0, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + However, if you set a custom package order like this: + package_orderers: + - type: custom + packages: + splunge: ['2', '1.1+<1.4'] + Then the preferred versions, from most to least preferred, will be: + [2.1, 2.0, 1.2, 1.1, 3.2, 3.0, 1.4, 1.0] + Any version which does not match any of these expressions are sorted in + decreasing version order (like normal) and then appended to this list (so they + have lower priority). This provides an easy means to effectively set a + "default version." So if you do: + package_orderers: + - type: custom + packages: + splunge: ['3.0'] + resulting order is: + [3.0, 3.2, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + You may also include a single False or empty string in the list, in which case + all "other" versions will be placed at that spot. ie + package_orderers: + - type: custom + packages: + splunge: ['', '3+'] + yields: + [2.1, 2.0, 1.4, 1.2, 1.1, 1.0, 3.2, 3.0] + Note that you could also have gotten the same result by doing: + package_orderers: + - type: custom + packages: + splunge: ['<3'] + If a version matches more than one range expression, it will be placed at + the highest-priority matching spot, so: + package_orderers: + - type: custom + packages: + splunge: ['1.2+<=2.0', '1.1+<3'] + gives: + [2.0, 1.4, 1.2, 2.1, 1.1, 3.2, 3.0, 1.0] + Also note that this does not change the version sort order for any purpose but + determining solving priorities - for instance, even if version priorities is: + package_orderers: + - type: custom + packages: + splunge: [2, 3, 1] + The expression splunge-1+<3 would still match version 2. + """ + name = "custom" + + def __init__(self, packages): + """Create a reorderer. + Args: + packages: (Dict[str, List[VersionRange]]): packages that + this orderer should apply to, and the version priority ordering + for that package + """ + self.packages_dict = self._packages_from_pod(packages) + self._version_key_cache = {} + + def sort_key_implementation(self, package_name, version): + family_cache = self._version_key_cache.setdefault(package_name, {}) + key = family_cache.get(version) + if key is not None: + return key + + key = self.version_priority_key_uncached(package_name, version) + family_cache[version] = key + return key + + @property + def packages(self): + return iter(self.packages_dict) + + def __str__(self): + return str(self.packages_dict) + + def version_priority_key_uncached(self, package_name, version): + version_priorities = self.packages_dict[package_name] + + default_key = -1 + for sort_order_index, version_range in enumerate(version_priorities): + # in the config, version_priorities are given in decreasing + # priority order... however, we want a sort key that sorts in the + # same way that versions do - where higher values are higher + # priority - so we need to take the inverse of the index + priority_sort_key = len(version_priorities) - sort_order_index + if version_range in (False, ""): + if default_key != -1: + raise ValueError("version_priorities may only have one " + "False / empty value") + default_key = priority_sort_key + continue + if version_range.contains_version(version): + break + else: + # For now, we're permissive with the version_sort_order - it may + # contain ranges which match no actual versions, and if an actual + # version matches no entry in the version_sort_order, it is simply + # placed after other entries + priority_sort_key = default_key + return priority_sort_key, version + + @classmethod + def _packages_to_pod(cls, packages): + return dict((package, [str(v) for v in versions]) + for (package, versions) in packages.items()) + + @classmethod + def _packages_from_pod(cls, packages): + from rez.version import VersionRange + parsed_dict = {} + for package, versions in packages.items(): + new_versions = [] + num_false = 0 + for v in versions: + if v in ("", False): + v = False + num_false += 1 + else: + if not isinstance(v, VersionRange): + if isinstance(v, (int, float)): + v = str(v) + v = VersionRange(v) + new_versions.append(v) + if num_false > 1: + raise ConfigurationError("version_priorities for CustomPackageOrder may " + "only have one False / empty value") + parsed_dict[package] = new_versions + return parsed_dict + + def to_pod(self): + return dict(packages=self._packages_to_pod(self.packages_dict)) + + @classmethod + def from_pod(cls, data): + return cls(packages=data["packages"]) -class PackageOrderList(list): - """A list of package orderer. +class OrdererDict(Mapping, YamlDumpable): + """ + Mapping of package names to the way they should be sorted. """ + + def __init__(self, orderer_list): + self.list = [] + self.by_package = {} + + for orderer in orderer_list: + if not isinstance(orderer, PackageOrder): + orderer = from_pod(orderer) + self.list.append(orderer) + for package in orderer.packages: + # We allow duplicates (so we can have hierarchical configs, + # which can override each other) - earlier orderers win + if package in self.by_package: + continue + self.by_package[package] = orderer + + def to_yaml_pod(self): + return self.to_pod() + def to_pod(self): - data = [] - for f in self: - data.append(to_pod(f)) - return data + return [to_pod(x) for x in self.list] @classmethod def from_pod(cls, data): - flist = PackageOrderList() - for dict_ in data: - f = from_pod(dict_) - flist.append(f) - return flist + orderers = [] + for entry in data: + orderers.append(from_pod(entry)) + return cls(orderers) + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.list) + + def __getitem__(self, package): + return self.by_package[package] + + def __iter__(self): + return iter(self.by_package) + + def __len__(self): + return len(self.by_package) @cached_class_property def singleton(cls): - """Filter list as configured by rezconfig.package_filter.""" - return cls.from_pod(config.package_orderers) + return config.package_orderers or cls([]) def to_pod(orderer): @@ -511,6 +840,21 @@ def register_orderer(cls): return False +def get_orderer(package_name, orderers=None): + from rez.config import config + if orderers is None: + orderers = config.package_orderers + if not orderers: + orderers = {} + found_orderer = orderers.get(package_name) + if found_orderer is None: + found_orderer = orderers.get(DEFAULT_TOKEN) + if found_orderer is None: + # default ordering is version descending + found_orderer = SortedOrder(descending=True, packages=[DEFAULT_TOKEN]) + return found_orderer + + # registration of builtin orderers _orderers = {} for o in list(globals().values()): diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index bbf68fef05..7573c4913d 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -25,7 +25,7 @@ from rez import package_order from rez.packages import get_variant, iter_packages from rez.package_filter import PackageFilterList -from rez.package_order import PackageOrderList +from rez.package_order import OrdererDict from rez.package_cache import PackageCache from rez.shells import create_shell from rez.exceptions import ResolvedContextError, PackageCommandError, \ @@ -184,7 +184,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, package_filter (PackageFilterList): Filter used to exclude certain packages. Defaults to settings from :data:`package_filter`. Use :data:`rez.package_filter.no_filter` to remove all filtering. - package_orderers (list[PackageOrder]): Custom package ordering. + package_orderers (Optional[OrdererDict]): Custom package ordering. Defaults to settings from :data:`package_orderers`. add_implicit_packages (bool): If True, the implicit package list defined by :data:`implicit_packages` is appended to the request. @@ -234,8 +234,8 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, else package_filter) self.package_orderers = ( - PackageOrderList.singleton if package_orderers is None - else package_orderers + OrdererDict.singleton if package_orderers is None + else package_orderers or config.package_orderers ) # settings that affect context execution @@ -1527,9 +1527,10 @@ def _add(field): data["patch_locks"] = dict((k, v.name) for k, v in self.patch_locks) if _add("package_orderers"): - package_orderers = [package_order.to_pod(x) - for x in (self.package_orderers or [])] - data["package_orderers"] = package_orderers or None + if self.package_orderers: + data["package_orderers"] = self.package_orderers.to_pod() + else: + data["package_orderers"] = None if _add("package_filter"): data["package_filter"] = self.package_filter.to_pod() @@ -1687,7 +1688,7 @@ def _print_version(value): data = d.get("package_orderers") if data: - r.package_orderers = [package_order.from_pod(x) for x in data] + r.package_orderers = package_order.OrdererDict(data) else: r.package_orderers = None diff --git a/src/rez/resolver.py b/src/rez/resolver.py index 921871ce61..d5932a3ac0 100644 --- a/src/rez/resolver.py +++ b/src/rez/resolver.py @@ -61,7 +61,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None self.package_paths = package_paths self.timestamp = timestamp self.callback = callback - self.package_orderers = package_orderers + self.package_orderers = package_orderers or config.package_orderers self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity @@ -72,7 +72,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None # store hash of package orderers. This is used in the memcached key if package_orderers: - sha1s = ''.join(x.sha1 for x in package_orderers) + sha1s = ''.join(x.sha1 for x in package_orderers.values()) self.package_orderers_hash = sha1(sha1s.encode("utf8")).hexdigest() else: self.package_orderers_hash = '' diff --git a/src/rez/solver.py b/src/rez/solver.py index f7652cf5d2..3b21d18a28 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -417,6 +417,8 @@ def sort(self): here as a safety measure so that sorting is guaranteed repeatable regardless. """ + from rez.package_order import get_orderer + if self.sorted: return @@ -428,13 +430,19 @@ def key(variant): if not request.conflict: req = variant.requires_list.get(request.name) if req is not None: - requested_key.append((-i, req.range)) + orderer = get_orderer(req.name, + self.solver.package_orderers or {}) + range_key = orderer.sort_key(req.name, req.range) + requested_key.append((-i, range_key)) names.add(req.name) additional_key = [] for request in variant.requires_list: if not request.conflict and request.name not in names: - additional_key.append((request.range, request.name)) + orderer = get_orderer(request.name, + self.solver.package_orderers or {}) + range_key = orderer.sort_key(request.name, request.range) + additional_key.append((range_key, request.name)) if (VariantSelectMode[config.variant_select_mode] == VariantSelectMode.version_priority): k = (requested_key, @@ -804,6 +812,7 @@ def _split(i_entry, n_variants, common_fams=None): if not fams: # trivial case, split on first variant + self.entries[0].sort() return _split(0, 1) # find split point - first variant with no dependency shared with previous @@ -833,25 +842,21 @@ def sort_versions(self): The order is typically descending, but package order functions can change this. """ + from rez.package_order import get_orderer + if self.sorted: return - for orderer in (self.solver.package_orderers or []): - entries = orderer.reorder(self.entries, key=lambda x: x.package) - if entries is not None: - self.entries = entries - self.sorted = True - - if self.pr: - self.pr("sorted: %s packages: %s", self.package_name, repr(orderer)) - return + orderer = get_orderer(self.package_name, + self.solver.package_orderers or {}) + def sort_key(entry): + return orderer.sort_key(entry.package.name, entry.version) - # default ordering is version descending - self.entries = sorted(self.entries, key=lambda x: x.version, reverse=True) + self.entries = sorted(self.entries, key=sort_key, reverse=True) self.sorted = True if self.pr: - self.pr("sorted: %s packages: version descending", self.package_name) + self.pr("sorted: %s packages: %s", self.package_name, repr(orderer)) def dump(self): print(self.package_name) @@ -1928,7 +1933,7 @@ def __init__(self, package_requests, package_paths, context=None, """ self.package_paths = package_paths self.package_filter = package_filter - self.package_orderers = package_orderers + self.package_orderers = package_orderers or config.package_orderers self.callback = callback self.prune_unfailed = prune_unfailed self.package_load_callback = package_load_callback diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py index ef8c390c61..da040e1a31 100644 --- a/src/rez/tests/test_completion.py +++ b/src/rez/tests/test_completion.py @@ -50,7 +50,7 @@ def _eq(prefix, expected_completions): _eq("zzz", []) _eq("", ["bahish", "nada", "nopy", "pybah", "pydad", "pyfoo", "pymum", - "pyodd", "pyson", "pysplit", "python", "pyvariants", + "pyodd", "pyson", "pysplit", "python", "pyvariants", "reorderable", "test_variant_split_start", "test_variant_split_mid1", "test_variant_split_mid2", "test_variant_split_end", "missing_variant_requires"]) _eq("py", ["pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index ee742a8df0..791529b6a8 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -11,6 +11,7 @@ from rez.bundle_context import bundle_context from rez.bind import hello_world from rez.utils.platform_ import platform_ +from rez.config import config from rez.utils.filesystem import is_subdirectory import unittest import subprocess @@ -26,11 +27,19 @@ def setUpClass(cls): TempdirMixin.setUpClass() cls.packages_path = os.path.join(cls.root, "packages") + + cls.py_packages_path = cls.data_path("packages", "py_packages") + cls.solver_packages_path = cls.data_path("solver", "packages") + os.makedirs(cls.packages_path) hello_world.bind(cls.packages_path) cls.settings = dict( - packages_path=[cls.packages_path], + packages_path=[ + cls.packages_path, + cls.solver_packages_path, + cls.py_packages_path, + ], package_filter=None, implicit_packages=[], warn_untimestamped=False, @@ -186,6 +195,71 @@ def _test_bundle(path): _test_bundle(bundle_path3) + def test_orderer(self): + """Test a resolve with an orderer""" + from rez.package_order import CustomPackageOrder, OrdererDict, VersionSplitPackageOrder + from rez.version import Version + + orderers = OrdererDict([ + VersionSplitPackageOrder(packages=["python"], + first_version=Version("2.6"))]) + r = ResolvedContext(["python", "!python-2.6"], + package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ["python-2.5.2"]) + + # make sure serializing the orderer works + file = os.path.join(self.root, "test_orderers.rxt") + r.save(file) + r2 = ResolvedContext.load(file) + self.assertEqual(r, r2) + self.assertEqual(r.package_orderers, r2.package_orderers) + + # Test that forcibly omitting the first entry in a custom order choose the second + orderers = OrdererDict([ + CustomPackageOrder(packages={"multi": ["1.2", "2.0", "1.0", "1.1"]}), + ]) + r = ResolvedContext(["multi", "!multi-1.2"], package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ["multi-2.0"]) + + # Test that package orderers also apply to variants, not just requirements + + def test_orderer_fallback(self): + """Test that we fall back to sorted order for unspecified packages.""" + from rez.package_order import VersionSplitPackageOrder, OrdererDict + from rez.resolved_context import ResolvedContext + from rez.version import Version + + # Test that we correctly apply the orderer to the expected family + orderers = OrdererDict([ + VersionSplitPackageOrder(packages=["multi"], + first_version=Version("1.2"))]) + r = ResolvedContext(["multi"], package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ["multi-1.2"]) + + # Test that an unordered family will get the correct default behavior. + r = ResolvedContext(["python"], package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ["python-2.7.0"]) + + def test_orderer_variants(self): + """Test that package orderers apply to the variants of packacges, not just requires.""" + from rez.package_order import CustomPackageOrder, OrdererDict + # Verify that we get the latest python without an orderer + r = ResolvedContext(["pyvariants", "!nada"]) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ['python-2.7.0', 'pyvariants-2']) + + # Now verify that we correctly get python-2.6.8 when the orderer demands it. + orderers = OrdererDict([ + CustomPackageOrder(packages={"python": ["2.6"]}), + ]) + r = ResolvedContext(["pyvariants", "!nada"], package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ['python-2.6.8', 'pyvariants-2']) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 55f5254c69..3a37f9bede 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -42,6 +42,10 @@ 'pysplit-5', 'pysplit-6', 'pysplit-7', 'python-2.5.2', 'python-2.6.0', 'python-2.6.8', 'python-2.7.0', 'pyvariants-2', + 'reorderable-1.9.0', 'reorderable-1.9.1', 'reorderable-2.0.0', + 'reorderable-2.0.5', 'reorderable-2.0.6', 'reorderable-2.1.0', + 'reorderable-2.1.1', 'reorderable-2.1.5', 'reorderable-2.2.0', + 'reorderable-2.2.1', 'reorderable-3.0.0', 'reorderable-3.1.1', 'test_variant_split_start-1.0', 'test_variant_split_start-2.0', 'test_variant_split_mid1-1.0', 'test_variant_split_mid1-2.0', 'test_variant_split_mid2-1.0', 'test_variant_split_mid2-2.0', diff --git a/src/rez/tests/test_packages_order.py b/src/rez/tests/test_packages_order.py index 702963f5f4..dd0bf119ee 100644 --- a/src/rez/tests/test_packages_order.py +++ b/src/rez/tests/test_packages_order.py @@ -8,8 +8,9 @@ import json from rez.config import config -from rez.package_order import NullPackageOrder, PackageOrder, PerFamilyOrder, VersionSplitPackageOrder, \ - TimestampPackageOrder, SortedOrder, PackageOrderList, from_pod +from rez.package_order import (CustomPackageOrder, NullPackageOrder, OrdererDict, + PerFamilyOrder, VersionSplitPackageOrder, + TimestampPackageOrder, SortedOrder, from_pod, to_pod) from rez.packages import iter_packages from rez.tests.util import TestBase, TempdirMixin from rez.version import Version @@ -36,12 +37,31 @@ def tearDownClass(cls): TempdirMixin.tearDownClass() def _test_reorder(self, orderer, package_name, expected_order): - """Ensure ordered order package version as expected.""" + import json + from rez.utils.yaml import dump_yaml + from rez.vendor import yaml + it = iter_packages(package_name) descending = sorted(it, key=lambda x: x.version, reverse=True) - ordered = orderer.reorder(descending) or descending + + pod = to_pod(orderer) + + # ResolvedContext.write_to_buffer will require conversion to both + # json and yaml, so test both + as_json = json.dumps(pod) + from_json = json.loads(as_json) + as_yaml = dump_yaml(pod) + from_yaml = yaml.load(as_yaml) + self.assertEqual(from_yaml, from_json) + + orderer2 = from_pod(from_yaml) + for orderer_ in (orderer, orderer2): + def key(package): + return orderer_.sort_key(package.name, package.version) + + ordered = sorted(descending, key=key, reverse=True) result = [str(x.version) for x in ordered] - self.assertEqual(expected_order, result) + self.assertEqual(result, expected_order) def _test_pod(self, orderer): """Ensure an orderer integrity when serialized to pod.""" @@ -50,35 +70,12 @@ def _test_pod(self, orderer): self.assertEqual(orderer, actual) -class TestAbstractPackageOrder(TestBase): - """Test case for the abstract PackageOrder class""" - - def test_reorder(self): - """Validate reorder is not implemented""" - with self.assertRaises(NotImplementedError): - PackageOrder().reorder([]) - - def test_to_pod(self): - """Validate to_pod is not implemented""" - self.assertRaises(NotImplementedError, PackageOrder().to_pod) - - def test_str(self): - """Validate __str__ is not implemented""" - with self.assertRaises(NotImplementedError): - str(PackageOrder()) - - def test_eq(self): - """Validate __eq__ is not implemented""" - with self.assertRaises(NotImplementedError): - PackageOrder() == PackageOrder() - - class TestNullPackageOrder(_BaseTestPackagesOrder): """Test case for the NullPackageOrder class""" def test_repr(self): """Validate we can represent a VersionSplitPackageOrder as a string.""" - self.assertEqual("NullPackageOrder({})", repr(NullPackageOrder())) + self.assertEqual("NullPackageOrder(*)", repr(NullPackageOrder())) def test_comparison(self): """Validate we can compare VersionSplitPackageOrder together.""" @@ -97,7 +94,7 @@ def test_sha1(self): """Validate we can get a sha1 hash. """ self.assertEqual( - 'bf7c2fa4e6bd198c02adeea2c3a382cf57242051', NullPackageOrder().sha1 + 'd59689dc10283d497b200ad1d2be8e61d07e8c02', NullPackageOrder().sha1 ) @@ -126,10 +123,10 @@ def test_comparison(self): def test_repr(self): """Validate we can represent a SortedOrder as a string.""" - self.assertEqual("SortedOrder(True)", repr(SortedOrder(descending=True))) + self.assertEqual("SortedOrder(*, descending=True)", repr(SortedOrder(descending=True))) def test_pod(self): - """Validate we can save and load a SortedOrder to it's pod representation.""" + """Validate we can save and load a SortedOrder to its pod representation.""" self._test_pod(SortedOrder(descending=True)) @@ -174,15 +171,16 @@ def test_comparison(self): inst4 = PerFamilyOrder(order_dict={'foo': NullPackageOrder()}, default_order=None) self.assertTrue(inst1 == inst2) # __eq__ positive self.assertFalse(inst1 == inst3) # __eq__ negative (different order dict) - self.assertFalse(inst1 == inst4) # __eq__ negative (different default_order) self.assertTrue(inst1 != inst3) # __ne__ positive (different order dict) - self.assertTrue(inst1 != inst4) # __ne__ positive (different default order) + self.assertFalse(inst1 != inst4) # __eq__ negative (No default orderer defaults to NullPackageOrder) self.assertFalse(inst1 != inst2) # __ne__ negative def test_repr(self): """Validate we can represent a PerFamilyOrder as a string.""" inst = PerFamilyOrder(order_dict={"family1": VersionSplitPackageOrder(Version("2.6.0"))}) - self.assertEqual("PerFamilyOrder(([('family1', '2.6.0')], 'None'))", repr(inst)) + self.assertEqual("PerFamilyOrder([('family1', \"" + "VersionSplitPackageOrder(*, first_version='2.6.0')\")], " + "default_order=NullPackageOrder(*))", repr(inst)) def test_pod(self): """Validate we can save and load a PerFamilyOrder to it's pod representation.""" @@ -199,7 +197,7 @@ def test_pod(self): class TestVersionSplitPackageOrder(_BaseTestPackagesOrder): """Test case for the VersionSplitPackageOrder class""" - def test_reordere(self): + def test_reorder(self): """Validate package ordering with a VersionSplitPackageOrder""" orderer = VersionSplitPackageOrder(Version("2.6.0")) expected = ["2.6.0", "2.5.2", "2.7.0", "2.6.8"] @@ -220,10 +218,10 @@ def test_comparison(self): def test_repr(self): """Validate we can represent a VersionSplitPackageOrder as a string.""" inst = VersionSplitPackageOrder(first_version=Version("1,2,3")) - self.assertEqual("VersionSplitPackageOrder(1,2,3)", repr(inst)) + self.assertEqual("VersionSplitPackageOrder(*, first_version='1,2,3')", repr(inst)) def test_pod(self): - """Validate we can save and load a VersionSplitPackageOrder to it's pod representation.""" + """Validate we can save and load a VersionSplitPackageOrder to its pod representation.""" self._test_pod(VersionSplitPackageOrder(first_version=Version("1.2.3"))) @@ -287,18 +285,57 @@ def test_comparison(self): def test_repr(self): """Validate we can represent a TimestampPackageOrder as a string.""" inst = TimestampPackageOrder(timestamp=1, rank=2) - self.assertEqual(repr(inst), "TimestampPackageOrder((1, 2))") + self.assertEqual(repr(inst), "TimestampPackageOrder(*, timestamp=1, rank=2)") def test_pod(self): """Validate we can save and load a TimestampPackageOrder to pod representation.""" self._test_pod(TimestampPackageOrder(timestamp=3001, rank=3)) -class TestPackageOrdererList(_BaseTestPackagesOrder): - """Test cases for the PackageOrderList class.""" +class TestCustomOrder(_BaseTestPackagesOrder): + """Test case for the CustomPackageOrder class""" + + def test_reorder(self): + """Validate we can sort packages in a custom order.""" + # Test that a pretty random order can be applied + self._test_reorder(CustomPackageOrder({ + "python": ['2.6.8', '2.7.0', '2.6.0', '2.5.2'], + }), "python", ['2.6.8', '2.7.0', '2.6.0', '2.5.2']) + + # Test that version supersets apply correctly and that the rest are sorted descending + self._test_reorder(CustomPackageOrder({ + "python": ["2.6"], + }), "python", ['2.6.8', '2.6.0', '2.7.0', '2.5.2']) + + # Test that version supersets apply correctly to multiple subsets of versions + self._test_reorder(CustomPackageOrder({ + "python": ["2.6", "2.5"], + }), "python", ['2.6.8', '2.6.0', '2.5.2', '2.7.0']) + + def test_repr(self): + """Validate we can represent a CustomPackageOrder as a string.""" + self.assertEqual( + "CustomPackageOrder({" + "'python': [VersionRange('1.2'), VersionRange('5.6')], " + "'pymum': [VersionRange('2'), VersionRange('1')]})", + repr(CustomPackageOrder({ + "python": ["1.2", "5.6"], + "pymum": ["2", "1"], + }))) + + def test_pod(self): + """Validate we can save and load a CustomPackageOrder to its pod representation.""" + self._test_pod(CustomPackageOrder({ + "python": ["1.2", "5.6", "3.4"], + "pymum": ["2", "1", "3"], + })) + + +class TestOrdererDict(_BaseTestPackagesOrder): + """Test cases for the OrdererDict class.""" def test_singleton(self): - """Validate we can build a PackageOrderList object from configuration values.""" + """Validate we can build a OrdererDict object from configuration values.""" config.override("package_orderers", [ { "type": "per_family", @@ -311,33 +348,33 @@ def test_singleton(self): ] } ]) - expected = PackageOrderList() - expected.append(PerFamilyOrder(order_dict={ + + expected = OrdererDict([PerFamilyOrder(order_dict={ "python": VersionSplitPackageOrder(Version("2.9.9")) - })) + })]) # Clear @classproperty cache try: - delattr(PackageOrderList, '_class_property_singleton') + delattr(OrdererDict, '_class_property_singleton') except AttributeError: pass - self.assertEqual(expected, PackageOrderList.singleton) + self.assertEqual(expected, OrdererDict.singleton) def test_singleton_novalue(self): - """Validate we can build a PackageOrderList object from empty configuration values.""" + """Validate we can build a OrdererDict object from empty configuration values.""" config.override("package_orderers", None) # Clear @classproperty cache try: - delattr(PackageOrderList, '_class_property_singleton') + delattr(OrdererDict, '_class_property_singleton') except AttributeError: pass - self.assertEqual(PackageOrderList(), PackageOrderList.singleton) + self.assertEqual(OrdererDict([]), OrdererDict.singleton) def test_pod(self): - """Validate we can save and load a PackageOrdererList to pod representation.""" - inst = PackageOrderList(( + """Validate we can save and load a OrdererDict to pod representation.""" + inst = OrdererDict(( VersionSplitPackageOrder(Version("2.6.0")), PerFamilyOrder(order_dict={}, default_order=SortedOrder(descending=False)) )) diff --git a/src/rez/tests/test_solver.py b/src/rez/tests/test_solver.py index 4c9d44266f..01f17af8bc 100644 --- a/src/rez/tests/test_solver.py +++ b/src/rez/tests/test_solver.py @@ -5,10 +5,13 @@ """ test dependency resolving algorithm """ +from __future__ import print_function + import rez.exceptions from rez.version import Requirement from rez.solver import Solver, Cycle, SolverStatus from rez.config import config +from rez.exceptions import ConfigurationError import unittest from rez.tests.util import TestBase import itertools @@ -26,7 +29,7 @@ def setUpClass(cls): packages_path=cls.packages_path, package_filter=None) - def _create_solvers(self, reqs): + def _create_solvers(self, reqs, do_permutations=True): s1 = Solver(reqs, self.packages_path, optimised=True, @@ -37,20 +40,22 @@ def _create_solvers(self, reqs): verbosity=solver_verbosity) s_perms = [] - perms = itertools.permutations(reqs) - for reqs_ in perms: - s = Solver(reqs_, - self.packages_path, - optimised=True, - verbosity=solver_verbosity) - s_perms.append(s) + if do_permutations: + perms = itertools.permutations(reqs) + for reqs_ in perms: + s = Solver(reqs_, + self.packages_path, + optimised=True, + verbosity=Solver.max_verbosity) + s_perms.append(s) return (s1, s2, s_perms) - def _solve(self, packages, expected_resolve): + def _solve(self, packages, expected_resolve, do_permutations=True): print() reqs = [Requirement(x) for x in packages] - s1, s2, s_perms = self._create_solvers(reqs) + s1, s2, s_perms = self._create_solvers(reqs, + do_permutations=do_permutations) s1.solve() self.assertEqual(s1.status, SolverStatus.solved) @@ -77,9 +82,11 @@ def _solve(self, packages, expected_resolve): self.assertEqual(resolve2, resolve) print("checking that permutations also succeed...") - for s in s_perms: - s.solve() - self.assertEqual(s.status, SolverStatus.solved) + if do_permutations: + print("checking that permutations also succeed...") + for s in s_perms: + s.solve() + self.assertEqual(s.status, SolverStatus.solved) return s1 @@ -256,6 +263,484 @@ def test_12_missing_variant_requires(self): config.override("error_on_missing_variant_requires", False) self._solve(["missing_variant_requires"], ["nada[]", "missing_variant_requires-1[1]"]) + # re-prioritization tests + def test_13_reversed_str(self): + """Test setting a package to reversed-version sorting + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["python"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.7.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.5.2"], + ["python-2.6.0[]"]) + self._solve(["python-2.6"], + ["python-2.6.0[]"]) + self._solve(["python-2.6+<2.7"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + def test_14_reversed_list(self): + """Test setting a package to reversed-version sorting + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": ["python"]}]) + self._solve(["python"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.7.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.5.2"], + ["python-2.6.0[]"]) + self._solve(["python-2.6"], + ["python-2.6.0[]"]) + self._solve(["python-2.6+<2.7"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + def test_15_reversed_is_requirement(self): + """Test setting a package to reversed-version sorting, when it is a + requirement + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["pyfoo"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + self._solve(["pyfoo-3.0"], + ["python-2.5.2[]", "pyfoo-3.0.0[]", ".eek-3+"]) + self._solve(["pyfoo-3.1"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + self._solve(["pybah"], + ["python-2.5.2[]", "pybah-5[]"]) + self._solve(["pybah-4"], + ["python-2.6.0[]", "pybah-4[]", ".eek-1"]) + self._solve(["pybah-5"], + ["python-2.5.2[]", "pybah-5[]"]) + + def test_16_reversed_has_requirement(self): + """Test setting a package to reversed-version sorting, when it has a + requirement + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": ["pyfoo", "pybah"]}]) + self._solve(["pyfoo"], + ["python-2.5.2[]", "pyfoo-3.0.0[]", ".eek-3+"]) + self._solve(["pyfoo-3.0"], + ["python-2.5.2[]", "pyfoo-3.0.0[]", ".eek-3+"]) + self._solve(["pyfoo-3.1"], + ["python-2.6.8[]", "pyfoo-3.1.0[]"]) + self._solve(["pybah"], + ["python-2.6.8[]", "pybah-4[]", ".eek-1"]) + self._solve(["pybah-4"], + ["python-2.6.8[]", "pybah-4[]", ".eek-1"]) + self._solve(["pybah-5"], + ["python-2.5.2[]", "pybah-5[]"]) + + def _test_complete_ordering(self, request, expected_order): + exclude = [] + for next in expected_order: + self._solve(request + exclude, [next + '[]'], + do_permutations=False) + exclude.append('!{}'.format(next)) + + def test_17_timestamp_no_rank_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728472, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.5", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + def test_18_timestamp_no_rank_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728473, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.5", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + ]) + + + def test_19_timestamp_rank2_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 2, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-3.1.1", + "reorderable-3.0.0", + ]) + + + def test_20_timestamp_rank2_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 2, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-3.1.1", + "reorderable-3.0.0", + ]) + + def test_21_timestamp_rank3_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 3, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_22_timestamp_rank3_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 3, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_23_timestamp_rank4_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 4, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_24_timestamp_rank4_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 4, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + def test_25_direct_complete(self): + """Test setting of the version_priority in simple situations, where + the altered package is a direct request + """ + # test a complete ordering + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "2.5.2", "2.7.0", "2.6.8"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python<=2.6.0"], + ["python-2.7.0[]"]) + self._solve(["python", "!python-2.6.0", "!python-2.5.2", + "!python-2.7.0"], + ["python-2.6.8[]"]) + + # check that we can still request a lower-priority version + self._solve(["python-2.6.8"], + ["python-2.6.8[]"]) + + def test_26_direct_single(self): + """check that if you specify only one version, that version is highest + priority, rest are normal + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.8"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.7.0[]"]) + self._solve(["python<2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + # confirm that sorting for version ranges is still normal + self._solve(["python-2.6+<2.7"], + ["python-2.6.8[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + def test_27_empty_string(self): + """confirm that we can use empty string to match unmatched versions""" + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["", "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "", "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + def test_28_false(self): + """confirm that we can use False to match unmatched versions""" + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": [False, "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", False, "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + def test_29_requirement_1_deep(self): + """Test setting of the version_priority for a required package 1 level + deep + """ + # python 2.5 is preferred... + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": [2.5]}}]) + + # # so if we request python directly, we get 2.5... + self._solve(["python"], + ["python-2.5.2[]"]) + + # ...but if we request pyfoo, IT'S version is more important, and we + # get 2.6 + self._solve(["pyfoo"], + ["python-2.6.8[]", "pyfoo-3.1.0[]"]) + + # but if we make specifically python-2.6.0 prioritized, it will be used + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0"]}}]) + self._solve(["pyfoo"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + + def test_30_requirement_2_deep(self): + """Test setting of the version_priority for a required package 2 + levels deep + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "2.5"]}}]) + self._solve(["pyodd"], + ["python-2.5.2[]", "pybah-5[]", "pyodd-2[]"]) + self._solve(["pyodd-2"], + ["python-2.5.2[]", "pybah-5[]", "pyodd-2[]"]) + self._solve(["pyodd-1"], + ["python-2.6.0[]", "pyfoo-3.1.0[]", "pyodd-1[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"pybah": ["4"], + "pyfoo": ["3.0.0"], + "python": ["2.6.0"]}}]) + self._solve(["pyodd"], + ["python-2.6.0[]", "pybah-4[]", "pyodd-2[]", ".eek-1"]) + self._solve(["pyodd-2"], + ["python-2.6.0[]", "pybah-4[]", "pyodd-2[]", ".eek-1"]) + self._solve(["pyodd-1"], + ["python-2.5.2[]", "pyfoo-3.0.0[]", "pyodd-1[]", ".eek-3+"]) + + def test_31_multiple_false(self): + """Make sure that multiple False / empty values raises an error + """ + self.assertRaises(ConfigurationError, + config.override, + "package_orderers", + [{ + "type": "custom", + "packages": {"python": ["2.6.0", False, False, "2.5"]} + }] + ) + self.assertRaises(ConfigurationError, + config.override, + "package_orderers", + [{ + "type": "custom", + "packages": {"python": ["2.6.0", "", "", "2.5"]} + }] + ) + self.assertRaises(ConfigurationError, + config.override, + "package_orderers", + [{ + "type": "custom", + "packages": {"python": ["2.6.0", "", False, "2.5"]} + }] + ) + + def test_32_multiple_matches(self): + """Test that if matches more than one, higher-priority is used + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.7.0|2.6.8", "2.5", "2.6.8|2.6.0"]}}]) + self._solve(["python<2.7"], + ["python-2.6.8[]"]) + + def test_33_orderer_used_for_variants(self): + self._solve(["pyvariants"], + ["python-2.7.0[]", "pyvariants-2[0]"]) + + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["pyvariants"], + ["python-2.6.8[]", "pyvariants-2[2]"]) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/tests/test_version.py b/src/rez/tests/test_version.py index 72c26b612d..103e815eb9 100644 --- a/src/rez/tests/test_version.py +++ b/src/rez/tests/test_version.py @@ -504,6 +504,31 @@ def _confl(reqs, a, b): _confl(["foo", "~bah-5+", "bah-7..12", "bah-2"], "bah-7..12", "bah-2") + def test_reversed_sort_key(self): + # self.assertEqual(reverse_sort_key(5), reverse_sort_key(5)) + self.assertLessEqual(reverse_sort_key(Version("1.3")), + reverse_sort_key(Version("1.3"))) + self.assertLessEqual(reverse_sort_key(VersionRange("1.2+")), + reverse_sort_key(VersionRange("<0.8"))) + self.assertGreaterEqual(reverse_sort_key(VersionRange("1+<2.4")), + reverse_sort_key(VersionRange("1+<2.4"))) + self.assertGreaterEqual(reverse_sort_key(VersionRange("1+<2.4")), + reverse_sort_key(VersionRange("2.0+<3.5"))) + l = [3, 5, 6, 2, 8, 1, 4, 7, 9, 10, 0, 4] + self.assertEqual(sorted(l, key=reverse_sort_key), + [10, 9, 8, 7, 6, 5, 4, 4, 3, 2, 1, 0]) + self.assertEqual(sorted(l, key=reverse_sort_key, reverse=True), + [0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10]) + + self.assertEqual(str(reverse_sort_key("foo")), + "reverse(foo)") + self.assertEqual(repr(reverse_sort_key("foo")), + "reverse('foo')") + self.assertEqual(str(reverse_sort_key((3, 'foo'))), + "reverse((3, 'foo'))") + self.assertEqual(repr(reverse_sort_key((3, 'foo'))), + "reverse((3, 'foo'))") + if __name__ == '__main__': unittest.main() diff --git a/src/rez/utils/yaml.py b/src/rez/utils/yaml.py index 5cef870f97..387585ddea 100644 --- a/src/rez/utils/yaml.py +++ b/src/rez/utils/yaml.py @@ -7,11 +7,20 @@ from rez.vendor.yaml.dumper import SafeDumper from rez.version import Version from rez.version import Requirement +from abc import ABCMeta, abstractmethod from types import FunctionType, BuiltinFunctionType from inspect import getsourcelines from textwrap import dedent +class YamlDumpable(object, metaclass=ABCMeta): + """A class that can be sedrialized to yaml""" + + @abstractmethod + def to_yaml_pod(self): + raise NotImplementedError + + class _Dumper(SafeDumper): """Dumper which can serialise custom types such as Version, and keeps long strings nicely formatted in >/| block-style format. @@ -32,6 +41,9 @@ def represent_sourcecode(self, data): code = data.source return self.represent_str(code) + def represent_yaml_dumpable(self, data): + return self.represent_data(data.to_yaml_pod()) + _Dumper.add_representer(str, _Dumper.represent_str) _Dumper.add_representer(Version, _Dumper.represent_as_str) @@ -39,6 +51,7 @@ def represent_sourcecode(self, data): _Dumper.add_representer(FunctionType, _Dumper.represent_function) _Dumper.add_representer(BuiltinFunctionType, _Dumper.represent_builtin_function) _Dumper.add_representer(SourceCode, _Dumper.represent_sourcecode) +_Dumper.add_multi_representer(YamlDumpable, _Dumper.represent_yaml_dumpable) def dump_yaml(data, Dumper=_Dumper, default_flow_style=False): diff --git a/src/rez/version/_version.py b/src/rez/version/_version.py index 0dc702e54b..4bf85b4468 100644 --- a/src/rez/version/_version.py +++ b/src/rez/version/_version.py @@ -29,7 +29,10 @@ def __init__(self, value): self.value = value def __lt__(self, other): - return not (self.value < other.value) + return self.value > other.value + + def __eq__(self, other): + return self.value == other.value def __gt__(self, other): return not (self < other or self == other) @@ -41,10 +44,12 @@ def __ge__(self, other): return not self < other def __str__(self): - return "reverse(%s)" % str(self.value) + # enclose self.value in a tuple in case it IS a tuple + return "reverse(%s)" % (self.value,) def __repr__(self): - return "reverse(%r)" % self.value + # enclose self.value in a tuple in case it IS a tuple + return "reverse(%r)" % (self.value,) class VersionToken(_Comparable):