From 680dce348f6b2f02bb9350b192d375b41546e2f1 Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 11:17:09 -0700 Subject: [PATCH 01/10] fix for _ReversedComparable --- src/rez/tests/test_version.py | 5 ----- src/rez/vendor/version/test.py | 25 +++++++++++++++++++++++++ src/rez/vendor/version/version.py | 11 ++++++++--- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/rez/tests/test_version.py b/src/rez/tests/test_version.py index bdd90047f..13fdd135e 100644 --- a/src/rez/tests/test_version.py +++ b/src/rez/tests/test_version.py @@ -4,11 +4,6 @@ import rez.vendor.unittest2 as unittest from rez.vendor.version.test import TestVersionSchema - -class TestVersions(TestVersionSchema): - pass - - if __name__ == '__main__': unittest.main() diff --git a/src/rez/vendor/version/test.py b/src/rez/vendor/version/test.py index 63359eebf..da4c280e2 100644 --- a/src/rez/vendor/version/test.py +++ b/src/rez/vendor/version/test.py @@ -481,6 +481,31 @@ def _confl(reqs, a, b): _confl(["foo", "~bah-5+", "bah-7..12", "bah-2"], "bah-7..12", "bah-2") + def test_reversed_sort_key(self): + # self.assertEqual(reverse_sort_key(5), reverse_sort_key(5)) + self.assertLessEqual(reverse_sort_key(Version("1.3")), + reverse_sort_key(Version("1.3"))) + self.assertLessEqual(reverse_sort_key(VersionRange("1.2+")), + reverse_sort_key(VersionRange("<0.8"))) + self.assertGreaterEqual(reverse_sort_key(VersionRange("1+<2.4")), + reverse_sort_key(VersionRange("1+<2.4"))) + self.assertGreaterEqual(reverse_sort_key(VersionRange("1+<2.4")), + reverse_sort_key(VersionRange("2.0+<3.5"))) + l = [3, 5, 6, 2, 8, 1, 4, 7, 9, 10, 0, 4] + self.assertEqual(sorted(l, key=reverse_sort_key), + [10, 9, 8, 7, 6, 5, 4, 4, 3, 2, 1, 0]) + self.assertEqual(sorted(l, key=reverse_sort_key, reverse=True), + [0, 1, 2, 3, 4, 4, 5, 6, 7, 8, 9, 10]) + + self.assertEqual(str(reverse_sort_key("foo")), + "reverse(foo)") + self.assertEqual(repr(reverse_sort_key("foo")), + "reverse('foo')") + self.assertEqual(str(reverse_sort_key((3, 'foo'))), + "reverse((3, 'foo'))") + self.assertEqual(repr(reverse_sort_key((3, 'foo'))), + "reverse((3, 'foo'))") + if __name__ == '__main__': unittest.main() diff --git a/src/rez/vendor/version/version.py b/src/rez/vendor/version/version.py index ebb129a33..c2b1a9e4d 100644 --- a/src/rez/vendor/version/version.py +++ b/src/rez/vendor/version/version.py @@ -40,13 +40,18 @@ def __init__(self, value): self.value = value def __lt__(self, other): - return not (self.value < other.value) + return self.value > other.value + + def __eq__(self, other): + return self.value == other.value def __str__(self): - return "reverse(%s)" % str(self.value) + # enclose self.value in a tuple in case it IS a tuple + return "reverse(%s)" % (self.value,) def __repr__(self): - return "reverse(%r)" % self.value + # enclose self.value in a tuple in case it IS a tuple + return "reverse(%r)" % (self.value,) class VersionToken(_Comparable): From 5afb14172cd7c36126330c931fcc50b1a9b42af0 Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:01:58 -0700 Subject: [PATCH 02/10] fix for serializing of VersionSplitPackageOrder in contexts --- src/rez/package_order.py | 9 +++++++-- src/rez/tests/test_context.py | 22 ++++++++++++++++++++++ src/rez/tests/test_packages.py | 15 ++++++++++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 3d9f0321b..129bc5e40 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -1,6 +1,8 @@ from inspect import isclass from hashlib import sha1 +from rez.vendor.version.version import Version + class PackageOrder(object): """Package reorderer base class.""" @@ -41,6 +43,9 @@ def sha1(self): def __repr__(self): return "%s(%s)" % (self.__class__.__name__, str(self)) + def __eq__(self, other): + return type(self) == type(other) and str(self) == str(other) + class NullPackageOrder(PackageOrder): """An orderer that does not change the order - a no op. @@ -245,11 +250,11 @@ def to_pod(self): type: version_split first_version: "3.0.0" """ - return dict(first_version=self.first_version) + return dict(first_version=str(self.first_version)) @classmethod def from_pod(cls, data): - return cls(data["first_version"]) + return cls(Version(data["first_version"])) class TimestampPackageOrder(PackageOrder): diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 0b4202709..647257362 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -5,6 +5,7 @@ from rez.resolved_context import ResolvedContext from rez.bind import hello_world from rez.utils.platform_ import platform_ +from rez.config import config import rez.vendor.unittest2 as unittest import subprocess import os.path @@ -62,6 +63,27 @@ def test_serialize(self): r2 = ResolvedContext.load(file) self.assertEqual(r.resolved_packages, r2.resolved_packages) + def test_orderer(self): + """Test a resolve with an orderer""" + from rez.package_order import VersionSplitPackageOrder + from rez.vendor.version.version import Version + path = os.path.dirname(__file__) + packages_path = os.path.join(path, "data", "solver", "packages") + config.override("packages_path", + [packages_path]) + orderers = [VersionSplitPackageOrder(first_version=Version("2.6"))] + r = ResolvedContext(["python", "!python-2.6"], + package_orderers=orderers) + resolved = [x.qualified_package_name for x in r.resolved_packages] + self.assertEqual(resolved, ["python-2.5.2"]) + + # make sure serializing the orderer works + file = os.path.join(self.root, "test_orderers.rxt") + r.save(file) + r2 = ResolvedContext.load(file) + self.assertEqual(r, r2) + self.assertEqual(r.package_orderers, r2.package_orderers) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 37790e21e..3f1001fbe 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -326,11 +326,24 @@ def test_9(self): to_pod, from_pod def _test(orderer, package_name, expected_order): + from rez.vendor import simplejson + from rez.utils.yaml import dump_yaml + from rez.vendor import yaml + it = iter_packages(package_name) descending = sorted(it, key=lambda x: x.version, reverse=True) pod = to_pod(orderer) - orderer2 = from_pod(pod) + + # ResolvedContext.write_to_buffer will require conversion to both + # json and yaml, so test both + as_json = simplejson.dumps(pod) + from_json = simplejson.loads(as_json) + as_yaml = dump_yaml(pod) + from_yaml = yaml.load(as_yaml) + self.assertEqual(from_yaml, from_json) + + orderer2 = from_pod(from_yaml) for orderer_ in (orderer, orderer2): ordered = orderer_.reorder(descending) From f241b3fd0df4584ed310bb43af6669aaa0cf2e3b Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:02:33 -0700 Subject: [PATCH 03/10] always use orderer dict lookup by family for speed --- src/rez/package_order.py | 164 ++++++++++++++++++++++++++------- src/rez/resolved_context.py | 8 +- src/rez/resolver.py | 4 +- src/rez/solver.py | 19 ++-- src/rez/tests/test_context.py | 6 +- src/rez/tests/test_packages.py | 53 +++++++++-- 6 files changed, 191 insertions(+), 63 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 129bc5e40..d0ad5ad3e 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -1,8 +1,10 @@ from inspect import isclass from hashlib import sha1 +import collections from rez.vendor.version.version import Version +DEFAULT_TOKEN = "" class PackageOrder(object): """Package reorderer base class.""" @@ -17,12 +19,6 @@ def reorder(self, iterable, key=None): You can safely assume that the packages referred to by `iterable` are all versions of the same package family. - Note: - Returning None, and an unchanged `iterable` list, are not the same - thing. Returning None may cause rez to pass the package list to the - next orderer; whereas a package list that has been reordered (even - if the unchanged list is returned) is not passed onto another orderer. - Args: iterable: Iterable list of packages, or objects that contain packages. key (callable): Callable, where key(iterable) gives a `Package`. If @@ -33,9 +29,32 @@ def reorder(self, iterable, key=None): """ raise NotImplementedError + # override this if you have an orderer that doesn't store packages + # in the standard way + @property + def packages(self): + """Returns an iterable over the list of package family names that this + order applies to + + Returns: + (iterable(str|None)) Package families that this orderer is used for + """ + return self._packages + + @packages.setter + def packages(self, packages): + if isinstance(packages, basestring): + self._packages = [packages] + else: + self._packages = sorted(packages) + def to_pod(self): raise NotImplementedError + @classmethod + def from_pod(self, data): + raise NotImplementedError + @property def sha1(self): return sha1(repr(self)).hexdigest() @@ -56,23 +75,29 @@ class NullPackageOrder(PackageOrder): """ name = "no_order" + def __init__(self, packages): + self.packages = packages + def reorder(self, iterable, key=None): return list(iterable) def __str__(self): - return "{}" + return str(self.packages) def to_pod(self): """ Example (in yaml): type: no_order + packages: ["foo"] """ - return {} + return { + "packages": self.packages, + } @classmethod def from_pod(cls, data): - return cls() + return cls(packages=data["packages"]) class SortedOrder(PackageOrder): @@ -80,7 +105,8 @@ class SortedOrder(PackageOrder): """ name = "sorted" - def __init__(self, descending): + def __init__(self, packages, descending): + self.packages = packages self.descending = descending def reorder(self, iterable, key=None): @@ -89,29 +115,39 @@ def reorder(self, iterable, key=None): reverse=self.descending) def __str__(self): - return str(self.descending) + return str((self.packages, self.descending)) def to_pod(self): """ Example (in yaml): type: sorted + packages: ["foo", "bar"] descending: true """ - return {"descending": self.descending} + return { + "packages": self.packages, + "descending": self.descending + } @classmethod def from_pod(cls, data): - return cls(descending=data["descending"]) + return cls(packages=data["packages"], descending=data["descending"]) class PerFamilyOrder(PackageOrder): - """An orderer that applies different orderers to different package families. + """WARNING: this orderer is DEPRECATED. It was implemented for performance + reasons that are no longer required! + + An orderer that applies different orderers to different package families. """ name = "per_family" def __init__(self, order_dict, default_order=None): - """Create a reorderer. + """WARNING: this orderer is DEPRECATED. It was implemented for + performance reasons that are no longer required! + + Create a reorderer. Args: order_dict (dict of (str, `PackageOrder`): Orderers to apply to @@ -119,7 +155,14 @@ def __init__(self, order_dict, default_order=None): default_order (`PackageOrder`): Orderer to apply to any packages not specified in `order_dict`. """ + import warnings + warnings.warn("The %s orderer is deprecated - it was implemented for" + " performance reasons that are no longer required" + % type(self).__name__) + self.order_dict = order_dict.copy() + if default_order is None: + default_order = NullPackageOrder(DEFAULT_TOKEN) self.default_order = default_order def reorder(self, iterable, key=None): @@ -131,14 +174,13 @@ def reorder(self, iterable, key=None): key = key or (lambda x: x) package = key(item) - orderer = self.order_dict.get(package.name) - if orderer is None: - orderer = self.default_order - if orderer is None: - return None - + orderer = self.order_dict.get(package.name, self.default_order) return orderer.reorder(iterable, key) + @property + def packages(self): + return iter(self.order_dict) + def __str__(self): items = sorted((x[0], str(x[1])) for x in self.order_dict.items()) return str((items, str(self.default_order))) @@ -189,10 +231,9 @@ def from_pod(cls, data): for d in data["orderers"]: d = d.copy() - fams = d.pop("packages") orderer = from_pod(d) - for fam in fams: + for fam in orderer.packages: order_dict[fam] = orderer d = data.get("default_order") @@ -210,12 +251,14 @@ class VersionSplitPackageOrder(PackageOrder): """ name = "version_split" - def __init__(self, first_version): + def __init__(self, packages, first_version): """Create a reorderer. Args: + packages: (str or list of str): packages that this orderer should apply to first_version (`Version`): Start with versions <= this value. """ + self.packages = packages self.first_version = first_version def reorder(self, iterable, key=None): @@ -241,20 +284,23 @@ def reorder(self, iterable, key=None): return below + above def __str__(self): - return str(self.first_version) + return str((self.packages, self.first_version)) def to_pod(self): """ Example (in yaml): type: version_split + packages: ["foo", "bar"] first_version: "3.0.0" """ - return dict(first_version=str(self.first_version)) + return dict(packages=self.packages, + first_version=str(self.first_version)) @classmethod def from_pod(cls, data): - return cls(Version(data["first_version"])) + return cls(packages=data["packages"], + first_version=Version(data["first_version"])) class TimestampPackageOrder(PackageOrder): @@ -296,15 +342,17 @@ class TimestampPackageOrder(PackageOrder): """ name = "soft_timestamp" - def __init__(self, timestamp, rank=0): + def __init__(self, packages, timestamp, rank=0): """Create a reorderer. Args: + packages: (str or list of str): packages that this orderer should apply to timestamp (int): Epoch time of timestamp. Packages before this time are preferred. rank (int): If non-zero, allow version changes at this rank or above past the timestamp. """ + self.packages = packages self.timestamp = timestamp self.rank = rank @@ -324,8 +372,9 @@ def reorder(self, iterable, key=None): else: break - if first_after is None: # all packages are before T - return None + if first_after is None: + # all packages are before T, just use version descending + return descending before = descending[first_after + 1:] after = list(reversed(descending[:first_after + 1])) @@ -368,23 +417,58 @@ def reorder(self, iterable, key=None): return before + after_ def __str__(self): - return str((self.timestamp, self.rank)) + return str((self.packages, self.timestamp, self.rank)) def to_pod(self): """ Example (in yaml): type: soft_timestamp + packages: ["foo", "bar"] timestamp: 1234567 rank: 3 """ - return dict(timestamp=self.timestamp, + return dict(packages=self.packages, + timestamp=self.timestamp, rank=self.rank) @classmethod def from_pod(cls, data): - return cls(timestamp=data["timestamp"], - rank=data["rank"]) + return cls(packages=data["packages"], + timestamp=data["timestamp"], + rank=data.get("rank", 0)) + + +class OrdererDict(collections.Mapping): + def __init__(self, orderer_list): + self.list = [] + self.by_package = {} + + for orderer in orderer_list: + if not isinstance(orderer, PackageOrder): + orderer = from_pod(orderer) + self.list.append(orderer) + for package in orderer.packages: + # We allow duplicates (so we can have hierarchical configs, + # which can override each other) - earlier orderers win + if package in self.by_package: + continue + self.by_package[package] = orderer + + def to_pod(self): + return [to_pod(x) for x in self.list] + + def __repr__(self): + return '%s(%r)' % (type(self).__name__, self.list) + + def __getitem__(self, package): + return self.by_package[package] + + def __iter__(self): + return iter(self.by_package) + + def __len__(self): + return len(self.by_package) def to_pod(orderer): @@ -417,6 +501,18 @@ def register_orderer(cls): return False +def get_orderer(package_name, orderers=None): + if not orderers: + orderers = {} + found_orderer = orderers.get(package_name) + if found_orderer is None: + found_orderer = orderers.get(DEFAULT_TOKEN) + if found_orderer is None: + # default ordering is version descending + found_orderer = SortedOrder([DEFAULT_TOKEN], descending=True) + return found_orderer + + # registration of builtin orderers _orderers = {} for o in globals().values(): diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index 52abbe500..ac2a5487f 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -1252,8 +1252,10 @@ def to_dict(self): serialize_version = '.'.join(str(x) for x in ResolvedContext.serialize_version) patch_locks = dict((k, v.name) for k, v in self.patch_locks) - package_orderers_list = [package_order.to_pod(x) - for x in (self.package_orderers or [])] + if self.package_orderers: + package_orderers_list = self.package_orderers.to_pod() + else: + package_orderers_list = None if self.graph_string and self.graph_string.startswith('{'): graph_str = self.graph_string # already in compact format @@ -1400,7 +1402,7 @@ def _print_version(value): data = d.get("package_orderers") if data: - r.package_orderers = [package_order.from_pod(x) for x in data] + r.package_orderers = package_order.OrdererDict(data) else: r.package_orderers = None diff --git a/src/rez/resolver.py b/src/rez/resolver.py index f7ce98b01..4f68a5feb 100644 --- a/src/rez/resolver.py +++ b/src/rez/resolver.py @@ -50,6 +50,8 @@ def __init__(self, context, package_requests, package_paths, package_filter=None caching: If True, cache(s) may be used to speed the resolve. If False, caches will not be used. """ + from rez.package_order import OrdererDict + self.context = context self.package_requests = package_requests self.package_paths = package_paths @@ -64,7 +66,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None # store hash of package orderers. This is used in the memcached key if package_orderers: - sha1s = ''.join(x.sha1 for x in package_orderers) + sha1s = ''.join(x.sha1 for x in package_orderers.itervalues()) self.package_orderers_hash = sha1(sha1s).hexdigest() else: self.package_orderers_hash = '' diff --git a/src/rez/solver.py b/src/rez/solver.py index ca15000c2..6fbb2e86e 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -891,25 +891,18 @@ def sort_versions(self): The order is typically descending, but package order functions can change this. """ + from rez.package_order import get_orderer + if self.sorted: return - for orderer in (self.solver.package_orderers or []): - entries = orderer.reorder(self.entries, key=lambda x: x.package) - if entries is not None: - self.entries = entries - self.sorted = True - - if self.pr: - self.pr("sorted: %s packages: %s", self.package_name, repr(orderer)) - return - - # default ordering is version descending - self.entries = sorted(self.entries, key=lambda x: x.version, reverse=True) + orderer = get_orderer(self.package_name, + self.solver.package_orderers or {}) + self.entries = orderer.reorder(self.entries, key=lambda x: x.package) self.sorted = True if self.pr: - self.pr("sorted: %s packages: version descending", self.package_name) + self.pr("sorted: %s packages: %s", self.package_name, repr(orderer)) def dump(self): print self.package_name diff --git a/src/rez/tests/test_context.py b/src/rez/tests/test_context.py index 647257362..85aeeb16e 100644 --- a/src/rez/tests/test_context.py +++ b/src/rez/tests/test_context.py @@ -65,13 +65,15 @@ def test_serialize(self): def test_orderer(self): """Test a resolve with an orderer""" - from rez.package_order import VersionSplitPackageOrder + from rez.package_order import VersionSplitPackageOrder, OrdererDict from rez.vendor.version.version import Version path = os.path.dirname(__file__) packages_path = os.path.join(path, "data", "solver", "packages") config.override("packages_path", [packages_path]) - orderers = [VersionSplitPackageOrder(first_version=Version("2.6"))] + orderers = OrdererDict([ + VersionSplitPackageOrder(packages=["python"], + first_version=Version("2.6"))]) r = ResolvedContext(["python", "!python-2.6"], package_orderers=orderers) resolved = [x.qualified_package_name for x in r.resolved_packages] diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 3f1001fbe..8e69ecc2b 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -323,7 +323,7 @@ def test_9(self): """test package orderers.""" from rez.package_order import NullPackageOrder, PerFamilyOrder, \ VersionSplitPackageOrder, TimestampPackageOrder, SortedOrder, \ - to_pod, from_pod + to_pod, from_pod, get_orderer, OrdererDict def _test(orderer, package_name, expected_order): from rez.vendor import simplejson @@ -350,29 +350,62 @@ def _test(orderer, package_name, expected_order): result = [str(x.version) for x in ordered] self.assertEqual(result, expected_order) - null_orderer = NullPackageOrder() - split_orderer = VersionSplitPackageOrder(Version("2.6.0")) - timestamp_orderer = TimestampPackageOrder(timestamp=3001, rank=3) + null_orderer = NullPackageOrder("pysplit") + split1_orderer = VersionSplitPackageOrder("python", Version("2.6.0")) + # test when split version is between actual versions + # (also tests that multiple orderers of same type, but different + # settings, are handled correctly) + split2_orderer = VersionSplitPackageOrder("multi", Version("1.3")) + # test when split version is > all versions + split3_orderer = VersionSplitPackageOrder("pydad", Version("5")) + timestamp_orderer = TimestampPackageOrder("timestamped", + timestamp=3001, rank=3) + default_orderer = SortedOrder("", descending=False) expected_null_result = ["7", "6", "5"] - expected_split_result = ["2.6.0", "2.5.2", "2.7.0", "2.6.8"] + expected_split1_result = ["2.6.0", "2.5.2", "2.7.0", "2.6.8"] + expected_split2_result = ["1.2", "1.1", "1.0", "2.0"] + expected_split3_result = ["3", "2", "1"] expected_timestamp_result = ["1.1.1", "1.1.0", "1.0.6", "1.0.5", "1.2.0", "2.0.0", "2.1.5", "2.1.0"] + expected_default_result = ["1", "2", "3"] _test(null_orderer, "pysplit", expected_null_result) - _test(split_orderer, "python", expected_split_result) + _test(split1_orderer, "python", expected_split1_result) + _test(split2_orderer, "multi", expected_split2_result) + _test(split3_orderer, "pydad", expected_split3_result) _test(timestamp_orderer, "timestamped", expected_timestamp_result) + _test(default_orderer, "pymum", expected_default_result) fam_orderer = PerFamilyOrder( order_dict=dict(pysplit=null_orderer, - python=split_orderer, + python=split1_orderer, + multi=split2_orderer, + pydad=split3_orderer, timestamped=timestamp_orderer), - default_order=SortedOrder(descending=False)) + default_order=default_orderer) _test(fam_orderer, "pysplit", expected_null_result) - _test(fam_orderer, "python", expected_split_result) + _test(fam_orderer, "python", expected_split1_result) + _test(fam_orderer, "multi", expected_split2_result) + _test(fam_orderer, "pydad", expected_split3_result) _test(fam_orderer, "timestamped", expected_timestamp_result) - _test(fam_orderer, "pymum", ["1", "2", "3"]) + _test(fam_orderer, "pymum", expected_default_result) + + orderers = OrdererDict([null_orderer, split1_orderer, split2_orderer, + split3_orderer, timestamp_orderer, + default_orderer]) + + def _test_orderer_dict(orderer_dict, package_name, expected_order): + orderer = get_orderer(package_name, orderer_dict) + _test(orderer, package_name, expected_order) + + _test_orderer_dict(orderers, "pysplit", expected_null_result) + _test_orderer_dict(orderers, "python", expected_split1_result) + _test_orderer_dict(orderers, "multi", expected_split2_result) + _test_orderer_dict(orderers, "pydad", expected_split3_result) + _test_orderer_dict(orderers, "timestamped", expected_timestamp_result) + _test_orderer_dict(orderers, "pymum", expected_default_result) class TestMemoryPackages(TestBase): From 994303878e6ae01b31efbec59c48bd821a28863e Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:02:37 -0700 Subject: [PATCH 04/10] ability to configure package orderers in rezconfig --- src/rez/config.py | 11 + src/rez/package_order.py | 3 + src/rez/resolved_context.py | 2 +- src/rez/resolver.py | 2 +- src/rez/rezconfig.py | 25 ++ src/rez/solver.py | 2 +- .../tests/data/solver/packages/reorderable.py | 46 +++ src/rez/tests/test_completion.py | 2 +- src/rez/tests/test_packages.py | 4 + src/rez/tests/test_solver.py | 318 +++++++++++++++++- 10 files changed, 397 insertions(+), 18 deletions(-) create mode 100644 src/rez/tests/data/solver/packages/reorderable.py diff --git a/src/rez/config.py b/src/rez/config.py index 99103daea..c9936fb31 100644 --- a/src/rez/config.py +++ b/src/rez/config.py @@ -223,6 +223,16 @@ def _parse_env_var(self, value): return value +class PackageOrderers(Setting): + @cached_class_property + def schema(cls): + from rez.package_order import from_pod, OrdererDict + return Or(Schema(And([Use(from_pod)], + Use(OrdererDict))), None) + + _env_var_name = None + + config_schema = Schema({ "packages_path": PathList, "plugin_path": PathList, @@ -335,6 +345,7 @@ def _parse_env_var(self, value): "env_var_separators": Dict, "variant_select_mode": VariantSelectMode_, "package_filter": OptionalDictOrDictList, + "package_orderers": PackageOrderers, "new_session_popen_args": OptionalDict, # GUI settings diff --git a/src/rez/package_order.py b/src/rez/package_order.py index d0ad5ad3e..54edc1866 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -502,6 +502,9 @@ def register_orderer(cls): def get_orderer(package_name, orderers=None): + from rez.config import config + if orderers is None: + orderers = config.package_orderers if not orderers: orderers = {} found_orderer = orderers.get(package_name) diff --git a/src/rez/resolved_context.py b/src/rez/resolved_context.py index ac2a5487f..6ba14d5d3 100644 --- a/src/rez/resolved_context.py +++ b/src/rez/resolved_context.py @@ -198,7 +198,7 @@ def __init__(self, package_requests, verbosity=0, timestamp=None, self.package_filter = (PackageFilterList.singleton if package_filter is None else package_filter) - self.package_orderers = package_orderers + self.package_orderers = package_orderers or config.package_orderers # patch settings self.default_patch_lock = PatchLock.no_lock diff --git a/src/rez/resolver.py b/src/rez/resolver.py index 4f68a5feb..69d8195b7 100644 --- a/src/rez/resolver.py +++ b/src/rez/resolver.py @@ -57,7 +57,7 @@ def __init__(self, context, package_requests, package_paths, package_filter=None self.package_paths = package_paths self.timestamp = timestamp self.callback = callback - self.package_orderers = package_orderers + self.package_orderers = package_orderers or config.package_orderers self.package_load_callback = package_load_callback self.building = building self.verbosity = verbosity diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index 59dd6e905..fdf935d40 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -285,6 +285,31 @@ # foo-5+ | Same as range(foo-5+) package_filter = None +# Package orderers. One or more objects which can re-order the package's +# priority when resolving - ie, if we know that a group of package versions +# can all satisfy a request, this can affect which of those package versions is +# returned in the resolved context. +# You can specify multiple orderers; each orderer in turn is tested against +# a given package to see if it applies to it (usually this is controlled through +# the "packages" config item in the orderer's config). The first orderer that +# applies is then used to reorder that package's versions. +# +# Here's an example: +# +# package_orderers: +# - type: soft_timestamp +# packages: ["gcc", "foo"] +# timestamp: 1429830188 +# - type: sorted +# packages: ["baz"] +# descending: False +# - type: version_split +# packages: ['bar', 'bah'] +# first_version: '4.0.5' + + +package_orderers = None + # If True, unversioned packages are allowed. Solve times are slightly better if # this value is False. allow_unversioned_packages = True diff --git a/src/rez/solver.py b/src/rez/solver.py index 6fbb2e86e..b1c25f34f 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -1782,7 +1782,7 @@ def __init__(self, package_requests, package_paths, context=None, """ self.package_paths = package_paths self.package_filter = package_filter - self.package_orderers = package_orderers + self.package_orderers = package_orderers or config.package_orderers self.pr = _Printer(verbosity, buf=buf) self.optimised = optimised self.callback = callback diff --git a/src/rez/tests/data/solver/packages/reorderable.py b/src/rez/tests/data/solver/packages/reorderable.py new file mode 100644 index 000000000..3b3e27c3c --- /dev/null +++ b/src/rez/tests/data/solver/packages/reorderable.py @@ -0,0 +1,46 @@ +name = 'reorderable' + +versions = [ + "3.1.1", + "3.0.0", + "2.2.1", + "2.2.0", + "2.1.5", + "2.1.1", + "2.1.0", + "2.0.6", + "2.0.5", + "2.0.0", + "1.9.1", + "1.9.0", +] + +# Note - we've intentionally left out timestamps for 2.2.0 and 1.9.1 to +# make sure the system still works +version_overrides = { + "3.1.1": {"timestamp": 1470728488}, + "3.0.0": {"timestamp": 1470728486}, + "2.1.5": {"timestamp": 1470728484}, + "2.2.1": {"timestamp": 1470728482}, + "2.1.1": {"timestamp": 1470728480}, + "2.1.0": {"timestamp": 1470728478}, + "2.0.6": {"timestamp": 1470728476}, + "2.0.5": {"timestamp": 1470728474}, + "2.0.0": {"timestamp": 1470728472}, + "1.9.0": {"timestamp": 1470728470}, +} + +# Copyright 2013-2016 Allan Johns. +# +# This library is free software: you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation, either +# version 3 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . diff --git a/src/rez/tests/test_completion.py b/src/rez/tests/test_completion.py index b23ae1d88..08c97e076 100644 --- a/src/rez/tests/test_completion.py +++ b/src/rez/tests/test_completion.py @@ -48,7 +48,7 @@ def _eq(prefix, expected_completions): _eq("zzz", []) _eq("", ["bahish", "nada", "nopy", "pybah", "pydad", "pyfoo", "pymum", - "pyodd", "pyson", "pysplit", "python", "pyvariants"]) + "pyodd", "pyson", "pysplit", "python", "pyvariants", "reorderable"]) _eq("py", ["pybah", "pydad", "pyfoo", "pymum", "pyodd", "pyson", "pysplit", "python", "pyvariants"]) _eq("pys", ["pyson", "pysplit"]) diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index 8e69ecc2b..cf31808b1 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -30,6 +30,10 @@ 'pysplit-5', 'pysplit-6', 'pysplit-7', 'python-2.5.2', 'python-2.6.0', 'python-2.6.8', 'python-2.7.0', 'pyvariants-2', + 'reorderable-1.9.0', 'reorderable-1.9.1', 'reorderable-2.0.0', + 'reorderable-2.0.5', 'reorderable-2.0.6', 'reorderable-2.1.0', + 'reorderable-2.1.1', 'reorderable-2.1.5', 'reorderable-2.2.0', + 'reorderable-2.2.1', 'reorderable-3.0.0', 'reorderable-3.1.1', # packages from data/packages/py_packages and .../yaml_packages 'unversioned', 'unversioned_py', diff --git a/src/rez/tests/test_solver.py b/src/rez/tests/test_solver.py index ce2d394c5..31d883f29 100644 --- a/src/rez/tests/test_solver.py +++ b/src/rez/tests/test_solver.py @@ -4,6 +4,7 @@ from rez.vendor.version.requirement import Requirement from rez.solver import Solver, Cycle, SolverStatus from rez.config import config +from rez.exceptions import ConfigurationError import rez.vendor.unittest2 as unittest from rez.tests.util import TestBase import itertools @@ -20,7 +21,7 @@ def setUpClass(cls): packages_path=cls.packages_path, package_filter=None) - def _create_solvers(self, reqs): + def _create_solvers(self, reqs, do_permutations=True): s1 = Solver(reqs, self.packages_path, optimised=True, @@ -31,20 +32,22 @@ def _create_solvers(self, reqs): verbosity=Solver.max_verbosity) s_perms = [] - perms = itertools.permutations(reqs) - for reqs_ in perms: - s = Solver(reqs_, - self.packages_path, - optimised=True, - verbosity=Solver.max_verbosity) - s_perms.append(s) + if do_permutations: + perms = itertools.permutations(reqs) + for reqs_ in perms: + s = Solver(reqs_, + self.packages_path, + optimised=True, + verbosity=Solver.max_verbosity) + s_perms.append(s) return (s1, s2, s_perms) - def _solve(self, packages, expected_resolve): + def _solve(self, packages, expected_resolve, do_permutations=True): print reqs = [Requirement(x) for x in packages] - s1, s2, s_perms = self._create_solvers(reqs) + s1, s2, s_perms = self._create_solvers(reqs, + do_permutations=do_permutations) s1.solve() self.assertEqual(s1.status, SolverStatus.solved) @@ -62,10 +65,11 @@ def _solve(self, packages, expected_resolve): resolve2 = [str(x) for x in s2.resolved_packages] self.assertEqual(resolve2, resolve) - print "checking that permutations also succeed..." - for s in s_perms: - s.solve() - self.assertEqual(s.status, SolverStatus.solved) + if do_permutations: + print "checking that permutations also succeed..." + for s in s_perms: + s.solve() + self.assertEqual(s.status, SolverStatus.solved) return s1 @@ -212,6 +216,292 @@ def test_10_intersection_priority_mode(self): self._solve(["pyvariants", "python", "nada"], ["python-2.6.8[]", "nada[]", "pyvariants-2[1]"]) + # re-prioritization tests + + def test_11_reversed_str(self): + """Test setting a package to reversed-version sorting + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["python"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.7.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.5.2"], + ["python-2.6.0[]"]) + self._solve(["python-2.6"], + ["python-2.6.0[]"]) + self._solve(["python-2.6+<2.7"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + def test_12_reversed_list(self): + """Test setting a package to reversed-version sorting + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": ["python"]}]) + self._solve(["python"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.7.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python-2.5.2"], + ["python-2.6.0[]"]) + self._solve(["python-2.6"], + ["python-2.6.0[]"]) + self._solve(["python-2.6+<2.7"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + def test_13_reversed_is_requirement(self): + """Test setting a package to reversed-version sorting, when it is a + requirement + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["pyfoo"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + self._solve(["pyfoo-3.0"], + ["python-2.5.2[]", "pyfoo-3.0.0[]"]) + self._solve(["pyfoo-3.1"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + self._solve(["pybah"], + ["python-2.5.2[]", "pybah-5[]"]) + self._solve(["pybah-4"], + ["python-2.6.0[]", "pybah-4[]"]) + self._solve(["pybah-5"], + ["python-2.5.2[]", "pybah-5[]"]) + + + def test_14_reversed_has_requirement(self): + """Test setting a package to reversed-version sorting, when it has a + requirement + """ + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": ["pyfoo", "pybah"]}]) + self._solve(["pyfoo"], + ["python-2.5.2[]", "pyfoo-3.0.0[]"]) + self._solve(["pyfoo-3.0"], + ["python-2.5.2[]", "pyfoo-3.0.0[]"]) + self._solve(["pyfoo-3.1"], + ["python-2.6.8[]", "pyfoo-3.1.0[]"]) + self._solve(["pybah"], + ["python-2.6.8[]", "pybah-4[]"]) + self._solve(["pybah-4"], + ["python-2.6.8[]", "pybah-4[]"]) + self._solve(["pybah-5"], + ["python-2.5.2[]", "pybah-5[]"]) + + def _test_complete_ordering(self, request, expected_order): + exclude = [] + for next in expected_order: + self._solve(request + exclude, [next + '[]'], + do_permutations=False) + exclude.append('!{}'.format(next)) + + def test_15_timestamp_no_rank_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728472, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.5", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + def test_16_timestamp_no_rank_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728473, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.5", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + ]) + + + def test_17_timestamp_rank2_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 2, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-3.1.1", + "reorderable-3.0.0", + ]) + + + def test_18_timestamp_rank2_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 2, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-3.1.1", + "reorderable-3.0.0", + ]) + + def test_19_timestamp_rank3_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 3, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_20_timestamp_rank3_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 3, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.6", + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.1.5", + "reorderable-2.1.1", + "reorderable-2.1.0", + "reorderable-2.2.1", + "reorderable-2.2.0", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_21_timestamp_rank4_exact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728474, + "rank": 4, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + + def test_22_timestamp_rank4_inexact_timestamp(self): + config.override("package_orderers", + [{"type": "soft_timestamp", + "packages": ["reorderable"], + "timestamp": 1470728475, + "rank": 4, + }]) + self._test_complete_ordering( + ['reorderable'], + [ + "reorderable-2.0.5", + "reorderable-2.0.0", + "reorderable-1.9.1", + "reorderable-1.9.0", + "reorderable-2.0.6", + "reorderable-2.1.0", + "reorderable-2.1.1", + "reorderable-2.1.5", + "reorderable-2.2.0", + "reorderable-2.2.1", + "reorderable-3.0.0", + "reorderable-3.1.1", + ]) + + if __name__ == '__main__': unittest.main() From 5936abdb7a45666b5f22667f53f708b47d76b892 Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:03:18 -0700 Subject: [PATCH 05/10] add CustomPackageOrder --- src/rez/package_order.py | 183 +++++++++++++++++++++++++++++++++ src/rez/rezconfig.py | 4 + src/rez/tests/test_solver.py | 189 +++++++++++++++++++++++++++++++++++ 3 files changed, 376 insertions(+) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 54edc1866..68c62e491 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -3,9 +3,11 @@ import collections from rez.vendor.version.version import Version +from rez.exceptions import ConfigurationError DEFAULT_TOKEN = "" + class PackageOrder(object): """Package reorderer base class.""" name = None @@ -439,6 +441,187 @@ def from_pod(cls, data): rank=data.get("rank", 0)) +class CustomPackageOrder(PackageOrder): + """A package order that allows explicit specification of version ordering. + + Specified through the "packages" attributes, which should be a dict which + maps from a package family name to a list of version ranges to prioritize, + in decreasing priority order. + + As an example, consider a package splunge which has versions: + + [1.0, 1.1, 1.2, 1.4, 2.0, 2.1, 3.0, 3.2] + + By default, version priority is given to the higest version, so version + priority, from most to least preferred, is: + + [3.2, 3.0, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + + However, if you set a custom package order like this: + + package_orderers: + - type: custom + packages: + splunge: ['2', '1.1+<1.4'] + + Then the preferred versions, from most to least preferred, will be: + [2.1, 2.0, 1.2, 1.1, 3.2, 3.0, 1.4, 1.0] + + Any version which does not match any of these expressions are sorted in + decreasing version order (like normal) and then appended to this list (so they + have lower priority). This provides an easy means to effectively set a + "default version." So if you do: + + package_orderers: + - type: custom + packages: + splunge: ['3.0'] + + resulting order is: + + [3.0, 3.2, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + + You may also include a single False or empty string in the list, in which case + all "other" versions will be placed at that spot. ie + + package_orderers: + - type: custom + packages: + splunge: ['', '3+'] + + yields: + + [2.1, 2.0, 1.4, 1.2, 1.1, 1.0, 3.2, 3.0] + + Note that you could also have gotten the same result by doing: + + package_orderers: + - type: custom + packages: + splunge: ['<3'] + + If a version matches more than one range expression, it will be placed at + the highest-priority matching spot, so: + + package_orderers: + - type: custom + packages: + splunge: ['1.2+<=2.0', '1.1+<3'] + + gives: + [2.0, 1.4, 1.2, 2.1, 1.1, 3.2, 3.0, 1.0] + + Also note that this does not change the version sort order for any purpose but + determining solving priorities - for instance, even if version priorities is: + + package_orderers: + - type: custom + packages: + splunge: [2, 3, 1] + + The expression splunge-1+<3 would still match version 2. + """ + name = "custom" + + def __init__(self, packages): + """Create a reorderer. + + Args: + packages: (dict from str to list of VersionRange): packages that + this orderer should apply to, and the version priority ordering + for that package + """ + self.packages_dict = self._packages_from_pod(packages) + self._version_key_cache = {} + + def reorder(self, iterable, key=None): + key = key or (lambda x: x) + + def sort_key(x): + package = key(x) + return self.version_priority_key_cached(package.name, + package.version) + + return sorted(iterable, key=sort_key, reverse=True) + + @property + def packages(self): + return iter(self.packages_dict) + + def __str__(self): + return str(self.packages_dict) + + def version_priority_key_cached(self, package_name, version): + family_cache = self._version_key_cache.setdefault(package_name, {}) + key = family_cache.get(version) + if key is not None: + return key + + key = self.version_priority_key_uncached(package_name, version) + family_cache[version] = key + return key + + def version_priority_key_uncached(self, package_name, version): + version_priorities = self.packages_dict[package_name] + + default_key = -1 + for sort_order_index, range in enumerate(version_priorities): + # in the config, version_priorities are given in decreasing + # priority order... however, we want a sort key that sorts in the + # same way that versions do - where higher values are higher + # priority - so we need to take the inverse of the index + priority_sort_key = len(version_priorities) - sort_order_index + if range in (False, ""): + if default_key != -1: + raise ValueError("version_priorities may only have one " + "False / empty value") + default_key = priority_sort_key + continue + if range.contains_version(version): + break + else: + # For now, we're permissive with the version_sort_order - it may + # contain ranges which match no actual versions, and if an actual + # version matches no entry in the version_sort_order, it is simply + # placed after other entries + priority_sort_key = default_key + return priority_sort_key, version + + @classmethod + def _packages_to_pod(cls, packages): + return dict((package, [str(v) for v in versions]) + for (package, versions) in packages.iteritems()) + + @classmethod + def _packages_from_pod(cls, packages): + from rez.vendor.version.version import VersionRange + parsed_dict = {} + for package, versions in packages.iteritems(): + new_versions = [] + numFalse = 0 + for v in versions: + if v in ("", False): + v = False + numFalse += 1 + else: + if not isinstance(v, VersionRange): + if isinstance(v, (int, float)): + v = str(v) + v = VersionRange(v) + new_versions.append(v) + if numFalse > 1: + raise ConfigurationError("version_priorities for CustomPackageOrder may only have one False / empty value") + parsed_dict[package] = new_versions + return parsed_dict + + def to_pod(self): + return dict(packages=self._packages_to_pod(self.packages_dict)) + + @classmethod + def from_pod(cls, data): + return cls(packages=data["packages"]) + + class OrdererDict(collections.Mapping): def __init__(self, orderer_list): self.list = [] diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index fdf935d40..d638a4cc4 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -297,6 +297,10 @@ # Here's an example: # # package_orderers: +# - type: custom +# packages: +# gcc: ['3.7', '2.8'] +# foo: ['2.6', '2.8'] # - type: soft_timestamp # packages: ["gcc", "foo"] # timestamp: 1429830188 diff --git a/src/rez/tests/test_solver.py b/src/rez/tests/test_solver.py index 31d883f29..990fb9e98 100644 --- a/src/rez/tests/test_solver.py +++ b/src/rez/tests/test_solver.py @@ -502,6 +502,195 @@ def test_22_timestamp_rank4_inexact_timestamp(self): ]) + def test_23_direct_complete(self): + """Test setting of the version_priority in simple situations, where + the altered package is a direct request + """ + # test a complete ordering + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "2.5.2", "2.7.0", "2.6.8"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.5.2[]"]) + self._solve(["python", "!python<=2.6.0"], + ["python-2.7.0[]"]) + self._solve(["python", "!python-2.6.0", "!python-2.5.2", + "!python-2.7.0"], + ["python-2.6.8[]"]) + + # check that we can still request a lower-priority version + self._solve(["python-2.6.8"], + ["python-2.6.8[]"]) + + + def test_24_direct_single(self): + """check that if you specify only one version, that version is highest + priority, rest are normal + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.8"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.7.0[]"]) + self._solve(["python<2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python<2.6"], + ["python-2.5.2[]"]) + + # confirm that sorting for version ranges is still normal + self._solve(["python-2.6+<2.7"], + ["python-2.6.8[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + + def test_25_empty_string(self): + """confirm that we can use empty string to match unmatched versions""" + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["", "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "", "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + + def test_26_false(self): + """confirm that we can use False to match unmatched versions""" + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": [False, "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6.8"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", False, "2.7.0"]}}]) + self._solve(["python"], + ["python-2.6.0[]"]) + self._solve(["python", "!python-2.6.0"], + ["python-2.6.8[]"]) + self._solve(["python", "!python-2.6"], + ["python-2.5.2[]"]) + self._solve(["python>2.6.8"], + ["python-2.7.0[]"]) + + + def test_27_requirement_1_deep(self): + """Test setting of the version_priority for a required package 1 level + deep + """ + # python 2.5 is preferred... + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": [2.5]}}]) + + # # so if we request python directly, we get 2.5... + self._solve(["python"], + ["python-2.5.2[]"]) + + # ...but if we request pyfoo, IT'S version is more important, and we + # get 2.6 + self._solve(["pyfoo"], + ["python-2.6.8[]", "pyfoo-3.1.0[]"]) + + # but if we make specifically python-2.6.0 prioritized, it will be used + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0"]}}]) + self._solve(["pyfoo"], + ["python-2.6.0[]", "pyfoo-3.1.0[]"]) + + + def test_28_requirement_2_deep(self): + """Test setting of the version_priority for a required package 2 + levels deep + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "2.5"]}}]) + self._solve(["pyodd"], + ["python-2.5.2[]", "pybah-5[]", "pyodd-2[]"]) + self._solve(["pyodd-2"], + ["python-2.5.2[]", "pybah-5[]", "pyodd-2[]"]) + self._solve(["pyodd-1"], + ["python-2.6.0[]", "pyfoo-3.1.0[]", "pyodd-1[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"pybah": ["4"], + "pyfoo": ["3.0.0"], + "python": ["2.6.0"]}}]) + self._solve(["pyodd"], + ["python-2.6.0[]", "pybah-4[]", "pyodd-2[]"]) + self._solve(["pyodd-2"], + ["python-2.6.0[]", "pybah-4[]", "pyodd-2[]"]) + self._solve(["pyodd-1"], + ["python-2.5.2[]", "pyfoo-3.0.0[]", "pyodd-1[]"]) + + + def test_29_multiple_false(self): + """Make sure that multiple False / empty values raises an error + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", False, False, + "2.5"]}}]) + self.assertRaises(ConfigurationError, + self._solve, ["python"], ["python-2.6.0[]"]) + + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "", "", + "2.5"]}}]) + self.assertRaises(ConfigurationError, + self._solve, ["python"], ["python-2.6.0[]"]) + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.6.0", "", False, + "2.5"]}}]) + self.assertRaises(ConfigurationError, + self._solve, ["python"], ["python-2.6.0[]"]) + + + def test_30_multiple_matches(self): + """Test that if matches more than one, higher-priority is used + """ + config.override("package_orderers", + [{"type": "custom", + "packages": {"python": ["2.7.0|2.6.8", + "2.5", + "2.6.8|2.6.0"]}}]) + self._solve(["python<2.7"], + ["python-2.6.8[]"]) + + if __name__ == '__main__': unittest.main() From d03f61bd001c7461f54d5becf19f6dce6e555a4c Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:04:28 -0700 Subject: [PATCH 06/10] fix for dumping PackageOrder in config --- src/rez/cli/config.py | 4 ++-- src/rez/package_order.py | 15 +++++++++--- src/rez/tests/test_config.py | 45 ++++++++++++++++++++++++++++++++++-- src/rez/utils/yaml.py | 13 +++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/rez/cli/config.py b/src/rez/cli/config.py index 401015322..002d94f67 100644 --- a/src/rez/cli/config.py +++ b/src/rez/cli/config.py @@ -21,7 +21,7 @@ def setup_parser(parser, completions=False): def command(opts, parser, extra_arg_groups=None): from rez.config import config - from rez.utils.yaml import dump_yaml + from rez.utils.yaml import dump_yaml, YamlDumpable if opts.search_list: for filepath in config.filepaths: @@ -44,7 +44,7 @@ def command(opts, parser, extra_arg_groups=None): except KeyError: raise ValueError("no such setting: %r" % opts.FIELD) - if isinstance(data, (dict, list)): + if isinstance(data, (dict, list, YamlDumpable)): txt = dump_yaml(data).strip() print txt else: diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 68c62e491..8b800a868 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -2,13 +2,14 @@ from hashlib import sha1 import collections -from rez.vendor.version.version import Version from rez.exceptions import ConfigurationError +from rez.utils.yaml import YamlDumpable +from rez.vendor.version.version import Version DEFAULT_TOKEN = "" -class PackageOrder(object): +class PackageOrder(YamlDumpable): """Package reorderer base class.""" name = None @@ -50,6 +51,11 @@ def packages(self, packages): else: self._packages = sorted(packages) + def to_yaml_pod(self): + data = self.to_pod() + data['type'] = self.name + return data + def to_pod(self): raise NotImplementedError @@ -622,7 +628,7 @@ def from_pod(cls, data): return cls(packages=data["packages"]) -class OrdererDict(collections.Mapping): +class OrdererDict(collections.Mapping, YamlDumpable): def __init__(self, orderer_list): self.list = [] self.by_package = {} @@ -638,6 +644,9 @@ def __init__(self, orderer_list): continue self.by_package[package] = orderer + def to_yaml_pod(self): + return self.to_pod() + def to_pod(self): return [to_pod(x) for x in self.list] diff --git a/src/rez/tests/test_config.py b/src/rez/tests/test_config.py index 4b6e7e78b..4dca0cf8b 100644 --- a/src/rez/tests/test_config.py +++ b/src/rez/tests/test_config.py @@ -2,9 +2,10 @@ test configuration settings """ import rez.vendor.unittest2 as unittest -from rez.tests.util import TestBase +from rez.tests.util import TestBase, get_cli_output from rez.exceptions import ConfigurationError -from rez.config import Config, get_module_root_config +from rez.config import Config, get_module_root_config, config,\ + _create_locked_config from rez.system import system from rez.utils.data_utils import RO_AttrDictWrapper from rez.packages_ import get_developer_package @@ -215,6 +216,46 @@ def _data(self): finally: os.environ = old_environ + def test_7_command_line_config_version_priority(self): + """Check that the rez-config command-line tool works when using a + custom package version-priority""" + import rez.vendor.yaml as yaml + + ver_prio_in = [ + {"type": "custom", + "packages": {"foo": ["2"]}}, + {"type": "sorted", + "descending": False, + "packages": ["bar"]}, + {"type": "soft_timestamp", + "timestamp": 1479846074, + "packages": ["baz"]}, + ] + + ver_prio_out = [ + {"type": "custom", + "packages": {"foo": ["2"]}}, + {"type": "sorted", + "descending": False, + "packages": ["bar"]}, + {"type": "soft_timestamp", + "timestamp": 1479846074, + "packages": ["baz"], + "rank": 0}, + ] + + self.update_settings({"package_orderers": ver_prio_in}) + + output, exitcode = get_cli_output(['config']) + self.assertEqual(exitcode, 0) + parsed_out = yaml.load(output) + self.assertEqual(ver_prio_out, parsed_out['package_orderers']) + + output, exitcode = get_cli_output(['config', 'package_orderers']) + self.assertEqual(exitcode, 0) + parsed_out = yaml.load(output) + self.assertEqual(ver_prio_out, parsed_out) + if __name__ == '__main__': unittest.main() diff --git a/src/rez/utils/yaml.py b/src/rez/utils/yaml.py index b34277cf2..ff4006ff7 100644 --- a/src/rez/utils/yaml.py +++ b/src/rez/utils/yaml.py @@ -7,6 +7,15 @@ from types import FunctionType, BuiltinFunctionType from inspect import getsourcelines from textwrap import dedent +from abc import ABCMeta, abstractmethod + + +class YamlDumpable(object): + __metaclass__ = ABCMeta + + @abstractmethod + def to_yaml_pod(self): + raise NotImplementedError class _Dumper(SafeDumper): @@ -52,6 +61,9 @@ def represent_sourcecode(self, data): code = data.source return self.represent_str(code) + def represent_yaml_dumpable(self, data): + return self.represent_data(data.to_yaml_pod()) + _Dumper.add_representer(str, _Dumper.represent_str) _Dumper.add_representer(Version, _Dumper.represent_as_str) @@ -59,6 +71,7 @@ def represent_sourcecode(self, data): _Dumper.add_representer(FunctionType, _Dumper.represent_function) _Dumper.add_representer(BuiltinFunctionType, _Dumper.represent_builtin_function) _Dumper.add_representer(SourceCode, _Dumper.represent_sourcecode) +_Dumper.add_multi_representer(YamlDumpable, _Dumper.represent_yaml_dumpable) def dump_yaml(data, Dumper=_Dumper, default_flow_style=False): From a6b2faecde0dfbe32ffff2452ac0ce4119e6bab8 Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:15:39 -0700 Subject: [PATCH 07/10] reimplement PackageOrder classes to use a sort_key --- src/rez/package_order.py | 231 ++++++++++++++++----------------- src/rez/solver.py | 6 +- src/rez/tests/test_packages.py | 4 +- 3 files changed, 123 insertions(+), 118 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 8b800a868..eff1a3869 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -1,34 +1,35 @@ from inspect import isclass from hashlib import sha1 import collections +from abc import ABCMeta, abstractmethod from rez.exceptions import ConfigurationError from rez.utils.yaml import YamlDumpable -from rez.vendor.version.version import Version +from rez.vendor.version.version import _Comparable, _ReversedComparable, Version DEFAULT_TOKEN = "" class PackageOrder(YamlDumpable): """Package reorderer base class.""" + __metaclass__ = ABCMeta + name = None def __init__(self): pass - def reorder(self, iterable, key=None): - """Put packages into some order for consumption. - - You can safely assume that the packages referred to by `iterable` are - all versions of the same package family. + @abstractmethod + def sort_key(self, package_name, version): + """Returns a sort key usable for sorting these packages within the + same family Args: - iterable: Iterable list of packages, or objects that contain packages. - key (callable): Callable, where key(iterable) gives a `Package`. If - None, iterable is assumed to be a list of `Package` objects. + package_name: (str) The family name of the package we are sorting + verison: (Version) the version object you wish to generate a key for Returns: - List of `iterable` type, reordered. + Comparable object """ raise NotImplementedError @@ -56,6 +57,7 @@ def to_yaml_pod(self): data['type'] = self.name return data + @abstractmethod def to_pod(self): raise NotImplementedError @@ -73,6 +75,11 @@ def __repr__(self): def __eq__(self, other): return type(self) == type(other) and str(self) == str(other) + # need to implement this to avoid infinite recursion! + @abstractmethod + def __str__(self): + raise NotImplementedError + class NullPackageOrder(PackageOrder): """An orderer that does not change the order - a no op. @@ -86,8 +93,10 @@ class NullPackageOrder(PackageOrder): def __init__(self, packages): self.packages = packages - def reorder(self, iterable, key=None): - return list(iterable) + def sort_key(self, package_name, version): + # python's sort will preserve the order of items that compare equal, so + # to not change anything, we just return the same object for all... + return 0 def __str__(self): return str(self.packages) @@ -117,10 +126,18 @@ def __init__(self, packages, descending): self.packages = packages self.descending = descending - def reorder(self, iterable, key=None): - key = key or (lambda x: x) - return sorted(iterable, key=lambda x: key(x).version, - reverse=self.descending) + def sort_key(self, package_name, version): + # Note that the name "descending" can be slightly confusing - it + # indicates that the final ordering this Order gives should be + # version descending (ie, the default) - however, the sort_key itself + # returns it's results in "normal" ascending order (because it needs to + # be used "alongside" normally-sorted objects like versions). + # when the key is passed to sort(), though, it is always invoked with + # reverse=True... + if self.descending: + return version + else: + return _ReversedComparable(version) def __str__(self): return str((self.packages, self.descending)) @@ -173,17 +190,16 @@ def __init__(self, order_dict, default_order=None): default_order = NullPackageOrder(DEFAULT_TOKEN) self.default_order = default_order - def reorder(self, iterable, key=None): - try: - item = iter(iterable).next() - except: - return None - - key = key or (lambda x: x) - package = key(item) - - orderer = self.order_dict.get(package.name, self.default_order) - return orderer.reorder(iterable, key) + def sort_key(self, package_name, version): + orderer = self.order_dict.get(package_name) + if orderer is None: + if self.default_order is not None: + orderer = self.default_order + else: + # shouldn't get here, because applies_to should protect us... + raise RuntimeError("package family orderer %r does not apply to package family %r", + (self, package_name)) + return orderer.sort_key(package_name, version) @property def packages(self): @@ -269,27 +285,9 @@ def __init__(self, packages, first_version): self.packages = packages self.first_version = first_version - def reorder(self, iterable, key=None): - key = key or (lambda x: x) - - # sort by version descending - descending = sorted(iterable, key=lambda x: key(x).version, reverse=True) - - above = [] - below = [] - is_above = True - - for item in descending: - if is_above: - package = key(item) - is_above = (package.version > self.first_version) - - if is_above: - above.append(item) - else: - below.append(item) - - return below + above + def sort_key(self, package_name, version): + priority_key = 1 if version <= self.first_version else 0 + return (priority_key, version) def __str__(self): return str((self.packages, self.first_version)) @@ -364,65 +362,76 @@ def __init__(self, packages, timestamp, rank=0): self.timestamp = timestamp self.rank = rank - def reorder(self, iterable, key=None): - reordered = [] - first_after = None - key = key or (lambda x: x) + # dictionary mapping from package family to the first-version-after + # the given timestamp + self._cached_first_after = {} + self._cached_sort_key = {} + + def get_first_after(self, package_family): + first_after = self._cached_first_after.get(package_family, KeyError) + if first_after is KeyError: + first_after = self._calc_first_after(package_family) + self._cached_first_after[package_family] = first_after + return first_after - # sort by version descending - descending = sorted(iterable, key=lambda x: key(x).version, reverse=True) + def _calc_first_after(self, package_family): + from rez.packages_ import iter_packages + descending = sorted(iter_packages(package_family), + key=lambda p: p.version, + reverse=True) - for i, o in enumerate(descending): - package = key(o) + first_after = None + for i, package in enumerate(descending): if package.timestamp: if package.timestamp > self.timestamp: - first_after = i + first_after = package.version else: - break - + if not self.rank: + return first_after + # if we have rank, then we need to then go back UP the + # versions, until we find one whose trimmed version doesn't + # match. + # Note that we COULD do this by simply iterating through + # an ascending sequence, in which case we wouldn't have to + # "switch direction" after finding the first result after + # by timestamp... but we're making the assumption that the + # timestamp break will be closer to the higher end of the + # version, and that we'll therefore have to check fewer + # timestamps this way... + trimmed_version = package.version.trim(self.rank - 1) + first_after = None + for after_package in reversed(descending[:i]): + if after_package.version.trim(self.rank - 1) != trimmed_version: + return after_package.version + return first_after + return first_after + + def _calc_sort_key(self, package_name, version): + first_after = self.get_first_after(package_name) if first_after is None: - # all packages are before T, just use version descending - return descending - - before = descending[first_after + 1:] - after = list(reversed(descending[:first_after + 1])) - - if not self.rank: # simple case - return before + after - - # include packages after timestamp but within rank - if before and after: - package = key(before[0]) - first_prerank = package.version.trim(self.rank - 1) - - for i, o in enumerate(after): - package = key(o) - prerank = package.version.trim(self.rank - 1) - if prerank != first_prerank: - break - - if i: - before = list(reversed(after[:i])) + before - after = after[i:] + # all packages are before T + is_before = True + else: + is_before = int(version < first_after) # ascend below rank, but descend within - after_ = [] - postrank = [] - prerank = None - - for o in after: - package = key(o) - prerank_ = package.version.trim(self.rank - 1) - - if prerank_ == prerank: - postrank.append(o) + if is_before: + return (is_before, version) + else: + if self.rank: + return (is_before, + _ReversedComparable(version.trim(self.rank - 1)), + version.tokens[self.rank - 1:]) else: - after_.extend(reversed(postrank)) - postrank = [o] - prerank = prerank_ - - after_.extend(reversed(postrank)) - return before + after_ + return (is_before, _ReversedComparable(version)) + + def sort_key(self, package_name, version): + cache_key = (package_name, str(version)) + result = self._cached_sort_key.get(cache_key) + if result is None: + result = self._calc_sort_key(package_name, version) + self._cached_sort_key[cache_key] = result + return result def __str__(self): return str((self.packages, self.timestamp, self.rank)) @@ -540,24 +549,7 @@ def __init__(self, packages): self.packages_dict = self._packages_from_pod(packages) self._version_key_cache = {} - def reorder(self, iterable, key=None): - key = key or (lambda x: x) - - def sort_key(x): - package = key(x) - return self.version_priority_key_cached(package.name, - package.version) - - return sorted(iterable, key=sort_key, reverse=True) - - @property - def packages(self): - return iter(self.packages_dict) - - def __str__(self): - return str(self.packages_dict) - - def version_priority_key_cached(self, package_name, version): + def sort_key(self, package_name, version): family_cache = self._version_key_cache.setdefault(package_name, {}) key = family_cache.get(version) if key is not None: @@ -567,6 +559,13 @@ def version_priority_key_cached(self, package_name, version): family_cache[version] = key return key + @property + def packages(self): + return iter(self.packages_dict) + + def __str__(self): + return str(self.packages_dict) + def version_priority_key_uncached(self, package_name, version): version_priorities = self.packages_dict[package_name] diff --git a/src/rez/solver.py b/src/rez/solver.py index b1c25f34f..c9fb55949 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -862,8 +862,10 @@ def _split(i_entry, n_variants, common_fams=None): if not fams: # trivial case, split on first variant + self.entries[0].sort() return _split(0, 1) + # find split point - first variant with no dependency shared with previous prev = None for i, entry in enumerate(self.entries): @@ -898,7 +900,9 @@ def sort_versions(self): orderer = get_orderer(self.package_name, self.solver.package_orderers or {}) - self.entries = orderer.reorder(self.entries, key=lambda x: x.package) + def sort_key(entry): + return orderer.sort_key(entry.package.name, entry.version) + self.entries = sorted(self.entries, key=sort_key, reverse=True) self.sorted = True if self.pr: diff --git a/src/rez/tests/test_packages.py b/src/rez/tests/test_packages.py index cf31808b1..99a59ffa7 100644 --- a/src/rez/tests/test_packages.py +++ b/src/rez/tests/test_packages.py @@ -350,7 +350,9 @@ def _test(orderer, package_name, expected_order): orderer2 = from_pod(from_yaml) for orderer_ in (orderer, orderer2): - ordered = orderer_.reorder(descending) + def key(package): + return orderer_.sort_key(package.name, package.version) + ordered = sorted(descending, key=key, reverse=True) result = [str(x.version) for x in ordered] self.assertEqual(result, expected_order) From 6f0349d0dfb7854ccc8171ffd4fa2343f46e5fed Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:16:39 -0700 Subject: [PATCH 08/10] added test that orderers work with variants --- .../data/solver/packages/pyvariants/2/package.py | 2 +- src/rez/tests/test_solver.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/rez/tests/data/solver/packages/pyvariants/2/package.py b/src/rez/tests/data/solver/packages/pyvariants/2/package.py index 253470537..5b291b13d 100644 --- a/src/rez/tests/data/solver/packages/pyvariants/2/package.py +++ b/src/rez/tests/data/solver/packages/pyvariants/2/package.py @@ -1,4 +1,4 @@ name = "pyvariants" version = "2" -variants = [["python-2.7.0"], ["python-2.6.8", "nada"]] \ No newline at end of file +variants = [["python-2.7.0"], ["python-2.6.8", "nada"], ["python-2.6.8"]] \ No newline at end of file diff --git a/src/rez/tests/test_solver.py b/src/rez/tests/test_solver.py index 990fb9e98..8b06afaab 100644 --- a/src/rez/tests/test_solver.py +++ b/src/rez/tests/test_solver.py @@ -501,7 +501,6 @@ def test_22_timestamp_rank4_inexact_timestamp(self): "reorderable-3.1.1", ]) - def test_23_direct_complete(self): """Test setting of the version_priority in simple situations, where the altered package is a direct request @@ -690,6 +689,17 @@ def test_30_multiple_matches(self): self._solve(["python<2.7"], ["python-2.6.8[]"]) + def test_31_orderer_used_for_variants(self): + self._solve(["pyvariants"], + ["python-2.7.0[]", "pyvariants-2[0]"]) + + config.override("package_orderers", + [{"type": "sorted", + "descending": False, + "packages": "python"}]) + self._solve(["pyvariants"], + ["python-2.6.8[]", "pyvariants-2[2]"]) + if __name__ == '__main__': unittest.main() From 3ab3155bfb48699ea31e5d1db9937b43d2287a0c Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:17:12 -0700 Subject: [PATCH 09/10] fix to ensure that orderers are used when sorting variants --- src/rez/package_order.py | 50 +++++++++++++++++++++++++++++++++------- src/rez/solver.py | 12 ++++++++-- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index eff1a3869..6ead3fb85 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -19,8 +19,42 @@ class PackageOrder(YamlDumpable): def __init__(self): pass + def sort_key(self, package_name, version_like): + """Returns a sort key usable for sorting these packages within the + same family + + Args: + package_name: (str) The family name of the package we are sorting + verison_like: (Version|_LowerBound|_UpperBound|_Bound|VersionRange) + the version-like object you wish to generate a key for + + Returns: + Comparable object + """ + from rez.vendor.version.version import Version, _LowerBound, _UpperBound, _Bound, VersionRange + if isinstance(version_like, VersionRange): + return tuple(self.sort_key(package_name, bound) + for bound in version_like.bounds) + elif isinstance(version_like, _Bound): + return (self.sort_key(package_name, version_like.lower), + self.sort_key(package_name, version_like.upper)) + elif isinstance(version_like, _LowerBound): + inclusion_key = -2 if version_like.inclusive else -1 + return (self.sort_key(package_name, version_like.version), + inclusion_key) + elif isinstance(version_like, _UpperBound): + inclusion_key = 2 if version_like.inclusive else 1 + return (self.sort_key(package_name, version_like.version), + inclusion_key) + elif isinstance(version_like, Version): + # finally, the bit that we actually use the sort_key_implementation + # for... + return self.sort_key_implementation(package_name, version_like) + else: + raise TypeError(version_like) + @abstractmethod - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): """Returns a sort key usable for sorting these packages within the same family @@ -93,7 +127,7 @@ class NullPackageOrder(PackageOrder): def __init__(self, packages): self.packages = packages - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): # python's sort will preserve the order of items that compare equal, so # to not change anything, we just return the same object for all... return 0 @@ -126,7 +160,7 @@ def __init__(self, packages, descending): self.packages = packages self.descending = descending - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): # Note that the name "descending" can be slightly confusing - it # indicates that the final ordering this Order gives should be # version descending (ie, the default) - however, the sort_key itself @@ -190,7 +224,7 @@ def __init__(self, order_dict, default_order=None): default_order = NullPackageOrder(DEFAULT_TOKEN) self.default_order = default_order - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): orderer = self.order_dict.get(package_name) if orderer is None: if self.default_order is not None: @@ -199,7 +233,7 @@ def sort_key(self, package_name, version): # shouldn't get here, because applies_to should protect us... raise RuntimeError("package family orderer %r does not apply to package family %r", (self, package_name)) - return orderer.sort_key(package_name, version) + return orderer.sort_key_implementation(package_name, version) @property def packages(self): @@ -285,7 +319,7 @@ def __init__(self, packages, first_version): self.packages = packages self.first_version = first_version - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): priority_key = 1 if version <= self.first_version else 0 return (priority_key, version) @@ -425,7 +459,7 @@ def _calc_sort_key(self, package_name, version): else: return (is_before, _ReversedComparable(version)) - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): cache_key = (package_name, str(version)) result = self._cached_sort_key.get(cache_key) if result is None: @@ -549,7 +583,7 @@ def __init__(self, packages): self.packages_dict = self._packages_from_pod(packages) self._version_key_cache = {} - def sort_key(self, package_name, version): + def sort_key_implementation(self, package_name, version): family_cache = self._version_key_cache.setdefault(package_name, {}) key = family_cache.get(version) if key is not None: diff --git a/src/rez/solver.py b/src/rez/solver.py index c9fb55949..052332b88 100644 --- a/src/rez/solver.py +++ b/src/rez/solver.py @@ -487,6 +487,8 @@ def sort(self): here as a safety measure so that sorting is guaranteed repeatable regardless. """ + from rez.package_order import get_orderer + if self.sorted: return @@ -498,13 +500,19 @@ def key(variant): if not request.conflict: req = variant.requires_list.get(request.name) if req is not None: - requested_key.append((-i, req.range)) + orderer = get_orderer(req.name, + self.solver.package_orderers or {}) + range_key = orderer.sort_key(req.name, req.range) + requested_key.append((-i, range_key)) names.add(req.name) additional_key = [] for request in variant.requires_list: if not request.conflict and request.name not in names: - additional_key.append((request.range, request.name)) + orderer = get_orderer(request.name, + self.solver.package_orderers or {}) + range_key = orderer.sort_key(request.name, request.range) + additional_key.append((range_key, request.name)) if (VariantSelectMode[config.variant_select_mode] == VariantSelectMode.version_priority): From a56b8766b6b8e811938ab48879eafbb7e8ac1ab3 Mon Sep 17 00:00:00 2001 From: Paul Molodowitch Date: Fri, 31 Mar 2017 16:17:12 -0700 Subject: [PATCH 10/10] fix for comparing variants with different packages --- src/rez/package_order.py | 32 ++++++++++++++++++- .../solver/packages/pyvariants/2/package.py | 3 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/rez/package_order.py b/src/rez/package_order.py index 6ead3fb85..d3493c3dc 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -10,6 +10,32 @@ DEFAULT_TOKEN = "" +class FallbackComparable(_Comparable): + """First tries to compare objects using the main_comparable, but if that + fails, compares using the fallback_comparable object. + """ + + def __init__(self, main_comparable, fallback_comparable): + self.main_comparable = main_comparable + self.fallback_comparable = fallback_comparable + + def __eq__(self, other): + try: + return self.main_comparable == other.main_comparable + except Exception: + return self.fallback_comparable == other.fallback_comparable + + def __lt__(self, other): + try: + return self.main_comparable < other.main_comparable + except Exception: + return self.fallback_comparable < other.fallback_comparable + + def __repr__(self): + return '%s(%r, %r)' % (type(self).__name__, self.main_comparable, + self.fallback_comparable) + + class PackageOrder(YamlDumpable): """Package reorderer base class.""" __metaclass__ = ABCMeta @@ -49,7 +75,11 @@ def sort_key(self, package_name, version_like): elif isinstance(version_like, Version): # finally, the bit that we actually use the sort_key_implementation # for... - return self.sort_key_implementation(package_name, version_like) + # Need to use a FallbackComparable because we can compare versions + # of different packages... + return FallbackComparable( + self.sort_key_implementation(package_name, version_like), + version_like) else: raise TypeError(version_like) diff --git a/src/rez/tests/data/solver/packages/pyvariants/2/package.py b/src/rez/tests/data/solver/packages/pyvariants/2/package.py index 5b291b13d..cb42bd489 100644 --- a/src/rez/tests/data/solver/packages/pyvariants/2/package.py +++ b/src/rez/tests/data/solver/packages/pyvariants/2/package.py @@ -1,4 +1,5 @@ name = "pyvariants" version = "2" -variants = [["python-2.7.0"], ["python-2.6.8", "nada"], ["python-2.6.8"]] \ No newline at end of file +variants = [["python-2.7.0"], ["python-2.6.8", "nada"], ["python-2.6.8"], + ["nada"]] \ No newline at end of file