diff --git a/src/rez/package_order.py b/src/rez/package_order.py index a71dbac02..d650fcdb9 100644 --- a/src/rez/package_order.py +++ b/src/rez/package_order.py @@ -7,6 +7,7 @@ from typing import Dict, Iterable, List, Optional, Union from rez.config import config +from rez.exceptions import ConfigurationError from rez.utils.data_utils import cached_class_property from rez.version import Version, VersionRange from rez.version._version import _Comparable, _ReversedComparable, _LowerBound, _UpperBound, _Bound @@ -608,6 +609,179 @@ def from_pod(cls, data): ) +class CustomPackageOrder(PackageOrder): + """A package order that allows explicit specification of version ordering. + Specified through the "packages" attributes, which should be a dict which + maps from a package family name to a list of version ranges to prioritize, + in decreasing priority order. + + As an example, consider a package splunge which has versions: + [1.0, 1.1, 1.2, 1.4, 2.0, 2.1, 3.0, 3.2] + By default, version priority is given to the higest version, so version + priority, from most to least preferred, is: + [3.2, 3.0, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + + However, if you set a custom package order like this: + package_orderers: + - type: custom + packages: + splunge: ['2', '1.1+<1.4'] + + Then the preferred versions, from most to least preferred, will be: + [2.1, 2.0, 1.2, 1.1, 3.2, 3.0, 1.4, 1.0] + Any version which does not match any of these expressions are sorted in + decreasing version order (like normal) and then appended to this list (so they + have lower priority). This provides an easy means to effectively set a + "default version." So if you do: + package_orderers: + - type: custom + packages: + splunge: ['3.0'] + + resulting order is: + [3.0, 3.2, 2.1, 2.0, 1.4, 1.2, 1.1, 1.0] + You may also include a single False or empty string in the list, in which case + all "other" versions will be placed at that spot. ie + package_orderers: + - type: custom + packages: + splunge: ['', '3+'] + yields: + [2.1, 2.0, 1.4, 1.2, 1.1, 1.0, 3.2, 3.0] + + Note that you could also have gotten the same result by doing: + package_orderers: + - type: custom + packages: + splunge: ['<3'] + If a version matches more than one range expression, it will be placed at + the highest-priority matching spot, so: + package_orderers: + - type: custom + packages: + splunge: ['1.2+<=2.0', '1.1+<3'] + gives: + [2.0, 1.4, 1.2, 2.1, 1.1, 3.2, 3.0, 1.0] + + Also note that this does not change the version sort order for any purpose but + determining solving priorities - for instance, even if version priorities is: + package_orderers: + - type: custom + packages: + splunge: [2, 3, 1] + The expression splunge-1+<3 would still match version 2. + """ + name = "custom" + + def __init__(self, + packages: Dict[str, List[Union[str, VersionRange]]], + version_orderer: Optional[PackageOrder] = None, + ): + """ + Args: + packages: (Dict[str, List[VersionRange]]): packages that + this orderer should apply to, and the version priority ordering + for that package + version_orderer (Optional[PackageOrder]): + How versions are sorted within custom version ranges. + If not provided, will use the standard rez version sorting. + """ + super().__init__(list(packages)) + self.packages_dict = self._packages_from_pod(packages) + self.version_orderer = version_orderer + + self._version_key_cache = {} + + def sort_key_implementation(self, package_name, version): + family_cache = self._version_key_cache.setdefault(package_name, {}) + key = family_cache.get(version) + if key is not None: + return key + key = self._version_priority_key_uncached(package_name, version) + family_cache[version] = key + return key + + def __str__(self): + return str(self.packages_dict) + + def _version_priority_key_uncached(self, package_name, version: Version): + version_priorities = self.packages_dict[package_name] + + default_key = -1 + for sort_order_index, version_range in enumerate(version_priorities): + # in the config, version_priorities are given in decreasing + # priority order... however, we want a sort key that sorts in the + # same way that versions do - where higher values are higher + # priority - so we need to take the inverse of the index + priority_sort_key = len(version_priorities) - sort_order_index + if version_range in (False, ""): + if default_key != -1: + raise ValueError("version_priorities may only have one " + "False / empty value") + default_key = priority_sort_key + continue + if version_range.contains_version(version): + break + else: + # For now, we're permissive with the version_sort_order - it may + # contain ranges which match no actual versions, and if an actual + # version matches no entry in the version_sort_order, it is simply + # placed after other entries + priority_sort_key = default_key + + if self.version_orderer: + version_key = self.version_orderer.sort_key_implementation(package_name, version) + else: + version_key = version + + return priority_sort_key, version_key + + @staticmethod + def _packages_to_pod(packages: Dict[str, List[Union[str, VersionRange]]]) -> Dict[str, List[str]]: + return { + package: [str(v) for v in versions] + for (package, versions) in packages.items() + } + + @staticmethod + def _packages_from_pod( + packages: Dict[str, List[Union[str, VersionRange]]], + ) -> Dict[str, List[Union[str, VersionRange]]]: + parsed_dict = {} + for package, versions in packages.items(): + new_versions = [] + num_false = 0 + for v in versions: + if v in ("", False): + v = False + num_false += 1 + else: + if not isinstance(v, VersionRange): + if isinstance(v, (int, float)): + v = str(v) + v = VersionRange(v) + new_versions.append(v) + if num_false > 1: + raise ConfigurationError("version_priorities for CustomPackageOrder may " + "only have one False / empty value") + parsed_dict[package] = new_versions + return parsed_dict + + def to_pod(self): + return dict( + packages=self._packages_to_pod(self.packages_dict), + version_orderer=to_pod(self.version_orderer) if self.version_orderer else None, + ) + + @classmethod + def from_pod(cls, data): + version_orderer = data["version_orderer"] + return cls( + packages=data["packages"], + version_orderer=from_pod(version_orderer) if version_orderer else None, + ) + + class PackageOrderList(list): """A list of package orderer. """ diff --git a/src/rez/tests/test_packages_order.py b/src/rez/tests/test_packages_order.py index 1138b86fe..e314ebd11 100644 --- a/src/rez/tests/test_packages_order.py +++ b/src/rez/tests/test_packages_order.py @@ -8,8 +8,10 @@ import json from rez.config import config -from rez.package_order import NullPackageOrder, PackageOrder, PerFamilyOrder, VersionSplitPackageOrder, \ - TimestampPackageOrder, SortedOrder, PackageOrderList, from_pod +from rez.package_order import ( + NullPackageOrder, PackageOrder, PerFamilyOrder, VersionSplitPackageOrder, + TimestampPackageOrder, SortedOrder, CustomPackageOrder, + PackageOrderList, from_pod) from rez.packages import iter_packages from rez.tests.util import TestBase, TempdirMixin from rez.version import Version @@ -289,6 +291,146 @@ def test_pod(self): self._test_pod(TimestampPackageOrder(timestamp=3001, rank=3)) +class TestCustomPackageOrder(_BaseTestPackagesOrder): + """Test case for the CustomPackageOrder class""" + + def test_reorder(self): + """Validate we can sort packages in a custom order.""" + # Test that a pretty random order can be applied + self._test_reorder(CustomPackageOrder({ + "python": ['2.6.8', '2.7.0', '2.6.0', '2.5.2'], + }), "python", ['2.6.8', '2.7.0', '2.6.0', '2.5.2']) + + # Test that version supersets apply correctly and that the rest are sorted descending + self._test_reorder(CustomPackageOrder({ + "python": ["2.6"], + }), "python", ['2.6.8', '2.6.0', '2.7.0', '2.5.2']) + + # Test that version supersets apply correctly to multiple subsets of versions + self._test_reorder(CustomPackageOrder({ + "python": ["2.6", "2.5"], + }), "python", ['2.6.8', '2.6.0', '2.5.2', '2.7.0']) + + # Test that version supersets apply correctly to multiple subsets of versions + self._test_reorder(CustomPackageOrder({ + "python": ["<=2.7", "2.5"], + }), "python", ['2.6.8', '2.6.0', '2.5.2', '2.7.0']) + + def test_reorder_sorted_version_orderer(self): + """Validate we can sort packages in a custom order with sorted version sorting.""" + self._test_reorder(CustomPackageOrder({ + "python": ['2.6'], + }, + version_orderer=SortedOrder(descending=True), + ), "python", [ + # 2.6 from highest to lowest + '2.6.8', '2.6.0', + # Remaining from highest to lowest + '2.7.0', '2.5.2', + ]) + + self._test_reorder(CustomPackageOrder({ + "python": ['2.6'], + }, + version_orderer=SortedOrder(descending=False), + ), "python", [ + # 2.6 from lowest to highest + '2.6.0', '2.6.8', + # Remaining from lowest to highest + '2.5.2', '2.7.0', + ]) + + # FIXME: Enable this if we get pypa package ordering merged in + # def test_reorder_pypa_version_orderer(self): + # """Validate we can sort packages in a custom order with pypa version sorting.""" + # self._test_reorder(CustomPackageOrder({ + # "pypa": ['1+<1.1', "2", "1.1+<2"], + # }, + # version_orderer=PyPAPackageOrder(), + # ), "pypa", [ + # # 1+<1.1 release versions at the front + # '1.0.1', + # '1.0.0.post1', + # '1.0.0+local', + # '1.0.0', + # # Followed by 1+<1.1 prerelease versions + # '1.0.2.rc2', + # '1.0.2.rc1', + # '1.0.2.b1', + # '1.0.2.a1', + # '1.0.1.rc2', + # '1.0.1.rc1', + # '1.0.1.b2', + # '1.0.1.b1', + # '1.0.1.a1', + # '1.0.0.rc2', + # '1.0.0.rc1', + # '1.0.0.b1', + # '1.0.0.a2', + # '1.0.0.a1', + # # Followed by version 2 prerelease (no releases for 2.0) + # '2.0.0.a2', + # '2.0.0.a1', + # # Lastly, followed by 1.1 prerelease (no releases for 1.1) + # '1.1.0.b2', + # '1.1.0.b1', + # '1.1.0.a1', + # ]) + # + # # Test getting a pypa risk tolerance of beta changes things + # self._test_reorder(CustomPackageOrder({ + # "pypa": ['1+<1.1', "2", "1.1+<2"], + # }, + # version_orderer=PyPAPackageOrder(prerelease="b"), + # ), "pypa", [ + # # Release and prerelease up to beta for 1+<1.1 to the front + # '1.0.2.rc2', + # '1.0.2.rc1', + # '1.0.2.b1', + # '1.0.1', + # '1.0.1.rc2', + # '1.0.1.rc1', + # '1.0.1.b2', + # '1.0.1.b1', + # '1.0.0.post1', + # '1.0.0+local', + # '1.0.0', + # '1.0.0.rc2', + # '1.0.0.rc1', + # '1.0.0.b1', + # # Followed by 1+<1.1 prerelease below beta + # '1.0.2.a1', + # '1.0.1.a1', + # '1.0.0.a2', + # '1.0.0.a1', + # # Followed by version 2 + # '2.0.0.a2', + # '2.0.0.a1', + # # Lastly followed by 1.1 + # '1.1.0.b2', + # '1.1.0.b1', + # '1.1.0.a1', + # ]) + + def test_repr(self): + """Validate we can represent a CustomPackageOrder as a string.""" + self.assertEqual( + "CustomPackageOrder({" + "'python': [VersionRange('1.2'), VersionRange('5.6')], " + "'pymum': [VersionRange('2'), VersionRange('1')]})", + repr(CustomPackageOrder({ + "python": ["1.2", "5.6"], + "pymum": ["2", "1"], + }))) + + def test_pod(self): + """Validate we can save and load a CustomPackageOrder to its pod representation.""" + self._test_pod(CustomPackageOrder({ + "python": ["1.2", "5.6", "3.4"], + "pymum": ["2", "1", "3"], + })) + + class TestPackageOrdererList(_BaseTestPackagesOrder): """Test cases for the PackageOrderList class."""