From 218c61afe5506f6154ea8ff5415f64d7b8f95dfc Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 14 Mar 2023 20:01:32 +0530 Subject: [PATCH 01/25] Add support for NormalizedVersionRanges - support normalization of range expression from GitHub, Snyk, GitLab - Discrete range normalization for OSV, DEPS, VulerableCode Signed-off-by: Keshav Priyadarshi --- src/univers/normalized_range.py | 329 ++++++++++++++++++++++ src/univers/span.py | 484 ++++++++++++++++++++++++++++++++ 2 files changed, 813 insertions(+) create mode 100644 src/univers/normalized_range.py create mode 100644 src/univers/span.py diff --git a/src/univers/normalized_range.py b/src/univers/normalized_range.py new file mode 100644 index 00000000..453e4ad7 --- /dev/null +++ b/src/univers/normalized_range.py @@ -0,0 +1,329 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. + + +import operator +import re +from typing import List +from typing import Union + +import attr + +from univers.span import Span +from univers.version_range import VersionRange +from univers.versions import AlpineLinuxVersion +from univers.versions import ArchLinuxVersion +from univers.versions import ComposerVersion +from univers.versions import DebianVersion +from univers.versions import GenericVersion +from univers.versions import GentooVersion +from univers.versions import GolangVersion +from univers.versions import MavenVersion +from univers.versions import NginxVersion +from univers.versions import NugetVersion +from univers.versions import OpensslVersion +from univers.versions import PypiVersion +from univers.versions import RpmVersion +from univers.versions import RubygemsVersion +from univers.versions import SemverVersion + + +@attr.s(frozen=True, order=False, eq=True, hash=True) +class NormalizedVersionRanges: + """ + A NormalizedVersionRange represents a list of VersionRange resolved + from diffrent datsource. + """ + + # A tuple of VersionRange + version_ranges = attr.ib(type=tuple, default=attr.Factory(tuple)) + + def __str__(self): + return "(" + ", ".join([f"'{str(vers)}'" for vers in self.version_ranges]) + ")" + + @staticmethod + def get_span_boundry(comparator: str, version: str, version_map: dict): + """ + Return Span with Lower and Upper boundry limit. + """ + index = NormalizedVersionRanges.get_version_rank(version, version_map) + resolved_operator = OPERATOR_BY_COMPRATOR.get(comparator, operator.eq) + + if resolved_operator == operator.lt: + return Span(1, index - 1) + if resolved_operator == operator.gt: + return Span(index + 1, len(version_map)) + if resolved_operator == operator.ge: + return Span(index, len(version_map)) + if resolved_operator == operator.le: + return Span(1, index) + if resolved_operator == operator.eq: + return Span(index) + if resolved_operator == operator.ne: + return Span(1, index - 1).union(Span(index + 1, len(version_map))) + + @staticmethod + def get_version_range_from_span(total_span: Span, purl_type: str, reverse_version_map: dict): + """ + Return list containg VersionRange for all subspans in a Span. + """ + version_ranges = [] + list_of_span = total_span.subspans() + for span in list_of_span: + lower_bound = reverse_version_map[span.start] + upper_bound = reverse_version_map[span.end] + vers_exp = ( + f"vers:{purl_type}/{lower_bound}" + if lower_bound == upper_bound + else f"vers:{purl_type}/>={lower_bound}|<={upper_bound}" + ) + version_ranges.append(VersionRange.from_string(vers_exp)) + return version_ranges + + @staticmethod + def get_version_rank(version: str, version_map: dict): + """ + Return equivalent integer ranking for a version. + """ + try: + return version_map[strip_leading_v(version)] + except KeyError as err: + err.args = (f"{version} doesn't exist.",) + raise + + @staticmethod + def parse_constraint(constraint: str): + """ + Return operator and version from a constraint + For example: + >>> assert NormalizedVersionRanges.parse_constraint(">=7.0.0") == ('>=', '7.0.0') + >>> assert NormalizedVersionRanges.parse_constraint("=7.0.0") == ('=', '7.0.0') + >>> assert NormalizedVersionRanges.parse_constraint("[3.0.0") == ('[', '3.0.0') + >>> assert NormalizedVersionRanges.parse_constraint("3.1.25]") == (']', '3.1.25') + """ + if constraint.startswith(("<=", ">=", "==", "!=")): + return constraint[:2], constraint[2:] + + if constraint.startswith(("<", ">", "=", "[", "(")): + return constraint[0], constraint[1:] + + if constraint.endswith(("]", ")")): + return constraint[-1], constraint[:-1] + return None, constraint + + @staticmethod + def get_version_map(versions: List, purl_type: str): + """ + Return dict mapping version to integer. + """ + if purl_type not in VERSIONS_BY_PACKAGE_TYPE: + return + + version_type = VERSIONS_BY_PACKAGE_TYPE.get(purl_type) + sorted_versions = sorted([version_type(i) for i in versions]) + sorted_versions = [version.string for version in sorted_versions] + index = list(range(1, len(sorted_versions) + 1, 1)) + return dict(zip(sorted_versions, index)) + + @classmethod + def from_github(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): + """ + Return NormalizedVersionRanges computed from GithHub version range expression. + GitHub range_expression example:: + ">= 10.4.0, < 10.4.1" + "> 7.1.1" + """ + version_map = cls.get_version_map(all_versions, purl_type) + reverse_version_map = {value: key for key, value in version_map.items()} + + items = [range_expression] if isinstance(range_expression, str) else range_expression + total_span = None + for item in items: + gh_constraints = item.strip().replace(" ", "") + constraints = gh_constraints.split(",") + local_span = None + for constraint in constraints: + if not constraint: + continue + gh_comparator, gh_version = cls.parse_constraint(constraint) + span = cls.get_span_boundry(gh_comparator, strip_leading_v(gh_version), version_map) + local_span = span if not local_span else local_span.intersection(span) + + total_span = local_span if not total_span else total_span.union(local_span) + + version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) + return cls(version_ranges=version_ranges) + + @classmethod + def from_snyk(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): + """ + Return NormalizedVersionRanges computed from Snyk version range expression. + Snyk range_expression example:: + ">=4.0.0, <4.0.10.16" + " >=4.1.0, <4.4.15.7" + "[3.0.0,3.1.25) + """ + version_map = cls.get_version_map(all_versions, purl_type) + reverse_version_map = {value: key for key, value in version_map.items()} + + items = [range_expression] if isinstance(range_expression, str) else range_expression + total_span = None + for item in items: + delimiter = "," if "," in item else " " + if delimiter == ",": + snyk_constraints = item.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + else: + snyk_constraints = item.strip() + constraints = snyk_constraints.split(" ") + local_span = None + for constraint in constraints: + if not constraint: + continue + snyk_comparator, snyk_version = cls.parse_constraint(constraint) + if not snyk_version: + continue + span = cls.get_span_boundry( + snyk_comparator, strip_leading_v(snyk_version), version_map + ) + local_span = span if not local_span else local_span.intersection(span) + + total_span = local_span if not total_span else total_span.union(local_span) + + version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) + return cls(version_ranges=version_ranges) + + @classmethod + def from_gitlab(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): + """ + Return NormalizedVersionRanges computed from GitLab version range expression. + GitLab range_expression example:: + "[7.0.0,7.0.11),[7.2.0,7.2.4)" + "[7.0.0,7.0.11),[7.2.0,7.2.4)" + ">=4.0,<4.3||>=5.0,<5.2" + ">=0.19.0 <0.30.0" + ">=1.5,<1.5.2" + """ + + version_map = cls.get_version_map(all_versions, purl_type) + reverse_version_map = {value: key for key, value in version_map.items()} + + items = [range_expression] if isinstance(range_expression, str) else range_expression + global_span = None + for item in items: + gitlab_constraints = item.strip() + if gitlab_constraints.startswith(("[", "(")): + # transform "[7.0.0,7.0.11),[7.2.0,7.2.4)" -> [ "[7.0.0,7.0.11)", "[7.2.0,7.2.4)" ] + splitted = gitlab_constraints.split(",") + constraints = [f"{a},{b}" for a, b in zip(splitted[::2], splitted[1::2])] + delimiter = "," + + else: + # transform ">=4.0,<4.3||>=5.0,<5.2" -> [ ">=4.0,<4.3", ">=5.0,<5.2" ] + # transform ">=0.19.0 <0.30.0" -> [ ">=0.19.0 <0.30.0" ] + # transform ">=1.5,<1.5.2" -> [ ">=1.5,<1.5.2" ] + delimiter = "," if "," in gitlab_constraints else " " + constraints = gitlab_constraints.split("||") + total_span = None + for constraint in constraints: + local_span = None + for subcontraint in constraint.strip().split(delimiter): + if not subcontraint: + continue + gitlab_comparator, gitlab_version = cls.parse_constraint(subcontraint.strip()) + if not gitlab_version: + continue + span = cls.get_span_boundry( + gitlab_comparator, strip_leading_v(gitlab_version), version_map + ) + local_span = span if not local_span else local_span.intersection(span) + + total_span = local_span if not total_span else total_span.union(local_span) + global_span = total_span if not global_span else global_span.union(total_span) + + version_ranges = cls.get_version_range_from_span( + global_span, purl_type, reverse_version_map + ) + return cls(version_ranges=version_ranges) + + @classmethod + def from_discrete(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): + """ + Return NormalizedVersionRanges computed from discrete version range expression. + Discrete range_expression example:: + ["1.5","3.1.2","3.1-beta"] + """ + version_map = cls.get_version_map(all_versions, purl_type) + reverse_version_map = {value: key for key, value in version_map.items()} + + item = range_expression if isinstance(range_expression, str) else " ".join(range_expression) + discrete_versions = re.split("[ ,\n]+", item) + + rank_list = [] + for version in discrete_versions: + try: + rank_int = version_map[strip_leading_v(version)] + rank_list.append(rank_int) + except KeyError: + pass + + total_span = Span(rank_list) + + version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) + return cls(version_ranges) + + +def strip_leading_v(version: str): + """ + Return version without leading v. + """ + if not version.startswith("v"): + return version + return version[1:] + + +VERSIONS_BY_PACKAGE_TYPE = { + "alpine": AlpineLinuxVersion, + "alpm": ArchLinuxVersion, + "apache": SemverVersion, + "cargo": SemverVersion, + # "cocoapods": None, + "composer": ComposerVersion, + # "conan": None, + # "conda": None, + # "cran": None, + "deb": DebianVersion, + "ebuild": GentooVersion, + "gem": RubygemsVersion, + "generic": GenericVersion, + "github": SemverVersion, + "golang": GolangVersion, + "hex": SemverVersion, + "mattermost": SemverVersion, + "maven": MavenVersion, + "mozilla": SemverVersion, + "nginx": NginxVersion, + "npm": SemverVersion, + "nuget": NugetVersion, + "openssl": OpensslVersion, + "pypi": PypiVersion, + "rpm": RpmVersion, + # "swift": None, +} + +OPERATOR_BY_COMPRATOR = { + "<": operator.lt, + ">": operator.gt, + "=": operator.eq, + "<=": operator.le, + ">=": operator.ge, + "==": operator.eq, + "!=": operator.ne, + ")": operator.lt, + "]": operator.le, + "(": operator.gt, + "[": operator.ge, +} diff --git a/src/univers/span.py b/src/univers/span.py new file mode 100644 index 00000000..aeccb46e --- /dev/null +++ b/src/univers/span.py @@ -0,0 +1,484 @@ +# +# Copyright (c) 2010 Matt Chaput. All rights reserved. +# Modifications by nexB Copyright (c) nexB Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are +# those of the authors and should not be interpreted as representing official +# policies, either expressed or implied, of Matt Chaput. + +from collections.abc import Set +from itertools import count +from itertools import groupby + +from intbitset import intbitset + +""" +Ranges and intervals of integers using bitmaps. +Used as a compact and faster data structure for token and position sets. +""" + + +class Span(Set): + """ + Represent ranges of integers (such as tokens positions) as a set of integers. + A Span is hashable and not meant to be modified once created, like a frozenset. + It is equivalent to a sparse closed interval. + Originally derived and heavily modified from Whoosh Span. + """ + + def __init__(self, *args): + """ + Create a new Span from a start and end ints or an iterable of ints. + + First form: + Span(start int, end int) : the span is initialized with a range(start, end+1) + + Second form: + Span(iterable of ints) : the span is initialized with the iterable + + Spans are hashable and immutable. + + For example: + >>> s = Span(1) + >>> s.start + 1 + >>> s = Span([1, 2]) + >>> s.start + 1 + >>> s.end + 2 + >>> s + Span(1, 2) + + >>> s = Span(1, 3) + >>> s.start + 1 + >>> s.end + 3 + >>> s + Span(1, 3) + + >>> s = Span([6, 5, 1, 2]) + >>> s.start + 1 + >>> s.end + 6 + >>> s + Span(1, 2)|Span(5, 6) + >>> len(s) + 4 + + >>> Span([5, 6, 7, 8, 9, 10 ,11, 12]) == Span([5, 6, 7, 8, 9, 10 ,11, 12]) + True + >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) + True + >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span(5, 12)) + True + """ + len_args = len(args) + + if len_args == 0: + self._set = intbitset() + + elif len_args == 1: + # args0 is a single int or an iterable of ints + if isinstance(args[0], int): + self._set = intbitset(args) + else: + # some sequence or iterable + self._set = intbitset(list(args[0])) + + elif len_args == 2: + # args0 and args1 describe a start and end closed range + self._set = intbitset(range(args[0], args[1] + 1)) + + else: + # args0 is a single int or args is an iterable of ints + # args is an iterable of ints + self._set = intbitset(list(args)) + + @classmethod + def _from_iterable(cls, it): + return cls(list(it)) + + def __len__(self): + return len(self._set) + + def __iter__(self): + return iter(self._set) + + def __hash__(self): + return hash(tuple(self._set)) + + def __eq__(self, other): + return isinstance(other, Span) and self._set == other._set + + def __and__(self, *others): + return Span(self._set.intersection(*[o._set for o in others])) + + def __or__(self, *others): + return Span(self._set.union(*[o._set for o in others])) + + def union(self, *others): + """ + Return the union of this span with other spans as a new span. + (i.e. all positions that are in either spans.) + """ + return self.__or__(*others) + + def intersection(self, *others): + """ + Return the intersection of this span with other spans as a new span. + (i.e. all positions that are in both spans.) + """ + return self.__and__(*others) + + def difference(self, *others): + """ + Return the difference of two or more spans as a new span. + (i.e. all positions that are in this span but not the others.) + """ + return Span(self._set.difference(*[o._set for o in others])) + + def __repr__(self): + """ + Return a brief representation of this span by only listing contiguous + spans and not all items. + + For example: + >>> Span([1, 2, 3, 4, 5, 7, 8, 9, 10]) + Span(1, 5)|Span(7, 10) + """ + subspans_repr = [] + for subs in self.subspans(): + ls = len(subs) + if not ls: + subspans_repr.append("Span()") + elif ls == 1: + subspans_repr.append("Span(%d)" % subs.start) + else: + subspans_repr.append("Span(%d, %d)" % (subs.start, subs.end)) + return "|".join(subspans_repr) + + def __contains__(self, other): + """ + Return True if this span contains other span (where other is a Span, an + int or an ints set). + + For example: + >>> Span([5, 7]) in Span(5, 7) + True + >>> Span([5, 8]) in Span([5, 7]) + False + >>> 6 in Span([4, 5, 6, 7, 8]) + True + >>> 2 in Span([4, 5, 6, 7, 8]) + False + >>> 8 in Span([4, 8]) + True + >>> 5 in Span([4, 8]) + False + >>> set([4, 5]) in Span([4, 5, 6, 7, 8]) + True + >>> set([9]) in Span([4, 8]) + False + """ + if isinstance(other, Span): + return self._set.issuperset(other._set) + + if isinstance(other, int): + return self._set.__contains__(other) + + if isinstance(other, (set, frozenset)): + return self._set.issuperset(intbitset(other)) + + if isinstance(other, intbitset): + return self._set.issuperset(other) + + @property + def set(self): + return self._set + + def issubset(self, other): + return self._set.issubset(other._set) + + def issuperset(self, other): + return self._set.issuperset(other._set) + + @property + def start(self): + if not self._set: + raise TypeError("Empty Span has no start.") + return self._set[0] + + @property + def end(self): + if not self._set: + raise TypeError("Empty Span has no end.") + return self._set[-1] + + @classmethod + def sort(cls, spans): + """ + Return a new sorted sequence of spans given a sequence of spans. + The primary sort is on start. The secondary sort is on length. + If two spans have the same start, the longer span will sort first. + + For example: + >>> spans = [Span([5, 6, 7, 8, 9, 10]), Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] + >>> Span.sort(spans) + [Span(1, 2), Span(3, 6), Span(3, 5), Span(5, 10), Span(8, 10)] + + >>> spans = [Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] + >>> Span.sort(spans) + [Span(1, 2), Span(3, 6), Span(3, 5), Span(8, 10)] + + >>> spans = [Span([1, 2]), Span([4, 5]), Span([7, 8]), Span([11, 12])] + >>> Span.sort(spans) + [Span(1, 2), Span(4, 5), Span(7, 8), Span(11, 12)] + + >>> spans = [Span([1, 2]), Span([7, 8]), Span([5, 6]), Span([12, 13])] + >>> Span.sort(spans) + [Span(1, 2), Span(5, 6), Span(7, 8), Span(12, 13)] + + """ + key = lambda s: ( + s.start, + -len(s), + ) + return sorted(spans, key=key) + + def magnitude(self): + """ + Return the maximal length represented by this span start and end. The + magnitude is the same as the length for a contiguous span. It will be + greater than the length for a span with non-contiguous int items. + An empty span has a zero magnitude. + + For example: + >>> Span([4, 8]).magnitude() + 5 + >>> len(Span([4, 8])) + 2 + >>> len(Span([4, 5, 6, 7, 8])) + 5 + + >>> Span([4, 5, 6, 14 , 12, 128]).magnitude() + 125 + + >>> Span([4, 5, 6, 7, 8]).magnitude() + 5 + >>> Span([0]).magnitude() + 1 + >>> Span([0]).magnitude() + 1 + """ + if not self._set: + return 0 + return self.end - self.start + 1 + + def density(self): + """ + Return the density of this span as a ratio of its length to its + magnitude, a float between 0 and 1. A dense Span has all its integer + items contiguous and a maximum density of one. A sparse low density span + has some non-contiguous integer items. An empty span has a zero density. + + For example: + >>> Span([4, 8]).density() + 0.4 + >>> Span([4, 5, 6, 7, 8]).density() + 1.0 + >>> Span([0]).density() + 1.0 + >>> Span().density() + 0 + """ + if not self._set: + return 0 + return len(self) / self.magnitude() + + def overlap(self, other): + """ + Return the count of overlapping items between this span and other span. + + For example: + >>> Span([1, 2]).overlap(Span([5, 6])) + 0 + >>> Span([5, 6]).overlap(Span([5, 6])) + 2 + >>> Span([4, 5, 6, 7]).overlap(Span([5, 6])) + 2 + >>> Span([4, 5, 6]).overlap(Span([5, 6, 7])) + 2 + >>> Span([4, 5, 6]).overlap(Span([6])) + 1 + >>> Span([4, 5]).overlap(Span([6, 7])) + 0 + """ + return len(self & other) + + def resemblance(self, other): + """ + Return a resemblance coefficient as a float between 0 and 1. + 0 means the spans are completely different and 1 identical. + """ + if self._set.isdisjoint(other._set): + return 0 + if self._set == other._set: + return 1 + resemblance = self.overlap(other) / len(self | other) + return resemblance + + def containment(self, other): + """ + Return a containment coefficient as a float between 0 and 1. This is an + indication of how much of the other span is contained in this span. + - 1 means the other span is entirely contained in this span. + - 0 means that the other span is not contained at all this span. + """ + if self._set.isdisjoint(other._set): + return 0 + if self._set == other._set: + return 1 + containment = self.overlap(other) / len(other) + return containment + + def surround(self, other): + """ + Return True if this span surrounds other span. + This is different from containment. A span can surround another span region + and have no positions in common with the surrounded. + + For example: + >>> Span([4, 8]).surround(Span([4, 8])) + True + >>> Span([3, 9]).surround(Span([4, 8])) + True + >>> Span([5, 8]).surround(Span([4, 8])) + False + >>> Span([4, 7]).surround(Span([4, 8])) + False + >>> Span([4, 5, 6, 7, 8]).surround(Span([5, 6, 7])) + True + """ + return self.start <= other.start and self.end >= other.end + + def is_before(self, other): + return self.end < other.start + + def is_after(self, other): + return self.start > other.end + + def touch(self, other): + """ + Return True if self sequence is contiguous with other span without overlap. + + For example: + >>> Span([5, 7]).touch(Span([5])) + False + >>> Span([5, 7]).touch(Span([5, 8])) + False + >>> Span([5, 7]).touch(Span([7, 8])) + False + >>> Span([5, 7]).touch(Span([8, 9])) + True + >>> Span([8, 9]).touch(Span([5, 7])) + True + """ + return self.start == other.end + 1 or self.end == other.start - 1 + + def distance_to(self, other): + """ + Return the absolute positive distance from this span to other span. + Overlapping spans have a zero distance. + Non-overlapping touching spans have a distance of one. + + For example: + >>> Span([8, 9]).distance_to(Span([5, 7])) + 1 + >>> Span([5, 7]).distance_to(Span([8, 9])) + 1 + >>> Span([5, 6]).distance_to(Span([8, 9])) + 2 + >>> Span([8, 9]).distance_to(Span([5, 6])) + 2 + >>> Span([5, 7]).distance_to(Span([5, 7])) + 0 + >>> Span([4, 5, 6]).distance_to(Span([5, 6, 7])) + 0 + >>> Span([5, 7]).distance_to(Span([10, 12])) + 3 + >>> Span([1, 2]).distance_to(Span(range(4, 52))) + 2 + """ + if self.overlap(other): + return 0 + + if self.touch(other): + return 1 + + if self.is_before(other): + return other.start - self.end + else: + return self.start - other.end + + @staticmethod + def from_ints(ints): + """ + Return a sequence of Spans from an iterable of ints. A new Span is + created for each group of monotonously increasing int items. + + >>> Span.from_ints([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) + [Span(1, 12)] + >>> Span.from_ints([1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]) + [Span(1, 3), Span(5, 12)] + >>> Span.from_ints([0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13]) + [Span(0), Span(2, 3), Span(5, 11), Span(13)] + """ + ints = sorted(set(ints)) + groups = (group for _, group in groupby(ints, lambda group, c=count(): next(c) - group)) + return [Span(g) for g in groups] + + def subspans(self): + """ + Return a list of Spans creating one new Span for each set of contiguous + integer items. + + For example: + >>> span = Span(5, 6, 7, 8, 9, 10) | Span([1, 2]) | Span(3, 5) | Span(3, 6) | Span([8, 9, 10]) + >>> span.subspans() + [Span(1, 10)] + + When subspans are not touching they do not merge : + >>> span = Span([63, 64]) | Span([58, 58]) + >>> span.subspans() + [Span(58), Span(63, 64)] + + Overlapping subspans are merged as needed: + >>> span = Span([12, 17, 24]) | Span([15, 16, 17, 35]) | Span(58) | Span(63, 64) + >>> span.subspans() + [Span(12), Span(15, 17), Span(24), Span(35), Span(58), Span(63, 64)] + """ + return Span.from_ints(self) From 1a69d070968f55b612b385e92045686871e7d9ef Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 15 Mar 2023 17:37:07 +0530 Subject: [PATCH 02/25] Test NormalizedVersionRanges Signed-off-by: Keshav Priyadarshi --- tests/test_normalized_range.py | 1072 ++++++++++++++++++++++++++++++++ 1 file changed, 1072 insertions(+) create mode 100644 tests/test_normalized_range.py diff --git a/tests/test_normalized_range.py b/tests/test_normalized_range.py new file mode 100644 index 00000000..6d3db981 --- /dev/null +++ b/tests/test_normalized_range.py @@ -0,0 +1,1072 @@ +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. + +from unittest import TestCase + +from univers.normalized_range import NormalizedVersionRanges +from univers.version_constraint import VersionConstraint +from univers.version_range import PypiVersionRange +from univers.version_range import VersionRange +from univers.versions import PypiVersion + +all_versions_pypi = [ + "1.1.3", + "1.1.4", + "1.10", + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + "1.10.5", + "1.10.6", + "1.10.7", + "1.10.8", + "1.10a1", + "1.10b1", + "1.10rc1", + "1.11", + "1.11.1", + "1.11.10", + "1.11.11", + "1.11.12", + "1.11.13", + "1.11.14", + "1.11.15", + "1.11.16", + "1.11.17", + "1.11.18", + "1.11.2", + "1.11.20", + "1.11.21", + "1.11.22", + "1.11.23", + "1.11.24", + "1.11.25", + "1.11.26", + "1.11.27", + "1.11.28", + "1.11.29", + "1.11.3", + "1.11.4", + "1.11.5", + "1.11.6", + "1.11.7", + "1.11.8", + "1.11.9", + "1.11a1", + "1.11b1", + "1.11rc1", + "1.2", + "1.2.1", + "1.2.2", + "1.2.3", + "1.2.4", + "1.2.5", + "1.2.6", + "1.2.7", + "1.3", + "1.3.1", + "1.3.2", + "1.3.3", + "1.3.4", + "1.3.5", + "1.3.6", + "1.3.7", + "1.4", + "1.4.1", + "1.4.10", + "1.4.11", + "1.4.12", + "1.4.13", + "1.4.14", + "1.4.15", + "1.4.16", + "1.4.17", + "1.4.18", + "1.4.19", + "1.4.2", + "1.4.20", + "1.4.21", + "1.4.22", + "1.4.3", + "1.4.4", + "1.4.5", + "1.4.6", + "1.4.7", + "1.4.8", + "1.4.9", + "1.5", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.5.9", + "1.6", + "1.6.1", + "1.6.10", + "1.6.11", + "1.6.2", + "1.6.3", + "1.6.4", + "1.6.5", + "1.6.6", + "1.6.7", + "1.6.8", + "1.6.9", + "1.7", + "1.7.1", + "1.7.10", + "1.7.11", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "1.7.9", + "1.8", + "1.8.1", + "1.8.10", + "1.8.11", + "1.8.12", + "1.8.13", + "1.8.14", + "1.8.15", + "1.8.16", + "1.8.17", + "1.8.18", + "1.8.19", + "1.8.2", + "1.8.3", + "1.8.4", + "1.8.5", + "1.8.6", + "1.8.7", + "1.8.8", + "1.8.9", + "1.8a1", + "1.8b1", + "1.8b2", + "1.8c1", + "1.9", + "1.9.1", + "1.9.10", + "1.9.11", + "1.9.12", + "1.9.13", + "1.9.2", + "1.9.3", + "1.9.4", + "1.9.5", + "1.9.6", + "1.9.7", + "1.9.8", + "1.9.9", + "1.9a1", + "1.9b1", + "1.9rc1", + "1.9rc2", + "2.0", + "2.0.1", + "2.0.10", + "2.0.12", + "2.0.13", + "2.0.2", + "2.0.3", + "2.0.4", + "2.0.5", + "2.0.6", + "2.0.7", + "2.0.8", + "2.0.9", + "2.0a1", + "2.0b1", + "2.0rc1", + "2.1", + "2.1.1", + "2.1.10", + "2.1.11", + "2.1.12", + "2.1.13", + "2.1.14", + "2.1.15", + "2.1.2", + "2.1.3", + "2.1.4", + "2.1.5", + "2.1.7", + "2.1.8", + "2.1.9", + "2.1a1", + "2.1b1", + "2.1rc1", + "2.2", + "2.2.1", + "2.2.10", + "2.2.11", + "2.2.12", + "2.2.13", + "2.2.14", + "2.2.15", + "2.2.16", + "2.2.17", + "2.2.18", + "2.2.19", + "2.2.2", + "2.2.20", + "2.2.21", + "2.2.22", + "2.2.23", + "2.2.24", + "2.2.25", + "2.2.26", + "2.2.27", + "2.2.28", + "2.2.3", + "2.2.4", + "2.2.5", + "2.2.6", + "2.2.7", + "2.2.8", + "2.2.9", + "2.2a1", + "2.2b1", + "2.2rc1", + "3.0", + "3.0.1", + "3.0.10", + "3.0.11", + "3.0.12", + "3.0.13", + "3.0.14", + "3.0.2", + "3.0.3", + "3.0.4", + "3.0.5", + "3.0.6", + "3.0.7", + "3.0.8", + "3.0.9", + "3.0a1", + "3.0b1", + "3.0rc1", + "3.1", + "3.1.1", + "3.1.10", + "3.1.11", + "3.1.12", + "3.1.13", + "3.1.14", + "3.1.2", + "3.1.3", + "3.1.4", + "3.1.5", + "3.1.6", + "3.1.7", + "3.1.8", + "3.1.9", + "3.1a1", + "3.1b1", + "3.1rc1", + "3.2", + "3.2.1", + "3.2.10", + "3.2.11", + "3.2.12", + "3.2.13", + "3.2.14", + "3.2.15", + "3.2.16", + "3.2.17", + "3.2.18", + "3.2.2", + "3.2.3", + "3.2.4", + "3.2.5", + "3.2.6", + "3.2.7", + "3.2.8", + "3.2.9", + "3.2a1", + "3.2b1", + "3.2rc1", + "4.0", + "4.0.1", + "4.0.10", + "4.0.2", + "4.0.3", + "4.0.4", + "4.0.5", + "4.0.6", + "4.0.7", + "4.0.8", + "4.0.9", + "4.0a1", + "4.0b1", + "4.0rc1", + "4.1", + "4.1.1", + "4.1.2", + "4.1.3", + "4.1.4", + "4.1.5", + "4.1.6", + "4.1.7", + "4.1a1", + "4.1b1", + "4.1rc1", + "4.2a1", + "4.2b1", +] + + +class TestNormalizedVersionRanges(TestCase): + purl_type_pypi = "pypi" + + def test_NormalizedVersionRanges_from_github_type1(self): + range_expression = "> 1.11rc1" + + normalized_range = NormalizedVersionRanges.from_github( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.11")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.2b1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_github_type2(self): + range_expression = ">= 3.2.2, < 4.0.2" + purl_type = "pypi" + + normalized_range = NormalizedVersionRanges.from_github( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.2.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_github_type3(self): + range_expression = [">= 3.2.2, < 4.0.2", ">1.4.9, < 1.11rc1"] + + normalized_range = NormalizedVersionRanges.from_github( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.4.10")), + VersionConstraint(comparator="<=", version=PypiVersion(string="1.11b1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.2.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_snyk_type1(self): + range_expression = " >= 3.0.1, < 4.0.2" + + normalized_range = NormalizedVersionRanges.from_snyk( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_snyk_type2(self): + range_expression = "[3.0.1,4.0.2)" + + normalized_range = NormalizedVersionRanges.from_snyk( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_snyk_type3(self): + range_expression = ["[3.0.1,4.0.2)", " >= 3.0.1, < 4.0.2"] + + normalized_range = NormalizedVersionRanges.from_snyk( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_gitlab_type1(self): + range_expression = "[1.4,2.1rc1),[3.2,4.0rc1)" + + normalized_range = NormalizedVersionRanges.from_gitlab( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.4")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.1b1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0b1")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_gitlab_type2(self): + range_expression = ">=4.0,<4.2b1||>=1.2,<2.1" + + normalized_range = NormalizedVersionRanges.from_gitlab( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="4.0")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.2a1")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_gitlab_type3(self): + range_expression = ">=1.2 <2.1" + + normalized_range = NormalizedVersionRanges.from_gitlab( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_gitlab_type4(self): + range_expression = ">=1.5,<1.5.2" + + normalized_range = NormalizedVersionRanges.from_gitlab( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.5")), + VersionConstraint(comparator="<=", version=PypiVersion(string="1.5.1")), + ) + ) + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_gitlab_type5(self): + range_expression = [ + ">=1.2 <2.1", + ">=4.0,<4.2b1||>=1.2,<2.1", + "[1.4,2.1rc1),[3.2,4.0rc1)", + ">=1.5,<1.5.2", + ] + + normalized_range = NormalizedVersionRanges.from_gitlab( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="3.2")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.0b1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="4.0")), + VersionConstraint(comparator="<=", version=PypiVersion(string="4.2a1")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_discrete_for_DEPS(self): + range_expression = [ + "2.0", + "2.0rc1", + "2.0rc1-py2.4-linux-i686", + "2.0rc1-py2.4-linux-x86_64", + "2.0rc1-py2.4-macosx-10.3-i386", + "2.0rc1-py2.4-win32", + "2.0rc1-py2.5-linux-i686", + "2.0rc1-py2.5-linux-x86_64", + "2.0rc1-py2.5-macosx-10.3-i386", + "2.0rc1-py2.5-win32", + "2.1-py2.4-win322.1-py2.5-win32", + "2.1-py2.6-win32", + " 2.1.0", + "2.1.1", + "2.1.1-py2.4-win32", + "2.1.1-py2.5-win32", + "2.1.1-py2.6-win32", + "2.10.0 2.2.0", + "2.2.1", + "2.2.3", + "2.2.11", + "2.2.9", + ] + + normalized_range = NormalizedVersionRanges.from_discrete( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="2.0rc1")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.0")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator="=", version=PypiVersion(string="2.1.1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator="=", version=PypiVersion(string="2.2.1")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator="=", version=PypiVersion(string="2.2.3")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator="=", version=PypiVersion(string="2.2.9")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator="=", version=PypiVersion(string="2.2.11")), + ) + ), + ] + ) + assert normalized_range == expected + + def test_NormalizedVersionRanges_from_discrete_for_VulnerableCode(self): + range_expression = [ + "1.10", + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + "1.10.5", + "1.10.6", + "1.10.7", + "1.10.8", + "1.10a1", + "1.10b1", + "1.10rc1", + "1.11", + "1.11.0", + "1.11.1", + "1.11.10", + "1.11.11", + "1.11.12", + "1.11.13", + "1.11.14", + "1.11.15", + "1.11.16", + "1.11.17", + "1.11.18", + "1.11.2", + "1.11.3", + "1.11.4", + "1.11.5", + "1.11.6", + "1.11.7", + "1.11.8", + "1.11.9", + "1.11a1", + "1.11b1", + "1.11rc1", + "1.1.3", + "1.1.4", + "1.2", + "1.2.1", + "1.2.2", + "1.2.3", + "1.2.4", + "1.2.5", + "1.2.6", + "1.2.7", + "1.3", + "1.3.1", + "1.3.2", + "1.3.3", + "1.3.4", + "1.3.5", + "1.3.6", + "1.3.7", + "1.4", + "1.4.1", + "1.4.10", + "1.4.11", + "1.4.12", + "1.4.13", + "1.4.14", + "1.4.15", + "1.4.16", + "1.4.17", + "1.4.18", + "1.4.19", + "1.4.2", + "1.4.20", + "1.4.21", + "1.4.22", + "1.4.3", + "1.4.4", + "1.4.5", + "1.4.6", + "1.4.7", + "1.4.8", + "1.4.9", + "1.5", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.5.9", + "1.6", + "1.6.1", + "1.6.10", + "1.6.11", + "1.6.2", + "1.6.3", + "1.6.4", + "1.6.5", + "1.6.6", + "1.6.7", + "1.6.8", + "1.6.9", + "1.7", + "1.7.1", + "1.7.10", + "1.7.11", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "1.7.9", + "1.8", + "1.8.1", + "1.8.10", + "1.8.11", + "1.8.12", + "1.8.13", + "1.8.14", + "1.8.15", + "1.8.16", + "1.8.17", + "1.8.18", + "1.8.19", + "1.8.2", + "1.8.3", + "1.8.4", + "1.8.5", + "1.8.6", + "1.8.7", + "1.8.8", + "1.8.9", + "1.8a1", + "1.8b1", + "1.8b2", + "1.8c1", + "1.9", + "1.9.1", + "1.9.10", + "1.9.11", + "1.9.12", + "1.9.13", + "1.9.2", + "1.9.3", + "1.9.4", + "1.9.5", + "1.9.6", + "1.9.7", + "1.9.8", + "1.9.9", + "1.9a1", + "1.9b1", + "1.9rc1", + "1.9rc2", + "2.0", + "2.0.0", + "2.0.1", + "2.0.10", + "2.0.2", + "2.0.3", + "2.0.4", + "2.0.5", + "2.0.6", + "2.0.7", + "2.0.8", + "2.0.9", + "2.1", + "2.1.0", + "2.1.1", + "2.1.2", + "2.1.3", + "2.1.4", + "2.1.5", + ] + + normalized_range = NormalizedVersionRanges.from_discrete( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.1.3")), + VersionConstraint(comparator="<=", version=PypiVersion(string="1.11.18")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="2.0")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.0.10")), + ) + ), + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="2.1")), + VersionConstraint(comparator="<=", version=PypiVersion(string="2.1.5")), + ) + ), + ] + ) + assert normalized_range == expected + return normalized_range + + def test_NormalizedVersionRanges_from_discrete_for_OSV(self): + range_expression = [ + "0", + "1.0.1", + "1.0.2", + "1.0.3", + "1.0.4", + "1.1", + "1.1.1", + "1.1.2", + "1.1.3", + "1.1.4", + "1.10", + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + "1.10.5", + "1.10.6", + "1.10.7", + "1.10.8", + "1.10a1", + "1.10b1", + "1.10rc1", + "1.11", + "1.11.1", + "1.11.10", + "1.11.11", + "1.11.12", + "1.11.13", + "1.11.14", + "1.11.15", + "1.11.16", + "1.11.17", + "1.11.18", + "1.11.2", + "1.11.20", + "1.11.21", + "1.11.22", + "1.11.23", + "1.11.24", + "1.11.25", + "1.11.26", + "1.11.27", + "1.11.28", + "1.11.3", + "1.11.4", + "1.11.5", + "1.11.6", + "1.11.7", + "1.11.8", + "1.11.9", + "1.11a1", + "1.11b1", + "1.11rc1", + "1.2", + "1.2.1", + "1.2.2", + "1.2.3", + "1.2.4", + "1.2.5", + "1.2.6", + "1.2.7", + "1.3", + "1.3.1", + "1.3.2", + "1.3.3", + "1.3.4", + "1.3.5", + "1.3.6", + "1.3.7", + "1.4", + "1.4.1", + "1.4.10", + "1.4.11", + "1.4.12", + "1.4.13", + "1.4.14", + "1.4.15", + "1.4.16", + "1.4.17", + "1.4.18", + "1.4.19", + "1.4.2", + "1.4.20", + "1.4.21", + "1.4.22", + "1.4.3", + "1.4.4", + "1.4.5", + "1.4.6", + "1.4.7", + "1.4.8", + "1.4.9", + "1.5", + "1.5.1", + "1.5.10", + "1.5.11", + "1.5.12", + "1.5.2", + "1.5.3", + "1.5.4", + "1.5.5", + "1.5.6", + "1.5.7", + "1.5.8", + "1.5.9", + "1.6", + "1.6.1", + "1.6.10", + "1.6.11", + "1.6.2", + "1.6.3", + "1.6.4", + "1.6.5", + "1.6.6", + "1.6.7", + "1.6.8", + "1.6.9", + "1.7", + "1.7.1", + "1.7.10", + "1.7.11", + "1.7.2", + "1.7.3", + "1.7.4", + "1.7.5", + "1.7.6", + "1.7.7", + "1.7.8", + "1.7.9", + "1.8", + "1.8.1", + "1.8.10", + "1.8.11", + "1.8.12", + "1.8.13", + "1.8.14", + "1.8.15", + "1.8.16", + "1.8.17", + "1.8.18", + "1.8.19", + "1.8.2", + "1.8.3", + "1.8.4", + "1.8.5", + "1.8.6", + "1.8.7", + "1.8.8", + "1.8.9", + "1.8a1", + "1.8b1", + "1.8b2", + "1.8c1", + "1.9", + "1.9.1", + "1.9.10", + "1.9.11", + "1.9.12", + "1.9.13", + "1.9.2", + "1.9.3", + "1.9.4", + "1.9.5", + "1.9.6", + "1.9.7", + "1.9.8", + "1.9.9", + "1.9a1", + "1.9b1", + "1.9rc1", + "1.9rc2", + ] + normalized_range = NormalizedVersionRanges.from_discrete( + range_expression=range_expression, + purl_type=self.purl_type_pypi, + all_versions=all_versions_pypi, + ) + + expected = NormalizedVersionRanges( + version_ranges=[ + PypiVersionRange( + constraints=( + VersionConstraint(comparator=">=", version=PypiVersion(string="1.1.3")), + VersionConstraint(comparator="<=", version=PypiVersion(string="1.11.28")), + ) + ) + ] + ) + assert normalized_range == expected From 3b3570cd22ee572fe8bdaa84a4054bd4ffcbed65 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 15 Mar 2023 19:51:04 +0530 Subject: [PATCH 03/25] Add ABOUT and LICENSE file for spans.py Signed-off-by: Keshav Priyadarshi --- src/univers/spans.py.ABOUT | 16 ++++++++++++++++ src/univers/spans.py.LICENSE | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 src/univers/spans.py.ABOUT create mode 100644 src/univers/spans.py.LICENSE diff --git a/src/univers/spans.py.ABOUT b/src/univers/spans.py.ABOUT new file mode 100644 index 00000000..27ad909c --- /dev/null +++ b/src/univers/spans.py.ABOUT @@ -0,0 +1,16 @@ +about_resource: spans.py +version: 2.4.1 +name: Whoosh Spans +vcs_url: hg+https://bitbucket.org/mchaput/whoosh@72e06bd0aac8 +home_url: https://bitbucket.org/mchaput/whoosh +owner: Matt Chaput +copyright: Copyright (c) 2011 Matt Chaput. +download_url: https://bitbucket.org/mchaput/whoosh/get/2.4.1.tar.gz +description: Manage spans of tokens and text. + This file was originally copied and modified from Whoosh. + +license_expression: bsd-simplified + +licenses: +- file: spans.py.LICENSE + key: bsd-simplified diff --git a/src/univers/spans.py.LICENSE b/src/univers/spans.py.LICENSE new file mode 100644 index 00000000..b0266325 --- /dev/null +++ b/src/univers/spans.py.LICENSE @@ -0,0 +1,26 @@ +Copyright 2011 Matt Chaput. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are +those of the authors and should not be interpreted as representing official +policies, either expressed or implied, of Matt Chaput. From d88b80c405765e4a8482dafd5a62dc8c78517fe1 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 3 Apr 2023 18:17:12 +0530 Subject: [PATCH 04/25] Fallback to builtin set when intbitset is not installed Signed-off-by: Keshav Priyadarshi --- src/univers/span.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/univers/span.py b/src/univers/span.py index aeccb46e..022fd88b 100644 --- a/src/univers/span.py +++ b/src/univers/span.py @@ -31,7 +31,13 @@ from itertools import count from itertools import groupby -from intbitset import intbitset +try: + from intbitset import intbitset + + spanset = intbitset +except: + spanset = set + """ Ranges and intervals of integers using bitmaps. @@ -99,24 +105,24 @@ def __init__(self, *args): len_args = len(args) if len_args == 0: - self._set = intbitset() + self._set = spanset() elif len_args == 1: # args0 is a single int or an iterable of ints if isinstance(args[0], int): - self._set = intbitset(args) + self._set = spanset(args) else: # some sequence or iterable - self._set = intbitset(list(args[0])) + self._set = spanset(list(args[0])) elif len_args == 2: # args0 and args1 describe a start and end closed range - self._set = intbitset(range(args[0], args[1] + 1)) + self._set = spanset(range(args[0], args[1] + 1)) else: # args0 is a single int or args is an iterable of ints # args is an iterable of ints - self._set = intbitset(list(args)) + self._set = spanset(list(args)) @classmethod def _from_iterable(cls, it): @@ -211,9 +217,9 @@ def __contains__(self, other): return self._set.__contains__(other) if isinstance(other, (set, frozenset)): - return self._set.issuperset(intbitset(other)) + return self._set.issuperset(spanset(other)) - if isinstance(other, intbitset): + if isinstance(other, spanset): return self._set.issuperset(other) @property @@ -230,12 +236,16 @@ def issuperset(self, other): def start(self): if not self._set: raise TypeError("Empty Span has no start.") + if isinstance(self._set, set): + return sorted(self._set)[0] return self._set[0] @property def end(self): if not self._set: raise TypeError("Empty Span has no end.") + if isinstance(self._set, set): + return sorted(self._set)[-1] return self._set[-1] @classmethod From 5ab9b3a15e0095a8186ca3870cb9f9eade83833d Mon Sep 17 00:00:00 2001 From: Omkar Phansopkar Date: Wed, 18 Oct 2023 15:42:56 +0530 Subject: [PATCH 05/25] Added docs server script, dark mode & copybutton for docs Signed-off-by: Omkar Phansopkar --- .github/workflows/docs-ci.yml | 3 --- docs/Makefile | 8 ++++++++ docs/make.bat | 12 ++++++++++++ docs/scripts/doc8_style_check.sh | 0 docs/source/conf.py | 4 ++++ setup.cfg | 3 +++ 6 files changed, 27 insertions(+), 3 deletions(-) mode change 100644 => 100755 docs/scripts/doc8_style_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 511b7c28..ada779bf 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -20,9 +20,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Give permission to run scripts - run: chmod +x ./docs/scripts/doc8_style_check.sh - - name: Install Dependencies run: pip install -e .[docs] diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf1..788b0396 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,6 +5,7 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source BUILDDIR = build @@ -14,6 +15,13 @@ help: .PHONY: help Makefile +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/docs/make.bat b/docs/make.bat index 6247f7e2..4a3c1a48 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -7,11 +7,16 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +if "%SPHINXAUTOBUILD%" == "" ( + set SPHINXAUTOBUILD=sphinx-autobuild +) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help +if "%1" == "docs" goto docs + %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. @@ -28,6 +33,13 @@ if errorlevel 9009 ( %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end +:docs +@echo +@echo Starting up the docs server... +@echo +%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% +goto end + :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh old mode 100644 new mode 100755 diff --git a/docs/source/conf.py b/docs/source/conf.py index 918d62c1..54e5e665 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,6 +30,10 @@ extensions = [ "sphinx.ext.intersphinx", "sphinx_reredirects", + 'sphinx_rtd_theme', + "sphinx_rtd_dark_mode", + "sphinx.ext.extlinks", + "sphinx_copybutton", ] diff --git a/setup.cfg b/setup.cfg index d6c7da7d..bd0e58a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,4 +62,7 @@ docs = sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 doc8>=0.11.2 + sphinx-autobuild + sphinx-rtd-dark-mode>=1.3.0 + sphinx-copybutton From 0a9d983650bf042a5bd2c277711b637979e566f1 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 20 Nov 2023 16:46:54 -0800 Subject: [PATCH 06/25] Update CSS to widen page and handle mobile #84 Reference: https://github.com/nexB/skeleton/issues/84 Signed-off-by: John M. Horan --- docs/source/_static/theme_overrides.css | 363 +----------------- .../_static/theme_overrides_SUPERSEDED.css | 353 +++++++++++++++++ docs/source/conf.py | 15 +- 3 files changed, 380 insertions(+), 351 deletions(-) create mode 100644 docs/source/_static/theme_overrides_SUPERSEDED.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index 9662d63a..de5ae433 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -1,353 +1,26 @@ -body { - color: #000000; -} - -p { - margin-bottom: 10px; -} - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -.custom_header_01 { - color: #cc0000; - font-size: 22px; - font-weight: bold; - line-height: 50px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 20px; - margin-top: 20px; -} - -h5 { - font-size: 18px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 15px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #000000; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff99; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - background: #006bb3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - background: #cce6ff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #99ccff; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - background: #ffe6cc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - +/* this is the container for the pages */ .wy-nav-content { max-width: 100%; - padding-right: 100px; - padding-left: 100px; - background-color: #f2f2f2; -} - -div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - border: 1px solid #ffff99; - background: #ffff99; - font-size: 100%; - font-weight: normal; - border-radius: 4px; - padding: 2px 0px; - margin: auto 2px; - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - overflow: visible; - white-space: pre-wrap; - color: #000000; -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - background-color: #f8f8f8; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #ffffff; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -span.pre { - color: #e01e5a; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + padding: 0px 40px 0px 0px; margin-top: 0px; - margin-bottom: 0px; } -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 0px; - line-height: 24px; +.wy-nav-content-wrap { + border-right: solid 1px; } -/* fix extra vertical spacing in page toctree */ -.rst-content .toctree-wrapper ul li ul, article ul li ul { - margin-top: 0; - margin-bottom: 0; -} - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #000000; - margin-top: 10px; - padding-top: 5px; -} - -.indextable ul li { - font-size: 14px; - margin-bottom: 5px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; -} - -div.genindex-jumpbox { - margin-bottom: 10px; -} - -/* rst image classes */ - -.clear-both { - clear: both; - } - -.float-left { - float: left; - margin-right: 20px; -} - -img { - border: solid 1px #e8e8e8; -} - -/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ -.img-title { - color: #000000; - /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ - line-height: 3.0; - font-style: italic; - font-weight: 600; -} - -.img-title-para { - color: #000000; - margin-top: 20px; - margin-bottom: 0px; - font-style: italic; - font-weight: 500; -} - -.red { - color: red; +div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } } diff --git a/docs/source/_static/theme_overrides_SUPERSEDED.css b/docs/source/_static/theme_overrides_SUPERSEDED.css new file mode 100644 index 00000000..9662d63a --- /dev/null +++ b/docs/source/_static/theme_overrides_SUPERSEDED.css @@ -0,0 +1,353 @@ +body { + color: #000000; +} + +p { + margin-bottom: 10px; +} + +.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* todo */ +.custom-admonition-todo .admonition-title { + color: #000000; + background: #cce6ff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #99ccff; +} +div.custom-admonition-todo.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #99ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* examples */ +.custom-admonition-examples .admonition-title { + color: #000000; + background: #ffe6cc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #d8d8d8; +} +div.custom-admonition-examples.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066cc; + text-decoration: none; +} +.wy-nav-content-wrap a:hover { + color: #0099cc; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + color: #000000; + background-color: transparent; + font-style: normal; + font-family: "Courier New"; + font-size: 13px; + padding: 3px 3px 3px 3px; +} + +.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + color: #000000; +} + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, .rst-content tt, .rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 3px; + font-size: 13px; + background-color: #ffffff; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #e01e5a; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, .rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + } + +.float-left { + float: left; + margin-right: 20px; +} + +img { + border: solid 1px #e8e8e8; +} + +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 54e5e665..7771ff09 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,7 @@ extensions = [ "sphinx.ext.intersphinx", "sphinx_reredirects", - 'sphinx_rtd_theme', + "sphinx_rtd_theme", "sphinx_rtd_dark_mode", "sphinx.ext.extlinks", "sphinx_copybutton", @@ -47,7 +47,10 @@ intersphinx_mapping = { "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), - "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), + "scancode-workbench": ( + "https://scancode-workbench.readthedocs.io/en/develop/", + None, + ), } @@ -82,7 +85,9 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = ["_static/theme_overrides.css"] +html_css_files = [ + "theme_overrides.css", +] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. @@ -108,6 +113,4 @@ # -- Options for LaTeX output ------------------------------------------------- -latex_elements = { - 'classoptions': ',openany,oneside' -} \ No newline at end of file +latex_elements = {"classoptions": ",openany,oneside"} From 4e36fc601eaa17bde0d2a4bebfed70d7bde28e7c Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 16 Jan 2024 12:22:54 -0800 Subject: [PATCH 07/25] Delete theme_overrides_SUPERSEDED.css as no longer needed #84 Reference: https://github.com/nexB/skeleton/issues/84 Signed-off-by: John M. Horan --- .../_static/theme_overrides_SUPERSEDED.css | 353 ------------------ 1 file changed, 353 deletions(-) delete mode 100644 docs/source/_static/theme_overrides_SUPERSEDED.css diff --git a/docs/source/_static/theme_overrides_SUPERSEDED.css b/docs/source/_static/theme_overrides_SUPERSEDED.css deleted file mode 100644 index 9662d63a..00000000 --- a/docs/source/_static/theme_overrides_SUPERSEDED.css +++ /dev/null @@ -1,353 +0,0 @@ -body { - color: #000000; -} - -p { - margin-bottom: 10px; -} - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -.custom_header_01 { - color: #cc0000; - font-size: 22px; - font-weight: bold; - line-height: 50px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 20px; - margin-top: 20px; -} - -h5 { - font-size: 18px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 15px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #000000; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff99; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - background: #006bb3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - background: #cce6ff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #99ccff; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - background: #ffe6cc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -.wy-nav-content { - max-width: 100%; - padding-right: 100px; - padding-left: 100px; - background-color: #f2f2f2; -} - -div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - border: 1px solid #ffff99; - background: #ffff99; - font-size: 100%; - font-weight: normal; - border-radius: 4px; - padding: 2px 0px; - margin: auto 2px; - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - overflow: visible; - white-space: pre-wrap; - color: #000000; -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - background-color: #f8f8f8; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #ffffff; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -span.pre { - color: #e01e5a; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { - margin-top: 0px; - margin-bottom: 0px; -} - -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 0px; - line-height: 24px; -} - -/* fix extra vertical spacing in page toctree */ -.rst-content .toctree-wrapper ul li ul, article ul li ul { - margin-top: 0; - margin-bottom: 0; -} - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #000000; - margin-top: 10px; - padding-top: 5px; -} - -.indextable ul li { - font-size: 14px; - margin-bottom: 5px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; -} - -div.genindex-jumpbox { - margin-bottom: 10px; -} - -/* rst image classes */ - -.clear-both { - clear: both; - } - -.float-left { - float: left; - margin-right: 20px; -} - -img { - border: solid 1px #e8e8e8; -} - -/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ -.img-title { - color: #000000; - /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ - line-height: 3.0; - font-style: italic; - font-weight: 600; -} - -.img-title-para { - color: #000000; - margin-top: 20px; - margin-bottom: 0px; - font-style: italic; - font-weight: 500; -} - -.red { - color: red; -} From 7d74b8a3c98761293cd133d543e4d58a525dc7bf Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 18 Jan 2024 17:11:14 +0530 Subject: [PATCH 08/25] Fix top padding for rst content Signed-off-by: Ayan Sinha Mahapatra --- docs/source/_static/theme_overrides.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index de5ae433..5863ccf5 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -12,7 +12,7 @@ div.rst-content { max-width: 1300px; border: 0; - padding: 0px 80px 10px 80px; + padding: 10px 80px 10px 80px; margin-left: 50px; } From 008d521aec51e5983f6d6a2adc4efa7fd92159cf Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 19 Feb 2024 15:21:45 +0530 Subject: [PATCH 09/25] Update CI runners and python version Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 8 ++++---- azure-pipelines.yml | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index ada779bf..8c2abfe9 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: max-parallel: 4 diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 95857301..d2206c87 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,10 +21,10 @@ on: jobs: build-pypi-distribs: name: Build and publish library to PyPI - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: @@ -47,7 +47,7 @@ jobs: name: Create GH release needs: - build-pypi-distribs - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Download built archives @@ -67,7 +67,7 @@ jobs: name: Create PyPI release needs: - create-gh-release - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Download built archives diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 764883de..373b78cd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macOS-11 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macOS-12 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,15 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +67,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 124da3dcef0d95a6f6aa76ed849f47ada25b83e2 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 15:11:21 +0530 Subject: [PATCH 10/25] Replace deprecated macos CI runners Replace macos-11 runners with macos-14 runners. Reference: https://github.com/actions/runner-images?tab=readme-ov-file#available-images Reference: https://github.com/nexB/skeleton/issues/89 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 373b78cd..c2a3b522 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,24 +25,24 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos11_cpython - image_name: macOS-11 + job_name: macos12_cpython + image_name: macOS-12 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos12_cpython - image_name: macOS-12 + job_name: macos13_cpython + image_name: macOS-13 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos13_cpython - image_name: macOS-13 + job_name: macos14_cpython_arm64 + image_name: macOS-14 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -50,8 +50,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython - image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + image_name: macOS-14-large + python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs From be4e14d414cf4f7112b529dc71f7abccc9dcf24a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 16:00:40 +0530 Subject: [PATCH 11/25] Update minimum required python version to 3.8 Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bd0e58a7..a8e20c5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = From f0bac8c828f2d112459888d64d950951d4f62cf1 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 19 Jul 2024 23:32:23 +0530 Subject: [PATCH 12/25] Support both PURL and GitLab schema in from_gitlab_native Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 37138aae..e07873c7 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1122,7 +1122,10 @@ class MattermostVersionRange(VersionRange): def from_gitlab_native(gitlab_scheme, string): - purl_scheme = PURL_TYPE_BY_GITLAB_SCHEME[gitlab_scheme] + purl_scheme = gitlab_scheme + if gitlab_scheme not in PURL_TYPE_BY_GITLAB_SCHEME.values(): + purl_scheme = PURL_TYPE_BY_GITLAB_SCHEME[gitlab_scheme] + vrc = RANGE_CLASS_BY_SCHEMES[purl_scheme] supported_native_implementations = [ ConanVersionRange, From 629d03ad8d1d299956be8e071dc64743a600293b Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 19 Jul 2024 23:33:04 +0530 Subject: [PATCH 13/25] Use native impl for Maven and NuGet in from_gitlab_native Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index e07873c7..01670449 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1129,6 +1129,8 @@ def from_gitlab_native(gitlab_scheme, string): vrc = RANGE_CLASS_BY_SCHEMES[purl_scheme] supported_native_implementations = [ ConanVersionRange, + MavenVersionRange, + NugetVersionRange, ] if vrc in supported_native_implementations: return vrc.from_native(string) From 1c70ea5ce3efb4bff78cbe756f4aaf04cdf00c05 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 19 Jul 2024 23:35:55 +0530 Subject: [PATCH 14/25] Use proper splitter for composer in from_gitlab_native - Bug fix: GitLab advisory uses both comma and space as separator for composer range. Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 9 +++++++-- tests/data/composer_gitlab.json | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 01670449..6ab7d2c5 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1137,8 +1137,13 @@ def from_gitlab_native(gitlab_scheme, string): constraint_items = [] constraints = [] split = " " - split_by_comma_schemes = ["pypi", "composer"] - if purl_scheme in split_by_comma_schemes: + if purl_scheme == "pypi": + split = "," + + # GitLab advisory for composer uses both `,` and space for separating constraints. + # https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/8ba4872b659cf5a306e0d47abdd0e428948bf41c/packagist/illuminate/cookie/GHSA-2867-6rrm-38gr.yml + # https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/8ba4872b659cf5a306e0d47abdd0e428948bf41c/packagist/contao-components/mediaelement/CVE-2016-4567.yml + if purl_scheme == "composer" and "," in string: split = "," pipe_separated_constraints = string.split("||") for pipe_separated_constraint in pipe_separated_constraints: diff --git a/tests/data/composer_gitlab.json b/tests/data/composer_gitlab.json index 61c5dbca..60fe38a2 100644 --- a/tests/data/composer_gitlab.json +++ b/tests/data/composer_gitlab.json @@ -7533,7 +7533,7 @@ "test_index": 1256, "scheme": "packagist", "gitlab_native": "<=3.8.0||>=4.0.0-alpha <=4.0.0-rc2", - "expected_vers": "vers:composer/<=3.8.0|>=4.0.0-alpha--4.0.0-rc2" + "expected_vers": "vers:composer/<=3.8.0|>=4.0.0-alpha|<=4.0.0-rc2" }, { "test_index": 1257, From d109a1bb2ee36022919dca1bcb16a3def4807753 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Sat, 20 Jul 2024 00:03:39 +0530 Subject: [PATCH 15/25] Support splitting bracket notation ranges Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 6ab7d2c5..0009a5cb 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -446,7 +446,7 @@ def from_native(cls, string): return cls(constraints=constraints) -def split_req(string, comparators, default=None, strip=""): +def split_req(string, comparators, comparators_rear={}, default=None, strip=""): """ Return a tuple of (vers comparator, version) strings given an common version requirement``string`` such as "> 2.3" or "<= 2.3" using the ``comparators`` @@ -467,6 +467,8 @@ def split_req(string, comparators, default=None, strip=""): >>> assert split_req(">= 2.3", comparators=comps) == (">=", "2.3",) >>> assert split_req("<= 2.3", comparators=comps) == ("<=", "2.3",) >>> assert split_req("(< = 2.3 )", comparators=comps, strip=")(") == ("<=", "2.3",) + >>> comps_rear = {")": "<", "]": "<="} + >>> assert split_req(" 2.3 ]", comparators=comps, comparators_rear=comps_rear) == ("<=", "2.3",) With a default, we return the default comparator:: @@ -487,6 +489,12 @@ def split_req(string, comparators, default=None, strip=""): version = constraint_string.lstrip(native_comparator) return vers_comparator, version + # Some bracket notation comparators starts from end. + for native_comparator, vers_comparator in comparators_rear.items(): + if constraint_string.endswith(native_comparator): + version = constraint_string.rstrip(native_comparator) + return vers_comparator, version + if default: return default, constraint_string @@ -749,7 +757,8 @@ def from_native(cls, string): comparator = ">" constraints.append( VersionConstraint( - comparator=comparator, version=cls.version_class(str(lower_bound)) + comparator=comparator, + version=cls.version_class(str(lower_bound)), ) ) @@ -760,7 +769,8 @@ def from_native(cls, string): comparator = "<" constraints.append( VersionConstraint( - comparator=comparator, version=cls.version_class(str(upper_bound)) + comparator=comparator, + version=cls.version_class(str(upper_bound)), ) ) @@ -1136,6 +1146,7 @@ def from_gitlab_native(gitlab_scheme, string): return vrc.from_native(string) constraint_items = [] constraints = [] + split = " " if purl_scheme == "pypi": split = "," @@ -1145,6 +1156,7 @@ def from_gitlab_native(gitlab_scheme, string): # https://gitlab.com/gitlab-org/security-products/gemnasium-db/-/blob/8ba4872b659cf5a306e0d47abdd0e428948bf41c/packagist/contao-components/mediaelement/CVE-2016-4567.yml if purl_scheme == "composer" and "," in string: split = "," + pipe_separated_constraints = string.split("||") for pipe_separated_constraint in pipe_separated_constraints: space_seperated_constraints = pipe_separated_constraint.split(split) From 169a6a1402c0b33e8ffcbbabda1a86839788dec5 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 22 Jul 2024 19:09:19 +0530 Subject: [PATCH 16/25] Add support for version range from snyk advisory Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 90 ++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 0009a5cb..207167fd 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -4,6 +4,9 @@ # # Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. +from typing import List +from typing import Union + import attr import semantic_version from packaging.specifiers import InvalidSpecifier @@ -1222,7 +1225,7 @@ def build_constraint_from_github_advisory_string(scheme: str, string: str): return VersionConstraint(comparator=comparator, version=version) -def build_range_from_github_advisory_constraint(scheme: str, string: str): +def build_range_from_github_advisory_constraint(scheme: str, string: Union[str, List]): """ Github has a special syntax for version ranges. For example: @@ -1231,7 +1234,7 @@ def build_range_from_github_advisory_constraint(scheme: str, string: str): Github native version range looks like: ``>= 1.0.0, < 1.0.1`` - Return a VersionRange built from a ``string`` single github-native + Return a VersionRange built from a ``string`` single or multiple github-native version relationship string. For example:: @@ -1245,14 +1248,91 @@ def build_range_from_github_advisory_constraint(scheme: str, string: str): >>> vr = build_range_from_github_advisory_constraint("pypi","= 9.0") >>> assert str(vr) == "vers:pypi/9.0" """ - constraint_strings = string.split(",") + if isinstance(string, str): + string = [string] + constraints = [] vrc = RANGE_CLASS_BY_SCHEMES[scheme] - for constraint in constraint_strings: - constraints.append(build_constraint_from_github_advisory_string(scheme, constraint)) + for item in string: + constraint_strings = item.split(",") + + for constraint in constraint_strings: + constraints.append(build_constraint_from_github_advisory_string(scheme, constraint)) return vrc(constraints=constraints) +vers_by_snyk_native_comparators = { + "==": "=", + "=": "=", + "!=": "!=", + "<=": "<=", + ">=": ">=", + "<": "<", + ">": ">", + "(": ">", + "[": ">=", +} + +vers_by_snyk_native_comparators_rear = { + ")": "<", + "]": "<=", +} + + +def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]): + """ + Return a VersionRange built from a ``string`` single or multiple snyk + version relationship string. + Snyk version range looks like: + ">=4.0.0, <4.0.10.16" + ">=4.1.0 <4.4.15.7" + "[3.0.0,3.1.25)" + "(,9.21]" + "[1.4.5,)" + + For example:: + + >>> vr = build_range_from_snyk_advisory_string("pypi", ">=4.0.0, <4.0.10") + >>> assert str(vr) == "vers:pypi/>=4.0.0|<4.0.10" + >>> vr = build_range_from_snyk_advisory_string("composer", ">=4.1.0 <4.4.15.7") + >>> assert str(vr) == "vers:composer/>=4.1.0|<4.4.15.7" + >>> vr = build_range_from_snyk_advisory_string("pypi", "(,9.21]") + >>> assert str(vr) == "vers:pypi/<=9.21" + """ + # https://security.snyk.io/package/golang/github.com%2Fmattermost%2Fmattermost%2Fserver%2Fpublic%2Fmodel + # >=9.6.0-rc1 <9.8.1-rc1 + version_constraints = [] + vrc = RANGE_CLASS_BY_SCHEMES[scheme] + + if isinstance(string, str): + string = [string] + + for item in string: + delimiter = "," if "," in item else " " + if delimiter == ",": + snyk_constraints = item.strip().replace(" ", "") + constraints = snyk_constraints.split(",") + else: + snyk_constraints = item.strip() + constraints = snyk_constraints.split(" ") + + for constraint in constraints: + comparator, version = split_req( + string=constraint, + comparators=vers_by_snyk_native_comparators, + comparators_rear=vers_by_snyk_native_comparators_rear, + ) + if comparator and version: + version = vrc.version_class(version) + version_constraints.append( + VersionConstraint( + comparator=comparator, + version=version, + ) + ) + return vrc(constraints=version_constraints) + + RANGE_CLASS_BY_SCHEMES = { "npm": NpmVersionRange, "deb": DebianVersionRange, From e7d7c554f19b25479b55aaeef26dc0548fde478d Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 22 Jul 2024 19:41:21 +0530 Subject: [PATCH 17/25] Add support for version range from discrete versions Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 207167fd..f5a4c9e4 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1294,8 +1294,8 @@ def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]) >>> vr = build_range_from_snyk_advisory_string("pypi", ">=4.0.0, <4.0.10") >>> assert str(vr) == "vers:pypi/>=4.0.0|<4.0.10" - >>> vr = build_range_from_snyk_advisory_string("composer", ">=4.1.0 <4.4.15.7") - >>> assert str(vr) == "vers:composer/>=4.1.0|<4.4.15.7" + >>> vr = build_range_from_snyk_advisory_string("golang", ">=9.6.0-rc1 <9.8.1-rc1") + >>> assert str(vr) == "vers:golang/>=9.6.0-rc1|<9.8.1-rc1" >>> vr = build_range_from_snyk_advisory_string("pypi", "(,9.21]") >>> assert str(vr) == "vers:pypi/<=9.21" """ @@ -1333,6 +1333,35 @@ def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]) return vrc(constraints=version_constraints) +def build_range_from_discrete_version_string(scheme: str, string: Union[str, List]): + """ + Return VersionRange computed from discrete versions. + Discrete version range looks like: + ["1.5","3.1.2","3.1-beta"] + + For example:: + + # >>> vr = build_constraints_from_discrete_version_string("pypi", ["1.5","3.1.2","3.1-beta"]) + # >>> assert str(vr) == "vers:pypi/1.5|3.1-beta|3.1.2" + # >>> vr = build_constraints_from_discrete_version_string("pypi","9.21") + # >>> assert str(vr) == "vers:pypi/9.21" + """ + version_constraints = [] + vrc = RANGE_CLASS_BY_SCHEMES[scheme] + + if isinstance(string, str): + string = [string] + + for item in string: + version = item.strip().lstrip("vV") + version_constraints.append( + VersionConstraint( + version=version, + ) + ) + return vrc(constraints=version_constraints) + + RANGE_CLASS_BY_SCHEMES = { "npm": NpmVersionRange, "deb": DebianVersionRange, From f8a6d707e05275c9817481699d47c18a486dc8ec Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 23 Jul 2024 23:58:10 +0530 Subject: [PATCH 18/25] Refactor NormalizedVersionRange Signed-off-by: Keshav Priyadarshi --- src/univers/normalized_range.py | 381 +++-------- src/univers/version_range.py | 4 + tests/test_normalized_range.py | 1092 ++----------------------------- 3 files changed, 143 insertions(+), 1334 deletions(-) diff --git a/src/univers/normalized_range.py b/src/univers/normalized_range.py index 453e4ad7..8367943f 100644 --- a/src/univers/normalized_range.py +++ b/src/univers/normalized_range.py @@ -5,325 +5,132 @@ # Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. -import operator -import re from typing import List -from typing import Union import attr from univers.span import Span +from univers.version_constraint import VersionConstraint +from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange -from univers.versions import AlpineLinuxVersion -from univers.versions import ArchLinuxVersion -from univers.versions import ComposerVersion -from univers.versions import DebianVersion -from univers.versions import GenericVersion -from univers.versions import GentooVersion -from univers.versions import GolangVersion -from univers.versions import MavenVersion -from univers.versions import NginxVersion -from univers.versions import NugetVersion -from univers.versions import OpensslVersion -from univers.versions import PypiVersion -from univers.versions import RpmVersion -from univers.versions import RubygemsVersion -from univers.versions import SemverVersion @attr.s(frozen=True, order=False, eq=True, hash=True) -class NormalizedVersionRanges: +class NormalizedVersionRange: """ - A NormalizedVersionRange represents a list of VersionRange resolved - from diffrent datsource. + A normalized_range is a summation of the largest contiguous version ranges. + + For example, for an npm package with the version range "vers:npm/<=2.0.0|>=3.0.0|<3.1.0|4.0.0" + and available package versions ["1.0.0", "2.0.0", "3.0.0", "3.1.0", "4.0.0"], the normalized range + would be "vers:npm/>=1.0.0|<=3.0.0|4.0.0". """ - # A tuple of VersionRange - version_ranges = attr.ib(type=tuple, default=attr.Factory(tuple)) + normalized_range = attr.ib(type=VersionRange, default=None) def __str__(self): - return "(" + ", ".join([f"'{str(vers)}'" for vers in self.version_ranges]) + ")" - - @staticmethod - def get_span_boundry(comparator: str, version: str, version_map: dict): - """ - Return Span with Lower and Upper boundry limit. - """ - index = NormalizedVersionRanges.get_version_rank(version, version_map) - resolved_operator = OPERATOR_BY_COMPRATOR.get(comparator, operator.eq) - - if resolved_operator == operator.lt: - return Span(1, index - 1) - if resolved_operator == operator.gt: - return Span(index + 1, len(version_map)) - if resolved_operator == operator.ge: - return Span(index, len(version_map)) - if resolved_operator == operator.le: - return Span(1, index) - if resolved_operator == operator.eq: - return Span(index) - if resolved_operator == operator.ne: - return Span(1, index - 1).union(Span(index + 1, len(version_map))) - - @staticmethod - def get_version_range_from_span(total_span: Span, purl_type: str, reverse_version_map: dict): - """ - Return list containg VersionRange for all subspans in a Span. - """ - version_ranges = [] - list_of_span = total_span.subspans() - for span in list_of_span: - lower_bound = reverse_version_map[span.start] - upper_bound = reverse_version_map[span.end] - vers_exp = ( - f"vers:{purl_type}/{lower_bound}" - if lower_bound == upper_bound - else f"vers:{purl_type}/>={lower_bound}|<={upper_bound}" - ) - version_ranges.append(VersionRange.from_string(vers_exp)) - return version_ranges - - @staticmethod - def get_version_rank(version: str, version_map: dict): - """ - Return equivalent integer ranking for a version. - """ - try: - return version_map[strip_leading_v(version)] - except KeyError as err: - err.args = (f"{version} doesn't exist.",) - raise - - @staticmethod - def parse_constraint(constraint: str): - """ - Return operator and version from a constraint - For example: - >>> assert NormalizedVersionRanges.parse_constraint(">=7.0.0") == ('>=', '7.0.0') - >>> assert NormalizedVersionRanges.parse_constraint("=7.0.0") == ('=', '7.0.0') - >>> assert NormalizedVersionRanges.parse_constraint("[3.0.0") == ('[', '3.0.0') - >>> assert NormalizedVersionRanges.parse_constraint("3.1.25]") == (']', '3.1.25') - """ - if constraint.startswith(("<=", ">=", "==", "!=")): - return constraint[:2], constraint[2:] - - if constraint.startswith(("<", ">", "=", "[", "(")): - return constraint[0], constraint[1:] - - if constraint.endswith(("]", ")")): - return constraint[-1], constraint[:-1] - return None, constraint - - @staticmethod - def get_version_map(versions: List, purl_type: str): - """ - Return dict mapping version to integer. - """ - if purl_type not in VERSIONS_BY_PACKAGE_TYPE: - return - - version_type = VERSIONS_BY_PACKAGE_TYPE.get(purl_type) - sorted_versions = sorted([version_type(i) for i in versions]) - sorted_versions = [version.string for version in sorted_versions] - index = list(range(1, len(sorted_versions) + 1, 1)) - return dict(zip(sorted_versions, index)) + return str(self.normalized_range) @classmethod - def from_github(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): + def from_vers(cls, vers_range: VersionRange, all_versions: List): """ - Return NormalizedVersionRanges computed from GithHub version range expression. - GitHub range_expression example:: - ">= 10.4.0, < 10.4.1" - "> 7.1.1" + Return NormalizedVersionRange computed from vers_range and all the available version of package. """ - version_map = cls.get_version_map(all_versions, purl_type) - reverse_version_map = {value: key for key, value in version_map.items()} + version_class = vers_range.version_class + versions = sorted([version_class(i.lstrip("vV")) for i in all_versions]) - items = [range_expression] if isinstance(range_expression, str) else range_expression + bounded_span = None total_span = None - for item in items: - gh_constraints = item.strip().replace(" ", "") - constraints = gh_constraints.split(",") - local_span = None - for constraint in constraints: - if not constraint: - continue - gh_comparator, gh_version = cls.parse_constraint(constraint) - span = cls.get_span_boundry(gh_comparator, strip_leading_v(gh_version), version_map) - local_span = span if not local_span else local_span.intersection(span) - - total_span = local_span if not total_span else total_span.union(local_span) - - version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) - return cls(version_ranges=version_ranges) - - @classmethod - def from_snyk(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): - """ - Return NormalizedVersionRanges computed from Snyk version range expression. - Snyk range_expression example:: - ">=4.0.0, <4.0.10.16" - " >=4.1.0, <4.4.15.7" - "[3.0.0,3.1.25) - """ - version_map = cls.get_version_map(all_versions, purl_type) - reverse_version_map = {value: key for key, value in version_map.items()} + for constraint in vers_range.constraints: + local_span = get_region(constraint=constraint, versions=versions) - items = [range_expression] if isinstance(range_expression, str) else range_expression - total_span = None - for item in items: - delimiter = "," if "," in item else " " - if delimiter == ",": - snyk_constraints = item.strip().replace(" ", "") - constraints = snyk_constraints.split(",") - else: - snyk_constraints = item.strip() - constraints = snyk_constraints.split(" ") - local_span = None - for constraint in constraints: - if not constraint: - continue - snyk_comparator, snyk_version = cls.parse_constraint(constraint) - if not snyk_version: - continue - span = cls.get_span_boundry( - snyk_comparator, strip_leading_v(snyk_version), version_map - ) - local_span = span if not local_span else local_span.intersection(span) + if bounded_span and constraint.comparator in ("<", "<="): + local_span = bounded_span.intersection(local_span) + elif constraint.comparator in (">", ">="): + bounded_span = local_span + continue total_span = local_span if not total_span else total_span.union(local_span) + bounded_span = None - version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) - return cls(version_ranges=version_ranges) - - @classmethod - def from_gitlab(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): - """ - Return NormalizedVersionRanges computed from GitLab version range expression. - GitLab range_expression example:: - "[7.0.0,7.0.11),[7.2.0,7.2.4)" - "[7.0.0,7.0.11),[7.2.0,7.2.4)" - ">=4.0,<4.3||>=5.0,<5.2" - ">=0.19.0 <0.30.0" - ">=1.5,<1.5.2" - """ - - version_map = cls.get_version_map(all_versions, purl_type) - reverse_version_map = {value: key for key, value in version_map.items()} - - items = [range_expression] if isinstance(range_expression, str) else range_expression - global_span = None - for item in items: - gitlab_constraints = item.strip() - if gitlab_constraints.startswith(("[", "(")): - # transform "[7.0.0,7.0.11),[7.2.0,7.2.4)" -> [ "[7.0.0,7.0.11)", "[7.2.0,7.2.4)" ] - splitted = gitlab_constraints.split(",") - constraints = [f"{a},{b}" for a, b in zip(splitted[::2], splitted[1::2])] - delimiter = "," - - else: - # transform ">=4.0,<4.3||>=5.0,<5.2" -> [ ">=4.0,<4.3", ">=5.0,<5.2" ] - # transform ">=0.19.0 <0.30.0" -> [ ">=0.19.0 <0.30.0" ] - # transform ">=1.5,<1.5.2" -> [ ">=1.5,<1.5.2" ] - delimiter = "," if "," in gitlab_constraints else " " - constraints = gitlab_constraints.split("||") - total_span = None - for constraint in constraints: - local_span = None - for subcontraint in constraint.strip().split(delimiter): - if not subcontraint: - continue - gitlab_comparator, gitlab_version = cls.parse_constraint(subcontraint.strip()) - if not gitlab_version: - continue - span = cls.get_span_boundry( - gitlab_comparator, strip_leading_v(gitlab_version), version_map - ) - local_span = span if not local_span else local_span.intersection(span) - - total_span = local_span if not total_span else total_span.union(local_span) - global_span = total_span if not global_span else global_span.union(total_span) + # If '<' or '<=' is the last constraint. + if bounded_span: + total_span = bounded_span if not total_span else total_span.union(bounded_span) - version_ranges = cls.get_version_range_from_span( - global_span, purl_type, reverse_version_map + normalized_version_range = get_version_range_from_span( + span=total_span, purl_type=vers_range.scheme, versions=versions ) - return cls(version_ranges=version_ranges) + return cls(normalized_range=normalized_version_range) - @classmethod - def from_discrete(cls, range_expression: Union[str, List], purl_type: str, all_versions: List): - """ - Return NormalizedVersionRanges computed from discrete version range expression. - Discrete range_expression example:: - ["1.5","3.1.2","3.1-beta"] - """ - version_map = cls.get_version_map(all_versions, purl_type) - reverse_version_map = {value: key for key, value in version_map.items()} - item = range_expression if isinstance(range_expression, str) else " ".join(range_expression) - discrete_versions = re.split("[ ,\n]+", item) - - rank_list = [] - for version in discrete_versions: - try: - rank_int = version_map[strip_leading_v(version)] - rank_list.append(rank_int) - except KeyError: - pass +def get_version_range_from_span(span: Span, purl_type: str, versions: List): + """ + Return VersionRange computed from the span and all the versions available for package. + + For example:: + >>> from univers.versions import SemverVersion as SV + >>> versions = [SV("1.0"), SV("1.1"), SV("1.2"), SV("1.3")] + >>> span = Span(0,1).union(Span(3)) + >>> vr = get_version_range_from_span(span, "npm", versions) + >>> assert str(vr) == "vers:npm/>=1.0.0|<=1.1.0|1.3.0" + """ - total_span = Span(rank_list) + version_range_class = RANGE_CLASS_BY_SCHEMES[purl_type] + version_constraints = [] + spans = span.subspans() + for subspan in spans: + lower_bound = versions[subspan.start] + upper_bound = versions[subspan.end] + if lower_bound == upper_bound: + version_constraints.append( + VersionConstraint( + version=lower_bound, + ) + ) + continue + version_constraints.append(VersionConstraint(comparator=">=", version=lower_bound)) + version_constraints.append( + VersionConstraint( + comparator="<=", + version=upper_bound, + ) + ) - version_ranges = cls.get_version_range_from_span(total_span, purl_type, reverse_version_map) - return cls(version_ranges) + return version_range_class(constraints=version_constraints) -def strip_leading_v(version: str): +def get_region(constraint: VersionConstraint, versions: List): """ - Return version without leading v. + Return a Span representing the region covered by the constraint on + the given universe of versions. + + For example:: + >>> from univers.versions import SemverVersion as SV + >>> versions = [SV("1.0"), SV("1.1"), SV("1.2"), SV("1.3")] + >>> constraint = VersionConstraint(comparator="<", version=SV("1.2")) + >>> get_region(constraint, versions) + Span(0, 1) """ - if not version.startswith("v"): - return version - return version[1:] - - -VERSIONS_BY_PACKAGE_TYPE = { - "alpine": AlpineLinuxVersion, - "alpm": ArchLinuxVersion, - "apache": SemverVersion, - "cargo": SemverVersion, - # "cocoapods": None, - "composer": ComposerVersion, - # "conan": None, - # "conda": None, - # "cran": None, - "deb": DebianVersion, - "ebuild": GentooVersion, - "gem": RubygemsVersion, - "generic": GenericVersion, - "github": SemverVersion, - "golang": GolangVersion, - "hex": SemverVersion, - "mattermost": SemverVersion, - "maven": MavenVersion, - "mozilla": SemverVersion, - "nginx": NginxVersion, - "npm": SemverVersion, - "nuget": NugetVersion, - "openssl": OpensslVersion, - "pypi": PypiVersion, - "rpm": RpmVersion, - # "swift": None, -} -OPERATOR_BY_COMPRATOR = { - "<": operator.lt, - ">": operator.gt, - "=": operator.eq, - "<=": operator.le, - ">=": operator.ge, - "==": operator.eq, - "!=": operator.ne, - ")": operator.lt, - "]": operator.le, - "(": operator.gt, - "[": operator.ge, -} + try: + index = 0 + if str(constraint.version) != "0": + index = versions.index(constraint.version) + except ValueError as err: + err.args = (f"'{constraint.version}' doesn't exist in the versions list.",) + raise + + last_index = len(versions) - 1 + comparator = constraint.comparator + + if comparator == "<": + return Span(0, index - 1) + if comparator == ">": + return Span(index + 1, last_index) + if comparator == ">=": + return Span(index, last_index) + if comparator == "<=": + return Span(0, index) + if comparator == "=": + return Span(index) + if comparator == "!=": + return Span(0, last_index).difference(Span(index)) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index f5a4c9e4..5ac34a5a 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1354,6 +1354,10 @@ def build_range_from_discrete_version_string(scheme: str, string: Union[str, Lis for item in string: version = item.strip().lstrip("vV") + if version == "0": + continue + + version = vrc.version_class(version) version_constraints.append( VersionConstraint( version=version, diff --git a/tests/test_normalized_range.py b/tests/test_normalized_range.py index 6d3db981..5eea7c89 100644 --- a/tests/test_normalized_range.py +++ b/tests/test_normalized_range.py @@ -6,1067 +6,65 @@ from unittest import TestCase -from univers.normalized_range import NormalizedVersionRanges +from univers.normalized_range import NormalizedVersionRange +from univers.normalized_range import get_region +from univers.normalized_range import get_version_range_from_span +from univers.span import Span from univers.version_constraint import VersionConstraint -from univers.version_range import PypiVersionRange from univers.version_range import VersionRange -from univers.versions import PypiVersion +from univers.versions import SemverVersion -all_versions_pypi = [ - "1.1.3", - "1.1.4", - "1.10", - "1.10.1", - "1.10.2", - "1.10.3", - "1.10.4", - "1.10.5", - "1.10.6", - "1.10.7", - "1.10.8", - "1.10a1", - "1.10b1", - "1.10rc1", - "1.11", - "1.11.1", - "1.11.10", - "1.11.11", - "1.11.12", - "1.11.13", - "1.11.14", - "1.11.15", - "1.11.16", - "1.11.17", - "1.11.18", - "1.11.2", - "1.11.20", - "1.11.21", - "1.11.22", - "1.11.23", - "1.11.24", - "1.11.25", - "1.11.26", - "1.11.27", - "1.11.28", - "1.11.29", - "1.11.3", - "1.11.4", - "1.11.5", - "1.11.6", - "1.11.7", - "1.11.8", - "1.11.9", - "1.11a1", - "1.11b1", - "1.11rc1", - "1.2", - "1.2.1", - "1.2.2", - "1.2.3", - "1.2.4", - "1.2.5", - "1.2.6", - "1.2.7", - "1.3", - "1.3.1", - "1.3.2", - "1.3.3", - "1.3.4", - "1.3.5", - "1.3.6", - "1.3.7", - "1.4", - "1.4.1", - "1.4.10", - "1.4.11", - "1.4.12", - "1.4.13", - "1.4.14", - "1.4.15", - "1.4.16", - "1.4.17", - "1.4.18", - "1.4.19", - "1.4.2", - "1.4.20", - "1.4.21", - "1.4.22", - "1.4.3", - "1.4.4", - "1.4.5", - "1.4.6", - "1.4.7", - "1.4.8", - "1.4.9", - "1.5", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.5.9", - "1.6", - "1.6.1", - "1.6.10", - "1.6.11", - "1.6.2", - "1.6.3", - "1.6.4", - "1.6.5", - "1.6.6", - "1.6.7", - "1.6.8", - "1.6.9", - "1.7", - "1.7.1", - "1.7.10", - "1.7.11", - "1.7.2", - "1.7.3", - "1.7.4", - "1.7.5", - "1.7.6", - "1.7.7", - "1.7.8", - "1.7.9", - "1.8", - "1.8.1", - "1.8.10", - "1.8.11", - "1.8.12", - "1.8.13", - "1.8.14", - "1.8.15", - "1.8.16", - "1.8.17", - "1.8.18", - "1.8.19", - "1.8.2", - "1.8.3", - "1.8.4", - "1.8.5", - "1.8.6", - "1.8.7", - "1.8.8", - "1.8.9", - "1.8a1", - "1.8b1", - "1.8b2", - "1.8c1", - "1.9", - "1.9.1", - "1.9.10", - "1.9.11", - "1.9.12", - "1.9.13", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "1.9.7", - "1.9.8", - "1.9.9", - "1.9a1", - "1.9b1", - "1.9rc1", - "1.9rc2", - "2.0", - "2.0.1", - "2.0.10", - "2.0.12", - "2.0.13", - "2.0.2", - "2.0.3", - "2.0.4", - "2.0.5", - "2.0.6", - "2.0.7", - "2.0.8", - "2.0.9", - "2.0a1", - "2.0b1", - "2.0rc1", - "2.1", - "2.1.1", - "2.1.10", - "2.1.11", - "2.1.12", - "2.1.13", - "2.1.14", - "2.1.15", - "2.1.2", - "2.1.3", - "2.1.4", - "2.1.5", - "2.1.7", - "2.1.8", - "2.1.9", - "2.1a1", - "2.1b1", - "2.1rc1", - "2.2", - "2.2.1", - "2.2.10", - "2.2.11", - "2.2.12", - "2.2.13", - "2.2.14", - "2.2.15", - "2.2.16", - "2.2.17", - "2.2.18", - "2.2.19", - "2.2.2", - "2.2.20", - "2.2.21", - "2.2.22", - "2.2.23", - "2.2.24", - "2.2.25", - "2.2.26", - "2.2.27", - "2.2.28", - "2.2.3", - "2.2.4", - "2.2.5", - "2.2.6", - "2.2.7", - "2.2.8", - "2.2.9", - "2.2a1", - "2.2b1", - "2.2rc1", - "3.0", - "3.0.1", - "3.0.10", - "3.0.11", - "3.0.12", - "3.0.13", - "3.0.14", - "3.0.2", - "3.0.3", - "3.0.4", - "3.0.5", - "3.0.6", - "3.0.7", - "3.0.8", - "3.0.9", - "3.0a1", - "3.0b1", - "3.0rc1", - "3.1", - "3.1.1", - "3.1.10", - "3.1.11", - "3.1.12", - "3.1.13", - "3.1.14", - "3.1.2", - "3.1.3", - "3.1.4", - "3.1.5", - "3.1.6", - "3.1.7", - "3.1.8", - "3.1.9", - "3.1a1", - "3.1b1", - "3.1rc1", - "3.2", - "3.2.1", - "3.2.10", - "3.2.11", - "3.2.12", - "3.2.13", - "3.2.14", - "3.2.15", - "3.2.16", - "3.2.17", - "3.2.18", - "3.2.2", - "3.2.3", - "3.2.4", - "3.2.5", - "3.2.6", - "3.2.7", - "3.2.8", - "3.2.9", - "3.2a1", - "3.2b1", - "3.2rc1", - "4.0", - "4.0.1", - "4.0.10", - "4.0.2", - "4.0.3", - "4.0.4", - "4.0.5", - "4.0.6", - "4.0.7", - "4.0.8", - "4.0.9", - "4.0a1", - "4.0b1", - "4.0rc1", - "4.1", - "4.1.1", - "4.1.2", - "4.1.3", - "4.1.4", - "4.1.5", - "4.1.6", - "4.1.7", - "4.1a1", - "4.1b1", - "4.1rc1", - "4.2a1", - "4.2b1", -] +class TestNormalizedVersionRange(TestCase): + purl_type = "pypi" + all_versions = versions = [ + "1.0", + "1.1", + "1.2", + "1.3", + "2.0", + "3.0", + ] + versions = [SemverVersion(i) for i in all_versions] -class TestNormalizedVersionRanges(TestCase): - purl_type_pypi = "pypi" + def test_get_region(self): - def test_NormalizedVersionRanges_from_github_type1(self): - range_expression = "> 1.11rc1" + constraint1 = VersionConstraint(comparator="<=", version=SemverVersion("1.0")) + constraint2 = VersionConstraint(comparator="!=", version=SemverVersion("1.1")) + constraint3 = VersionConstraint(comparator=">", version=SemverVersion("1.3")) - normalized_range = NormalizedVersionRanges.from_github( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, + assert get_region(constraint=constraint1, versions=self.versions) == Span(0) + assert get_region(constraint=constraint2, versions=self.versions) == Span(0).union( + Span(2, 5) ) + assert get_region(constraint=constraint3, versions=self.versions) == Span(4, 5) - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.11")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.2b1")), - ) - ) - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_github_type2(self): - range_expression = ">= 3.2.2, < 4.0.2" - purl_type = "pypi" + def test_get_version_range_from_span(self): + span1 = Span(1) + span2 = Span(1, 4) + span3 = Span(1, 4).intersection(Span(3, 5)) + span4 = Span(0).union(Span(2, 3)).union(Span(5)) - normalized_range = NormalizedVersionRanges.from_github( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, + vr1 = get_version_range_from_span( + span=span1, purl_type=self.purl_type, versions=self.versions ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.2.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), - ) - ) - ] + vr2 = get_version_range_from_span( + span=span2, purl_type=self.purl_type, versions=self.versions ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_github_type3(self): - range_expression = [">= 3.2.2, < 4.0.2", ">1.4.9, < 1.11rc1"] - - normalized_range = NormalizedVersionRanges.from_github( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, + vr3 = get_version_range_from_span( + span=span3, purl_type=self.purl_type, versions=self.versions ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.4.10")), - VersionConstraint(comparator="<=", version=PypiVersion(string="1.11b1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.2.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), - ) - ), - ] + vr4 = get_version_range_from_span( + span=span4, purl_type=self.purl_type, versions=self.versions ) - assert normalized_range == expected - def test_NormalizedVersionRanges_from_snyk_type1(self): - range_expression = " >= 3.0.1, < 4.0.2" - - normalized_range = NormalizedVersionRanges.from_snyk( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), - ) - ) - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_snyk_type2(self): - range_expression = "[3.0.1,4.0.2)" - - normalized_range = NormalizedVersionRanges.from_snyk( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), - ) - ) - ] - ) - assert normalized_range == expected + assert str(vr1) == "vers:pypi/1.1.0" + assert str(vr2) == "vers:pypi/>=1.1.0|<=2.0.0" + assert str(vr3) == "vers:pypi/>=1.3.0|<=2.0.0" + assert str(vr4) == "vers:pypi/1.0.0|>=1.2.0|<=1.3.0|3.0.0" - def test_NormalizedVersionRanges_from_snyk_type3(self): - range_expression = ["[3.0.1,4.0.2)", " >= 3.0.1, < 4.0.2"] - - normalized_range = NormalizedVersionRanges.from_snyk( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.0.1")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0.1")), - ) - ) - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_gitlab_type1(self): - range_expression = "[1.4,2.1rc1),[3.2,4.0rc1)" - - normalized_range = NormalizedVersionRanges.from_gitlab( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.4")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.1b1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0b1")), - ) - ), - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_gitlab_type2(self): - range_expression = ">=4.0,<4.2b1||>=1.2,<2.1" - - normalized_range = NormalizedVersionRanges.from_gitlab( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="4.0")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.2a1")), - ) - ), - ] - ) - assert normalized_range == expected + def test_NormalizedVersionRange_from_vers(self): + vr = VersionRange.from_string("vers:pypi/<=1.3|>=2.0|<3.0.0") + nvr = NormalizedVersionRange.from_vers(vers_range=vr, all_versions=self.all_versions) - def test_NormalizedVersionRanges_from_gitlab_type3(self): - range_expression = ">=1.2 <2.1" - - normalized_range = NormalizedVersionRanges.from_gitlab( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), - ) - ), - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_gitlab_type4(self): - range_expression = ">=1.5,<1.5.2" - - normalized_range = NormalizedVersionRanges.from_gitlab( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.5")), - VersionConstraint(comparator="<=", version=PypiVersion(string="1.5.1")), - ) - ) - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_gitlab_type5(self): - range_expression = [ - ">=1.2 <2.1", - ">=4.0,<4.2b1||>=1.2,<2.1", - "[1.4,2.1rc1),[3.2,4.0rc1)", - ">=1.5,<1.5.2", - ] - - normalized_range = NormalizedVersionRanges.from_gitlab( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.1rc1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="3.2")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.0b1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="4.0")), - VersionConstraint(comparator="<=", version=PypiVersion(string="4.2a1")), - ) - ), - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_discrete_for_DEPS(self): - range_expression = [ - "2.0", - "2.0rc1", - "2.0rc1-py2.4-linux-i686", - "2.0rc1-py2.4-linux-x86_64", - "2.0rc1-py2.4-macosx-10.3-i386", - "2.0rc1-py2.4-win32", - "2.0rc1-py2.5-linux-i686", - "2.0rc1-py2.5-linux-x86_64", - "2.0rc1-py2.5-macosx-10.3-i386", - "2.0rc1-py2.5-win32", - "2.1-py2.4-win322.1-py2.5-win32", - "2.1-py2.6-win32", - " 2.1.0", - "2.1.1", - "2.1.1-py2.4-win32", - "2.1.1-py2.5-win32", - "2.1.1-py2.6-win32", - "2.10.0 2.2.0", - "2.2.1", - "2.2.3", - "2.2.11", - "2.2.9", - ] - - normalized_range = NormalizedVersionRanges.from_discrete( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="2.0rc1")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.0")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator="=", version=PypiVersion(string="2.1.1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator="=", version=PypiVersion(string="2.2.1")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator="=", version=PypiVersion(string="2.2.3")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator="=", version=PypiVersion(string="2.2.9")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator="=", version=PypiVersion(string="2.2.11")), - ) - ), - ] - ) - assert normalized_range == expected - - def test_NormalizedVersionRanges_from_discrete_for_VulnerableCode(self): - range_expression = [ - "1.10", - "1.10.1", - "1.10.2", - "1.10.3", - "1.10.4", - "1.10.5", - "1.10.6", - "1.10.7", - "1.10.8", - "1.10a1", - "1.10b1", - "1.10rc1", - "1.11", - "1.11.0", - "1.11.1", - "1.11.10", - "1.11.11", - "1.11.12", - "1.11.13", - "1.11.14", - "1.11.15", - "1.11.16", - "1.11.17", - "1.11.18", - "1.11.2", - "1.11.3", - "1.11.4", - "1.11.5", - "1.11.6", - "1.11.7", - "1.11.8", - "1.11.9", - "1.11a1", - "1.11b1", - "1.11rc1", - "1.1.3", - "1.1.4", - "1.2", - "1.2.1", - "1.2.2", - "1.2.3", - "1.2.4", - "1.2.5", - "1.2.6", - "1.2.7", - "1.3", - "1.3.1", - "1.3.2", - "1.3.3", - "1.3.4", - "1.3.5", - "1.3.6", - "1.3.7", - "1.4", - "1.4.1", - "1.4.10", - "1.4.11", - "1.4.12", - "1.4.13", - "1.4.14", - "1.4.15", - "1.4.16", - "1.4.17", - "1.4.18", - "1.4.19", - "1.4.2", - "1.4.20", - "1.4.21", - "1.4.22", - "1.4.3", - "1.4.4", - "1.4.5", - "1.4.6", - "1.4.7", - "1.4.8", - "1.4.9", - "1.5", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.5.9", - "1.6", - "1.6.1", - "1.6.10", - "1.6.11", - "1.6.2", - "1.6.3", - "1.6.4", - "1.6.5", - "1.6.6", - "1.6.7", - "1.6.8", - "1.6.9", - "1.7", - "1.7.1", - "1.7.10", - "1.7.11", - "1.7.2", - "1.7.3", - "1.7.4", - "1.7.5", - "1.7.6", - "1.7.7", - "1.7.8", - "1.7.9", - "1.8", - "1.8.1", - "1.8.10", - "1.8.11", - "1.8.12", - "1.8.13", - "1.8.14", - "1.8.15", - "1.8.16", - "1.8.17", - "1.8.18", - "1.8.19", - "1.8.2", - "1.8.3", - "1.8.4", - "1.8.5", - "1.8.6", - "1.8.7", - "1.8.8", - "1.8.9", - "1.8a1", - "1.8b1", - "1.8b2", - "1.8c1", - "1.9", - "1.9.1", - "1.9.10", - "1.9.11", - "1.9.12", - "1.9.13", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "1.9.7", - "1.9.8", - "1.9.9", - "1.9a1", - "1.9b1", - "1.9rc1", - "1.9rc2", - "2.0", - "2.0.0", - "2.0.1", - "2.0.10", - "2.0.2", - "2.0.3", - "2.0.4", - "2.0.5", - "2.0.6", - "2.0.7", - "2.0.8", - "2.0.9", - "2.1", - "2.1.0", - "2.1.1", - "2.1.2", - "2.1.3", - "2.1.4", - "2.1.5", - ] - - normalized_range = NormalizedVersionRanges.from_discrete( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.1.3")), - VersionConstraint(comparator="<=", version=PypiVersion(string="1.11.18")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="2.0")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.0.10")), - ) - ), - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="2.1")), - VersionConstraint(comparator="<=", version=PypiVersion(string="2.1.5")), - ) - ), - ] - ) - assert normalized_range == expected - return normalized_range - - def test_NormalizedVersionRanges_from_discrete_for_OSV(self): - range_expression = [ - "0", - "1.0.1", - "1.0.2", - "1.0.3", - "1.0.4", - "1.1", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.10", - "1.10.1", - "1.10.2", - "1.10.3", - "1.10.4", - "1.10.5", - "1.10.6", - "1.10.7", - "1.10.8", - "1.10a1", - "1.10b1", - "1.10rc1", - "1.11", - "1.11.1", - "1.11.10", - "1.11.11", - "1.11.12", - "1.11.13", - "1.11.14", - "1.11.15", - "1.11.16", - "1.11.17", - "1.11.18", - "1.11.2", - "1.11.20", - "1.11.21", - "1.11.22", - "1.11.23", - "1.11.24", - "1.11.25", - "1.11.26", - "1.11.27", - "1.11.28", - "1.11.3", - "1.11.4", - "1.11.5", - "1.11.6", - "1.11.7", - "1.11.8", - "1.11.9", - "1.11a1", - "1.11b1", - "1.11rc1", - "1.2", - "1.2.1", - "1.2.2", - "1.2.3", - "1.2.4", - "1.2.5", - "1.2.6", - "1.2.7", - "1.3", - "1.3.1", - "1.3.2", - "1.3.3", - "1.3.4", - "1.3.5", - "1.3.6", - "1.3.7", - "1.4", - "1.4.1", - "1.4.10", - "1.4.11", - "1.4.12", - "1.4.13", - "1.4.14", - "1.4.15", - "1.4.16", - "1.4.17", - "1.4.18", - "1.4.19", - "1.4.2", - "1.4.20", - "1.4.21", - "1.4.22", - "1.4.3", - "1.4.4", - "1.4.5", - "1.4.6", - "1.4.7", - "1.4.8", - "1.4.9", - "1.5", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.5.9", - "1.6", - "1.6.1", - "1.6.10", - "1.6.11", - "1.6.2", - "1.6.3", - "1.6.4", - "1.6.5", - "1.6.6", - "1.6.7", - "1.6.8", - "1.6.9", - "1.7", - "1.7.1", - "1.7.10", - "1.7.11", - "1.7.2", - "1.7.3", - "1.7.4", - "1.7.5", - "1.7.6", - "1.7.7", - "1.7.8", - "1.7.9", - "1.8", - "1.8.1", - "1.8.10", - "1.8.11", - "1.8.12", - "1.8.13", - "1.8.14", - "1.8.15", - "1.8.16", - "1.8.17", - "1.8.18", - "1.8.19", - "1.8.2", - "1.8.3", - "1.8.4", - "1.8.5", - "1.8.6", - "1.8.7", - "1.8.8", - "1.8.9", - "1.8a1", - "1.8b1", - "1.8b2", - "1.8c1", - "1.9", - "1.9.1", - "1.9.10", - "1.9.11", - "1.9.12", - "1.9.13", - "1.9.2", - "1.9.3", - "1.9.4", - "1.9.5", - "1.9.6", - "1.9.7", - "1.9.8", - "1.9.9", - "1.9a1", - "1.9b1", - "1.9rc1", - "1.9rc2", - ] - normalized_range = NormalizedVersionRanges.from_discrete( - range_expression=range_expression, - purl_type=self.purl_type_pypi, - all_versions=all_versions_pypi, - ) - - expected = NormalizedVersionRanges( - version_ranges=[ - PypiVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=PypiVersion(string="1.1.3")), - VersionConstraint(comparator="<=", version=PypiVersion(string="1.11.28")), - ) - ) - ] - ) - assert normalized_range == expected + assert str(nvr) == "vers:pypi/>=1.0|<=2.0" From d2904b0e92804ceb940e1a950eb70a8930c0e703 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 24 Jul 2024 15:27:05 +0530 Subject: [PATCH 19/25] Add multi vers test for normalized version range Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 2 -- tests/test_normalized_range.py | 36 ++++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 5ac34a5a..9c786d85 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1299,8 +1299,6 @@ def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]) >>> vr = build_range_from_snyk_advisory_string("pypi", "(,9.21]") >>> assert str(vr) == "vers:pypi/<=9.21" """ - # https://security.snyk.io/package/golang/github.com%2Fmattermost%2Fmattermost%2Fserver%2Fpublic%2Fmodel - # >=9.6.0-rc1 <9.8.1-rc1 version_constraints = [] vrc = RANGE_CLASS_BY_SCHEMES[scheme] diff --git a/tests/test_normalized_range.py b/tests/test_normalized_range.py index 5eea7c89..495bf042 100644 --- a/tests/test_normalized_range.py +++ b/tests/test_normalized_range.py @@ -18,20 +18,20 @@ class TestNormalizedVersionRange(TestCase): purl_type = "pypi" all_versions = versions = [ - "1.0", - "1.1", - "1.2", - "1.3", - "2.0", - "3.0", + "1.0.0", + "1.1.0", + "1.2.0", + "1.3.0", + "2.0.0", + "3.0.0", ] versions = [SemverVersion(i) for i in all_versions] def test_get_region(self): - constraint1 = VersionConstraint(comparator="<=", version=SemverVersion("1.0")) - constraint2 = VersionConstraint(comparator="!=", version=SemverVersion("1.1")) - constraint3 = VersionConstraint(comparator=">", version=SemverVersion("1.3")) + constraint1 = VersionConstraint(comparator="<=", version=SemverVersion("1.0.0")) + constraint2 = VersionConstraint(comparator="!=", version=SemverVersion("1.1.0")) + constraint3 = VersionConstraint(comparator=">", version=SemverVersion("1.3.0")) assert get_region(constraint=constraint1, versions=self.versions) == Span(0) assert get_region(constraint=constraint2, versions=self.versions) == Span(0).union( @@ -64,7 +64,19 @@ def test_get_version_range_from_span(self): assert str(vr4) == "vers:pypi/1.0.0|>=1.2.0|<=1.3.0|3.0.0" def test_NormalizedVersionRange_from_vers(self): - vr = VersionRange.from_string("vers:pypi/<=1.3|>=2.0|<3.0.0") - nvr = NormalizedVersionRange.from_vers(vers_range=vr, all_versions=self.all_versions) + vr1 = VersionRange.from_string("vers:pypi/<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") + nvr1 = NormalizedVersionRange.from_vers(vers_range=vr1, all_versions=self.all_versions) - assert str(nvr) == "vers:pypi/>=1.0|<=2.0" + vr2 = VersionRange.from_string("vers:pypi/>=1.0.0|<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") + nvr2 = NormalizedVersionRange.from_vers(vers_range=vr2, all_versions=self.all_versions) + + vr3 = VersionRange.from_string("vers:pypi/<=1.3.0|3.0.0") + nvr3 = NormalizedVersionRange.from_vers(vers_range=vr3, all_versions=self.all_versions) + + vr4 = VersionRange.from_string("vers:pypi/<2.0.0|3.0.0") + nvr4 = NormalizedVersionRange.from_vers(vers_range=vr4, all_versions=self.all_versions) + + assert str(nvr1) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" + assert str(nvr2) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" + assert str(nvr3) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" + assert str(nvr4) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" From 5a32eda73f67236bb7317b706f0ededfd72bf0e6 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 24 Jul 2024 19:42:49 +0530 Subject: [PATCH 20/25] Fix the edge case resulting in incorrect `contains` resolution - Pairwise constraint evaluation misses the case when filtered constraints contains only one item - Fixes https://github.com/nexB/univers/issues/137 Signed-off-by: Keshav Priyadarshi --- src/univers/version_constraint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/univers/version_constraint.py b/src/univers/version_constraint.py index 1cc87552..356aa3a8 100644 --- a/src/univers/version_constraint.py +++ b/src/univers/version_constraint.py @@ -506,6 +506,10 @@ def contains_version(version, constraints): if not constraints: return False + # If we end up with constraints list contains only one item. + if len(constraints) == 1: + return version in constraints[0] + # Iterate over the current and next contiguous constraints pairs (aka. pairwise) # in the second list. # For each current and next constraint: From c833b97b4ede18ccf2e5b49e6a6b27b1abed2e25 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 24 Jul 2024 19:55:59 +0530 Subject: [PATCH 21/25] Refactor VersionRange normalization without Span Signed-off-by: Keshav Priyadarshi --- src/univers/normalized_range.py | 136 --------- src/univers/span.py | 494 -------------------------------- src/univers/spans.py.ABOUT | 16 -- src/univers/spans.py.LICENSE | 26 -- src/univers/version_range.py | 31 ++ tests/test_normalized_range.py | 82 ------ tests/test_version_range.py | 29 +- 7 files changed, 59 insertions(+), 755 deletions(-) delete mode 100644 src/univers/normalized_range.py delete mode 100644 src/univers/span.py delete mode 100644 src/univers/spans.py.ABOUT delete mode 100644 src/univers/spans.py.LICENSE delete mode 100644 tests/test_normalized_range.py diff --git a/src/univers/normalized_range.py b/src/univers/normalized_range.py deleted file mode 100644 index 8367943f..00000000 --- a/src/univers/normalized_range.py +++ /dev/null @@ -1,136 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. - - -from typing import List - -import attr - -from univers.span import Span -from univers.version_constraint import VersionConstraint -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import VersionRange - - -@attr.s(frozen=True, order=False, eq=True, hash=True) -class NormalizedVersionRange: - """ - A normalized_range is a summation of the largest contiguous version ranges. - - For example, for an npm package with the version range "vers:npm/<=2.0.0|>=3.0.0|<3.1.0|4.0.0" - and available package versions ["1.0.0", "2.0.0", "3.0.0", "3.1.0", "4.0.0"], the normalized range - would be "vers:npm/>=1.0.0|<=3.0.0|4.0.0". - """ - - normalized_range = attr.ib(type=VersionRange, default=None) - - def __str__(self): - return str(self.normalized_range) - - @classmethod - def from_vers(cls, vers_range: VersionRange, all_versions: List): - """ - Return NormalizedVersionRange computed from vers_range and all the available version of package. - """ - version_class = vers_range.version_class - versions = sorted([version_class(i.lstrip("vV")) for i in all_versions]) - - bounded_span = None - total_span = None - for constraint in vers_range.constraints: - local_span = get_region(constraint=constraint, versions=versions) - - if bounded_span and constraint.comparator in ("<", "<="): - local_span = bounded_span.intersection(local_span) - elif constraint.comparator in (">", ">="): - bounded_span = local_span - continue - - total_span = local_span if not total_span else total_span.union(local_span) - bounded_span = None - - # If '<' or '<=' is the last constraint. - if bounded_span: - total_span = bounded_span if not total_span else total_span.union(bounded_span) - - normalized_version_range = get_version_range_from_span( - span=total_span, purl_type=vers_range.scheme, versions=versions - ) - return cls(normalized_range=normalized_version_range) - - -def get_version_range_from_span(span: Span, purl_type: str, versions: List): - """ - Return VersionRange computed from the span and all the versions available for package. - - For example:: - >>> from univers.versions import SemverVersion as SV - >>> versions = [SV("1.0"), SV("1.1"), SV("1.2"), SV("1.3")] - >>> span = Span(0,1).union(Span(3)) - >>> vr = get_version_range_from_span(span, "npm", versions) - >>> assert str(vr) == "vers:npm/>=1.0.0|<=1.1.0|1.3.0" - """ - - version_range_class = RANGE_CLASS_BY_SCHEMES[purl_type] - version_constraints = [] - spans = span.subspans() - for subspan in spans: - lower_bound = versions[subspan.start] - upper_bound = versions[subspan.end] - if lower_bound == upper_bound: - version_constraints.append( - VersionConstraint( - version=lower_bound, - ) - ) - continue - version_constraints.append(VersionConstraint(comparator=">=", version=lower_bound)) - version_constraints.append( - VersionConstraint( - comparator="<=", - version=upper_bound, - ) - ) - - return version_range_class(constraints=version_constraints) - - -def get_region(constraint: VersionConstraint, versions: List): - """ - Return a Span representing the region covered by the constraint on - the given universe of versions. - - For example:: - >>> from univers.versions import SemverVersion as SV - >>> versions = [SV("1.0"), SV("1.1"), SV("1.2"), SV("1.3")] - >>> constraint = VersionConstraint(comparator="<", version=SV("1.2")) - >>> get_region(constraint, versions) - Span(0, 1) - """ - - try: - index = 0 - if str(constraint.version) != "0": - index = versions.index(constraint.version) - except ValueError as err: - err.args = (f"'{constraint.version}' doesn't exist in the versions list.",) - raise - - last_index = len(versions) - 1 - comparator = constraint.comparator - - if comparator == "<": - return Span(0, index - 1) - if comparator == ">": - return Span(index + 1, last_index) - if comparator == ">=": - return Span(index, last_index) - if comparator == "<=": - return Span(0, index) - if comparator == "=": - return Span(index) - if comparator == "!=": - return Span(0, last_index).difference(Span(index)) diff --git a/src/univers/span.py b/src/univers/span.py deleted file mode 100644 index 022fd88b..00000000 --- a/src/univers/span.py +++ /dev/null @@ -1,494 +0,0 @@ -# -# Copyright (c) 2010 Matt Chaput. All rights reserved. -# Modifications by nexB Copyright (c) nexB Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR IMPLIED -# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# The views and conclusions contained in the software and documentation are -# those of the authors and should not be interpreted as representing official -# policies, either expressed or implied, of Matt Chaput. - -from collections.abc import Set -from itertools import count -from itertools import groupby - -try: - from intbitset import intbitset - - spanset = intbitset -except: - spanset = set - - -""" -Ranges and intervals of integers using bitmaps. -Used as a compact and faster data structure for token and position sets. -""" - - -class Span(Set): - """ - Represent ranges of integers (such as tokens positions) as a set of integers. - A Span is hashable and not meant to be modified once created, like a frozenset. - It is equivalent to a sparse closed interval. - Originally derived and heavily modified from Whoosh Span. - """ - - def __init__(self, *args): - """ - Create a new Span from a start and end ints or an iterable of ints. - - First form: - Span(start int, end int) : the span is initialized with a range(start, end+1) - - Second form: - Span(iterable of ints) : the span is initialized with the iterable - - Spans are hashable and immutable. - - For example: - >>> s = Span(1) - >>> s.start - 1 - >>> s = Span([1, 2]) - >>> s.start - 1 - >>> s.end - 2 - >>> s - Span(1, 2) - - >>> s = Span(1, 3) - >>> s.start - 1 - >>> s.end - 3 - >>> s - Span(1, 3) - - >>> s = Span([6, 5, 1, 2]) - >>> s.start - 1 - >>> s.end - 6 - >>> s - Span(1, 2)|Span(5, 6) - >>> len(s) - 4 - - >>> Span([5, 6, 7, 8, 9, 10 ,11, 12]) == Span([5, 6, 7, 8, 9, 10 ,11, 12]) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) - True - >>> hash(Span([5, 6, 7, 8, 9, 10 ,11, 12])) == hash(Span(5, 12)) - True - """ - len_args = len(args) - - if len_args == 0: - self._set = spanset() - - elif len_args == 1: - # args0 is a single int or an iterable of ints - if isinstance(args[0], int): - self._set = spanset(args) - else: - # some sequence or iterable - self._set = spanset(list(args[0])) - - elif len_args == 2: - # args0 and args1 describe a start and end closed range - self._set = spanset(range(args[0], args[1] + 1)) - - else: - # args0 is a single int or args is an iterable of ints - # args is an iterable of ints - self._set = spanset(list(args)) - - @classmethod - def _from_iterable(cls, it): - return cls(list(it)) - - def __len__(self): - return len(self._set) - - def __iter__(self): - return iter(self._set) - - def __hash__(self): - return hash(tuple(self._set)) - - def __eq__(self, other): - return isinstance(other, Span) and self._set == other._set - - def __and__(self, *others): - return Span(self._set.intersection(*[o._set for o in others])) - - def __or__(self, *others): - return Span(self._set.union(*[o._set for o in others])) - - def union(self, *others): - """ - Return the union of this span with other spans as a new span. - (i.e. all positions that are in either spans.) - """ - return self.__or__(*others) - - def intersection(self, *others): - """ - Return the intersection of this span with other spans as a new span. - (i.e. all positions that are in both spans.) - """ - return self.__and__(*others) - - def difference(self, *others): - """ - Return the difference of two or more spans as a new span. - (i.e. all positions that are in this span but not the others.) - """ - return Span(self._set.difference(*[o._set for o in others])) - - def __repr__(self): - """ - Return a brief representation of this span by only listing contiguous - spans and not all items. - - For example: - >>> Span([1, 2, 3, 4, 5, 7, 8, 9, 10]) - Span(1, 5)|Span(7, 10) - """ - subspans_repr = [] - for subs in self.subspans(): - ls = len(subs) - if not ls: - subspans_repr.append("Span()") - elif ls == 1: - subspans_repr.append("Span(%d)" % subs.start) - else: - subspans_repr.append("Span(%d, %d)" % (subs.start, subs.end)) - return "|".join(subspans_repr) - - def __contains__(self, other): - """ - Return True if this span contains other span (where other is a Span, an - int or an ints set). - - For example: - >>> Span([5, 7]) in Span(5, 7) - True - >>> Span([5, 8]) in Span([5, 7]) - False - >>> 6 in Span([4, 5, 6, 7, 8]) - True - >>> 2 in Span([4, 5, 6, 7, 8]) - False - >>> 8 in Span([4, 8]) - True - >>> 5 in Span([4, 8]) - False - >>> set([4, 5]) in Span([4, 5, 6, 7, 8]) - True - >>> set([9]) in Span([4, 8]) - False - """ - if isinstance(other, Span): - return self._set.issuperset(other._set) - - if isinstance(other, int): - return self._set.__contains__(other) - - if isinstance(other, (set, frozenset)): - return self._set.issuperset(spanset(other)) - - if isinstance(other, spanset): - return self._set.issuperset(other) - - @property - def set(self): - return self._set - - def issubset(self, other): - return self._set.issubset(other._set) - - def issuperset(self, other): - return self._set.issuperset(other._set) - - @property - def start(self): - if not self._set: - raise TypeError("Empty Span has no start.") - if isinstance(self._set, set): - return sorted(self._set)[0] - return self._set[0] - - @property - def end(self): - if not self._set: - raise TypeError("Empty Span has no end.") - if isinstance(self._set, set): - return sorted(self._set)[-1] - return self._set[-1] - - @classmethod - def sort(cls, spans): - """ - Return a new sorted sequence of spans given a sequence of spans. - The primary sort is on start. The secondary sort is on length. - If two spans have the same start, the longer span will sort first. - - For example: - >>> spans = [Span([5, 6, 7, 8, 9, 10]), Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(5, 10), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([3, 4, 5]), Span([3, 4, 5, 6]), Span([8, 9, 10])] - >>> Span.sort(spans) - [Span(1, 2), Span(3, 6), Span(3, 5), Span(8, 10)] - - >>> spans = [Span([1, 2]), Span([4, 5]), Span([7, 8]), Span([11, 12])] - >>> Span.sort(spans) - [Span(1, 2), Span(4, 5), Span(7, 8), Span(11, 12)] - - >>> spans = [Span([1, 2]), Span([7, 8]), Span([5, 6]), Span([12, 13])] - >>> Span.sort(spans) - [Span(1, 2), Span(5, 6), Span(7, 8), Span(12, 13)] - - """ - key = lambda s: ( - s.start, - -len(s), - ) - return sorted(spans, key=key) - - def magnitude(self): - """ - Return the maximal length represented by this span start and end. The - magnitude is the same as the length for a contiguous span. It will be - greater than the length for a span with non-contiguous int items. - An empty span has a zero magnitude. - - For example: - >>> Span([4, 8]).magnitude() - 5 - >>> len(Span([4, 8])) - 2 - >>> len(Span([4, 5, 6, 7, 8])) - 5 - - >>> Span([4, 5, 6, 14 , 12, 128]).magnitude() - 125 - - >>> Span([4, 5, 6, 7, 8]).magnitude() - 5 - >>> Span([0]).magnitude() - 1 - >>> Span([0]).magnitude() - 1 - """ - if not self._set: - return 0 - return self.end - self.start + 1 - - def density(self): - """ - Return the density of this span as a ratio of its length to its - magnitude, a float between 0 and 1. A dense Span has all its integer - items contiguous and a maximum density of one. A sparse low density span - has some non-contiguous integer items. An empty span has a zero density. - - For example: - >>> Span([4, 8]).density() - 0.4 - >>> Span([4, 5, 6, 7, 8]).density() - 1.0 - >>> Span([0]).density() - 1.0 - >>> Span().density() - 0 - """ - if not self._set: - return 0 - return len(self) / self.magnitude() - - def overlap(self, other): - """ - Return the count of overlapping items between this span and other span. - - For example: - >>> Span([1, 2]).overlap(Span([5, 6])) - 0 - >>> Span([5, 6]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6, 7]).overlap(Span([5, 6])) - 2 - >>> Span([4, 5, 6]).overlap(Span([5, 6, 7])) - 2 - >>> Span([4, 5, 6]).overlap(Span([6])) - 1 - >>> Span([4, 5]).overlap(Span([6, 7])) - 0 - """ - return len(self & other) - - def resemblance(self, other): - """ - Return a resemblance coefficient as a float between 0 and 1. - 0 means the spans are completely different and 1 identical. - """ - if self._set.isdisjoint(other._set): - return 0 - if self._set == other._set: - return 1 - resemblance = self.overlap(other) / len(self | other) - return resemblance - - def containment(self, other): - """ - Return a containment coefficient as a float between 0 and 1. This is an - indication of how much of the other span is contained in this span. - - 1 means the other span is entirely contained in this span. - - 0 means that the other span is not contained at all this span. - """ - if self._set.isdisjoint(other._set): - return 0 - if self._set == other._set: - return 1 - containment = self.overlap(other) / len(other) - return containment - - def surround(self, other): - """ - Return True if this span surrounds other span. - This is different from containment. A span can surround another span region - and have no positions in common with the surrounded. - - For example: - >>> Span([4, 8]).surround(Span([4, 8])) - True - >>> Span([3, 9]).surround(Span([4, 8])) - True - >>> Span([5, 8]).surround(Span([4, 8])) - False - >>> Span([4, 7]).surround(Span([4, 8])) - False - >>> Span([4, 5, 6, 7, 8]).surround(Span([5, 6, 7])) - True - """ - return self.start <= other.start and self.end >= other.end - - def is_before(self, other): - return self.end < other.start - - def is_after(self, other): - return self.start > other.end - - def touch(self, other): - """ - Return True if self sequence is contiguous with other span without overlap. - - For example: - >>> Span([5, 7]).touch(Span([5])) - False - >>> Span([5, 7]).touch(Span([5, 8])) - False - >>> Span([5, 7]).touch(Span([7, 8])) - False - >>> Span([5, 7]).touch(Span([8, 9])) - True - >>> Span([8, 9]).touch(Span([5, 7])) - True - """ - return self.start == other.end + 1 or self.end == other.start - 1 - - def distance_to(self, other): - """ - Return the absolute positive distance from this span to other span. - Overlapping spans have a zero distance. - Non-overlapping touching spans have a distance of one. - - For example: - >>> Span([8, 9]).distance_to(Span([5, 7])) - 1 - >>> Span([5, 7]).distance_to(Span([8, 9])) - 1 - >>> Span([5, 6]).distance_to(Span([8, 9])) - 2 - >>> Span([8, 9]).distance_to(Span([5, 6])) - 2 - >>> Span([5, 7]).distance_to(Span([5, 7])) - 0 - >>> Span([4, 5, 6]).distance_to(Span([5, 6, 7])) - 0 - >>> Span([5, 7]).distance_to(Span([10, 12])) - 3 - >>> Span([1, 2]).distance_to(Span(range(4, 52))) - 2 - """ - if self.overlap(other): - return 0 - - if self.touch(other): - return 1 - - if self.is_before(other): - return other.start - self.end - else: - return self.start - other.end - - @staticmethod - def from_ints(ints): - """ - Return a sequence of Spans from an iterable of ints. A new Span is - created for each group of monotonously increasing int items. - - >>> Span.from_ints([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 12)] - >>> Span.from_ints([1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12]) - [Span(1, 3), Span(5, 12)] - >>> Span.from_ints([0, 2, 3, 5, 6, 7, 8, 9, 10, 11, 13]) - [Span(0), Span(2, 3), Span(5, 11), Span(13)] - """ - ints = sorted(set(ints)) - groups = (group for _, group in groupby(ints, lambda group, c=count(): next(c) - group)) - return [Span(g) for g in groups] - - def subspans(self): - """ - Return a list of Spans creating one new Span for each set of contiguous - integer items. - - For example: - >>> span = Span(5, 6, 7, 8, 9, 10) | Span([1, 2]) | Span(3, 5) | Span(3, 6) | Span([8, 9, 10]) - >>> span.subspans() - [Span(1, 10)] - - When subspans are not touching they do not merge : - >>> span = Span([63, 64]) | Span([58, 58]) - >>> span.subspans() - [Span(58), Span(63, 64)] - - Overlapping subspans are merged as needed: - >>> span = Span([12, 17, 24]) | Span([15, 16, 17, 35]) | Span(58) | Span(63, 64) - >>> span.subspans() - [Span(12), Span(15, 17), Span(24), Span(35), Span(58), Span(63, 64)] - """ - return Span.from_ints(self) diff --git a/src/univers/spans.py.ABOUT b/src/univers/spans.py.ABOUT deleted file mode 100644 index 27ad909c..00000000 --- a/src/univers/spans.py.ABOUT +++ /dev/null @@ -1,16 +0,0 @@ -about_resource: spans.py -version: 2.4.1 -name: Whoosh Spans -vcs_url: hg+https://bitbucket.org/mchaput/whoosh@72e06bd0aac8 -home_url: https://bitbucket.org/mchaput/whoosh -owner: Matt Chaput -copyright: Copyright (c) 2011 Matt Chaput. -download_url: https://bitbucket.org/mchaput/whoosh/get/2.4.1.tar.gz -description: Manage spans of tokens and text. - This file was originally copied and modified from Whoosh. - -license_expression: bsd-simplified - -licenses: -- file: spans.py.LICENSE - key: bsd-simplified diff --git a/src/univers/spans.py.LICENSE b/src/univers/spans.py.LICENSE deleted file mode 100644 index b0266325..00000000 --- a/src/univers/spans.py.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -Copyright 2011 Matt Chaput. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are -those of the authors and should not be interpreted as representing official -policies, either expressed or implied, of Matt Chaput. diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 9c786d85..7f89949a 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -230,6 +230,37 @@ def __eq__(self, other): and self.constraints == other.constraints ) + def normalize(self, known_versions: List[str]): + """ + Return a new VersionRange normalized and simplified using the universe of + ``known_versions`` list of version strings. + """ + versions = sorted([self.version_class(i) for i in known_versions]) + + resolved = [] + contiguous = [] + for kv in versions: + if self.__contains__(kv): + contiguous.append(kv) + elif contiguous: + resolved.append(contiguous) + contiguous = [] + + if contiguous: + resolved.append(contiguous) + + version_constraints = [] + for contiguous_segment in resolved: + lower_bound = contiguous_segment[0] + upper_bound = contiguous_segment[-1] + if lower_bound == upper_bound: + version_constraints.append(VersionConstraint(version=lower_bound)) + else: + version_constraints.append(VersionConstraint(comparator=">=", version=lower_bound)) + version_constraints.append(VersionConstraint(comparator="<=", version=upper_bound)) + + return self.__class__(constraints=version_constraints) + def from_cve_v4(data, scheme): """ diff --git a/tests/test_normalized_range.py b/tests/test_normalized_range.py deleted file mode 100644 index 495bf042..00000000 --- a/tests/test_normalized_range.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. -# SPDX-License-Identifier: Apache-2.0 -# -# Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. - -from unittest import TestCase - -from univers.normalized_range import NormalizedVersionRange -from univers.normalized_range import get_region -from univers.normalized_range import get_version_range_from_span -from univers.span import Span -from univers.version_constraint import VersionConstraint -from univers.version_range import VersionRange -from univers.versions import SemverVersion - - -class TestNormalizedVersionRange(TestCase): - purl_type = "pypi" - all_versions = versions = [ - "1.0.0", - "1.1.0", - "1.2.0", - "1.3.0", - "2.0.0", - "3.0.0", - ] - versions = [SemverVersion(i) for i in all_versions] - - def test_get_region(self): - - constraint1 = VersionConstraint(comparator="<=", version=SemverVersion("1.0.0")) - constraint2 = VersionConstraint(comparator="!=", version=SemverVersion("1.1.0")) - constraint3 = VersionConstraint(comparator=">", version=SemverVersion("1.3.0")) - - assert get_region(constraint=constraint1, versions=self.versions) == Span(0) - assert get_region(constraint=constraint2, versions=self.versions) == Span(0).union( - Span(2, 5) - ) - assert get_region(constraint=constraint3, versions=self.versions) == Span(4, 5) - - def test_get_version_range_from_span(self): - span1 = Span(1) - span2 = Span(1, 4) - span3 = Span(1, 4).intersection(Span(3, 5)) - span4 = Span(0).union(Span(2, 3)).union(Span(5)) - - vr1 = get_version_range_from_span( - span=span1, purl_type=self.purl_type, versions=self.versions - ) - vr2 = get_version_range_from_span( - span=span2, purl_type=self.purl_type, versions=self.versions - ) - vr3 = get_version_range_from_span( - span=span3, purl_type=self.purl_type, versions=self.versions - ) - vr4 = get_version_range_from_span( - span=span4, purl_type=self.purl_type, versions=self.versions - ) - - assert str(vr1) == "vers:pypi/1.1.0" - assert str(vr2) == "vers:pypi/>=1.1.0|<=2.0.0" - assert str(vr3) == "vers:pypi/>=1.3.0|<=2.0.0" - assert str(vr4) == "vers:pypi/1.0.0|>=1.2.0|<=1.3.0|3.0.0" - - def test_NormalizedVersionRange_from_vers(self): - vr1 = VersionRange.from_string("vers:pypi/<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") - nvr1 = NormalizedVersionRange.from_vers(vers_range=vr1, all_versions=self.all_versions) - - vr2 = VersionRange.from_string("vers:pypi/>=1.0.0|<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") - nvr2 = NormalizedVersionRange.from_vers(vers_range=vr2, all_versions=self.all_versions) - - vr3 = VersionRange.from_string("vers:pypi/<=1.3.0|3.0.0") - nvr3 = NormalizedVersionRange.from_vers(vers_range=vr3, all_versions=self.all_versions) - - vr4 = VersionRange.from_string("vers:pypi/<2.0.0|3.0.0") - nvr4 = NormalizedVersionRange.from_vers(vers_range=vr4, all_versions=self.all_versions) - - assert str(nvr1) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" - assert str(nvr2) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" - assert str(nvr3) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" - assert str(nvr4) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" diff --git a/tests/test_version_range.py b/tests/test_version_range.py index 5449acb9..a8320bda 100644 --- a/tests/test_version_range.py +++ b/tests/test_version_range.py @@ -43,7 +43,7 @@ def test_VersionRange_to_string(self): assert str(version_range) == "vers:pypi/>=0.0.0|0.0.1|0.0.2|0.0.3|0.0.4|0.0.5|0.0.6" def test_VersionRange_pypi_does_not_contain_basic(self): - vers = "vers:pypi/0.0.2|0.0.6|>=0.0.0|0.0.1|0.0.4|0.0.5|0.0.3" + vers = "vers:pypi/0.0.2|0.0.6|>=3.0.0|0.0.1|0.0.4|0.0.5|0.0.3" version_range = VersionRange.from_string(vers) assert not version_range.contains(PypiVersion("2.0.3")) @@ -456,3 +456,30 @@ def test_mattermost_version_range(): VersionConstraint(comparator=">=", version=SemverVersion("5.0")), ] ) == VersionRange.from_string("vers:mattermost/>=5.0") + + +def test_version_range_normalize_case1(): + known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + + vr = VersionRange.from_string("vers:pypi/<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") + nvr = vr.normalize(known_versions=known_versions) + + assert str(nvr) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" + + +def test_version_range_normalize_case2(): + known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + + vr = VersionRange.from_string("vers:pypi/<=1.3.0|3.0.0") + nvr = vr.normalize(known_versions=known_versions) + + assert str(nvr) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" + + +def test_version_range_normalize_case3(): + known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + + vr = VersionRange.from_string("vers:pypi/<2.0.0|3.0.0") + nvr = vr.normalize(known_versions=known_versions) + + assert str(nvr) == "vers:pypi/>=1.0.0|<=1.3.0|3.0.0" From 8f0d7276e13686d93fee5313641b9803f4d4cb15 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 24 Jul 2024 22:15:02 +0530 Subject: [PATCH 22/25] Add function to parse bracket notation constraints Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 60 +++++++++++++++++++++--------------- tests/test_version_range.py | 45 +++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 27 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 7f89949a..0532c525 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -480,7 +480,7 @@ def from_native(cls, string): return cls(constraints=constraints) -def split_req(string, comparators, comparators_rear={}, default=None, strip=""): +def split_req(string, comparators, default=None, strip=""): """ Return a tuple of (vers comparator, version) strings given an common version requirement``string`` such as "> 2.3" or "<= 2.3" using the ``comparators`` @@ -501,8 +501,6 @@ def split_req(string, comparators, comparators_rear={}, default=None, strip=""): >>> assert split_req(">= 2.3", comparators=comps) == (">=", "2.3",) >>> assert split_req("<= 2.3", comparators=comps) == ("<=", "2.3",) >>> assert split_req("(< = 2.3 )", comparators=comps, strip=")(") == ("<=", "2.3",) - >>> comps_rear = {")": "<", "]": "<="} - >>> assert split_req(" 2.3 ]", comparators=comps, comparators_rear=comps_rear) == ("<=", "2.3",) With a default, we return the default comparator:: @@ -523,12 +521,6 @@ def split_req(string, comparators, comparators_rear={}, default=None, strip=""): version = constraint_string.lstrip(native_comparator) return vers_comparator, version - # Some bracket notation comparators starts from end. - for native_comparator, vers_comparator in comparators_rear.items(): - if constraint_string.endswith(native_comparator): - version = constraint_string.rstrip(native_comparator) - return vers_comparator, version - if default: return default, constraint_string @@ -1300,14 +1292,35 @@ def build_range_from_github_advisory_constraint(scheme: str, string: Union[str, ">=": ">=", "<": "<", ">": ">", - "(": ">", - "[": ">=", } -vers_by_snyk_native_comparators_rear = { - ")": "<", - "]": "<=", -} + +def split_req_bracket_notation(string): + """ + Return a tuple of (vers comparator, version) strings given an bracket notation + version requirement ``string`` such as "(2.3" or "3.9]" + + For example:: + + >>> assert split_req_bracket_notation(" 2.3 ]") == ("<=", "2.3") + >>> assert split_req_bracket_notation("( 3.9") == (">", "3.9") + """ + comparators_front = {"(": ">", "[": ">="} + comparators_rear = {")": "<", "]": "<="} + + constraint_string = remove_spaces(string).strip() + + for native_comparator, vers_comparator in comparators_front.items(): + if constraint_string.startswith(native_comparator): + version = constraint_string.lstrip(native_comparator) + return vers_comparator, version + + for native_comparator, vers_comparator in comparators_rear.items(): + if constraint_string.endswith(native_comparator): + version = constraint_string.rstrip(native_comparator) + return vers_comparator, version + + raise ValueError(f"Unknown comparator in version requirement: {string!r} ") def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]): @@ -1346,11 +1359,13 @@ def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]) constraints = snyk_constraints.split(" ") for constraint in constraints: - comparator, version = split_req( - string=constraint, - comparators=vers_by_snyk_native_comparators, - comparators_rear=vers_by_snyk_native_comparators_rear, - ) + if any(comp in constraint for comp in "[]()"): + comparator, version = split_req_bracket_notation(string=constraint) + else: + comparator, version = split_req( + string=constraint, + comparators=vers_by_snyk_native_comparators, + ) if comparator and version: version = vrc.version_class(version) version_constraints.append( @@ -1382,10 +1397,7 @@ def build_range_from_discrete_version_string(scheme: str, string: Union[str, Lis string = [string] for item in string: - version = item.strip().lstrip("vV") - if version == "0": - continue - + version = item.strip() version = vrc.version_class(version) version_constraints.append( VersionConstraint( diff --git a/tests/test_version_range.py b/tests/test_version_range.py index a8320bda..03387862 100644 --- a/tests/test_version_range.py +++ b/tests/test_version_range.py @@ -20,6 +20,8 @@ from univers.version_range import OpensslVersionRange from univers.version_range import PypiVersionRange from univers.version_range import VersionRange +from univers.version_range import build_range_from_discrete_version_string +from univers.version_range import build_range_from_snyk_advisory_string from univers.version_range import from_gitlab_native from univers.versions import InvalidVersion from univers.versions import NugetVersion @@ -85,6 +87,11 @@ def test_VersionRange_contains_version_in_between(self): version_range = VersionRange.from_string(vers) assert version_range.contains(PypiVersion("1.5")) + def test_VersionRange_contains_filterd_constraint_edge_case(self): + vers = "vers:pypi/<=1.3.0|3.0.0" + version_range = VersionRange.from_string(vers) + assert version_range.contains(PypiVersion("1.0.0")) + def test_VersionRange_from_string_pypi(self): vers = "vers:pypi/0.0.2|0.0.6|0.0.0|0.0.1|0.0.4|0.0.5|0.0.3" version_range = VersionRange.from_string(vers) @@ -458,8 +465,40 @@ def test_mattermost_version_range(): ) == VersionRange.from_string("vers:mattermost/>=5.0") +def test_build_range_from_snyk_advisory_string(): + expression = [">=4.0.0, <4.0.10", ">7.0.0, <8.0.1"] + vr = build_range_from_snyk_advisory_string("pypi", expression) + expected = "vers:pypi/>=4.0.0|<4.0.10|>7.0.0|<8.0.1" + + assert str(vr) == expected + + +def test_build_range_from_snyk_advisory_string_bracket(): + expression = ["[3.0.0,3.1.25)", "[1.0.0,1.0.5)"] + vr = build_range_from_snyk_advisory_string("nuget", expression) + expected = "vers:nuget/>=1.0.0|<1.0.5|>=3.0.0|<3.1.25" + + assert str(vr) == expected + + +def test_build_range_from_snyk_advisory_string_spaced(): + expression = [">=4.1.0 <4.4.1", ">2.1.0 <=3.2.7"] + vr = build_range_from_snyk_advisory_string("composer", expression) + expected = "vers:composer/>2.1.0|<=3.2.7|>=4.1.0|<4.4.1" + + assert str(vr) == expected + + +def test_build_range_from_discrete_version_string(): + expression = ["4.1.0", " 4.4.1", "2.1.0 ", " 3.2.7 "] + vr = build_range_from_discrete_version_string("pypi", expression) + expected = "vers:pypi/2.1.0|3.2.7|4.1.0|4.4.1" + + assert str(vr) == expected + + def test_version_range_normalize_case1(): - known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + known_versions = ["4.0.0", "3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] vr = VersionRange.from_string("vers:pypi/<=1.1.0|>=1.2.0|<=1.3.0|3.0.0") nvr = vr.normalize(known_versions=known_versions) @@ -468,7 +507,7 @@ def test_version_range_normalize_case1(): def test_version_range_normalize_case2(): - known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + known_versions = ["4.0.0", "3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] vr = VersionRange.from_string("vers:pypi/<=1.3.0|3.0.0") nvr = vr.normalize(known_versions=known_versions) @@ -477,7 +516,7 @@ def test_version_range_normalize_case2(): def test_version_range_normalize_case3(): - known_versions = ["3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] + known_versions = ["4.0.0", "3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] vr = VersionRange.from_string("vers:pypi/<2.0.0|3.0.0") nvr = vr.normalize(known_versions=known_versions) From 3d0de11f8ff4cfd28e717bf3f4b8e13f1391949f Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 25 Jul 2024 00:03:09 +0530 Subject: [PATCH 23/25] Use from_versions for getting vers from discrete versions Signed-off-by: Keshav Priyadarshi --- src/univers/version_range.py | 30 ------------------------------ tests/test_version_range.py | 9 --------- 2 files changed, 39 deletions(-) diff --git a/src/univers/version_range.py b/src/univers/version_range.py index 0532c525..d2bff302 100644 --- a/src/univers/version_range.py +++ b/src/univers/version_range.py @@ -1377,36 +1377,6 @@ def build_range_from_snyk_advisory_string(scheme: str, string: Union[str, List]) return vrc(constraints=version_constraints) -def build_range_from_discrete_version_string(scheme: str, string: Union[str, List]): - """ - Return VersionRange computed from discrete versions. - Discrete version range looks like: - ["1.5","3.1.2","3.1-beta"] - - For example:: - - # >>> vr = build_constraints_from_discrete_version_string("pypi", ["1.5","3.1.2","3.1-beta"]) - # >>> assert str(vr) == "vers:pypi/1.5|3.1-beta|3.1.2" - # >>> vr = build_constraints_from_discrete_version_string("pypi","9.21") - # >>> assert str(vr) == "vers:pypi/9.21" - """ - version_constraints = [] - vrc = RANGE_CLASS_BY_SCHEMES[scheme] - - if isinstance(string, str): - string = [string] - - for item in string: - version = item.strip() - version = vrc.version_class(version) - version_constraints.append( - VersionConstraint( - version=version, - ) - ) - return vrc(constraints=version_constraints) - - RANGE_CLASS_BY_SCHEMES = { "npm": NpmVersionRange, "deb": DebianVersionRange, diff --git a/tests/test_version_range.py b/tests/test_version_range.py index 03387862..1d5848c1 100644 --- a/tests/test_version_range.py +++ b/tests/test_version_range.py @@ -20,7 +20,6 @@ from univers.version_range import OpensslVersionRange from univers.version_range import PypiVersionRange from univers.version_range import VersionRange -from univers.version_range import build_range_from_discrete_version_string from univers.version_range import build_range_from_snyk_advisory_string from univers.version_range import from_gitlab_native from univers.versions import InvalidVersion @@ -489,14 +488,6 @@ def test_build_range_from_snyk_advisory_string_spaced(): assert str(vr) == expected -def test_build_range_from_discrete_version_string(): - expression = ["4.1.0", " 4.4.1", "2.1.0 ", " 3.2.7 "] - vr = build_range_from_discrete_version_string("pypi", expression) - expected = "vers:pypi/2.1.0|3.2.7|4.1.0|4.4.1" - - assert str(vr) == expected - - def test_version_range_normalize_case1(): known_versions = ["4.0.0", "3.0.0", "1.0.0", "2.0.0", "1.3.0", "1.1.0", "1.2.0"] From 594baf59ea535fb09c77949130998d3a3cb0c57e Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 25 Jul 2024 00:50:10 +0530 Subject: [PATCH 24/25] Set `shell` param to False while running code style tests Signed-off-by: Keshav Priyadarshi --- tests/test_codestyle.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_codestyle.py b/tests/test_codestyle.py index d63b20ee..c0fbd5e0 100644 --- a/tests/test_codestyle.py +++ b/tests/test_codestyle.py @@ -5,14 +5,18 @@ # Visit https://aboutcode.org and https://github.com/nexB/univers for support and download. import subprocess +import sys import unittest +import pytest + +@pytest.mark.skipif(sys.platform == "darwin", reason="Test does not run on macOS13 and older.") class BaseTests(unittest.TestCase): def test_codestyle(self): args = "black --check -l 100 setup.py src tests" try: - subprocess.check_output(args.split()) + subprocess.check_output(args.split(), shell=False) except subprocess.CalledProcessError as e: print("===========================================================") print(e.output) @@ -25,7 +29,7 @@ def test_codestyle(self): args = "isort --check-only src tests setup.py" try: - subprocess.check_output(args.split()) + subprocess.check_output(args.split(), shell=False) except Exception as e: print("===========================================================") print(e.output) From b12572de72ce47f793fb8c70cf0178c22aaedc57 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Thu, 25 Jul 2024 00:58:20 +0530 Subject: [PATCH 25/25] Use only macOS-14 image for macOS 14 CI Signed-off-by: Keshav Priyadarshi --- azure-pipelines.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f9562c98..b7e21c8e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,16 +53,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython - image_name: macOS-14-large - python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] - test_suites: - all: | - source venv/bin/activate - pytest -n 2 -vvs - - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython