From a7e3dd38fb30b6802a608d3fef2bf2b88fe4fb0c Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Fri, 22 Dec 2023 20:57:22 +1100 Subject: [PATCH 1/6] Filter architectures first --- mybox/package/github.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/mybox/package/github.py b/mybox/package/github.py index ca2ec9b..d271ad6 100644 --- a/mybox/package/github.py +++ b/mybox/package/github.py @@ -87,23 +87,21 @@ def environment_filters( for signature_hint in [".asc", ".sig", "sha256", "sha512", ".yml"]: yield cls.excludes_(signature_hint) - for other_os_hint in [".exe", ".dmg"]: - yield cls.excludes_(other_os_hint) - for os_hint in target_os.switch( - linux=[ - cls.includes_("linux"), - cls.includes_("gnu"), - cls.excludes_("musl"), - ], - macos=[cls.includes_(hint) for hint in ["macos", "darwin", "osx"]], - ): - yield os_hint - for arch, synonyms in ARCHITECTURE_FILTERS.items(): method = cls.includes_ if arch == target_arch else cls.excludes_ for synonym in [arch, *synonyms]: yield method(synonym) + for os_hint in target_os.switch( + linux=["linux"], macos=["macos", "darwin", "osx"] + ): + yield cls.includes_(os_hint) + if target_os.switch(linux=True, macos=False): + yield cls.includes_("gnu") + yield cls.excludes_("musl") + for other_os_hint in [".exe", ".dmg"]: + yield cls.excludes_(other_os_hint) + def all_filters( self, *, target_os: OS, target_arch: Architecture ) -> Iterator[Callable[[str], bool]]: From daffdf77a220e8898fb0255da348f6f4350a759c Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 27 Dec 2023 20:58:04 +1100 Subject: [PATCH 2/6] Tweak rules to choose files from releases --- mybox/package/github.py | 4 ++-- tests/package/test_github.py | 24 ++++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mybox/package/github.py b/mybox/package/github.py index d271ad6..4c16bef 100644 --- a/mybox/package/github.py +++ b/mybox/package/github.py @@ -82,10 +82,10 @@ async def latest_release(self) -> GitHubRelease: def environment_filters( cls, *, target_os: OS, target_arch: Architecture ) -> Iterator[Callable[[str], bool]]: - for hint in [".tar.gz"]: - yield cls.includes_(hint) for signature_hint in [".asc", ".sig", "sha256", "sha512", ".yml"]: yield cls.excludes_(signature_hint) + for system_package_hint in [".deb", ".rpm"]: + yield cls.excludes_(system_package_hint) for arch, synonyms in ARCHITECTURE_FILTERS.items(): method = cls.includes_ if arch == target_arch else cls.excludes_ diff --git a/tests/package/test_github.py b/tests/package/test_github.py index 407f32a..34eccfb 100644 --- a/tests/package/test_github.py +++ b/tests/package/test_github.py @@ -153,22 +153,26 @@ async def check_installed(self): ] +class TestGitHubCLI(PackageTestBase): + async def constructor_args(self) -> PackageArgs: + return {"repo": "cli/cli", "binary": "gh", "strip": 1} + + async def check_installed_command(self): + return ["gh", "--version"] + + check_installed_output = "gh version" + + class TestJQ(PackageTestBase): async def constructor_args(self) -> PackageArgs: return { - "repo": "stedolan/jq", - "include": "linux64", - "binary": "jq-linux64", - "raw": True, + "repo": "jqlang/jq", + "binary": "jq", + "raw": "jq", "raw_executable": True, } async def check_installed_command(self): - return ["jq-linux64", "--version"] + return ["jq", "--version"] check_installed_output = "jq-" - - async def check_applicable(self) -> None: - await super().check_applicable() - if not (await self.driver.os()).switch(linux=True, macos=False): - pytest.skip("This test is only applicable on Linux.") From 92f45d54fa307856ce49d4caf72475d49c05f743 Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 27 Dec 2023 21:00:22 +1100 Subject: [PATCH 3/6] More architectures --- mybox/package/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mybox/package/github.py b/mybox/package/github.py index 4c16bef..afdbc6d 100644 --- a/mybox/package/github.py +++ b/mybox/package/github.py @@ -53,11 +53,12 @@ class GitHubRelease: ARCHITECTURE_FILTERS: dict[str, list[str]] = { - "x86_64": ["amd64", "x64"], "arm64": ["aarch64", "arm"], "i386": ["i686", "x86"], - "powerpc64": ["ppc64"], + "mips": [], + "powerpc": ["ppc"], "s390x": [], + "x86_64": ["amd64", "x64"], } From 7c4f6264efad5d04a7bd3e4037fd191d3a2bedef Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 27 Dec 2023 21:06:59 +1100 Subject: [PATCH 4/6] Tweak OS filters --- mybox/package/github.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/mybox/package/github.py b/mybox/package/github.py index afdbc6d..569fa94 100644 --- a/mybox/package/github.py +++ b/mybox/package/github.py @@ -61,6 +61,12 @@ class GitHubRelease: "x86_64": ["amd64", "x64"], } +OS_FILTERS: dict[str, list[str]] = { + "darwin": ["macos", "osx"], + "linux": [], + "windows": [], +} + class GitHubPackage(ArchivePackage, Filters): repo: str @@ -85,23 +91,26 @@ def environment_filters( ) -> Iterator[Callable[[str], bool]]: for signature_hint in [".asc", ".sig", "sha256", "sha512", ".yml"]: yield cls.excludes_(signature_hint) - for system_package_hint in [".deb", ".rpm"]: + for system_package_hint in [".deb", ".rpm", ".dmg", ".exe"]: yield cls.excludes_(system_package_hint) + for os_name, synonyms in OS_FILTERS.items(): + method = ( + cls.includes_ + if os_name == target_os.switch(linux="linux", macos="darwin") + else cls.excludes_ + ) + for synonym in [os_name, *synonyms]: + yield method(synonym) + for arch, synonyms in ARCHITECTURE_FILTERS.items(): method = cls.includes_ if arch == target_arch else cls.excludes_ for synonym in [arch, *synonyms]: yield method(synonym) - for os_hint in target_os.switch( - linux=["linux"], macos=["macos", "darwin", "osx"] - ): - yield cls.includes_(os_hint) if target_os.switch(linux=True, macos=False): yield cls.includes_("gnu") yield cls.excludes_("musl") - for other_os_hint in [".exe", ".dmg"]: - yield cls.excludes_(other_os_hint) def all_filters( self, *, target_os: OS, target_arch: Architecture From ed0a93625f75234368410a494a13356887aad56c Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 27 Dec 2023 21:43:48 +1100 Subject: [PATCH 5/6] Filter own architecture first, then others --- mybox/filters.py | 30 +++++++++++++++++++++++------- mybox/package/github.py | 29 +++++++++++------------------ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/mybox/filters.py b/mybox/filters.py index a4cab1e..9a6d362 100644 --- a/mybox/filters.py +++ b/mybox/filters.py @@ -5,30 +5,46 @@ from .utils import T, allow_singular_none +Filter = Callable[[T], bool] + class Filters(BaseModel): @staticmethod - def includes_(substring: str) -> Callable[[str], bool]: + def includes_(substring: str) -> Filter[str]: return lambda x: substring in x.lower() @staticmethod - def excludes_(substring: str) -> Callable[[str], bool]: + def excludes_(substring: str) -> Filter[str]: return lambda x: substring not in x @staticmethod - def startswith(prefix: str) -> Callable[[str], bool]: + def startswith(prefix: str) -> Filter[str]: return lambda x: x.startswith(prefix) @staticmethod - def endswith(suffix: str) -> Callable[[str], bool]: + def endswith(suffix: str) -> Filter[str]: return lambda x: x.endswith(suffix) @staticmethod - def regex_(regex: str) -> Callable[[str], bool]: + def regex_(regex: str) -> Filter[str]: regex_compiled = re.compile(regex) return lambda x: regex_compiled.match(x) is not None + @classmethod + def from_synonyms( + cls, items: dict[str, list[str]], key: str + ) -> Iterator[Filter[str]]: + yield cls.includes_(key) + for synonym in items.get(key, []): + yield cls.includes_(synonym) + + for other, synonyms in items.items(): + if other == key: + continue + for synonym in [other, *synonyms]: + yield cls.excludes_(synonym) + prefixes: list[str] = Field(default_factory=list, alias="prefix") prefixes_val = allow_singular_none("prefixes") @@ -44,7 +60,7 @@ def regex_(regex: str) -> Callable[[str], bool]: regex: list[str] = Field(default_factory=list) regex_val = allow_singular_none("regex") - def filters(self) -> Iterator[Callable[[str], bool]]: + def filters(self) -> Iterator[Filter[str]]: for prefix in self.prefixes: yield self.startswith(prefix) for suffix in self.suffixes: @@ -57,7 +73,7 @@ def filters(self) -> Iterator[Callable[[str], bool]]: yield self.regex_(regex) -def choose(candidates: list[T], filters: Iterator[Callable[[T], bool]]) -> T: +def choose(candidates: list[T], filters: Iterator[Filter[T]]) -> T: if len(candidates) == 0: raise ValueError("No candidates to choose from.") while len(candidates) > 1: diff --git a/mybox/package/github.py b/mybox/package/github.py index 569fa94..aab830a 100644 --- a/mybox/package/github.py +++ b/mybox/package/github.py @@ -1,12 +1,12 @@ import os from dataclasses import dataclass from subprocess import CalledProcessError -from typing import Any, Callable, Iterator, Optional +from typing import Any, Iterator, Optional import requests from ..driver import OS, Architecture -from ..filters import Filters, choose +from ..filters import Filter, Filters, choose from ..utils import async_cached, async_cached_lock, run_output from .archive import ArchivePackage @@ -88,25 +88,18 @@ async def latest_release(self) -> GitHubRelease: @classmethod def environment_filters( cls, *, target_os: OS, target_arch: Architecture - ) -> Iterator[Callable[[str], bool]]: + ) -> Iterator[Filter[str]]: for signature_hint in [".asc", ".sig", "sha256", "sha512", ".yml"]: yield cls.excludes_(signature_hint) + for system_package_hint in [".deb", ".rpm", ".dmg", ".exe"]: yield cls.excludes_(system_package_hint) - for os_name, synonyms in OS_FILTERS.items(): - method = ( - cls.includes_ - if os_name == target_os.switch(linux="linux", macos="darwin") - else cls.excludes_ - ) - for synonym in [os_name, *synonyms]: - yield method(synonym) + yield from cls.from_synonyms( + OS_FILTERS, target_os.switch(linux="linux", macos="darwin") + ) - for arch, synonyms in ARCHITECTURE_FILTERS.items(): - method = cls.includes_ if arch == target_arch else cls.excludes_ - for synonym in [arch, *synonyms]: - yield method(synonym) + yield from cls.from_synonyms(ARCHITECTURE_FILTERS, target_arch) if target_os.switch(linux=True, macos=False): yield cls.includes_("gnu") @@ -114,7 +107,7 @@ def environment_filters( def all_filters( self, *, target_os: OS, target_arch: Architecture - ) -> Iterator[Callable[[str], bool]]: + ) -> Iterator[Filter[str]]: yield from self.filters() yield from self.environment_filters( target_os=target_os, target_arch=target_arch @@ -124,8 +117,8 @@ async def artifact(self) -> GitHubReleaseArtifact: candidates = (await self.latest_release()).assets def candidate_filter( - name_filter: Callable[[str], bool] - ) -> Callable[[GitHubReleaseArtifact], bool]: + name_filter: Filter, + ) -> Filter[GitHubReleaseArtifact]: return lambda candidate: name_filter(candidate.name) target_os = await self.driver.os() From aa8f97f27b62286e903674f3962ae91bc430acb0 Mon Sep 17 00:00:00 2001 From: Alexey Kotlyarov Date: Wed, 27 Dec 2023 22:04:15 +1100 Subject: [PATCH 6/6] Test for Filters --- tests/test_filters.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_filters.py b/tests/test_filters.py index cf5fe89..2f46742 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -2,7 +2,12 @@ import pytest -from mybox.filters import choose +from mybox.filters import Filters, choose + + +def test_filters_attributes(): + filters = Filters(prefix=["a", "b"], suffix="c", include="2", exclude="3") + assert choose(["a1c", "a2c", "a23c", "b1d", "a1e"], filters.filters()) == "a2c" class TestChoose: