From 6a39d04b40eabf9285d21f2c73f9d88489f0e6ab Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 21 Jul 2024 17:23:15 +0200 Subject: [PATCH 1/3] CI: add a job for running tests under MSVC CPython with GCC as the default compiler The tests currently assume everywhere that there is only one compiler per platform, and while it would be possible to parametrize all the tests it would make things more complex and we'd also have to decide which compiler is required for running the tests and which one is optional etc. To avoid all this introduce a DISTUTILS_TEST_DEFAULT_COMPILER env var which can be used to override the default compiler type for the whole test run. This keeps the tests as is and makes sure all tests run against the alternative compiler. Also add it to pass_env for tox, so it gets passed to pytest, if set. The added CI job installs an ucrt targeting GCC via MSYS2, and forces the MSVC CPython to use it via DISTUTILS_TEST_DEFAULT_COMPILER=mingw32. --- .github/workflows/main.yml | 22 ++++++++++++++++++++++ conftest.py | 23 +++++++++++++++++++++++ tox.ini | 2 ++ 3 files changed, 47 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 70d70bc6..2f4ec6b4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -163,6 +163,28 @@ jobs: source /tmp/venv/bin/activate pytest + test_msvc_python_mingw: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Install tox + run: python -m pip install tox + - name: Install GCC + uses: msys2/setup-msys2@v2 + with: + msystem: ucrt64 + install: mingw-w64-ucrt-x86_64-cc + - name: Run + run: | + $env:MSYS2_ROOT = msys2 -c 'cygpath -m /' + $env:PATH = "$env:MSYS2_ROOT/ucrt64/bin;$env:PATH" + $env:DISTUTILS_TEST_DEFAULT_COMPILER = "mingw32" + tox + ci_setuptools: # Integration testing with setuptools strategy: diff --git a/conftest.py b/conftest.py index 6639aa65..fd6cb6d6 100644 --- a/conftest.py +++ b/conftest.py @@ -162,3 +162,26 @@ def disable_macos_customization(monkeypatch): from distutils import sysconfig monkeypatch.setattr(sysconfig, '_customize_macos', lambda: None) + + +@pytest.fixture(autouse=True, scope="session") +def monkey_patch_get_default_compiler(): + """ + Monkey patch distutils get_default_compiler to allow overriding the + default compiler. Mainly to test mingw32 with a MSVC Python. + """ + from distutils import ccompiler + + default_compiler = os.environ.get("DISTUTILS_TEST_DEFAULT_COMPILER") + + if default_compiler is not None: + + def patched_get_default_compiler(*args, **kwargs): + return default_compiler + + original = ccompiler.get_default_compiler + ccompiler.get_default_compiler = patched_get_default_compiler + yield + ccompiler.get_default_compiler = original + else: + yield diff --git a/tox.ini b/tox.ini index d4bcc416..54835876 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ setenv = PYTHONWARNDEFAULTENCODING = 1 # pypa/distutils#99 VIRTUALENV_NO_SETUPTOOLS = 1 +pass_env = + DISTUTILS_TEST_DEFAULT_COMPILER commands = pytest {posargs} usedevelop = True From 943fc8253fb136bbe94064aa7524ca9c8684b4b6 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 21 Jul 2024 18:13:03 +0200 Subject: [PATCH 2/3] mingw: make get_msvcr() a noop This was added back in the day to make mingw use the same CRT as CPython (https://bugs.python.org/issue870382), but at least with newer mingw-w64 and ucrt switching the CRT at "runtime" isn't supported anymore. To build a compatible extension you have to use a ucrt mingw-w64 build, so things match up and link against the same CRT. CPython 3.5+ uses ucrt (see https://wiki.python.org/moin/WindowsCompilers), so anything besides that is no longer relevant, which only leaves vcruntime140. Since it's not clear what linking against vcruntime140 solves, and there have been reports of it needing to be patched out: * https://github.com/pypa/setuptools/issues/4101 * https://github.com/pypa/distutils/issues/204#issuecomment-1420892028 let's just make it return nothing. Keep get_msvcr() around for now to avoid breaking code which patched it. Fixes #204 --- distutils/cygwinccompiler.py | 40 ++--------------------- distutils/tests/test_cygwinccompiler.py | 42 ------------------------- 2 files changed, 2 insertions(+), 80 deletions(-) diff --git a/distutils/cygwinccompiler.py b/distutils/cygwinccompiler.py index 7b812fd0..f3e593a4 100644 --- a/distutils/cygwinccompiler.py +++ b/distutils/cygwinccompiler.py @@ -9,13 +9,11 @@ import copy import os import pathlib -import re import shlex import sys import warnings from subprocess import check_output -from ._collections import RangeMap from .errors import ( CCompilerError, CompileError, @@ -26,42 +24,10 @@ from .unixccompiler import UnixCCompiler from .version import LooseVersion, suppress_known_deprecation -_msvcr_lookup = RangeMap.left( - { - # MSVC 7.0 - 1300: ['msvcr70'], - # MSVC 7.1 - 1310: ['msvcr71'], - # VS2005 / MSVC 8.0 - 1400: ['msvcr80'], - # VS2008 / MSVC 9.0 - 1500: ['msvcr90'], - # VS2010 / MSVC 10.0 - 1600: ['msvcr100'], - # VS2012 / MSVC 11.0 - 1700: ['msvcr110'], - # VS2013 / MSVC 12.0 - 1800: ['msvcr120'], - # VS2015 / MSVC 14.0 - 1900: ['vcruntime140'], - 2000: RangeMap.undefined_value, - }, -) - def get_msvcr(): - """Include the appropriate MSVC runtime library if Python was built - with MSVC 7.0 or later. - """ - match = re.search(r'MSC v\.(\d{4})', sys.version) - try: - msc_ver = int(match.group(1)) - except AttributeError: - return [] - try: - return _msvcr_lookup[msc_ver] - except KeyError: - raise ValueError(f"Unknown MS Compiler version {msc_ver} ") + """No longer needed, but kept for backward compatibility.""" + return [] _runtime_library_dirs_msg = ( @@ -109,8 +75,6 @@ def __init__(self, verbose=False, dry_run=False, force=False): linker_so=(f'{self.linker_dll} -mcygwin {shared_option}'), ) - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. self.dll_libraries = get_msvcr() @property diff --git a/distutils/tests/test_cygwinccompiler.py b/distutils/tests/test_cygwinccompiler.py index 2e1640b7..677bc0ac 100644 --- a/distutils/tests/test_cygwinccompiler.py +++ b/distutils/tests/test_cygwinccompiler.py @@ -71,50 +71,8 @@ def test_check_config_h(self): assert check_config_h()[0] == CONFIG_H_OK def test_get_msvcr(self): - # [] - sys.version = ( - '2.6.1 (r261:67515, Dec 6 2008, 16:42:21) ' - '\n[GCC 4.0.1 (Apple Computer, Inc. build 5370)]' - ) assert get_msvcr() == [] - # MSVC 7.0 - sys.version = ( - '2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1300 32 bits (Intel)]' - ) - assert get_msvcr() == ['msvcr70'] - - # MSVC 7.1 - sys.version = ( - '2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1310 32 bits (Intel)]' - ) - assert get_msvcr() == ['msvcr71'] - - # VS2005 / MSVC 8.0 - sys.version = ( - '2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1400 32 bits (Intel)]' - ) - assert get_msvcr() == ['msvcr80'] - - # VS2008 / MSVC 9.0 - sys.version = ( - '2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1500 32 bits (Intel)]' - ) - assert get_msvcr() == ['msvcr90'] - - sys.version = ( - '3.10.0 (tags/v3.10.0:b494f59, Oct 4 2021, 18:46:30) ' - '[MSC v.1929 32 bit (Intel)]' - ) - assert get_msvcr() == ['vcruntime140'] - - # unknown - sys.version = ( - '2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.2000 32 bits (Intel)]' - ) - with pytest.raises(ValueError): - get_msvcr() - @pytest.mark.skipif('sys.platform != "cygwin"') def test_dll_libraries_not_none(self): from distutils.cygwinccompiler import CygwinCCompiler From 267fb5f5a75aa303e8ed04b4416242249d39ed3e Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sun, 21 Jul 2024 21:08:19 +0200 Subject: [PATCH 3/3] Remove unused RangeMap Its last use in cygwinccompiler was just removed. --- distutils/_collections.py | 145 -------------------------------------- 1 file changed, 145 deletions(-) diff --git a/distutils/_collections.py b/distutils/_collections.py index d11a8346..863030b3 100644 --- a/distutils/_collections.py +++ b/distutils/_collections.py @@ -1,11 +1,7 @@ from __future__ import annotations import collections -import functools import itertools -import operator -from collections.abc import Mapping -from typing import Any # from jaraco.collections 3.5.1 @@ -60,144 +56,3 @@ def __contains__(self, other): def __len__(self): return len(list(iter(self))) - - -# from jaraco.collections 5.0.1 -class RangeMap(dict): - """ - A dictionary-like object that uses the keys as bounds for a range. - Inclusion of the value for that range is determined by the - key_match_comparator, which defaults to less-than-or-equal. - A value is returned for a key if it is the first key that matches in - the sorted list of keys. - - One may supply keyword parameters to be passed to the sort function used - to sort keys (i.e. key, reverse) as sort_params. - - Create a map that maps 1-3 -> 'a', 4-6 -> 'b' - - >>> r = RangeMap({3: 'a', 6: 'b'}) # boy, that was easy - >>> r[1], r[2], r[3], r[4], r[5], r[6] - ('a', 'a', 'a', 'b', 'b', 'b') - - Even float values should work so long as the comparison operator - supports it. - - >>> r[4.5] - 'b' - - Notice that the way rangemap is defined, it must be open-ended - on one side. - - >>> r[0] - 'a' - >>> r[-1] - 'a' - - One can close the open-end of the RangeMap by using undefined_value - - >>> r = RangeMap({0: RangeMap.undefined_value, 3: 'a', 6: 'b'}) - >>> r[0] - Traceback (most recent call last): - ... - KeyError: 0 - - One can get the first or last elements in the range by using RangeMap.Item - - >>> last_item = RangeMap.Item(-1) - >>> r[last_item] - 'b' - - .last_item is a shortcut for Item(-1) - - >>> r[RangeMap.last_item] - 'b' - - Sometimes it's useful to find the bounds for a RangeMap - - >>> r.bounds() - (0, 6) - - RangeMap supports .get(key, default) - - >>> r.get(0, 'not found') - 'not found' - - >>> r.get(7, 'not found') - 'not found' - - One often wishes to define the ranges by their left-most values, - which requires use of sort params and a key_match_comparator. - - >>> r = RangeMap({1: 'a', 4: 'b'}, - ... sort_params=dict(reverse=True), - ... key_match_comparator=operator.ge) - >>> r[1], r[2], r[3], r[4], r[5], r[6] - ('a', 'a', 'a', 'b', 'b', 'b') - - That wasn't nearly as easy as before, so an alternate constructor - is provided: - - >>> r = RangeMap.left({1: 'a', 4: 'b', 7: RangeMap.undefined_value}) - >>> r[1], r[2], r[3], r[4], r[5], r[6] - ('a', 'a', 'a', 'b', 'b', 'b') - - """ - - def __init__( - self, - source, - sort_params: Mapping[str, Any] = {}, - key_match_comparator=operator.le, - ): - dict.__init__(self, source) - self.sort_params = sort_params - self.match = key_match_comparator - - @classmethod - def left(cls, source): - return cls( - source, sort_params=dict(reverse=True), key_match_comparator=operator.ge - ) - - def __getitem__(self, item): - sorted_keys = sorted(self.keys(), **self.sort_params) - if isinstance(item, RangeMap.Item): - result = self.__getitem__(sorted_keys[item]) - else: - key = self._find_first_match_(sorted_keys, item) - result = dict.__getitem__(self, key) - if result is RangeMap.undefined_value: - raise KeyError(key) - return result - - def get(self, key, default=None): - """ - Return the value for key if key is in the dictionary, else default. - If default is not given, it defaults to None, so that this method - never raises a KeyError. - """ - try: - return self[key] - except KeyError: - return default - - def _find_first_match_(self, keys, item): - is_match = functools.partial(self.match, item) - matches = list(filter(is_match, keys)) - if matches: - return matches[0] - raise KeyError(item) - - def bounds(self): - sorted_keys = sorted(self.keys(), **self.sort_params) - return (sorted_keys[RangeMap.first_item], sorted_keys[RangeMap.last_item]) - - # some special values for the RangeMap - undefined_value = type('RangeValueUndefined', (), {})() - - class Item(int): - "RangeMap Item" - - first_item = Item(0) - last_item = Item(-1)