From cd2ed917801b93fb46d1dcf19dd480e5146932d8 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 18 Mar 2024 19:48:50 +0800 Subject: [PATCH 01/82] gh-115538: Emit warning when use bool as fd in _io.WindowsConsoleIO (GH-116925) --- Lib/test/test_winconsoleio.py | 6 ++++++ .../Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst | 2 ++ Modules/_io/winconsoleio.c | 7 +++++++ 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index 209e4464e1a5c0..a10d63dfdc9753 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -43,6 +43,9 @@ def test_open_fd(self): self.assertEqual(0, f.fileno()) f.close() # multiple close should not crash f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(1, 'w') @@ -55,6 +58,9 @@ def test_open_fd(self): self.assertEqual(1, f.fileno()) f.close() f.close() + with self.assertWarns(RuntimeWarning): + with ConIO(False): + pass try: f = ConIO(2, 'w') diff --git a/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst b/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst new file mode 100644 index 00000000000000..fda2ebf7593ed5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-17-18-12-39.gh-issue-115538.PBiRQB.rst @@ -0,0 +1,2 @@ +:class:`_io.WindowsConsoleIO` now emit a warning if a boolean value is +passed as a filedescriptor argument. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 54e15555417287..ec5c298066a587 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -298,6 +298,13 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, self->fd = -1; } + if (PyBool_Check(nameobj)) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "bool is used as a file descriptor", 1)) + { + return -1; + } + } fd = PyLong_AsInt(nameobj); if (fd < 0) { if (!PyErr_Occurred()) { From e2fcaf19d302b05d3466807bad0a61f39db2a51b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 18 Mar 2024 13:24:24 +0100 Subject: [PATCH 02/82] gh-115874: Don't use module state in teedataobject tp_dealloc (#116204) Co-authored-by: Brandt Bucher --- Lib/test/test_itertools.py | 10 +++++++++- Modules/itertoolsmodule.c | 8 +++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 9af0730ea98004..95e67911db6a7f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,7 +1,7 @@ import doctest import unittest from test import support -from test.support import threading_helper +from test.support import threading_helper, script_helper from itertools import * import weakref from decimal import Decimal @@ -1699,6 +1699,14 @@ def test_tee(self): self.pickletest(proto, a, compare=ans) self.pickletest(proto, b, compare=ans) + def test_tee_dealloc_segfault(self): + # gh-115874: segfaults when accessing module state in tp_dealloc. + script = ( + "import typing, copyreg, itertools; " + "copyreg.buggy_tee = itertools.tee(())" + ) + script_helper.assert_python_ok("-c", script) + # Issue 13454: Crash when deleting backward iterator from tee() def test_tee_del_backward(self): forward, backward = tee(repeat(None, 20000000)) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index d377c2c57006aa..44b92f8dcffe4d 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -815,10 +815,9 @@ teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) } static void -teedataobject_safe_decref(PyObject *obj, PyTypeObject *tdo_type) +teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_IS_TYPE(obj, tdo_type) && - Py_REFCNT(obj) == 1) { + while (obj && Py_REFCNT(obj) == 1) { PyObject *nextlink = ((teedataobject *)obj)->nextlink; ((teedataobject *)obj)->nextlink = NULL; Py_SETREF(obj, nextlink); @@ -837,8 +836,7 @@ teedataobject_clear(teedataobject *tdo) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; - itertools_state *state = get_module_state_by_cls(Py_TYPE(tdo)); - teedataobject_safe_decref(tmp, state->teedataobject_type); + teedataobject_safe_decref(tmp); return 0; } From 2a4cbf17af19a01d942f9579342f77c39fbd23c4 Mon Sep 17 00:00:00 2001 From: Jakub Stasiak Date: Mon, 18 Mar 2024 13:57:00 +0100 Subject: [PATCH 03/82] GH-65056: Improve the IP address' is_global/is_private documentation (GH-113186) * GH-65056: Improve the IP address' is_global/is_private documentation It wasn't clear what the semantics of is_global/is_private are and, when one gets to the bottom of it, it's not quite so simple (hence the exceptions listed). Co-authored-by: Petr Viktorin --- Doc/library/ipaddress.rst | 27 +++++++++++++++--- Lib/ipaddress.py | 58 ++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 7b07a34bd57575..73f4960082617b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -178,15 +178,34 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_private - ``True`` if the address is allocated for private networks. See + ``True`` if the address is defined as not globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exceptions: + + * ``is_private`` is ``False`` for the shared address space (``100.64.0.0/10``) + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + + ``is_private`` has value opposite to :attr:`is_global`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. attribute:: is_global - ``True`` if the address is allocated for public networks. See + ``True`` if the address is defined as globally reachable by iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ - (for IPv6). + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the shared address space + (``100.64.0.0/10`` range) where they are both ``False``. .. versionadded:: 3.4 diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index e398cc138308d9..7d6edcf2478a82 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1333,18 +1333,38 @@ def is_reserved(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return any(self in net for net in self._constants._private_networks) @property @functools.lru_cache() def is_global(self): + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: + + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. + """ return self not in self._constants._public_network and not self.is_private @property @@ -2049,13 +2069,19 @@ def is_site_local(self): @property @functools.lru_cache() def is_private(self): - """Test if this address is allocated for private networks. + """``True`` if the address is defined as not globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exceptions: - Returns: - A boolean, True if the address is reserved per - iana-ipv6-special-registry, or is ipv4_mapped and is - reserved in the iana-ipv4-special-registry. + * ``is_private`` is ``False`` for ``100.64.0.0/10`` + * For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_private == address.ipv4_mapped.is_private + ``is_private`` has value opposite to :attr:`is_global`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ ipv4_mapped = self.ipv4_mapped if ipv4_mapped is not None: @@ -2064,12 +2090,18 @@ def is_private(self): @property def is_global(self): - """Test if this address is allocated for public networks. + """``True`` if the address is defined as globally reachable by + iana-ipv4-special-registry_ (for IPv4) or iana-ipv6-special-registry_ + (for IPv6) with the following exception: - Returns: - A boolean, true if the address is not reserved per - iana-ipv6-special-registry. + For IPv4-mapped IPv6-addresses the ``is_private`` value is determined by the + semantics of the underlying IPv4 addresses and the following condition holds + (see :attr:`IPv6Address.ipv4_mapped`):: + + address.is_global == address.ipv4_mapped.is_global + ``is_global`` has value opposite to :attr:`is_private`, except for the ``100.64.0.0/10`` + IPv4 range where they are both ``False``. """ return not self.is_private From f6cdc6b4a191b75027de342aa8b5d344fb31313e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 14:54:45 +0100 Subject: [PATCH 04/82] Revert "gh-96844: Improve error message of list.remove (gh-106455)" (#116956) This reverts commit 217f47d6e5e56bca78b8556e910cd00890f6f84a. --- Doc/library/doctest.rst | 6 +++--- Lib/test/test_xml_etree.py | 2 +- Objects/listobject.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 1bfcd69f72df2e..835a3a76806148 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -430,10 +430,10 @@ Simple example:: >>> [1, 2, 3].remove(42) Traceback (most recent call last): File "", line 1, in - ValueError: 42 is not in list + ValueError: list.remove(x): x not in list -That doctest succeeds if :exc:`ValueError` is raised, with the ``42 is not in list`` -detail as shown. +That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): +x not in list`` detail as shown. The expected output for an exception must start with a traceback header, which may be either of the following two lines, indented the same as the first line of diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 14df482ba6c207..3f01a79cc05efd 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -329,7 +329,7 @@ def test_simpleops(self): self.serialize_check(element, '') # 5 with self.assertRaises(ValueError) as cm: element.remove(subelement) - self.assertIn('not in list', str(cm.exception)) + self.assertEqual(str(cm.exception), 'list.remove(x): x not in list') self.serialize_check(element, '') # 6 element[0:0] = [subelement, subelement, subelement] self.serialize_check(element[1], '') diff --git a/Objects/listobject.c b/Objects/listobject.c index 6f919ce02b3ce2..096043bb3d3c51 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3194,7 +3194,7 @@ list_remove_impl(PyListObject *self, PyObject *value) else if (cmp < 0) return NULL; } - PyErr_Format(PyExc_ValueError, "%R is not in list", value); + PyErr_SetString(PyExc_ValueError, "list.remove(x): x not in list"); return NULL; } From 52ef4430a9b3e212fe9200675cddede77b90785b Mon Sep 17 00:00:00 2001 From: kernc Date: Mon, 18 Mar 2024 16:13:02 +0100 Subject: [PATCH 05/82] gh-71765: Fix inspect.getsource() on empty file (GH-20809) * bpo-27578: Fix inspect.getsource() on empty file For modules from empty files, `inspect.getsource()` now returns an empty string, and `inspect.getsourcelines()` returns a list of one empty string, fixing the expected invariant. As indicated by `exec('')`, empty strings are valid Python source code. Co-authored-by: Oleg Iarygin --- Lib/linecache.py | 4 +++- Lib/test/test_inspect/test_inspect.py | 14 +++++++++++++- Lib/test/test_linecache.py | 4 ++++ .../2020-06-11-16-20-33.bpo-27578.CIA-fu.rst | 3 +++ 4 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index 04c8f45a6c60ca..b97999fc1dc909 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -137,7 +137,9 @@ def updatecache(filename, module_globals=None): lines = fp.readlines() except (OSError, UnicodeDecodeError, SyntaxError): return [] - if lines and not lines[-1].endswith('\n'): + if not lines: + lines = ['\n'] + elif not lines[-1].endswith('\n'): lines[-1] += '\n' size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c3a9dc998e38d0..21d9f96c8c460e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -35,7 +35,7 @@ from test.support import cpython_only from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN +from test.support.os_helper import TESTFN, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python from test.support import has_subprocess_support, SuppressCrashReport from test import support @@ -730,6 +730,18 @@ def test_getsourcefile(self): finally: del linecache.cache[co.co_filename] + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + def test_getfile(self): self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e42df3d9496bc8..8ac521d72ef13e 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -83,6 +83,10 @@ def test_getlines(self): class EmptyFile(GetLineTestsGoodData, unittest.TestCase): file_list = [] + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, ['\n']) + class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): file_list = ['\n'] diff --git a/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst b/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst new file mode 100644 index 00000000000000..df58a7ede45521 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-06-11-16-20-33.bpo-27578.CIA-fu.rst @@ -0,0 +1,3 @@ +:func:`inspect.getsource` (and related functions) work with +empty module files, returning ``'\n'`` (or reasonable equivalent) +instead of raising ``OSError``. Patch by Kernc. From c80d2d3263b3caf579777fd2a98399aeb3497f23 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 18 Mar 2024 11:50:32 -0400 Subject: [PATCH 06/82] gh-116877: Update `wheel` to version `0.43.0` (#116878) --- .../wheeldata/wheel-0.40.0-py3-none-any.whl | Bin 64545 -> 0 bytes .../wheeldata/wheel-0.43.0-py3-none-any.whl | Bin 0 -> 65775 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl create mode 100644 Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl diff --git a/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.40.0-py3-none-any.whl deleted file mode 100644 index 410132385bba4d3919508eb280a9095a796f3c7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 64545 zcmZ6SV{oQX*QKA>d15>1*fu)u*tTt_W81dvq+{E*la6g?dcOGv@BG+R_x)>E?Oo^8 zs%xDkF9imU1^@t{0bCxossi_dINbmKc>Wdgzj8A-F|lUQ)3dO(aMshKxA%~u8J|%J zij}(6${u_gMk15Bme6EZDeC$Vf$Z=bmaSN2Z-Ri?$K2}%^fKxT;ks_6J5)wMNrLz zz<59l*nI`^EZCi$dXJF=j&P-e2^y2&so5Zjr``rkAk3&Ju$<>;gYfxN9 z7+r><7L|if$wu#RTZV?k(HNT-&Ut2@7J!6{vHXAm7PH%w-+l@}6bu7HWNeT_q_m%h z>*8=iiyM!!c&>W4`%F&Fosn13Nk8Q7g&IJ}Z;gWYA?CETja*Wq+4=js!LFF+iS$x20f38M`dOf5W4pbu=hO7a$rZcuQBUaFoz|41*l@TS{ljV8YEW`|jEX zsj%^c#*F&hZv31?aOOqUJR$Xxpu8uWq+f~&_3yU_e1&u`BkBcyzu+e-s(`-^!zsOgxzPC1G z0q0x@0|pu_os4Yo*z{$Fnq8@o=YqBOuo6NMZvK(VriZUfYQ=Z*C(><+W=FNi_O7Ia zpi(#WD^>~DN+1zcREN-U$R@3*Ukwbu_7(e&n~h9?EuUI4TxRU{Xu0=oS_1Zso5vy? zEh1a--u6Kgl=!IcxCQ3t%jswAdzCBY*Z;S*CI!?mO-ukFO9lY=pU3Qf9VkO%3nyp2 z|N8#7ttnm|w+&9$>o0W9$#haj5{E=inFiaX>XBPxZ}#xtC}s;9u@l7-68jk>8Uam( z4qH3hUJL*f0M*b&_ad-G1c3Sc4wuI_!^2tbv1yOd*ui$pbwAMILQuzwDw9pOPEHef zv%9MvZ3;?GzfNnXu1#@%kTK8-$j^ z!z(i@p3Swa5eR3qPIKGQmwfi1iHeofplwX{vBz8yRjmrzogSd)dA_CXv^*pPfZ;B=76?!rwQl~)UJ@$KCxvy-g4o?cC%4f{j7M`tWPXO@a{o`4>`4-B<>H zMJq%V?49GsD$I#p{IH89HmWcY_@&jGBjQbNESt-WEpZn58G%L{jMD4dhoT*=g93v?d`?0S))M6id+}f zz3ELPG}D+8?d{S|j>Ox$`!BYEzj+Wv<$qA9zj#3+`fLu@uCv&zYF1nhB47arKnTpQ zp+5FJmk+J}N>h>9Jq-sqtzt`!*x+H;8iC-5iSCg&m3+)rtjps{F=+LT4Z^gMoeAzU zj){oj>rmkVfUv?}je~<>wV4`_$)Vc0_eP+Z{=?kukcP8^w};2uDQRz6R}uFI-D46` zrA;FwKp(9;A-mu0RAs~KCM|y(Yzn9^aFR^3?HqDWlS{BvcQ~Je$p!zU$aB<$W)4r37oD1*Z=L(YrfbWB7hKkUP0^9%36%-)oL@brhit-!GnFB>}lzp7*fe2B(s)7)6oZfa{%99vlvHv1#?JF~tP>yb1)kx|hh=d?3w6zSGh@wf`mFcqd_4(Rg zhPk0y!kC>N0fAfVXvRNjh{{`H`S(I4q-Nj!9|Kfgs95XR7r7RwKWR6yf+SuUy1X{u z_k4)7vMIrV&!+>QM|<7BZ#<#SMm6$%4h1P3bH>NG_os;V`FghD%9)Az4BYTqmGf+c z$lJuOlg&8rNDP|!yxA%%Rzj~@SQtLM($T;44+=3?{8_NgoMkzcW!(qqHlRC_#b-y+))Cl~Cd3(^(fqE@p!9g2M{}s7X9V6ulXCzoP1RWhY2>-X>ZkN~$6< z*~H4hT@?IlEq?{|qAfYe1Cqy+Nu-xEnIHdh(}EAo%&o1+#lehWL#K#2?2yF<>%&Pm zlKwjWt#V$Ltw^81U5S(K;}NcNo3Yl`06wRx4ts;yQPZPAyI`T zE6guW3V@jFA3vug5*H|HTl{^mfPgc~y*ELqnAXrwO}R??ok1!^Yk3pe(qK04Gl;PD zY*7ztf&;3-Il7g^cE}L`#lvY5$Z#zf3i)fb9XRcfvDk}&iqE)Pp!R{K7@v`ZG^?+M zPap}mh9?5s)OaqQ-w67*4DzIE6YAQ2{Tl~=-58Ufph)1v>Yx+zle}A~b}3roJGd8$ zcSj~;Z&n*re6XfNewg}bfK`<~cJ)(uceTxbRHUO@a<+I*S!n`^_e?)f+yg}8e42>O z1000;=A*J34si ze0QS11bR_n%S>x=UL(2Zxnk4@o&>kXoQU|k|6urZ@i%fs3C3Un_N>^jRQV}p-a45L zjIk?(#4(m7$&zQHQ3f&~Ye?nE-9x{6pJHh*@;g8;^yd6v#LqnB;YfbS1Z^VC7I@R1 zV&JTh!W9e0l8}KMxOTO8^Lvt!9RiL8n=(8zN>dr9h{3&^mD&6QXS@llvvYd6O5wu( z3U3fvLUR2|!FX{suZeQfv{!`B3qCw1&K&Ct+v6y&38>edfUvG}u*wzAjI3hA7sJNP zl`v=M+!cdaSf+r5r6WG==v?F$T1X@_6D*o3JBp|4)`E~IE$n6-?JPPjB5u+Wox~@i z4?MtfQH-61J2G$b)GhT&BE6X7rRB8#i`yiBBaUdK1o)>yNma{XIl1(t6IF6xf+c%; z16bspC1#m+rOdV&K$Q}hs5nCsZs!i_^b>hyr zal&dgPgn36mrTA`c5;?oG&VXG9U!Z|;GUMv7xg^#B5dJUB7QzNKf5!wh1hTmB#sTD za7oCZ%5*+i&hJ8MV-i_QIC4REhL!~LnFs`&-%Lf9EVuZ>P1pZEY!@(YaB*yn5gqJd zLtb>WnGTlYE4#IT$zc%EN!)AN9wLNy-mYi5KqjJQ1NM=UTj3VpLcTe}EA||_1)YiT zVe4DijbkusVC^gu9l^ZQE(QGcMyJqn6(x5-2%B5`0U;bmXp)c}d7d%>%|&SN1E&>@ z$Q(|&JX;9Insp!l2#$52w{yppb}ozC=m_UaNwTjA~1 zeZe2<;W1(*FJmdiUy9RVJJ!;o?H0aIQqQNnvw_Ky~ByEyY5^yxAZHTm;^7gSG!%|Pbf%pFYia{fr!gfe8ORS~^h8Nz*)8E&g>H>oHpnswW z<*63zaUmgnKQrq;^@m1HAU+@|q>W0zY1Emhac|$>r(L`@Nn( zmQQCCeXKwiPnYo@jmbj@RLjm>?=U-f3iEgkc4$dd2oK1HE(*Ofi>cEBVTFvMs9vQ( znK95Z0SY*gOp85T3yn{4o(=>=?PTm>#sOX?Q}xgxC70Gp*?}(oI?rp12K?=(^Q`qz zPEl90mxHHL;qAJ}p6j}R^icGmZN{TZb|1dN$V}n&MSqe*H3lLMo(BR0QOj_pL5&4t z1@TY|KVJ}8?2{99?e2DCJCmpA7igoM$DtQb700WDs!42V%R2cs9Kf|?kH35$=X)TzFR`nO$3WfsZWEs(8QJBP19@R5ihX-#C!NxohW$88G3*#@R$3Gh=&Wau&i%@4mhP6vax+pC1~7H`H=|pd z7e|t}tJr}F$&Yo3nJKU41>zuEX_ZNa?WvfiqT9a*r;&36UZfCYwwSvAWN!@Qtyp;I zUJA?XM1~{&fOaj(V@BM<{ge6i)=jxzbJtt!Ca~P+leTq^zgxHt%74m7&^f1H5G!mQ z-F($?^>98wJFa17vFwgIQH6Nx1+IeG-IjklkFi;f;n=S_D$btG2I$tD3uXn@jtGv! zNQ;XbX7Qxs#GafUx1Y2@weZW=nu+;6oZfv$<|KsI(SW8N&^t^Upw=Bi$lh)Ra(NX` ze0Sg^+?9D%_U$k|!wh(CMbXv-SZ=;@_z`*9A`xlb{-QOto9*24_zaoXN7|><^K5~S zLIj3hEt>)}9oMe-5PG1Zq2#xn?blIfGwGy`x$BS68zfiVcB-Api<=DYmKXnBRT#2$ zw~JLqYpu8I)C>oU=O~D@D{mj(#k|*o%_jwMkT7W4I#34MDtMjg!4DDwTPKM;^r<0z zs#k>Eq^}M)^Z;F-ERyWSAuf15pdi+T3vJIg>)BR&pn2wPNMh+mz{$S&M@QRWP{#5G zbmwQCUsA0c!X!Mb&qeA?c4oMk)-oIRL|#c1uDn^+mY0!J;`8pbpi=cpsl3w?I`)8m zl-n44Ue0ZW*ZoQ0$2yA>DI;h3h?sW3u^PeYw{+LyzD}W)Hv#{B*R)$4n}jKUcX;0HZ3z!-og;{!?bR|=Us?_etbFwwD%4{c z=);b_F^*F2&;{GmYCy1R_g5tMfd>-8}ouk^K)D$b;B=WDFB?>@S zIuc@HJk-?!yJ~x~a=JOJCmF|T7~B6C@T>p0ca*fDe`GG~lX}Phr8k|`AoC%EfzHKn zG!uYiXTaKd%#}m z1co|M>j4EittMhk(2;<^*}F)ER^W_V_W558P~~6rF?u9xw<+sA(`y1fl{YaVd(^M^A6h_|tB5M)w;b1AqB zq-U5igf8SYW?p?p>9==71g49;RBED6JiEcRD?)m%)ByC903fMaT8guG_w2O<4oa#t zUZG(YpVzoQ#YMq%otNLMWVq{2*WgqnIat?D0X)s<42yMFer16$%PTX4P5*@qS}89a zM7u~{JuX-MN%(wX4IbbwC?~4`|9OOG2vTH6mXEg8>*j5`(ipX~aJN^{5wESB@i3&b zk`dd~$Zko|kM(UmO@^-z?tt}i1Y6u2_efXq`eDp;g!YpNRXGwJ=9LU5GoV!z6@F80 z#N_&OZR~WUGI3_5RKT#!)zRnnd~L+-;q~D42mJiV0Yhp}mham~p@zE>ASsU$lDcdl z@7EO~DPu;6n9T*+&-Q5~M=q8sHyoXcFCkC@)H^~oo zV))L(@OtZ3m1C5Xdk%zOyTq=lO{B+#5ja%o{c+id#OZlii!f63$!ML=NHD*eGDULf z8T?6C91D;?z=ggEi5u0_A1z}DX6<+rSWXKPRrX}l=^ac)ug_FN9p9(Csv^q~^g9s} z(H|5&cSue~cpDwUCefv2Co%6-*&{GZXAn-m($>Uiad81pV)R%5t05T#tuaD*yAAiZRH+xmfInme zv7Is&L&Gj2F^|UmVM&i`uD%xfUr-OSBvZsRPRVf`1nM2`k(!3M!teJzZY2(?6+PwW z5mQRe1wx)J*ND9eG-mx7i$eK#DRI)XjP>NwN|jvv)8~j=zuoQ48-EnE2!a>gMZxUH z9MX}NHpz6`^%qZ&PeL#^g#9P9c%v`AD&M%w1`WA&a#Y3PlB6#*cXl->w|#*9ZVr5! zu5<1DQ+LqURP-AVw{_a|TLuV5$syE!e^~kFzTVese`9<4r}eB>hfwZrn%T%ieCJGl z5lilGXZ3aq(qo;E*R6|-gPG$Rk}p!Uj14X~>m0_2zK4uC$g0##YN={A??}P`a7<-A zc^oSa+hV&QMH{JYhcX9p`ap%q<^hB7E#6p;NLJIG5^Jx)D{(OWnQ~$@oRM!hka;A7 z`|35eiU}uvVLwuEOOE*M^}y-h+l_+>qZwYZvQzMkig1Zft%|l+ z!g6aCKuK5pH{qad|Io=1iM5@G6ZeGM$gO|?I`6ngz2)+~nb8PA*9C2bGLMh4y~)OE z)_JBFBluT@DIuVH;cD1FtT%U*R!@TG_|lc`G?~#ajon&#O|D4 zXr8ckiP&IgT)oBi79ot)WZ?%Tkvc5A_yctmXwumTK_%A|9Y3!*SURMx?tYB?&g;zv z;G%F{uT0aX)|_*vau`hSJGVrD!_61%8YDEnQo}?r%BaUP=iHbl!m)3V@gjm@1TRFH z;U0_Kq>nm+BcGET2=(2`=!T^QkMT|!#(?7;8z-BPP!%9Zp}s)5t#Sdkbv{lw?Z|$x z1N}J|BR%4dtYm&LxT_vs@YC8N9UQ&#K(>~40^E_Oc&R*kQQ0B(`a7W$ho~p*R?(Mhj6@RZZ)>-511L=7RcF!CfDB1%c_vz|5D(XcxX=3o!gwA_%xMMl-`Jz7n_ zUSF2h!bj*AXA&FL4xBLHat>w5uX>3DecFX;EZBw5MyT`?M-|sup;ZMjti5s@3>(p2 ziGM{f6I@6s&p61Aq@qY_B%*WVdH7S)Kw)j8Ji~Jp4hVW*=7Asq6$~sMH8^wOoP&)k zm_i76XntwN?D4qzg59h?t#8fK5$%E3G-{7>=2zymozjIXgr}1iBwUi1>}h*$JF`-i zJiMOdHg0tI=$zAmQRaJu{(L*PA77w}q$w4MA(rn!DFQN#ZmjY5G`dCqmV_X7HJY^ z<0P?}wZj$$(0SNTV1kAlZX#!!WE}6I^#jjVvc&ZRrSqsS$h9|>14cWgg3@3pxn|Zl zK}pfps3}$TMeM&x_U>q4_;ftdz+u_R zt(`^aj-@Wd^#F|*(makStbtH|eW-2vfb`^%h#H4#Nc!3PHP~=}Mw)2<`(t=q|ErPhty5w7wSu0|Gd@^k%yZ z6x><7UU4T=pY^&j`Ym^?<#&Xv*R<19f~7KNgcbOkTp8vVQ*IQEA(!`cL=9ZUO2{-0 zjW~_LNbfW7M$m=O1Ehp`UP>hRnpQ6&52W&ftZykcHA4gX+!CxZNqQr$_ZE)6u*r6X zcq~Z~>IWw~DOK+25Q|-&@^bQFJ>*;fY%_u-@au{~FeR1ulm_(UZC;4W%uv>93-o6# zBki)wpQbq}4YiDX8kUkDMY;8r{%}@)qgR7fKM>ms05D$yNMCh=b$}+XuRZPv$8{L9F|eS-CC&E9o!0cgOAex%8)BNh_?EsB?0@faY2s z!0>iO(VBY0!&=UkTQ>t=h|~FEc>RhdW;WSchh+Ix9g5v5B1Rqshnc8$ocJfzhLQOU zgjPDhoUKp@2Nz%@AX#I~Ez@TqmqTf%dr6>;+7Lt0xw=czRl!XK!txlh2N8d4l3?*c zZke16yYy?Co);aG;oN1H*u7ZPJ*x(#0-s55+#j`run}c*h+hoU1&GE{p*a?v6nhL` zOC`)M?~F%vA1Vqh>_H>DWGipJgxl_TO)cHdwy~b0mauddr-N#I{)tHmH~ec^7X#ie zra?Y%_2*%&28Q|6Qu3buw?lG%b@OjQ2f?NtWkU14-`cp^}Y4=I5$8$&W($Pbz+EmrrSS@p=5+=?Ys7ZYieV+xO*r*1&%;l+XKPWEqSi{-+qj?wDt;XG zfsqaCPPyES!1)VR&8NK=Zbx(9Hu^(hBQxw

#cQ?q)_wuV;9k0I6*mdzNny^gk+* z-7k%Q%m}aMRKv!(xW@oW7$(;e@Dr>D@1>sCGA zUtT9X11P%!IEPj+^^K$kXGl9fT0#H<%v^lQM8ZGLf0jK!ljI!pmjG>)UlkvP z&}4sWJBYvwvpJC;<6OB;CxhZXoAQWtCjXo-=G{JzPOr~pV6 zCb6M3`2sq7EVf~Le@$n}!kkV7_sS)zkMo!V#mzwQ-fok+AU|&%)KdcNZb7=dn`$4M z!2`|PsR?(Z(q3uJ1|JGj&v3Q7Ym~(gY#f0aNKmfMgPWAJ5fZJTsw4z~PO`Hc$vxdhuWE!sN0=!c|y$B24csGg-7=7R}w-wEl`q>KjYu?W6IX&A9ie zjK$w_Q-`m&nf9|D6f;OmPwvT^g>ygcPl$Idl8;i9wQnM%bhi3~o`Bi>faS}TP3>>z zFtlZN32hB8EH-D_E4vB|H7i#DIRlBDnQ@t&54j&*_bQQBS&0^pSaIc)1mnq;*2Kcv zvx_Iuxk;O`T4x|kIs@9Iyvz*{uNKsM*(tHH@5V-zQLSMS8|^Kj`v=iJmciJ<2!v>q zsXbuaMpt`eHb|EX_-V8WZL(r)#rAhnj@aHCX&|lPJ{88#il>^nBN@o2Z`0&zt%`NG ztW_S4Nj$#p>k(D8&N{Qb$kT)*LBXUbG#4n5%=VF>pxayeDUj)NqWG}4qsb=_2 zoiLA#uj?B{6Dpig?pjucAqzzCWq zM1#Cew6hgG>XH<(?BLL{`3Li}XCPQ&6!w_N;Gs=2d1le%#HapDn#kQ+fWaHO8=vRC z0OYgp&M*hUenOY`UC}3bRy}pIl+PNikgWkL1$4wF_uTfe+qLqH(n(vF3n` zfnz z7jNkR@?uj)dqb0XXg{(Nk>Lv=oGjwI3iU6mw1`h6O;c(%;5dmkuDU+PsW?5HY)hgP zm`%?{_N+Q>Xa?MAV~C+S0iSfrd$jWz4t>DrX9FXO;|qu9Hyb$i%s-2f6k!YTv1X8R2zjLnf{LE(f8 zO&CrC0yZ{z_}aY#XT>rV1O!BoP|a5mEiG+?CM{W^W8s^wr|kAqqgx$?)l*5a;Z_z~ z`ddq&m5Rbk`FPgS4Z21uND6;Us&p#&3-J!F<^nJ0{v1D>U=Vl5azNUsq?eS{GDW)^ z&fv84*DsiA*Q_+G|AEb;ZGxmv3g+Rd|f)&6J06Df^<4fkA4zT|p;Jkpa)|(yPX@XmIA}izHvy zdf+J;`ax<*E?gFS7sSKIQ6M~BHy$AY*=sy?%i1LK*G%$_3qRSHK9=O$8m2Kwlm?uQ z3MH9K?9C;PR0@GXPt1VHKDT4|O-TlK2zK=VIiXKeR?@16w*AErsQ~W9*&d+NWT|J@o)sOP)$k zop_WZan(;NL;cin`=kK9?tQKB zD~V8Jth0k3^$z|oWt6?4=gj?(ZGC@PctyZYbK7y74~M>(viKrli~Le8a}X=_bf+&x z4F&EJm?4xNE_; zDM#yJ;to?>^@dGimaw!x_lq_f84k(9=|adzh&_^o$ra{`QyPkV2H=yIxK$Z4iLfoF z5wpt(84-nVuFXcvp{@;=Dh1gWdHojBk#9z?-@j;FbzoEwL6>`@W}E%nFlsm%HzHHn zev*Q`c9Ig#ZnG!2ueTpotW@7ECFo}MsZs<^Tlv!%kduH&8&3xGvJq?R{Ww+8OVI^R z7|i@it=WgU>|;gYjD_sJ0F7B)qMPf+uje7NJBJx!gJiQf0FN?tXuJAxZiqmNfa6wO z5j$=1Qy;&~=5P)n*V-${B~8~mDriF~^o)`O(=syeXMYzsp(4*Q9G=sKrxh(J>xYt? zh#k+~t-|9`=4Kf4wy#tkM!jab8-r4c5$pl2T#_H3i zw_a|*EJ<8YVv=~ z!c#6;S3$C6ffq!QbT6c^grF@JuVY1_OOv#R!42{ABUmVfR7Lz*iSb#c-t3X=#C)(G z_44g*u&4;?h|I^z=M9Y%Jpo4CzScMWw+q1>XU9FpL!Kp4{2J{J%Qk!fTW_9E5mX*6 z#QV%5jH{uxq}xTP30*`%NORFXnC}UduXHiW8ac@g3Ow%s46UCM4ybM4b+NKCF+D`9p&F6>C$s{5BX-Ypu-?NA7B=vUh8fMb4kYL zFrp6J2?4Ev#pJ%9Z)|ar$#0#BWD^fpuFuSK>=vQKW0H4*q76<^0YiH60OM;;l+RkI z9f#i!kSB3{GlwMvvL|0OS)Efm%!8ChvZuhCWu6${6&pAzIxkEx+$-#Llix-749M!q zPQIQ6g|%Y*P~;1OH!T2nDESPSXiBgKv_@hPwT(v!mIDR~%KlwxXQLX_vLvG&DkJxj zAH+Oe>XNGG#xlhw@J}E3ZZ-R%Wu&O^y-N<12<3cPMqBc`w^30|sEYR*$2H~Y>ChG@RNix%$#{f^-&q{%B zRwJt_`h{y6VQ662Gp3bfhc;Fhq-fu->7y{<8G%6fv%(faZ+gLkB{?S+&Er-tS-){` zDCJX5O7oN7YJM&6pKf4g@4vDAM%(scc59ZDwcw=Knd)T=Wk)wdo8XAL+f@_6hfHz3 z!1%ScC*YzpzVASstFbb z{SB2Ic`W?6wRo*3ph}2kUtHDb&14_UFr$FdD9_L6DI6~6P_q6oAR!&HVs(BR1d5SE z4lYS?HMCptl*Ow62(|h;$@zT0V&?kX{!J|^zJnJ3Wd=fa2O`E<98?Qp&L2+T zJQ8tG+|$fKuGjZXqbCdV5dRo%rmCJVFQ+^zz;S*hb<5c*BHZqXHDwLi@Mw1q$n?{} zCFm+}i?AH=(Ejo|ai=!0Htst${l6f zv`FJ5IF)BAOdn|{?*wPV;&(pQOx(9<@qEllf{&!_u0}tOOWs|BgbOq)dr^FLF1qr8 zsIhX@of{lXvHzH#^C_Na8gDnK_DBv9ZMg|r&1kN}nJHLze~)~vGDc6y#GCWIJiJKQaAVH121bBo+;j*c>4GRD?nnu1r( zkuJ*{Uczm7Oy2rUwA&N^3!G-rxU%V#IB_u55CBHVU25mIfE|b$309|7oVVRXSmrBuJyw@ zE8pD4P;f1;lW6V=(G=L4=UCJ(mHo_^4Mug71_ETIHFRR1A0J}j1cn7 z(}=bzeVaw|h{AJm-9BuqUVKUZcu8sgdK{4+I({UXFA>l^I52pAGr8nycQv?rQBl_= zaiMno*p^99i+1p+So^WS4_I%@H|H3O7jf&L9sTcHsRhKIQZSZ zm!EsJG70^5QfX%TYa>9>>Z$s4wNU4i@~2DW;R;p~YgUIdJddN7`(1FXsfZ{fudX&| z2~Ynlrp`>CX2Gv8wCV={WC#v9&0@E}$DzYs%ja5@`i9F!u~w`K7dyw{H+$z0kF16) z!P;w0X=!Lk!ZCu!woePo56<@&2VD-?a+uWyW}td6aS6#*STrmPNQ(MYH~_(|BAQ4JwZj5L9{e1w5|b-N^Z0kzuM zo~n$GS|Fzudad|8)7~Rr3r8b^#L&%=G2RSO9XwPF_PJ=TjZajm;xWq(PTV}k~rKsJDdY!^4XRV zU5oJQlUB!OIHs=`jJZU~S1J^*4)H}XzDv#fze^~KPxYXv-N99Qag+mnP(Xno=k*-L zm+fb)uGE}mB%@_$E0a)0>x`fthU3a)bE@xTjy;#CVQFEl)rK@tuAZvQJCX!NxYw@F znpWhcKhh9065Y&lqeh`NcO2aRRco;6-ChPxqdHQ`-&V9!aj*a+p_Zb4Xt=JD))1Ro z--(&&FC5C$)77Dm@xH^Q+36L3wAU{3E_t?IPBInBVro1Glcf#X?bnr8qh+CC?!J?x zW5zCK)h6JVG@nFW&n$qDoigP)#AkR?4VXX7I8;Ed_4n#mlBe4H0gG>9tkFnIESk+w z?kBopEGap}70%)b!D2ft6v`8XuBI@IGKC~06iHc{V6C2Fv92pfZ;l;aR*?6E{wbzN zFA>M6amj@A`uAdBV`xA1=4C5$;l^9&sE#_!Ll>0JO-vtp#KFf+FXuC;XzvE|=yhmh zM_{aL2B_!h#TlHtjyQfKT#|&hyVA-$=Vk`G0)*H z;mp(X6L9w*j1y23IQ;y3zGj^=n^a|v4yq_mYyevEld}B)7 zB*=tz`Go=^4dsOjLhdY$4#Z}(f;se|F9~qT) zY%wD$zY9)-;yRRoV?PH-zWIx3j}$%}e66^gbb-g~fk)|x@6UBnt$o9^G= zj$C*?uEpF#<`in>E$UB1@KUrv(ZR^df*tN0>1uB*=S$)_2vn%#} z1}G4CbwHb-*n!Diu)%3uXjS(}b5IDrnXBX9UrRaHQQ-n^K$(aa6M-U0zump=_3$7c zOkfY{@prfuM&}>8oJ1CzzJkmc5ffedqFkHboezgS37UKJGTFI>2JR#rM@>-s%eSU2dl0ih(fR*BN*815~G_JZ(KX4)xr**V{ zgmwWie`==^8jKKi?0V$XRi2#iTMf?6)~2$cdlU~GU3N!|=g?h6@Qo;iBM(=8luhzy zbo0Mw8puc9T&fXMdn5U+x&G)PA+JHc8V{bCCx6mX&|fjTRBc&=94^!; zke=eZGKo*M{xY=~?Fp9JV&Nu^WDpral;BkDgQ8`_x-}t+*N!@}i4r>QLc&ZE^P#Go z0<4@$Yv#MmVmJy8qBVN`H{ZuW^Q)Y4i5i3=CEOiMB>2K|wPXBAjfk z$(WCf2>z@Ht>Vg!0~;X&on(9kSv+VNkzTN1ys=7Ja)yJQOcNY<|GE4lr2rv_CqE}K z^Xb(!43-CK$kDZ~F}@rKPrS&p&%|?kBguz3G{zI$H#& z&QhU-=iFc0kK%M&@VYx0VGr9WG{e(sVvM0B&$GcS1)Ev^QP|yUQ(v43x)N?YnRei@ zAktY=XC4UCig%b72bZjPv0t!B<~n5Q8sZXOY=afK3c=a$#A+`L`*@TkFAu`|*zK&f zC)i^7{G8~kgSPwl%f)}2sSBxBOrt(4#uWZIkXX;Z)s`1yHX#k!R# z{x^@5)mzhL4H(E7L=(;03fAcwbP$v1b>8fz{cbXDMdaOsBNSL-+ku)1o60AuUa&)F zn`=g+`7CHSZ!_bI>zOP5a(-zYFx{FnR4**+evAJ0M-c=r5@I4*LN;GRB9CjOE9{IW zY27}5x#(2nO_?GB#sp=8vc7(%LFU zI3@Sl$Md1;i-IeCANS^P>`<$-!-!7(^(-wWDM zZuBfQet(^_jDbJ4ynx49a+`vSN}j&l<(J@g6VS$9@j8CY6%o4>ZMQ^URjJ>cuI+*$u;1M6$ZQCz-H@Q$eul%7oU92zCV; zHl5&8xcrQn+dei_Jx&_!YMQ0X=vz>FeGHax@wN)e?CIm`@XVlbEA&M?U!l1Ur3MjiBak?x%EhoDuKk%Km5R;!uv zdHk}ngK#WKyyISwgQlc;Kw^LUlS?7(w4qPbYp&u){mRtgnalPz+gND^eASg7_^QAP z7x})fjjOQYgLsJq=U%yDgM|I5nvGTgHz=IJ48U@6#hg7|w70nMX;Oe>b1qFZb4FGX zwt^76UCL2ns(S-$-nr_toKjaiHv^!ycsEM}OC`BC0gw))?>|UfC=4bBSsN$)CO=xp zOq6<$6!&=x$HPN}eqMwPdcQuh9BLmud43*uY(Tkk%NAF#0nZP>$MaGJ^{qs$4(mXh3tiqM7`Zm`<$GP-Yrg*~+r|t~DCi{l9QpDWKuZfP5 z4x$81GSjR*bdO%(ibb@Nr3Qz-~H!JBj> zCBt@Vt4Y%7^08lde-i(t+WN2E!n<>cnoS(}38GzO=U5C4z{lxSR{Qssf3SaD&BmsA z6g`mmd%rm9Kt{hDQ{X~}+i|zYujVEO#_5joMIDg&3H6@}F>G$W{tFxc&;|kk=>JC{ z8d+QXzxnqrH7UCTcBJleH6(CQbsMs#6#zfk)&mHuXxv3XDkv!396NG4ngn(D_iwY1 zBC-v~<+dtd$YA#hMI&MNy_UMM%jz(oPdH;Eb_CH}QmK^IgWA7isnk=@Zu>)m0d`A^Z6F zI&Md=C9|D)P=@%4pCRh$Rm|rvp#)kKX2S;W0 ze7Cz!M017|_*xARU3);21Kcml=oKg_k_7Y3!gN?$uZ)Nm?byN@@!ghfpMV zppQ1!bWI62^ylMK?D5&mdk-P;mVL$71i@gWae&1u#+I*=&jkCKie<&mG26$S9DTJ8 z*=5UWV=zsd2_jrj!&|K}yjES)8C~_-hme%fkXi5pfqQ+DcGHk64 zHwB5-Y)!Up+qP|+r)}HTY1_7K+qP|=w)?a({r&&w-if%gANJd>sE3N&D|6+_3iPO( zaPJ2vlHlI+!_#_{YbaoF)=dQ!0V&n=y30wu%(v@4-;mISo>paYLdx*J-TP}f>;BW| zLv4#z!7dh^v{%C)TC`gG14ltr?MR4EDm*y#*mJ>~ZqScL2fso+j&-bWbmc#8{>R%$ z0luQ=nl{KPpRWYW9D?zwX$>?_WnI*gS0R0=cS5W4%uVTiv`h%g{a1$)i|p~63WIB8 zwRI>J%pUXNx#@c9RQ-0jl6pKd4w%0%9L>hP6K zp{8{d4&e{!BHT|F`+b9b6_v&R3@LeRPQJ#;OLM@G*GI7MO{Ln)fKkDY_t;?35L_m! zE;)Aj=DmlhOJDNVv`}qf4~lYxy9sfx2L6fg&b4J&=<5VnuqGwAyU~}k!llcQv@uii z&|WLG=X^b^gaKz+WH3ak8(^LW z^-3Gwr|`1i9s@{%qah&vD1bo+g|R|++}im}uH#1*!+E76GLU>{7R+Md>-+mIM@LAB zJmA~IyS2?fZJz)eIeibaf3^?V=*q)svBtglUEsd=d&r*=Q_vC{XI!b2Ybe9u$0{um zlXjA($QIJYYFEc-P*-sLvE~_4iQ1_`Y~Il0e-=u%4JC{AYF{}Rns2I;Rbii$IrGb6 zq5w1~pE5$VK7dYYDWm_HWD|I$q16c0FN(0 zWX?Aejno@Xws#)Wei{!5Oe3GS%0a%~fSU~@aB0&M19q?JH}Jm#W{n5HGBOMRz#_^2 zoo(e!kx|%pT|7aB3nl`bUBZyx$diBLtIH<}jj*`7No~#K=aB#3V#B2w7F@#lR z%NAZJlJ=6+=l$Q^vr|3xlII6tlo161@x*RsrYANh(d*5B1*=v&doHw^b)kLC&s&fi zD$eCxJ8Gs3_g5?v*&8>PT2!u$-DAJITl{0fdzjSq_sn+SaP2~Vfdo1c>-lsodyg?%QLg@2! zKAX%_)_WwML#y`qB-s36*y>TvIdACn=JD?I_3#wF?da7&`vlYLJu*3I#u@WPd}{VY zlKlPh5n9{p`^lq4gB~fWC}lLnrF7nr*lO|E)0Du6)If7lc={xTQml$}x)#O>} z?osL5K+o+^Nwqv7XA{hd1Hpx4M^qMWA?*Sn>q<9|Z6e}y{^XvrtU4A=Ml_;W?&ht% zsg_)0%d5Y*63j$4N{A)K0@On#Q_J1%Z*-~e7>wV^GsRi;=aQ#Vzsem>c{ zndx;vvyEPZ)M}9(_Go9oo8H)B$s9eB{z3}T=S!)ez8gSNc z=m~@ZM8>7c4Vd2*sx)^8Qgi%7CQegKOV^O=H-WxvjFxtEKCoW^iCI7K!x8cI9?KA8 z*Li}QvZDJlumM+xkN;G@}^%Fqkwe*Ztd8xsE z9%e#HE7+G-d31!WM~Ad+1$@Jj7SWhBM1ce8HE(75m$# zvh39Po3hUC*o@a*mj^=;v?d9^deZv1O1%zMG~8=js4rSCqfCCJqhB~0*y5}@MA>PwqkwyT>p*eTGZx-rKu-NoGt z=8jZY`!SMQIDxpK_$;!-7mqpNO_MaBG7o5J%<*z%%H5~JLK`H9gI!fc*92kol#jxg z2QpjKnEjkAZ^@zp*J@7>WR7CtGW+6Y87eU~n2ZWJ+cns6=j~nKKVr4V zsv!8r_esJ+&&+Z?G_qjr8IA!0WM<>MzOGs;GgIk%}1YA3? zr(Lu;F~xI?jBJXcSZ)A=V|C6*HpN%_Lph7qr(jCGf^y|KG2Q3h{t3XBYCmBsiwrzJ zQgvNO$>%G4ofsCxq)X|6>C?aUQxYtv&7y1`6)*u?C27pD#b5VIV^St2;v;Sn7AoZm zYIzw(HPz4XAuoan${G~2ITO$UwWLcAPdbox(d25`Rfm!xz-dfXg|V8ftQ1w?vZ?KJ z?l9a*J&VAceIY=b>4s&z*TKIyjyz zg>z!$Hv)8F+-7z)B z#8@$j5L#+Pv&Q2THszcsLx5bC1^`Vq`l3m?()CK;(;kJ1dP?xLv=x4Sn}zVEkzZY% znm(^P{e-9ORyTHPSl3dN)p6!e>RH$l2Pi%}ZDI6(tyw>C8kTPl&W-HNM`bBo&plXElP2Df+8 znVk?Dur;jX9d_Pk5HqAQ5>~gfjH12gl@M-SkPdmjA%4XjT9~>{ouz{jo*AnG+K}f{ z$UGK=SPe%WY}-h^Z<7iTcb4qKm&M5H&N1Lu2i^ILLCTAOcJMHiKvfET&O zwWk`QT=D{Kq52M-i`;s1P0Y_mBg15sL&B>rRf%k5VgBqe*nN#Z8$oXwTZ1DExPF1tt< zu!}wI+5Yx>=yk|@$Q`!MkzDp18^lcxqK#tZ$`V%Kh3`5zZ`)*c0KrhgNO2=!%x|KrOkoS3yoOAr!Ohm#3xKQU%Dl zN9BGwX0Q-VFQ1%5|4E#zwl-Fgd*=Aa--(qDGh#5RG4}(?axm~zCQ@t5h!^{Qil}!i z6KlC|?)8xP^rzysj9-p%9+5$BTrkH6^Yz(~ivjd;e*zu(cfN27Z6^D*(3cBb7n8kO zc8i?a=3B0|!@;2$+VG=tt?y!Mx$yNJ$?Ob@dv1k309}v5@j}=AodVgL~X@l9sg;cNPEpocv zoS&YGIKCfk)lR8=mLof~<1Q<@Hh8Avc_gKD<729Ojf)k$Z8Tk42{y`vm40ep4(D3x zBU6I1N2)SsANpt!F6* z3m!>6u=%gAA7Mm8Qwl$ncG+EbkCSR8?p)d{Dp%@F972H>ZycPb2P{{9P^;lXS;xZ@H)=abcGM~492xwBbb8o;RW{%R>j~h_bJYN5aiF=p&S&IdpuY|Bt@MI zV)jKSQpif3$!lQ-6X)W5EX`MPA+IJTuQGQvrdDvhvhOYeMK*uUIX1X6kE&lpYY}Q+ zbT(fMzVatna%TtBpWfBc4fqN?klJxk2m2F>#3?z$P7{3s#^5JC77bxzfp7G|J&}%MUcL^V~mv{c`#u_3Ei?C=mPWI1Q6qUs`}ox@W3pw8q!; zXS(7~Pxu~0c)wZrevMckJtQ4Q_kJ+9z#N|`cW7o2gZaWc0tCGHv)mUj zjf%^fU`aC7gCk*lCSi0YL2PzFGjPV2J__4VKP5rua~FKl0t!|%O)BMws!Oe1Dtg5U zUlBICG`1)%+|tm6|8_jNZP>?=q9Y5{Z`HX#9nE2?>Bf}k?1E1c&^k0KIJst3O zRB9cmIwxilApC77C z7*(Up`9&b+g28K&)m=7u1S2$AAy4?SYfFP$bu*78?%OC!^JXz(pavDsep@t>>B9t3 zVv38rnMrGXL4N_*q*VHv@_`p;4LRu^Tj`u)nc83pr(Cm6#Y;;kug8`iAKTxt*M#1} zm-w0PaB91AayT=1>wAd)DW(A%H^P#_1*IT2*lY^i3Bcwo%5ID(Co(@Oez*D1?4_~e zM}7C$**u!up$8_hcRan-hSINq)Yornd^h_OQ|2a{UUXxlr`%I&7gFVW65G|JNs4@K z;)$8r;8tE2fPx+=L^|WR4gh@YJdQHZPWI;re;&TNWY4u9? zz1GE)&Vc3X8g5_4=j7L1bJY6sEJdco3N;x1%xfl(*@oX+2}-*0SW-i*O`w=Yij|v* z32~G?Q>m~%p{mNGR9Zt(-{eqi{N4^$vg}oqn$yk_7hnWl@}}qqX>6?VrJf84$)%`% zw@-7Aw2d+q`d!j!=tDy-nlb+7XpQ+pP{hm!;&Yy*jL58Lt;k~#uxEJC%5>u9+1shl zn$_Eh#gJ+?x!;LnFx#?61!L5T%(89x*q9Xm80#j`+WuZP1#Oml8^?@3-9`ozVdmqf!P*r*n8KUy`mH~dq=9X{W;tIsEQEcgr`y*pwVl!$pIk#q-Y zN|SgXp~GxMCHO6(CCILNx*d?Br=9brNEh)waN_edCK?e2<0Dv%fKyDb2T^dLj0+7F zzGKMkYZ#y3*;BJX_w524$IRKr-(maUl6uVIxb~ z5r%}j8mG!8c-iJL;!Wdi9V2%VdlVPhsb~<1J@;})Q4c$yjX)+$w9|J-1=(er9tM(c zuw$8LdYa4g@n#2!BHd@D!;kq#lGkML99=8=CKZ+QkrY z5~may5E+B!x5x1UGvf0{3D{+Wf)cgUJs#p|UabkkBB9@AD$?G4m*}S7+JiHrJGfQI zi8sy$ah!N#5)kVzR;tZv*E>*%suLLY5Mb!7O&}$i zB38v2`aI!^y4BXKE*;RYCg4}?zCUiM`BVIIWme+?v|g_ce0n`K-}WHSJrvEo{V{x9 z^H`Q`NT*^iQ`<+%J@>Ae`jNPK*9rkE85Rp2BoREO0GJcBdrlM5&XUC;Kv!LfCwNLe zo{&sSfxmISw)1x1WtXw*fTi%G7UcC4-S#e9F|jJo+VHFRrPyVb*Z2mW3kPI-iCy>_ z7GE8U&`(&tamkycP8bwQjW!GbaHf4M_%r5Hm6j74D1Yal=Fz8-2a@f+BN?D9+R-UY zI&gkXn@sYDzq7{re5U;!9i)y+T7rMVg+Byvw5}&@;%*PhA2gM^tJ!!`^^4Z}8ul*l z%n(Yb%X02aT)k*l{r89lukg%L#2>HYColj2^1t+8U2Oj+AJ9?mkI7^}_z4Clrcxm= zk@Ye!BqA-XNR_fFwrw6A^LMa^W^0f&@0jRy8xcsUg2x!K+v)N!N@~Q#fwP1N92^96 zUe!Vuz%mMi*}%#riD2x8k*$n{-UIiUAs32*HLP@woIFDY?@=I-Ou#~gDM?+@#=>ZvmUNg@*#+@J@%7pPjZFQsw0El5R1Wg$Iw* zOwif}WT$;fO4;T`%9;nM)ZP4}KH`M2IXbbj4#jLs+toK>-^pov9^c zX5?gMr=(k(S(#Rt_kjLwp`UO6_ZEIyWBlFMQY3Fj_ZfQbA@ zjLAbsJTl1RfcPTfXUQuF$kH(qlULfY+_0n6nQeY!;G2OPR)@J zu!H`tJ>)1ZjXx6k$b*dS?zK?rt4~UQ9EIyoF6@)DzhD3G8gb`9e8c#rZlC?cd(C#9 z{51D_*Ok4JuQ=nYW^P`y%--~oy~q-yrqj>eDe~nI+VA1je%{5$n9kw#NutC91An@m z-5+zrRJq}s7|ZDl{d`-^grM|~j-38h8zZHFf$(A`F}kXQZiRA11}1*sMAx1@$r-SM zfxs@@ERAdiV|^IkBzBAspNP>AqsU?|8qB&9rd{QRY-Au05=&Rw2&y@870=c7r3Q6F zJj(2Z@7LBv7E-Fo`#ot(Xnmj`vQ|VDZ9hW%-p+cNMCTE-awYDrYR!c=1`N2|n8}Gf7 zpoXwc;U$$b>V_dnG_t1rX9feEk#7Vf6yTeA1mzTfc!`I#Smo0?Ks`bbRmkJPZ|^kW zJ)}!Rmz_YzSBB?#@a$};YH?ASo!~lDviWuoH%~zTtOp;X;&Vsqk&miFM6J1r8-Q^t^rEhr!4EmbxR?$5CA@O=FYPS0gG2cJEHcl~x!@3Tb$nJYZzybY%o z8uQgRbttuMAxu{?6s@(!@KefTPH899G9b`C7RUoAS0_w&oE#~0#SQ9u#%KnxV@Fzx zjk4h+Psg=J8kXwzD7x{%T_e$iYQ;X-7NsG$vlZGXXFyh)a%b^Az3T+lJ^Y&l5Go*F zcBy4GvuGKz(+2_TLZ+Kw^?~3^SOvw(F2c%x;{vaOTmjM#V9??iV9dD5kQfMtLaYcG z{Cd5b1BM5X8ocr6Ntk}KBK7Ejp5eP+a7uLVEYvmPu{nP<8`A_&v#^L(Vfjyk=&wvq z$dr4E%EXLgOf8v{k8N(!vpFCZ_p8dL3d{`E`#dSkufoI22J%rr^4H8}FgajF0yG-vIqe2dS!LX26fe?2m zM|7JK9p27uz~S@dhc;5*IyYXGaX_-!$=7(O+t+w3`i)$Tuj#1uhx>27mE4uI?~CbTyEhJR4o>#)5a7c zDC%e!G+|x}Jm7d#YkL=;l#`;rnJ#3PhqXLZ(+_omOcW&0a+R?p$@t?%-j5!5wP?V1 z6afu;4-zb;nS)v|L(nY;9&MdrO1cbX37|X%PW?iPeK#UwQL$AZkQ1r_p+sh9LcQaY zK_6oW#vWJ@#Z*fLV@vbqMNqn9eb9>n9&zAoVd9RY205SejzgR*z3hJ*?CX^)YfKWw zo_c_s6q7s#z;L7K(tgoMLKnt7LCD)9(eGix8`qnTjwV}qmU=DHGXJE85C#Y5NcLPz zFL_T6t+mzXnCWbAj~~)9qa*53J;AR$KO3z)+}CPuI=miU0kh8853(NZR}Px-9DWa? zp?lTBvHz_*zz>F36Y-P689Q-WUk#ab$E#|g6r+1YusQ52=I@Ehg4OZngb2mPFpexm z@N8iMa7_V=hkQTc>EU{Wf5x4IElvHyI2o`aOmeDt)s4r`m75=deredBqCesn9t&}y zKb%;sAcGb${1MAy=3XiAo6ZY?22nqd$mCh_tq)H%5BdOLtr|dqCh;c@7KTEvxJhbJ zl2TBSCG!`azlJ)AiXh8CVVPzz&t4WV=nI`z%3gwLX0mn-NQ(1w_+1NtR==uLGUEbV zLS_^Zfd2s8h_(MX=t{pAk83grkiNY%9tx?eZs0r9ElUt>V;s7W9ja8eoktH?6daJJ zRZ3l~NdHa@KX-f2-R=zCZ=S!qI@-EE91GzOs$>#5FAkwBOH$i-Suv@Kf^&gkqYrWT zf;^6+emJ>+QZw?ttfN>g;VbuDY$J$(nYBeA56f?jYz2192ZJUK`Y&z`eTM>_cG<(X zBV@Fr4v#m_?NJVz6YKF|qCa)|amfn1XI<5=M`o3r;b$&=dQIXX0N)fO&tq%2MVdWo z`$iz!5uhYoArCuKW_Z#O_Bu(B(D1R-_%8JMje`Dt{Rxkp9q6#6!%Ha(tDQK)ddR63 z?M1ob0AC`MG0ib9$*?Z4Di6677JZE&=r)n50~Q+M{lAXONF#J7T%slyhRI}weP21^ zXWmT^Xo>bbJDK!P=!fI(I8pjdios_l7wC=SlY!goN6R~=injqewF^iF4CFTIDhrb7 zMnv}KvbtQlyuFpqvck9J#eNR@3n+9K>a^U0{!X4(z1<(ft6yO#jy?`9XE0VY?r8#y zD~*S2o=@9cEiqmX-P$JcG+^6I$0EW;DYS}m1ld`sqACqbCQ~aL#aA@;%4u$sf=x*u z#Hon&_}KbrTj8N(XrVbLf6E*B5d> zY(Vu+0jpc1VGJz!3dK7IDGw|JZbFCPh~!x&L0`(%G0rj`?^D;UPtT*mX=)(l;nfLH zMlmalN?tO&VITv}UxSd_sl+06G>r$QAOh-yMMC3q@-f{=Em=TFpaxB5j7CQ;f%VkF z0n7VH5Yx8;F$RA5dPzKZMZ5=Vr?l%^Xs9=ZCX2Kz&L^aJsMt5KQ z=VIws7i1MTCW^~@^rOCPM{4)jCfT48Y6FE@RS1N6xEb<~C#e6V& zmx#R5BvDFmKROc9?;Ro`C3Of9GSjlE$d@yfNrQ+`@IuC78Kkyaza>IY-z4cUhrK{! z!3}NkE}MnGCzWjaboAni45Up+qD&)Zg|^(oUO>r@h-h-}p&7q}-50_FVF;PXJB1P* z#~Tjn;Wy9BuM)s5=ar`-20usv!S>V(so0)!Ov`x1S^5(lMWNV_`${kOzrBZ%^9n{p%fZe4kwXT}W7FHiulS;4P zTvSW8PW5kj$=6L0XPdeV){5Nwk{y48ej6)ZN8_>%MUTBgmg!IH(p1cgrZ&)FGAfeK zze&-zPO&PWy&`y8tIa&wx&y=IrVvAUpf~3Sz4I1UWtn)2yYZ!PI5j|&@gA+l_!-Z? zz&xq`de>>PBnutWnPy8{Bw8zF58KDo>m*xOa1S%P5XI%p#4H0&Ro5}Q%(X`H#nu>R zHnj20djXy1j52Ei>jbLrF|?~kj7G&9^mz~I7Fp;7<%L0UOa}n-+bbZt6C_a)OaMrx z5eJ&}@%Sc<^QvS8chewHGWIaIl$~yqySl-mW|seP)ZYSO8Hm&;9Y7=s1uE`nT1wA_ zMX18dl-$BmT53c!Wvy`_Xgpt+UnXS4y^$MD^s!@d2*K+_o13+CyV+Z=wB1J-d=>~} z9A_TO?uWnWU3?C_u^a8Iz4_w&h%MEd@2^mc*?bx>*nBut-JNIw<@G$(`IFr^I`tUX z!B=)8u8gZ;-(S?h;pyqMdgfneqKQ*4PpiSlZEH3Y%0y{Q^P9PV#b}TZ`btjw!;YK% z!wSJ`l83g@zp&UoM7Z@iV`#w25@tn5nSMf=f3J=XXKRSOt>|J)&-D)-r8i*{b#yDa zGAC%ZO^f2xJlE>S36O1>-NQ+eIxa34y3Y*a)t?k?|_=~FdY2jQ%yHPF0~e$W+yAE zy@W^kuV$A=S=|V!OH{SDfaSb=h?-n2F6R&%oZv}b%A%Yz8!csxOIL3-t9MsNv#2?D zj-k4+YHH0^7i&`TMwqFsXaI}c3Oa@uuk4WWNjOb#4^_ZKdhDFKBmFCD3LTm*D?^fbGVpQR#6$w4rxAbeL|`6@)s~5?qbo4`Ez{V+ot&tuUQPw*W=Nz^hGdu zc=olwV^HdITWG8M&|w4{ZlXQ1%_T8^tfZD+(x})hG)Y7a?FB5q5R?I)z0wLeuXZ$Z zRv9Dw78lptv66RA94565Lx8Y-7A{Ir9=@ppaJ(orUxddYC&&9GXvRJZbIpW;SMp@8 z^K05thitIpv*+0c`m?5YY-457#hK%@L(D3Fb7#0iT zqnc4@QERQ{cV%M@Bo_yPdX+7zU@qTb?jhp)z2U8mS?;w`yP^lsR9wYoaxFQx&lG-( zEZP+43tp;|xG(4lpxIjExV)?tJ{8?^95f%Wi7pjnr~hI%3}v*>-rYE z2apS>#engn9pSj`x56f_Ivi@HzBU;7g z$FP9#V|jL1e!6p#++W{s$g(4GL)RH85h~jP6P5gXXD_~lJ z;I7QeH>ZWU*S2hwsrfRsD`Het&AWc7ZSrB?LxJDp9 z6a}}{oo;rYtERM%-)15|7VHA4)H%2!AisTic4&!{C7`z2t@F_xMC3gWcm%0M z1{q|>uzKFL^I5c_ErL2@SG(=5rZN=!PJ5!Xr4`m~Y0ALqFuSh$b+k?&_)ge)&0rAP zZT;WV>E$KqoeElX@dzY))#M=J*#BI8_FP`D|!)npjNJ3Mwrc zeB%YOe>6#b=Em={-iliLVR|U@8|@PZXYC=&Fyu1vTQZk3X}udmk_u!B_1-l0xhwYr z=ZVNd9n|}=t|yI57x@&%t}AaIywLE@3R*qur@==z^au4?H*#kua4I@Rs7ztG)fb!B zMRVv4U1o<;J#vhSeR1~(lCB~WQpMws>jA;{XD8`%MAtof)DFIz0SsQ*4t|4tI0ASB zFyVDs4*{4TA(n&E?dt`H4}5`OD7^}BiQukAD6Jr2GKoIm{`Okv60AV)O<127z>-1)^pr*qu)Fg7E$2Y(p~x(ydhW zu{O8;{J`srODXxZLRYQWA$VRkFp|)?;m216eF=%yUt{Lh%HFyP zecDvC;-F%Fhr-d^)0jOr%keN^Pe@IJHT{mcM0E`*9DD+v>Y#;3z6a0j7#IL~(+e$Y zh?o&I(`iPaLm|6JgjA0h)V2Bw!KYPVAE9$0awW%X~0j_sqTJ=(yC}(IW5|esR+V#}f zm8v`;Y+3(e3);JXbTugsIUV7o%jO#5pskv31QF&tOV(wKyfKQ%b^ds(1?bI9qcxz% z&)znm^D*$pzb2A&9etBydg?>^wmEQ>?ZKe}p>vG-uINSOKE>W=jE zAkp(175yg&1RZpQJOXH?%_O3re=dRuR--8eAFF48rOq#wSEICB7fWYSxQgd*m^q6C zX0n~RHdXv?WfCF?7|uZm0D#sX37Pmm$|Uv%j!q_yKN`uj=C<8t3!3k$UVk}!3e$3bT_~W~ zwr;D7no)W7y4*VgNPy*HID*E41eeA2%6se0tuPU#;upn=SFQh0!kCjgpYKUr!_E6m zqW7nS(ujS`M391vLcO4s_JMcw!!mLKscvy53i8Ht*D6X^>vjs}&Ol&L z;xq5=GcPgj1ZCtL%lg4zM{pV<7HZB=lE^qu`{coH za?VF1)vZIGPxjODwJzLVY#kk%!`J0ucR-QlwiAy^ntT-5NLD;|2tKPWK%GS%69*iH zEE}T7IAd-e0sVLcUSCKjC)QNom!zIF{}9I#vs*x{VX?S@>JpN)D+yZ$2FUh1I7QnJ zbYG}*4m8anM(tz$5{@sd$Dz4Z-Is+`@BY6Q3I2a{q7st~72;&6)b{_T#R*}b8v)_) znQ`o)L4%JvsZKBvkz>FJKG+?#oV_YZ_o_W)CV%^x<0Co7LY_kLq2{~joZ(9P+btzQ z4r-+l7u$E+f*3Ci2z zOO}rtPAllr54)C%2Lq{f4Jv_8GsFBdvhOg%e1h3wR8+C8rHl3C7a8ZN<<*BPai4_ZT?pq*-G%twx|9%7oLsL;vX=a_~^+ zfQ9i=HWF6@6Jq5)>@h$psBJnAOlFu$$T-!2tI3}AD4lq&ayn1!$RBLir{H#hK(qp! zpoNcCH-CKd&Y9DL9dmZL8V26MVk&wSgGyP9R<{`jVT*Iy#$nG_^Ce$RJg*A7E+!|u z*Kbx`!6>*kHFPSm(4F^j(%2<;%dj7e6z0xRTmy)6RnekY!l0Y2szUUXS>d8?O6u9b z#NMOU1{cp1upG@IpD0-*qj<1n`y;vjtyZDf9hPV zNSYOL0E?Treou>!K+lw|6R9!bA>)}UP`6)*%bs#l(R|axTJy3+Z8IeE%4hJK3x<6I zaI^OiTKz(Q09Kv55_gL0gnn&s_1Aj{urpp@^n zs)`U_2ZTad)!&9)Zk|OugimB|j9hPWB+jdeuGMGW;cYD_ExGKI?Fo-KL%wwC9ZI{& zu}-!5s24&8gqXKCx1xtY=QTU+j=?VGD5h$lC6uWK)*s1}Yc$wHMeT155Pgrm@ zX?!qu^>BHxeJ{4v4~V?$H2Lu4^L*HJ5683cC>c5UQPsxDAI58ME+oZhw@#=M?Zii@ zn``WfQex}A87l>PCIAZVwbs+;(WG~gCUFs47%!@vTI_@l6aD2V2{qch7xldo0zqiU z(raD~V*36FxEG=SEwXx)aQ~yPf4So)YjQ@Hi%ILcDVrRU;lWSop@+$GsM`cH!J|;l z6xDVZ>lRyStG?~4_PQ1c!u4zr+ATY0HigHi>KRT8z$#2~!L*RM9tuy~TYqmh_p!4) znYA$iY0=RttkX$Q9c1Yq6jL2n%VSz)$Ymu@!_DPyO0p`>tpbq&8$7l>IgeB1%dabg zRi zM{?|!R`*|EI%lW;X}ld3Yp<>vzo;S&OtUrqp6)@3hi={jZw{REdrBA*v3AFnHFYwH z6p?}s4!(;(I5kAT#2K8UgK-(-u-;amwz2>1e#O+*)W~D*yaLs&Vy+h60nua?e2{bB z3R-P5!iVF)N*J@SZz=lqisTG)YM}*p>4xX^j5})II*YRPj%l!O#Lg-v(f-6D7jtcY zaATjfpER*5EC#_tGSo#YHJUeUk^7iIGjm&#?ap8y!Pq1tD<=91>)obqE5M;xbN(wo z-?j)eM$?MRVNG}KYBCnmq2pe#3+96(Ltd1TSG)tl(jN zbyACObl&PQzdOTthgs{DW-l&(Q^sVSES^+E2`)_T*^jkXqj|FWcXFJ<$}B*HM8Yrz z9+tCez_{97R(+oHGZPvbi?X>Ymyg9+qBtJDU|8j^6 zW6-d?_oBlkw;vwM8?P@uxWdTMob-b(cy&JfE+5X^`1JzI)%;nNM9i#Z_B2&$Boa|1 zDv6M2%xfgyd(xUOW7J$E#3z8B@Qc^@H_;=$1N8R-kC2oi@$>h;1wSzq-Y4k)L};b{ zn9RxlV-4iw?C4_T?BZzhk4bSlk=N=cTr0%wJ7Q1+$6n?P!#p3K5=1B^s79!Mwk3k7 zeW5+hS5HGOf`6gkI2>*LzB|_W;11@gxGoMleD6=yAEY7)MUAOERZOfrB>y=a+S9Gx zu3tPZizT#?`^eF#nxM??GI)l2R1YS#ozK#;K-?m|Z5qAREuG6A!pEyoD@zUAkT9wn zPz!&K+}qjW&W%Xp5tvBmiev)_BuHD)R;bXW<}7?G>@oeYG5_?X31f^9MKa|M9$?a> zKRGeODA}L{rP^aSCy=(9uv4N$C&!l@x;x?raZ-A$w zc|mkx`tK?u4nUcNSE3N-hUJHoN4-XxR;?N{wvTt0dx6uCJ#&yy(9DGCviSOn$1{ra;~D+WOxxMc%EZ>f^S^4uTx)n_lMveI_I^qly8e-5&BFb>@%#e=~+{0}&fA~_y ziv$| zlc&-5y2{*ditF2lW|YBx#z-d!DS`E`5!+@%O-knOy+;3rE)4p89U_H8Z)KIeK_GXdkn={0HC4=S5qyrsIw9VKn#xJkZz`Z*6$M=Vl zzPFw1%2n;{OM9=RcIVl!;D?KM5(~bx&gA{UD)cixcRxV>Em~^9ZKapYBUyr)UCZ1zNXk?sYfUEWdEy9s3yG7A$|rF z$3XULBNygb!@UFAzh3` zkNkfSc8<-0Kx=b7wr$(CZQHhO+qP}nwr$(?oXOluzT{RV@4wi)pVhsH>>3mtfV3MT z7Gn;RazaHBGR2F5PjLTABh6-+@{ z51l(d>;ba!yn_$R3)wx*_^Z#3z$B}@R0%d#s+rhX zJ&TbgzaSyyRgaT(n11F(S5CmIAMs3Mjmmz(c;QS<^@8f|s;d0zz6`-+fZx4S0uc@1 zSimY0aK20@mt^Bi^A<5Zr?lF8iQn1@VM_5xINaqbw5Ve&;F&YDyB@yNHa8Kiag=O` z(|Y<-e@kueTKR)DHI~BOTGeffekvqO!Nyf0+aYm8CS4^2gHoy#b`aw1&90T7OuPak zVNHy>S3_9SzhmAQ$&~y4d7?k{%yG~9ZIGEa-EnypSx-ROu4w=>Xg1gmsd))0H!3iO zSPTE%-VDf&JXwP~Uy-1QmT)6cG~u^&iPfn_W9qJwLzDKLmjZ9eDUIEAnlj0}vvn0aG1-wd}{%N9xk3)_U()&iVfgh%5Rd`Coj5#`|u5yiSgRe;3s z-dh4k0zGvi4c5h>XmuXA^LC!Gs=87UrR^cSB|v99pysrOaY%h>$*N%`EF9I7Y>eqx z;>5e-R9gEaLLrxpzEUEp=?8x#Wco68^QtD=xY(9pWIE=%lrV&KE|>uY2WN{P6)4cO zFPn9`*-#xGN?;02AiDuIDob_nU^if~#5@MO;h(h=rcYWU8G6_eA&oS*jz`%CwuVMmpc>! zdqzO@tbIMuZE-M_PkLOHb`q^u>_@}MW=P!jcd#7C3;uQjR))5}V;XI3FC0;Y91~L+ zRm^i7A_CD7b=NBd7_!!YQ|avpn>{{Il=*cv)nn>zjH2~(@7Z@O7u|Ou6tAmdgO-t+88`D}Mq}0|f7T_2D9riA17v@i*MBP1h z_%@g}aX90^@tlo$72>nwDBIR!2=cp*Yb!BKO~z6!Np4jZvB_wZv1Luv{&wmrP5tp6 zO`M(V&sf@t(v19M`)MRsQ9--OKkpC4Ldt1A1NZy%HoMvV%uKJa!AM%_*dqu$s#V#U zcm?p&rj#gkw;hHlW@ScU@4}h==u3@CW0_ebu-Sk8u)?-7u;uHq4$iwx)k%J~;N&Rz z^I+|mk<8v=c_9k?{Sm2d21jN6@!iMTK)N+EIbJt9DG$_s(UFp!Mg?2;=;K#<6;-(| zc6a{E$pCn~P$rD6IGT&n?b6(C5}2E)F7v<-9*0l!_c@s1r6TD{FX9~i?ayC~%4nF|dQLl>}`jTKPe(uJwFW zHnAY1%?>Jq{0DLYq3B zmhp@$u{Isc=8Hs6(?@_1e-+$pO>p3@(V3=D1w#zMV9eYC&Pid8VacPlaQQg zUS=QoM2F#tI6YP)x6aaHN1nMfxsQyBrfP<*^yxiGj8F){p?Z_#ks5CoLnkv$v zaZ+ZwJdosEGS5H(>h`m zqK4NxCZ{;RPX1gLGNe~9H64hsOJaQH=eg#eOg1bdi0LR^HGDr}Mgb0b>EkuajKrEB zn$C&Z3=RXp8ywG|sg7fe# z$OFu2ymWJ6u(66RXNnooHUijP{go9_PLYoxArJ@=3n!Bdt~+wqX2ehF0vN26SjggO zlZ?+pQ>K}|k3;gw28fmVq{C@H4ouKUeP9a4009Q@qQf8!<_R}LY4%fbnl^$h4sR@1 z447z8OpWUL&LY!zgDoJxUH>SG<{FZfumdi-*jA{8GD847 zuSk!JVla0cLNlMfVDNxMnLI1)kPKlWkXm=85ibQe3RYD(+i;_VoyoVzuc!lOaIPa_ zp^@CCdDAo17jGd!&xfp|il&LbhgQJ;Lh3gRIWe>gxxmC6^gMHhnD4{!`^kTQUhSDI zB0eseh{^qyb)2&x0-h_m#;>j&uOh)H}PiVqLiwnef-~m zOnBd2>L>2G$!yMnlnVDFNQ69s$t|C*%b(+gv)VrHSh@@vip)Jr|mdH6y>*?xO=tsfN{k8*t>!d1Zpvu7pARNL^nBgusY0xyl9 zxo?Bas$HX=RB)UuZk=mFar-)|G(KDbKYTyiZnMmsZ|%|xe72@QCtn294Q+yrHx<5$ zN%%p_=I-=TyuUPsng+?~zRIO0>oqS!H2UWF)N!7clMVzfn=OayG?$4f=lB7~680^S zZTTjAx>{H^kaJmXv8Mv%CXd(KMsk}HvcQf{Go2XF<4EzJ zK6^5m8{%=u)-=5fC3d<0k!yy;Q_z%+nY-6Lb{#T(z*v;T{gBj+^9aG_9T>L)=Y)?K zQn(B+n8+%42gh9gF0}Mw0Pir7xf3g^4DWq8Ib=cd(xxUUN2PyUPNohfkx$-o3X+s! zC><>jQ2R1^#gk>LRMB_OEXHft&Y=__O9GVKx_M3FW zxi`jJ&Kj!4f40{w!3FtKs5qqnM6yZLs}+5pAk3a;7K!|XC?#s8!#h2JA9#)Lff$nR z&$^5%({s5RF|?LPt0FeR>>QGbKeMu2C2V?h34 ztHx7-<&o0Ljx{yW2v@zx{ri+&=rtv;EGieR?)>!Ro1Ne16n~d>i{cXePB8{%R$lOQ zxiXS+GA^Z0hflp%MX*`|W^B3F@E#>-T$}@`MtmQ!=iBV_A_@Y9&eZ4L=xh093b4$8 zA~50yZP*MEVK8>Ape3jwmz=I7>4C;bKf2X#+BU1_l`D6j@IakG6rFifu{SC+up5^_ zAbW?psS&Yja{xKpx4BW^d>t*q`&KHhv)u#h{v>=H!cXCJqdU6*t?`5~K6=86Zt$|5 z;G)5cZM{<+@)$;+W?G_W;oB1=S-q~WTT+VP=A1!A)|OzpnQ0DTa5h(klkzp|-W$Ns z5eJCIHSb_86z>c9fsQ3CME_Vc`r=-(MqPYQC`tWKhyHPnA#u zP*SvwXFAlrnnj4RUB>r$vV=@r6MrIVy(2D100^Vb@n;YSz%sQ4KzMpX=1+5Z?6rKu zy@~4ALRMAKoQ#v?orqUqk>hc*AHBnvIif3rauhkKx5 ziaL?5AqH`F;1yd8+Vn9TTi6~pvUCM<9fQ-*?vTbGp!OY{DUiH3cY?||ySco30pbQO zF5(FXi#}1NjNkvI#X&Z1-2M-6G4K!VruhG5bxx*^u9i-wwx)J2|9R<#s!9KQ=_2&q z*Jn(HO9T*sfI^{cbJbyg8w|w4a?8Sp5iU?qO^XqsCOvr>fAqVDtpqPkRlrq~zRt4W z6UHA*t6@-xjsotBPE%Ekgh>xWTbK^AZqR~HyV?q*+Bt4D!1mGc(uCIQU>BZTLHNz_ zl?<318u;(!y=K7Jak^)xqvy}@@8`Kgm2NLfY0k%@S?o~|*DQITP^X5)R%J(N!-2NT zsE}#bitJL@7Otw2;x&a?4y%`{v2OShpAft3%b)UzLdVD+RyA4>Rx7cL)bVgk=a$?Z z+NE0m_M&F3>ulkft+LaN#v^*(p4h!FHJv-xrrss6F7Gp-@!rt4%MhkZt=9;J-v#UA+IS+)JhJvWYZQnCar472@5EVzrD|v)>EQ$ClA6RhtY$Aq=?kQ*HpaSH#%AI zIqoNV0t10x&LjGCa&>DBlam>Wx7snNX~&gT4pw0O0I# zJ?ZPp5&i0?vv9ms9dJC%7Y(nOESSP?@6o*AgaW{~T2RT5h6T!LoJVqi6;}^N_(Vw@ zo{QXT8M=)nP;}GoxJT2Oa?Cdswj5D#C(jc_hP)VHl(JdfJK zq$0=z`Q!3a*7z5@v|_tB?ztCR*7gili_lSRJiJU95VSjKTgnz`fM*n34PlgN_$Jys zbAmivuJ=*_)2k&cosp@Yd0vN5VI(a02oCxPZ^$AJRWnob@bxI7oGfE49bem;%fm}) z*j69o*IL137mPBn``cuSHD>JfBtc-xVbdA@0y8=vHn9G;pesL{AM2FV0z@gSES9fW z6iGh)wDW{J2gk1y;h}5zb^Pp6o7|L#kr=WZS*!(H&JOI*sbwaY&e|RLg4mzE<>wF8 z3-ip%hys^*Myp1Gu2F;NGR3z#BqlabjVB^_lO$6<{F+IPWd}_I*1g`DP5rx~lf`(( zY5S%{iR|Dc(DKp<6zvzVp4UzfsaI5;23(ij5XxW@zH|{Yu-`Sz!@_!ksPnc}WO)mG z8*)wwo_&=4d0#SJ@&0h+!~7 zpAEPB!Li~oi_AQimjxU+DKF6p^ZashH;#9xg z)NV9hPgNrnbP_QH1%d&HTGQIw_xqE<4(2nlHzKKqZv%<*BJ=L@{_+wXzPMf85=FNx zBl7UqCe=hSO-@uVQnO{syeQ_18gaEqM@7+5(w|40O!?#98*eRsFS1E0`sd()!3!t< zt(YW#vBQ4fryOi_5u5Ph@SQ%t-{bWxG>5U=+*EM|1}9A+%2ct`RdElfXSyh+Y}6<- zQcHwck=#;k6g*e%oFL9nQB^n6s}%!>d$vqRxlnDpsj7c6-jD)dx1oY*QCk72NmMycO?)-6LfHJaArqLShCQHa;$2) zpMMFS%p>IHP~3ovG`uiR2huH5&s1|ylneJ(arTb`Kb(svALeo^#D%2{mYLLx4J?;w zDyzV@UH3^hrFtl)i&7dAyC44YlTu2Zd`X=sDy2BSR~k* zP+|_7n=ZmkS$|@>7g=v$=NUljTbAnBW;q}^8Q$+)mO-+l-E{EC=L>=$d^_CJc69c zSJU>EB-8YwDKdUa%3No(RD-GNqE_3W$%X)*h*6AI83w!!gpSSyxeZ)WRo2;*EcNWg z{4*+2FTytAQf|X`+;n3+s&}UNsQ|b&l*2@*g%NexT5^bybP}R9(MkXSMH|#oLzaa;Jl;`$GX?8F?3CS^@l=mtEN7HEd)(MzY&cYnvrW!u#~uX@G~MK zB4r?U+Q9?!YjO ztYNT{8m6-XKBp{T*UlU)c&H7Brn-a5wJL8SATlhOb=E8CL*cbafo;n3S?3=kbQ4Gh z!y20$gc1hQJ+gHQs7^h&D+Ll^(KB-T+>|$>Dk#32D4dP#@tXRj%T%y1qn1#Ywu-{d ztS=oxd2(ka&hr$`SxpYh8X|8{DUN*8id_ImQ%MbQGJ905p2{}Y$8%EL1Ls6wNTvZ; zRZ&PQI$FcTfo_|Q70NB}5IBiFDDknyzo^u%IENI8Gd7IO3%{V?}o!`_USw&3`Ka{J6q7MS4 z!M_NoGTpNmY(@atdIka!JVH10D>v3yP7>fdEP{3hWV`(f@e=rWGtn?>K=pMZ48bS|uSI zE@0FY<{+YL0=UHi()6ewc(O>9re_{NG>E%*782piu^;H``iPDh0VZn%Dd`P}Taq_8 zgaB=ixfK|B(KE(_Ln!c-DHMLr2jyfuo@EV|20t$+{q=Z!7QV;wRA`{DLzMY>Q1(6< z&-A^=S+w$2W>s;PBneINXFEo=qes}}4p+jX3qPp=wf02-A-PX@SDKX3o!LAL%g;<3 zwVozv@9W)JmVx4Y{B$I&iCREq@raGa_>)fh)~)O94s|{Qw3zLdF#Yd>i| zV5Imh2|I+-dQE6={ZS#};!Wlv9ah85TxtPcRTS@d%d+chcF9{yRT&4gK$2H5c?QMt zF>HN}u5K<$D`sw}>d!v%x-dX@-IoT<7lBVTj9SVS0fd8Ues*izCDC|CUYAAC9$^{y z;*LD0d-~jL!qnSw%iJMI`G#I#W-u?M7tcR-`7{nD0?f7$l7l{g?{WyL~ZOk zm%ZErt=}&)j58;ZZM^NV)}rA2(uN8fsTT1ZuGaV>1c)6;+A+Kms16GH6Hw37<}pl2 zK^kfnE-E`&Ol;~CeDsEHh$(5JUpx39K5|h|r$+j(Uceokjryrbiwbv>8_|!HO##5$ z@w=Zgu3pQt6V12(av2ui?u~f7m}WNQ(LDvX^AX4GkQJf>3omRJ6#jtJ^V}WR`2cw0 zPXGIId(BQi5EaU=_xred&EME8?eIAW`1j{M>Q~;s+_99Ay)9)J{ahRI&a7v^)Zb+;1fZUOh_4w$R&`fOrMzqJSe~@F7I_Y zAf0ycrOxMnke%_MdgnEu@cjzeXrhXK0g`1Dlikbx`#33QxgA)UNX;0x3FFg6(bwX; zjTXeEf$6%eF|tUa_}-Cs9jn^^Y{*11?>ZBNrA|ayuC5DWtr8UTy5!+b+j3x5@YaMJ zrRg`q&xh;k6;|nCWo8yNo3Bg(0KD4JP{)iei@*G*eOnQ$a5*~hJ#AO7-?w3BXF$4V7xY^jR7DvUCWAm_}kU0*lYp`*RAOrh`BgI=^i4BUAi<99LB!9t0d2osk zZhr8G187fVKO59Ksq_=(hWSOigARM+{$Kr^IB9s5U<4Sa*S^P4`R%)?V=NjIY*^4N zO4ZW4bJ4$$x17n%S(*{l$x&_K5{yu1sZ~+EicsMJ=+4)^W>4sQ%8ejWJ#Kg3)X4$_(~)1 z)8SgeUjcg-%OF&;uMzX$#gAfzxx!dCn|F6?YaO{il;cF=F{$_<^Ds z`UaTzhC}kb&2h6z+Dm15t=Ln5V@ZsqNXox*3l5rGku)8;9eD`X8MsUHZl3fb_}K!W zCAH6w%j5F7A4szwUEVe!_3LFUVqE=6W0+T-V1R5to1+e)Mt26fHpkdD{%m z*ic{ffAnp6{EZ6d#%A&i{1oT42R5GK%yK1vN%06cUuVXZ86$%|?wy#;5WSG?^_mYqvEe=sgA>3&KkGQB_FjXAk@hARlyA zh{unBIrWmP!$~&gZ`h=l{n{&Wm@Htxkbpf>TFR(yh85X~_BAQRo+g&ntDlWLA1jyZ zuCSc7At*D(=*bs>X5%}IS&^~2(i$3N0RL0r}``R?Ek_l+}?iR}rgW2815P^cRaqXEUb zp+Y!!>KYdW+Yu0|JS!gcU`o0IxxuN|a)!uJCJ_|rIP7iJCQQT%Hm$6V(Xu$&B0u%Q z2y1k(WJreI8ANr>q|WYe;lwjOLVLZIRux;jdSv)3AgWwNk3V3zD30ZRloP$q9n;iY z)zO1ST%Bmw)Gti%265qdTVOiueI`+Lj;!BgU%ebaL7n#;Dq0)Nn=?@fgc8liDkwux z25r^32`6x#)qfDueReDn9mNeEm-er$A~0PJ%HiJ_O- ztmmHUV((e7x6)vyH8ElB-3p&ma{qlL^kR&+DseM;OoO_1YJ^Ig2Bq%^^Z~e z1ku*(t|6XJ$S`~Ec%kv+%McsNYtW1!Pd zA?B7j&B`+wV&Vg3xX*Vs!@k#HZAheIA1vx)yAXV z`J8$!AvyaLw>rZj+Gob-bU94WWFCvXL*11wSpLpxh z%KPG%ZpvzgN&_D5WuO=P_SKX)&b^4`BK?!++0{H;Nftw6~Ff6h&|(wrLX9KZVYH=(X$zKu$}~Da&e?408qu8<~nk1^NEq>tclkkof?Xdi?PT~Us9PqhfEROFrdKbZ&PHZ%xVsj zA{fX3F&7ZVt7-~>k^4=%CYo)^14m-dHlWMDsfnAKd|?wwmzD9fd^0qP>~U69(yVK; zg0fbdJ;>|RrNS0G0lRAS8dht9!cZNm-Gcs`|{S-LjW4DBy~j;5KKj{%j19kCLn5=o*2+Ly{ z+)`8ht*;6qf&Eo%S=dxB2(3ZNp|zR_LiDo6=QRm5ight#Nh%JPGtQx^iNJd{$7VtF z>5)XujQ{J`bsX|$13RzoaO9}1Zl{$^%qAmB86MBmxUX~8m z=z&>_DgU>ji@SpEO>w_%GHU&2USXeW{nhzi6@#_u4?SaAVY==XVAOi?pDMhqL}gW; z+T-$_m;mlqg$+%*VKog0eE0JKu(nrdFldqb{SC=4lOb#|dTsY!o%4N)~>G#g4h8@xdfm z(bn7L#OS&i~rm6gm#~ngd7>YO#HI;?vh+D z&X_q^)CGMV{6uxL`~s$xY|}~#YqnDvt)<;8)1Po-H=2H&s^GwRoU!^-;w*XE=0#4O zuHTYZrZ?5g)%`m4%2HJqO-WZFQbu9k1YaB;8a{$Y&a#h!9Yjk~T_8)ueuOvPG6aa+ z*-Vogd>Xw3Je`->MbH|dwn~M*J&KJ_VWp>c@h~nhn`5@CU z@cJVS^iD18S%)TBS?_ytf*D_GfRSo5*XZYhH~uJU&$9+VZj@z^i-6`-kwe<2l|sGY zUFD2TGwQ_P_jKYiLt|deeS`^nJNvii_xXK)(NZi9jnSBtA~{`0MR?RMZ&Nr^WUyG% z$TN-%8tsIKh#27a8=j+-o(B5t2ZE2QtAl-G9E#!B98*f3i`O5kM*TvZMW5fRYHVm< zKF-VhU0)D_*uxD!#|F!G#)&swV1eC%qB}`HBq{(~BrPkJlrT^F*sQVI744D?g_>eS1ME51EoA;TW6F%BUQTOo+X;f+k}~&rq(6 z!1ayL^MW}^&ztfEL}M=Fp>p6nDV29&>31DEyBZ4G_PopCYw_{%mFr-M=p5B+McbyJ z&nmA?te!4>{JO!*)L{*dBQzFu)wWFMEh90UdUn$sMj;o64LGJPmO{!+w*$FE6bSti z4Y_vPBl+SE4-R&C8y~-b^uBR?#IZ>y%IAzt*{#thamtuvRp#0bunm5Bjd}AC8PUSm zjY756UbPy^+`~HW%(M6j!9OdL{%Sm;+L_u4hOxGA)=N(j9pQ`N})fz={dd5Mr-lbk9h%>{iJ!{kd~bt$zA#QM1RaV zI(Ktk7FC{-ilGbajPcqyG=550ZD>q@yD`_cs%kxVwff7ok4{SjcC~I9%jglX8Ma$( z&||fAE|=_i>b>M9GxJ0@lwZxnQ7N;V>7Mdrp)DySVDJVSx?C!dRc|fFZ5rY|LWzkU z&SLd;W&}1b7xI^R(0_{ZH`);2bA7Gb^c>LaXlXCfPo?ac`a4oJ#x!=Re~MO=q30V3 z=&vC5_Ih~jdyY(#0^r7eH6$mY?fjg*XInXYN|n9dw)k zK@S+y>Kw*Evhxfb3w~}sP%JolZFPzGOWbRrpCsL>sa_=n8yQQL@Vu6Q6meI)0XtWD zZuA94$&v7T3w|g;zQ`;q*hL3oS@@x(1(B%qi`_26E!3x!L07K;Ze3#@a0tMD{bWpIyN$-=m;_&GW=u|)u=K-nw5AMH4T*7B zw!-smY%Ka6uQAd=^&Q_af)0P*n2#F+MGl47(jzLSK1p)JbVkCxDJ1Xls$O960?}?h zilrb9b{+GL$dZ*v))OTdmh$J9vQzr9#-)HJFZWzQ-rlmX2fFFfgzF{}5K5x8*_Hxm z1#ds^yS|2BFxs9yRD#-1s5s83R`tT<%(}KUUDDWO@QZUD3YCoMZCLyb%xp%h?ftta zO|{g@igC&wI6$m-Of#0j5g(2_e?NkD8Xej=*(I|RU{!?Or!eGjuIuW--*NK)Xg{4U z0g8HPog`!44>a);dqc5xQeR2NsO$SSN8`$lovumv`XMVC9t4i@u{4`@IMXt@p9}BO z;_CPWzTlj+k?VkT{bk`f)Ng`T*j6{3xIu8XO;*CZgF&H<$~+-*c5gE{>@1D48-U5H z<_oqONxz4BQ3}&;XfEv2!Yo_(1@s1S8bZ0&b9nwjW2j$DtlSFyqipGowB!osE{^)r zP}j%Ccyo^eIkF|g#Ls9TJ?L;3Vyt(p>JtmwT(TXs{JpV`?BuK#D8jN0deqZ1KmiZj zhGcJ|43L$oV-)*>(_^dB=v=yQLk5)W5QQl&lpQbKK>#~b!^DUFI z1VBQrD?qGH#xS=3(j;+_P1a*g)!(O6X-{eqy=1{I$(B22yyp z!9&Kei05nKmeE~XIs>Qt7uYS`ap5}^dkUn)Kw3GoKOKYv)-r(!cbNI(FERJ-i3H!Z z{k<(RNNee=EcJ^*cXl6*ZG8ihgBa~U;kZU9*JQXiTR>wf18_j|qW#ZMW;8DLC*1p7 zA>|Id5U2j$sBHNXnV&!CDm&O)_f|bS%pl?jb9Rb>^LMd-bX=a4>44PXqGI>b0dC6i zm;Adz%3!AM!!HvXMSbqNWBeLbRdf_H`MZ4MPI@h`{LT8?wTID46uW0VCy7bncG`EY zRf3Uo2Z@lvbCO_qSfP(Sx%G-xoV5c>U`NSz#Sc4;aXcxD>+duFB5p z&!t&2hwA5?{x1>9)>#-PS0s6hB4qRwjrBx1WSlX(DHpJ|(r?3O0#;3(G^wAa+6I8| z`W|ex=q@a{_J5Zp3;TrD=gG}IYHM%=?)F)$@z%$e<^s_0n>2H;{#y^9zjoTXR7woL zH@b7-3h}l2rwJA;qs^%3e{7@e$ev{NXRZ8aZThriKw5cSf5i1*^82!w3>p;m!_ktV zgrVN3T%__H(``T1)^yDCJ2&xObPqRavz7wsiPk#Z%({awJjj@}bF??=@pYrO2~#nu z(&umMa-9e=i2kI}m7oizka8mKs(H1cZLsuff3+WF=|)+8bC9~r>$L&h|F z^;DgR>xbG+SWo@Y(-2>hi83yzI$x(M^1`>U9W%*9Z=w-WXU;HKK4F`RI?4jqSjbu= znyxrWf{DcTZ)B%z!I88*5=QF?;kwZj<^OGGb^t0?&dM_86EqFT&jQ`IH{RHg{WXAWfi zZdTe{97RVJ$2e0FYuZ>HAklRVd1rCexC0?0@V2W4j4Pt9>F*r13)64b>wWYLxYJ&x z9Qy9&yzBafGwoTFnzQ$v>-rf+XWQJ_oJVB?4-DAFo42xI5HCiX`c^NM&&1X?=H?6} z)h2IG!5UQUm~vPBvzT+uS2B@rY(c3jccx`m``iv?$^y>&4`DG~H|-eh{!vqS(-zz` zM^7nKMHzqOm(|Q{d+PHr-IPmUUr)K!mTGgI0z-QbezR6d;%xRUG-CP#dC>`%tAR_8 z5*6fQX{{@c!TG_bid+9@&y}oSIefDHS&N(W{hSVc%~uK(a|XR`cN_cp`!T^g_v@C) z*w3whMk6!KFHP^Z4K{Agv`GbBm2axIUY}oOB#|8INP@HeV#-LUiIpSO_NAg!C`oi) z&ljCE8psBQn>I$EZC(fP zg4KfNA6O(=+XpRlCgRjZMn4;5ZrUXsWZdkk*uII)Yxz+rPEh?x9aSHQN!toK`GX(0m=Agx|xzIC8d;V z2%wXcN|d?Fw3IO&CDP`dR-gGhs5MPyeW#Jq)rg@0M6_GhM$bXfp*c&%@m=2gpe%dZ zcMsF5wa?i3B$U_lbRA&wnLRq5mX^!jm3|F1llqe+^{x9Zb(ch}6H?h%Wzg(5im!cz zZ7_s8D{VVt-iU9!pBP9OcyU72?6S2$lM|pc9ssFn^1KBk+ zw;6SSGiiL(GoIvx)>X0qt?B!XLMpMAQARDT2Mh8+>F0iZ_qh;L^2}KOt?o{zTO-ZO z%j>J#TWr&yJYA@%Q-U-2z9dRH?VnYTL|!A6w8!9-)SQ$aE$P(xtFb@E{NYtG5O!0( zCxK|*3)_Z#2djxg8sg@5(i<4L33=}FY<$=qJYOE}B?NPq`l&~yBQPk4-kEZ$M+30W zQ?Eg@Pou2ogyztcMhWxH%cyZC`d>Ic8ahEYGjEEO&R}5ZH*M8L^Mskcw5Qpb-prF7 z9lh5xfiANvJIbOnuq84Uo!U1J_v;9Sf{*7LC)elm;qVb3jLcf1Lu>f2e09(G(SzRX zB%zKU_2ZtG>+t6p0Rux``R*r(v`Qa8bVWgY8tDV0{AwNYZ|%{hOgv|8n%PG>Ou@=2 z%~)e_9uXzWibynvb}3P{U6_OJWJ;Yk?+J#4i;)Jp-oq2|?zOlg8odV!VBN8*XQsJU z6@U!{C$gsJ-Z#qpmADA%9zK)T2KSGT*TeB+Jop1U*XQBI`0~Z`Z+UPsuhDEKO*COJ z|2e};8X5wPWs?DXeGs8B_RT%c2EAFK7hg4?*|dQL7$5^&qq|6=IcrJ7d;g~QK7j9i z-TJlYaqh+Kq2);e)JcF)z>R&$M{e^va@+IGrU!U{xg`xLcP#xIN*_VNz3<9$kaS)u z(r$*XzEfV%RMG~s=GcJvX7l+)N8#TQcPT=K$_1iB*0a!-mP$TT-J5ds2zfW-%N z3|x^1)y4J(?4JS7U}sjoL+1fPCPgtt{U;YWAvTUT3`yjM3+mu-Ke&wx*zX*3okv*y z2tt=ifTkRM!z}$^JUFa}0Q`f4`>b;_H2|h`+6h<(C!DmPXkfd5nh#X14iGt@WzQ*e z0tpv!Mhb6Fup>V6bt2hKlsvsrOH^QYOz`yqsm24iqbDXh!;}-!4Wle6GO5YFu*igc zCk-9{hg4dQ3Iiq(RRjh<#Cim3AmAx!hsFZH3oYFehH#P9$&8#)%`iVQv+0i@;#-61 zekmpThvEJqSpd{i(yKg>7t*hKXrdWG%QV!cdt9NZEh8j~In+RtH1+EYF z0*lFLCsWoXOnD5+<$yS?#RE-M6I);HaKtiy7%~9u@Ii$?7%XfuoNQzM&1h-CA_Hwt zm+XAhkgF{7%UJ0d%`iiy0K2MtCGx~!|B3*t8O*omJv<`_QCxG$ddXFN>}B^;wnY_+ z{NN>G=?4~K%!)HqicNo#{}`7>^ke|{X$0Yg;PJUw{3$Ed>QURe9!q%DcKB#TTY9E; z3vq78<8;uh_YtDyz-Sb6fVhkULF@+ZCgFp((_Yz)wY%NAX*C?tJ(vq%JUgV+LyqHq zV|SxQ0`++Zdw7vxJ{Ed{&=JP}t>Q@Ci)(5%+Z&--wOfjOO z*hsWzRYVQWmGT^L7d%PLq8dQ;I!VUPIq<~=#MzUMQg4b=g@6PW>I}jB9;8qZRw&mE z4VpMGVmvTB(7+k>R#1;2Vh|^f31nUthp@HOe<*;-Ik6_gx(`o?Ykw_reQmEo1yJPQW)) z8Z78zUUHv7A!u?$p%>iIc~jlO(yc72hU?zaDq3*L>EZ9g%{xhN$&q1!^&p73AeE^F7E?ngE-;fT7_+SRjm0waS!$F`GD- z%B2+xhD0Hnlm>s@+?)Fp=(bc)p5al4&M^l|UzO<#pr7TEnIQsRW$8ZfK<(xC_&&W4 z)M^}pGz7!>>yxIQ;H7wll1^H}MeU#sXh8AIBnKPtrjC7Mo+!u%2I@0s+NPNl&3Z9t z=B&+$K;7b1P)~M5K^PLnv$*jg3#kXZ*%08v5Nbi7B9P&vkexHNqA;wNcB*v8>+c6B zU0X(I;01T8`Qt;GHXoXTnm#{%tbPpJ6;@thUwoT>oV%Oe?Z4S59Y&dGtB6FWE*WsO zq3B^ve&m3Pw(RcL6<2!ihy?J;0zBT0wFn|0EG|TsT}%z9%mXlwXA@!L*tFUT%y+ooUIN&hL7FW}bK8U#K;$`W|ZI(M1SeGv(Dz7{&!K>{8^c?CC zBEx0td@6hKdV|S_b=tETFA3O%IH1zpE+|A*%r(HPF918$mtqvHF28ny%9JsfT%rDJ z)fub}&HNH-8<3e1T|+1+<{CSBRoX^vg}}IYn?-<%Owns~3(iM-r1i$gcSd0*7&MI3 zl_q=Qg)9pfY-}zAGi@m)1Yjn;uGY(v0W>1c{R zgjK@_Wqe%|_LN<1*&x;`FUw`n#2IH9M6cN^pu-8CR%B>&OcJy*SXAefoR%UWUXA3q z4}6t#w%O5XoRyak+}U!Z2)V?UIuzT&xOH-%#UPBjFyGg`x?dMHk#|JJT`@d=|6Q{lN_9@Ns8KDT~>Kx8|sskpKF%w_ZX_w zzIn!~qO&+nj~LLBTlha(@IO{mAQE+TjqxymXX~XX3DalV7NBw?6v(7#b-Z zmTl$=bn3_I#iviF9<<~rG9e6TqoRo8lt!B{ucr_TCXSSK9fBp9$u&xhyDP-`Z-JAn zZx9d)8>#mJIDAQ~K6G|nR+(C_E||=mv{hZND;C-79br|?!i^I(h3&gT`R4{-s(6*gmcjl|C@jvq)f z|5st>0AAP9{r?l&Y8uj0h>mixuSR<8}Gt`c@7a zUqbdRXGocWaykaovWur{WrC|-S=jGu3t^tIkTrzRhA1Wt(r@)NoIz&U}&% zjSU6&I<1?w7DTDKp+ zxbdCFq?B3Mv>JEjv$xH^QS4(BoeppVbmkRb;Z=v=4Ep_;m?HRYaZZnExyL=}q`>vU zdSS*V)HcD?Sb8u;k9pioNpGIw_nhz18WugsqoVZHH*ZHb zr)$l|2AZ!r&4lQx)d5Q2+0=#01xz)?Arq5Ie}971Y4h(Gv8;97Mg7Q48g9cEVS5Rzc)5?KdTPkUC*YWY{D)0Hokt)FYA(;NyOe5I~H<*wWhk4iY(LC=$Wl`}?b9tlxg=9-~Mxrowfp z=}sofAvh1ee=$6QMDPyH)$w+MokJK;z?t?aPD4^a@$ofM)#NT?WYb+m(Wl(>$hBgY z`o!4tcEhhZZKiLE(Lr29i(jLDgX%+NE~ z`Rn;kNYrTUp|-w4+m)M+BDeM0U&J4MS{ZX5!z4U91UHc$!IBSrEQ~zx6W+9R6P18V%$g|8>o#c%$DzlO>nBqnK)~ZEQGm(!tUBc~ z8p72MlffJR*6t@zVutv&2r-Lx+blaqf{TSm4^LVaDujLqY>f0|#Xfxl@(H`C*--1JeE zlhZMDBPFvl`g^StM1RJIEe_uz`6eVDqu_X6m<7J{3ZEIzgB=0FpuU|2yDi}?zC%Ih z^-eJg(-N~K|G~Pl5j)4Lv5}*m;=|$)f+Nk*wtR)5r{a729(Bh>e3CJ6(7yhy)o)wh zgm5Of^C@>0KMyyb5*i~@yXp}?%6lJSZL!(=?UA$NHFPjna?dhjE^2Mk7%~_T<1D@1 zY{N3QE~i;bZT%q%r!`|kXXDMR4AKRjgn1-GMY8Ul0Vdbn3jJ9@_X?3I_osU zkiuNH_kJAILo1jI~e()G|`Bzxvv*xWK%GV@}HJnc5V z6ef@GST)ABRPzN!yn?URd7p&KH`zMu@FX!0U0VR!Ac4u*WP9%sRDCB@5x*ckuyuYb z(Dt^EsYokAGvEeJbq$>S=JXM6>qP6;M4@8hG;gv{i5H%G@JJH%Ex+$`)`lc|3(L>G zGynXVBdDm3n60&WaL@h%ak;tA{PJ4VAu9XAlsEj40teZ@GaqG30y{y0-g zjP8TUytcvOTu#&Lk*aw5;7t^SeAgPLTj|U@7$xhAql^e2UXdHe=gJRtE$bilNop%U z6qD|d3ViuRCszdT3m8r%VylcAd z>nm|vjZ*WgZj6_S92BG7_3NRJd(5Yc$}~T}Q#8f8a{jauMKH3_FN<(HS$aYmv>)t* zUM@%_S0;6;PBjguKQ^Fz787H*8!^i=nq(a!SJdAu%ZJrAQFnIs$Qy2>cb3W&m(7-W zVX){=yJr8)avtP@=}scs_gO_g^PO`C2ld<5U7B|+!#r^+i}d4MDd!o-MENTw2YYEg zFcnarRkUL{X|~bqP!yZmT5bn#8$WaGY>Aty<%&oMDm$aKJG^+iC71h{6}M! zuS>)iH!VATvmm;asi6>KD2F^{3AoN5jft2-N{1B#wTURvZD4$j@Yg>|NgEtAXaWqB zBnROR05QE%0I7KF8V@Xjw`s`q$m81&%TKe*1E1YAQ`vf2O6^p@L1pOEwvAIiDJNgwvjYTXNvq>r!xy*^cU+PqanuVQU2@ejQ)E&+zd# zU7OSrB9bD@8%}eoA!lhy9Zz6Oi2N|;@{Xrc=@M^9>RNPk@Tm#I_%$AxR&wwueHSAO zwkJ%$Es($RB>kcZ-K_$EQ(+{yNL?*evYRoABS=OC7DuDMhfUNv1#xis@SO#h;03?& z)}6R{I90U^J!tN27jlq2{Bd3Oi1MQPSa+>4_N_?dY{4dk41!@zH`zM!M^KS9mfaDw zb6?zd)43-qEXx^1bKjsfT6GmwPiUCiD=Aq&qn2KTf5sq~ej+r+(QgTTnOk?2TS zMl*@0Qfw*VpL&e|JIP@UszwGod$B&{r1)tSr{T=A>Q+}k)=s6#uj+|2P-7p8#)Rz9 zQaN{92i+G|BjW)5ys%`7$trQKnk7AYD~lRTLdu|yG8eU7$WFh><* zT6_?5j#RV2QZ_B&$bFeq$56X<*g1)(GExkc|+#vV$;+lGR(9l$BW0jI|kRG#oJe0 zV_Tld@&We`&uzIlubbtU(E1mH9CQBp4gi}4>w7uFr^ix+Gl2O;ksA^=7j+{>s1hf( z)S47sbj3nn&LQK;PCk+|=QkvS87k+aA2B%>yn#DGp#$4>-8E|~Le1#=!SifcN(=&^ zbx*94ru`SbPM`5z#C0Yw8y!FF>M5LbmYqPg4`!r5mcH0CxBF*dp|+p6dh<_k#mmoD zhaeue?~3np`6FIDQdi*kIKQ?1pxozsF8R?*Ya3VIJ#&KDn`A=?g%J zBlOD=Q)H2xG7LeW*3X$kjoQi%>Id^*Ge%LotO)LC6q6i$>V%tEG>L&YKPh0wcXn- zOQgyWeeBLPUST;N238;JCV8U*q$kNI@<%t1FMMKpP?rcqDBjU8ZN|cAvdW25-wJ5< z%?A#_mmZk*AUepfZfPMGqIauvLFCY~i)hA`zq1!dt8$w1rw7#7>J)M{)0O5<(a2hX z9-?*K%0*6serJOBSUM7zQ4t8CKn%Hf2K(#s)@2$pEFAE%Q7rKP$Ghq$S%o)niC;M6ysl3hW$XOl($I;l{Kl$p%9al)8Ls|L9o+4-|3 ze=&iBI&W|+0ozjC3u%G$Rry^8UT8kij+U$;P&B5Fk%$K^82{Dt>h}I+xh9P<--3A( zP7a4N&N#WOV0aJmaNK1i&$mTdj7XCfHmiX86hdeUPGnOGLmK^L-gM@K>HgE0??_-1 zj(?`<;#P%u6xo;>J=~i(`Z~!1i|{;NIWg646EgxbLZ|oxs|UeI{AmVu8bbSs&ONJ9 z3#zG?YGL0j#N}o0Fq30an($NKj^612bI#%-WO+5BMWvdL!dl!E%e*5A#w@g5nGWiK zclx^5#4Q-}o-Z)HH{P>UjLey}BJL=_hw8tb?)E($9eujK-if^kA%c^1e>N@{s-jq? zQ%GFmZ#lhzmkJ`qh$AjyXk-ahVDKqc&e$*Yq_F?yfv1MX+S&}*dT02seO2qxnDvq> z!Mhfol!FO!P`e0DR@UUiT7bew;5Q%uHT2p758r|E;p77vmYre+u4g?S>w(Ri@JA@# zw$l#1%2#@J!cTqf%D^j^A4K#R(RIVi{Y3&o>Okrf9W6CITC&9tEUffI zqSdcQx$UCOyU@2P6hrIYhNB8qLCXwB%M3d8w{HJI8&=sT z5Z=GY`{KKQ^LiPYJ5i$O3XpB$J2vF7DY^4-3$f_bCere!m@y?M?Jsi8C-KIlbB2kB zHHL*K2p$WbaRK=u%Us0~MAS?BfUvs@5f_}ABDf!WbSHzo1F2z4_;CQVhcJ=~WF#=# zTv(a#EXM6Ao&T|l24&0V$ITJVC}a%1_)|cOFOw&`=XmRnKsa!B%8X8Cu23QXt3X)z z0!IitEF}J%`v=$?OQBn*db3AJy1)&N@4S6_rwRgLDODX7;jgTG$Jg(cMZOc}LuFNB zUV=rYLcQq?Xo_gYfp;hWsMVp?H=z+6Z0G5bCP><0$6*FKey$TdYimn!>*ZIH@7dK> z;N%7IvN;4+vMq$ziOT6yH=nmXQ+*!Lp{ zP!(@q*Pgq8hKZ_?j|zqeNpQzpB#xiNkF1+oj8CFcH`kZ?8An-4npsAjYhBv{)Tg9eqsbLfCUj5OyVhZDt~Btn&IK@#m>cVO1c5gQaU zeS#U?0N^WI7$(;{x{--NTR*+_Eq|NGsF916_xJpG^s%vr@v;R;bp%ihJ)@QC|%BYBsqh#joHfDYxNBy{s;j;)yvDj&H6S??` z@@tMfkXSTPP?VMT9!0J>(;;6=c-ZaL&7XN_$EZ)qLpk`vpJxG?V-1(7-1Q^@JD;z@ zV_GFzEPfgjRwShn9jB`zl@i%qV*+2yX1-> z^fM5)_5<9!eMdWA+(Vs0EpYTKj}my5@cEnwV7NzUyeP=Gr{;|dKOdY@m?AjSSG7tA zNtsI)JKF8qCJVah$`yj`=Z+iYOHnptplFpX{^o}X(pD825m9BvLoxG@?R+}~Tt1gbVtkF-Fykcp#^6aj`R zjWS|8);;|ZdUD`)^$aRr$q&>CROXFtsr z9YA!VEf@%o10E!cEe;f!Jq|e?n`*%h*P!ZM39N7hR`*7|sGe>S`$sIe;WEO2mXK(W z@6u1skP(EoOJR<}?LL(f2U+ z&5_PagNIl}sO$IZ)~R6R_6Y`Q@#niJDPS29Et5=~!Q)(PP6e;n|dcB-&R? zsV8B}rIBeyotG3ic_Md$e?HM)BtTm3iR-|%U9n3+A zW2H4n8T`7=e^JeUsiU>N-fS>;k&MH?8QGFJu7NuHC47(RFxjzrril}lz$SFm(8t$d zcu;R&JVOi3;`U8Ubh;$-RSho=$YFtw2ZBsXQbqEmR!Ug6j<#%Q{U>uLzA@Z+-Vs_s z3yU`7?@NY!sBP^c9vlyCZwx+eZQx8nz}HOW5^9>Z9S`Znb_wyVJMEf$?KyGyaE-nO z&@Quf6!BMOsV8d$ipS>-j4}C%_TZi`K4C%4_u$2=keBXJ2IsNMr=I8jeXq>DIzRCrPEv;-n@Ro5eq1gg( zAKgph?q)Bh*3+3)cwsfINBen$z#Mgh9*yJ4%>9#wd+<{W1Uq+RFZ1D~SZ&iV)#ei3 z;Yy`^5Ep$psAJEsrZdQ2B-`KH-i48+a0uq|ZQTb@E_PC1D4^SSejr`y@`prtX-9T# z5T;8uBPqyx3TDRg{sPxAB0Y6{--Z{Dn(aP!<=Kwsa%(D^3C;&I-~5Ewtk-5x@l{m8 zzO23}4ITFf$MT$1fjg5m+uDRW-E0x!b0({r0OV30s^YQ3wB#x;*^&^`d_mhu;Z-)~ zY4N>6it@+j6kU6Uhl2>!xz_f6OP(!=QyGo$Vw&$`Wo6Uz7~=hIgATdk{r;0f--WOJ z#h5MkL*&pYq{G%J#)jWvS(&&@Y*1Mz@R66ID#dUN{xH*W+D6^Y0c!@~{HjH@V_%5N zTbd-fzmg1kBKV5r3VQvKkfG_QA7WLH0&0!hOY1n4JGg1J=Fr0Rnu~Q{rr*_fHh;OO z+upgD<45h?5)~*4`NI3gM40F31L-vBFDEDok9p>v?_v;M;%-PGW>Y?_FN-j9OidK- zxXk*(@xsB7A!ty?qFOT+W`z`#C=;~9%()TvN9{}HWAoJWt6piZwIO`mD29%v1h4HetE=Ns>G7jiYike>fKhRWnL6 zvRP-02O;c=)$kAw5e0%ZNP`~pQ;zf~c780`ZHZHE?x&3_=BJqy`To4=0Mc6$>VrA-(Xx@;Vy@bHM?Q&$9@*YkC!E>Tuci=P z7JexD?`&~nZZi`rilJ9s25-Vwt}j@Nr47ME<7LzwR-5n9&FwgyXwhn;4c6gz7n+_7*^zdgIG)B z36-xGWYOOzWN>bg#*o^kDs-mmL{7~a9GG+QQ$^^ZywJVjLYSNH7VB*xzYy(mAry}3GVIhG#0!FUrpB2 z*o0ZQ{%{^{Pg<`SL7#i{Q6ps`Jj3HY_F|UqS_kFbDJ>MU51>Qc8?m5r$10stQ5PN? zD!xxb%irP7ZwvA+W z5<+iI5D*J1sbY*4A}hemC=(++xlo#fRuwY6e2ezc1vl!GN6$qhw?c680D)ISg z&&QuAGN7s3EbD5IcK_5LG>ixQdAhon>cqn(Tep)UxUGC@JPxFM=a77gCwVKXc(DcC zMTunwHm|^I-qI8&`%>N;{bT=>yi=vft}WPB4s3#^dOF&B=1uu)+BI1j0;bgP^hCE* z-M);b!`^9BxSAALOFGU_+)|SRa}IHrkJ3%cqEzX+#myQfSWs@{u;XlnXoE$cnWTj| zIN80X^^}W9X$f4{*N&s220w-0q+Gz~le4JJb0wxk+_+_9xrg`4E~T@Fs4-V(W>ezq zT<7^}NYDgh;xZN%=ISlS4_RPnUTa<e)#;ez^#mbO&&`=+rONYZ?nn43^-q>t)1WnO&+g1Rbm4^|F7f+>9930vCk&%% zWR=|XWPVd|$`*ZS*ew$ab||ys{c=~q8LkZWXdBcqRC{WF%PD((`W3IOu1d>TDDZ9$ zk!r=2>YPiwAbl(S3HfDFU#hW7pOmCDZs$HvZV5bo7j2($&*7N`7LJW@1w61A{#MC@m#b%pUcmq}+H7M56ZySDJcDM4Feb?le`} z09hef8rOjL$U{p@gcp@yp%h_{rzwIYvAU!!J^e2#pl`cyc49#R02N?jO5)#^1pe0- zSt$`Qc@?prZs;lNIDBG6_rI&+995yowLRHkm2>H)T7V|pVI9i{Bljfc?48JaeZu28 zYLyeN^`+k!w`~sXt@n0LkDJRfuyDcU<*kps{Qhkg&D5INJ9IV1KagP!dBUN?`L*Cn z^QnS?kQ`pU`&Av6ZhIykQA3u(AeRXRBO)Z+V$XPDlobMcrXB2Xk z<3e2@gwN%utT}fuG2JLkqQBr93!ebBx)|S4u9df8o>R-W&NGP`{FgW8bBOrQIY!f~ z9GI~%LJ9I^C||&H?-qi~zJ+vnrZ$Lx*>y`gmf$hyOmG40LMo1FoJi@jT#Xa7Sf#_EZB>jmq5${vt!Clmivj7$IX}G7(?jhb9^(Q(zCaOM zKQUA=`uOY=jB=CWB`l5S!5tMZHkjRdlZ%?;!xMA+wtOqmjhxdt8zz298R|(!f}zR4 zuok~?oir4Rv#aa4V-GUyE7Bu=>N)ga*=sEel=s26Ed=v`Mwi{EjeXPy7f88=k37_G zE+MUzIxeHls&f$5v0zS{zm!#gXCT)n`IIaOj`8^(n0*91DZwqUD#G+~xbX4>7FE;|B5W6NS^%#`K(mU`Xx)ZLc?vCWp#|G-MHS2~625ZPQ3SbB zpHu6L;}=;~G2~U6Wdt%&gdgqDF`-Y5Mi<)5;&A=ey;mE3GFN}ng#0i)X_yNm@MZC5YeBLdG*K zI|`|G0JRYz4<$CZ?)gT`pZxr(G1*_E7h1i8o$18%Mye9U`alQ!0i;gPHUpz;$Nr=p z)<>l{q^Qw`=~$}AVoBOgI#sf4y4zu*A9B4kJo~~Ln>gyawF8|)c}3E|w?7)5zq{&n z%tGdvu07dxa&f)zgeM9~X+%%|M8@e|z|B{a;qVga=628IE)+EQQ&YOxt-&$f@Ds>JN>**|Oc`^fec!x$pyGe{c5f0}GOg$t)x; zflJ#w5!&1pG5GjlNP&G;j!QV?iuTYbIM1F0tl-nv2(>T#>v}1S_QT9d#(uDJ(R!3o z#0YZdN~KFV&T7LAG*g&FPUY3j2wrG9F1D82rCD6mX|WrO?$H_HQzT#boZw$TS{7=K zG`#k{=B>CnSF2j8Bib6~W03g`jC?ztntN~LqI)Uv-YXyMO~+L|TVTvC0q?@$EbU<< z%l#XMX6})@LOJ)%YIaJOvI31_LNLm5D)7yx$Qyh0O`wCFQ4?v^n?+NO)i(ON?bGOYTweQ{R{3sExLnL2fU!Mkr6?>ZixyvF z#1cumFwW*>14@sm?yKEk_U+=`pwU)^>IHONSYFmj4p?>Q1+h`O2ZE#$X9|k%x4WUt z>kdhWnhi*<1<`wp)GA(-kltvIrCTL@%&XjQN?c&|x|r)N#p|drXeW(o7!18O$J{@8 ze1>beJ`vt3I;@6!PRxuqk2icbha^cHA)jhbsNtkY9m<(kJe=j9N)3Jdxfhh$ifiaA zFrV4+3?9=PO^04w{9u9Lq>Y^RH#}iqShOG88)8j1DJ$6zJz~5E-HNQfKKwEENFYpKaaHt^@S%q0e@;w|Lo*Mj$z0)`{;Ozbb z{}siQ*!_66%WN+|-kQIiQhl%j*4ZKI+u5$Q)000_=L_QlXVyUl>bApQFYG_yPkYyZ z>+9`6DHFVZ9`VXzA_~f)6$?!t%QP_oQ|(<4rQgU81qsUq0|78pMq3?FLHCcj-lgl? z^X;XUsF>6Ht3J3cZ^iaDDnHFHAGSK2!M#yN>WK)D|6&3o?<`7ec=7G0mc*W{$v z^dx!O4N@}tp7DhdNBnN%F4K(z5rLvE_6glNQ-D1bb*Lke6Ka0|>oQjXbefIFzmD<4 z@@ptcQq#@m6SZAIR&hE*6~vl^F+&e?6T_Y+G8WTepOgk_px`Fb?#4k%K9{h#6k|R! z9A?3`XK-j)G&gRxrH90B?>F9PiIF#6O#6TY!WtxM$~pnHHz$vx+Y#p6QHBZ0y4?%| zZiN=LqdP*JM3N1eA5()p!vRU+@tK&cH!R0Pc5MgP#B&DxVDkOkA*1{OB2H=c7t+FZY z5_R)1yHaClUA}BLizh}eiEWVLw`MbWx5*1 zwv&&DJ_@QNQ{zBRS+-=(lTq5TXh?QI0+Ab!XaYGbyW2=5p9A;Cz7`^HK2?i%mQ zBaTdHBC}BGT2T_uiXUZUZ1KnHZ&pjkuVr`sx)&YeKQ7|WcPnzbR9%lXD5QB%k(fKt zjxy$nEubWp0^)nLp_d={@WRoO> zoRaT)D`2!WFrl4bk0ntjQ-Uj^N5A$fh&&`$WqAu-9q^_k&5FefyS2p4sDu}jDxtb- z;wa`m@!GRULLfhD1ieH!bojr2^uyWpwRb;AZ70gvL-AR5+)5vomb-#i;__e zsW67LOezYfepizn>~x&(=i+MV%BS%vhY7wI8ly8P_kBt+bGm}Qa)a)jfGrJWuK^#V zt;kgXtoK2z9LoIiqt1N~wP`1)@rU((xYgSI!y>=BEWet$@11B_q7cGFsai&LH~8Xy zlN$O4qA&ce-%5CJcT2fVCi+fiRB$ThB-M;ImW8uT(>L82b|BW*>smkDU!3tD7nR4B zmf+tJbX%V#iR14WaP`%I*Il#rc-v`Y#?|sTx`oKAc2?(Hj9;idi0Hp#hppv&;(M(N z8TC?UW*UUAJ~&7a42n&>42AdaP63Eyza?W0Yvx>K_ausbUjM=T&G<}Jpwh`aUrw;D zwPnASuLQ!sVOun6#l_pkS0^-Fq~(ksmiY2w?~bgoI3EOd$&wG@?5hRjdwnWuF@vkt za%5q#Sq$t`EwasEuupchI|9Qi`)_KC)|V?QM4d3gN0s7_PL6sBGF&BFkhR^U+fJb!j&q*ojor^BWQIH5a?rf|OBz8Zj-yRANV?9~Lc*d=xxxNzY3oFKCJ=SEo(m0+0X z@Xkq>eJSTCP)rdI{KOx%Q;dwCW9MzM@h!67*&=aetdyE47+FtK&*44#G5%ecb=rIP zLgEtE(Xnmy;<-pU8L&X3_s=n%5xB+Au#2+X8unsYF#~YA^qgPEBj7^bUs)4mtf=+u z-fOhwGg;GIUPva_Ar6JO-K&oEw|Z%C_THQ|uI%!v7TQcHKTdtmpOrdRao$R+!@}tV zgANTi(dSfe!v)!_`k6Kx1QZ<%_&6v48o=dcuP$&c_?8>^i3fN9SJ8hj1wlXGdRUs7 z*)kXySlL^-8W;eV&VMy86tJ%V0|5YrfX83Wt^Q1pU6Y#(P_w6obu5OO54i2_12DT0s7FPBaf8$8K654xhEC3)w4gmOByWdJdkP`4t z_rE0LpHo^ICHF7oj8*}_*Z43fn&Fz5gf$+Nr06;-d9LsO!KexdB zjp?I&BHk4Y0PqOBzN7xd^egLcrvIt>KY0FrdZy+TIM>!i0st`nV*kwlo4tvx)xULy zN-w}X8c;)Gmh`XZ;4Fwv^v?(r2YcXV4A;M@1h;KUem?L%o7KYagB z8=(Dv3b4=oUJ8PG^?w7no7tN>IGdUNTSML;Ch`3|hrm_)Z$@_Q)=5;OQ8cYapM=;z@2H^+?OAu88^oun6dmj6nO_*bHtt@%H6I|N@1 zQeq&@j`+Wc`+sw?AgILj-+Z()viGvJvUl_Rn{E(<7Z3GQ&tw%i9Dd5${#FWt5S{*& z>E>elPfTKHc&(;%s&JXe+B+3H}eNDAmLwtzlhKLO8r#< z;t#5D%D+(mq6hIS^Vie~e=zl$|AqMvKQF&-a{hw`GX5{L-?lve>fEmz-~Pby&i)JT zk8N;&2L4_2_je%3``!No{%?@z*I4kMp+E00ehuUP4h=%z`$y=Xg1Ud^{aqaGcOJ;j z!9Vi;6IJ__^lKFC50b;-KazfniTw)ub%Ol|%;@wVVgH?OE6P9sFIm940sj(!zgS*Z HKR^9HaRT!n diff --git a/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl b/Lib/test/wheeldata/wheel-0.43.0-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..67e2308717d675f953a9bc89d05046091cc9d4a8 GIT binary patch literal 65775 zcmZ6yV~i+JlP%h|ZQHhOowjYBwr$(CZQHhO+uirPc{g+Cn;(_z{i~8ysibPHRSMF; zASeI;01yBPnO5qz;d?q<|9;&60`XtCnwywd)9dM3*jhO0>CxG{OH)tGs)WVLQ`1t? zN=3)2jZM*x(Nif(OifPGOVW%_%~GEnM8wid(^7#o_&>BjNr8Egjd*}bX#*(yN1|eW z+iGAy0D#ed@jnyU7+Bc;S0WvSe%nC;nC?eZRS)yO6qC;JALxlLWmF=_W`e-nKm}~R z{CO5^)8t=MkDjwCD&ez`^Fp&T7Fc~)LSa*AmL%Sd5RX{oWE`ArsksKlW%$u$NNQ0z zxD>2(4)x zQ5LUNpLXBLskyTXDmv*$Tz%k!u=#CKFg}DFwziSWf2enly&LR`xqq10#c__R4oO^i zT4Sx9MK44|Suujj&>`&xMZ#J(s9VBE{z7ebPjO}EUYLo2^4tb0oKhnr@ zDUD|rx6DZ$rz>f_Wq%re{V&8+as-{c{-JdN1OR~Z|4QR%;^b`aWM^mXsAp|wW@ce) z_OD98O0srY0{_6Hrul17jLJ_if%lQFTJcvx^EnJ`#BVvBF@q61zwEbr2e87%0|Gtj zd#CYx3f74SQS*%0Q-b1=WQuM%Ce(kx9^{3j0MQMu*FHVUsn7>4eU9au#&M_w@BvvI z5E&>E@gOe3aMDKe(@e8TS^X+s(fZNaaLy97{?Hu074!_*IRQg|dKqr^3}|qhPUY9@ z_3iF?-)Hfh8&eoX7B!AOUJ`y+Y^l%x-d*yKUK{HFlz&Vm=Obs-V>=FUgXg*bqnXY;AI@78Znr zKdKtTMphb=+Hz{4EOXfo@WSg~(^eQx7nskp*$fwkg*$5~GqBG_l8LE5@8o!>E`9tN zelGO@#Wvk$S6}e0*(myLDUXhojHe$`;%|$r)IOTjJQ!H59~&T@bw_0vEk^usSiJj) z_Te(v7I#$9FeX|*BQ1Tq3cwqeoUBT_6!h0SGLj6$D_fBrZpV4oNrH}d;C*Usj68cj zm$!?hm)Da&+Z>&IgfFe-hFk8km@J`S5|#ed3FM$$x#W;X-4*`kh_H260d5rpa-bD0 z#V;?v)+YYoj|XOLFbt` zWPQEQ-7}Bv$EUcHj0YxJQd78XH?L^h$h+A8kp--Pr%Eq%NY;GmM}Y9>m|?H7e9SyB9X5jQu}=sS@?}DRc6+P(-g?CB+ftB zMb5+E5hU|Dth2phXV+sD>f6})>ErRCt|;4;If~we4-EkfaZV2(8I7*Z(FIP6)GKH;yPfGf zD?E&BJUe)Mcz7E0^iy<~bbs3|FAc&7=i=f4m=^)aQX|!$q}B_sHzRW!Mwo5^>-pRX zMz33s;#FgqkZlAa>8r~MhOXV(3cQ42%`qSF+YQ|D9?zV4^FF>Ptf0uyjpdV25Y=Mj zO>Bz0-VOh{)At>CVUBOrE1<8{6`{|$eZA0bMc_B>zJ#mZF7g`^l7$gOt}GG5zUMtp z@sm)HGc5>?hctxo)I0hbi>uqPEH!F8%PbR>0%X2Nl~ivdy5%MEpbCHakX#UxQfz3L zW>UZFrLDz&Il6m?r>~a8U4Tk(yT1rtohFC^>l~ICSaZq#Y+~o-@Vc4m;_>x{!_nF8 znIO|j?S9**%>0I&!fwlsNaDyb0BA4>$2tG%32oS)mP2ybO~pLRF-al$p7EN;=pKWH z0o5@k=-R|McqN)?;Iw83OjZ1AUw?kpVEG=@0Qj+Xd0mRth}#|Dk8eR4X+SyQOv~V@ zk3Sph%VX3g7=0ibn&Oxf(5Si&#+lhB)D0XuOPE`}C+r0+Da14DR@RK$2d}K-6Q$2- zxO>hm2WsB}HRxE>k5)C~AoR7TCH&UfCjiDx5CBwf@cDXoZ_qpFGC()R%20$P<5ztM z3k=U08kbHkssae#Jq~ae%4Ro|;hVAvKiaRHm>ISEVC|wG3)fY1(o6aGV7Ur8pYYs` zPKJKP6CWk2!~k!V0G~h#Y8HIO`2j+K&r6n1AXd+sjtyLpyu8Ct$7Ot?(3#3lz|=or zIE$2JD`1L>E55*AqV25C92O%KJ_oP4ohF%FYuI^9KtHH@|DNk-;e?1%=>}7dOw{rG zyKm!XV?!_4fM91B>|qL@zuTYoI$B=}*xgioP5;&d(fSZ9+V(XhMB^ zx+;C=v~e=GFj8*^x7XX#=in@{eb_i}n+8jn-Ufl_ zxcBbH;WqTs-`ZIUm1nCI{`E5S^QHfCFo@U;+8L(-NP|eYk#dhbiOB~o?tNz&P7dUp z8&=#D<Qp)Uk80Q!zO|Q=JuXM0gBUul5GtS7cLVel?*Q5N0j4jKWP+H2`%F9mDm3gXYoth+dO>z_@th07p*V zxyxR4I$~7JG??cakf(KUfX8pRXT|4ion3JOz+^LtD){4s>4_ zl&qf0K9u&vWf77IF*GoJ{U){Pb{UWSc8Yq@1B&4^7LRPSOr^oKH@RM|Z5`H2Oca~o z?GsNP`e6p%Z3Yy}q2SHJ3PESUoY0!uw&gCdW=VOODMb$F&!Rm|Ry|qJC_)tUTM#9Z zqA)AbkWM7pASAaSz@KYM%ay77$DGSI-p&|5uB6&igatz53OG2g{tmoQtCRIeZ+&4H z%0tA@5;&fhXq~}4hctgf>JIj-)IUYLqBo{@0H#R96B(@*jHsMvaw&?rkp`RK4;B!W z3S~3|9&Kl~CdD;YT@Kc~(wdati{lXLsx7d06rsveFXu9Iag!Sf4;R>qat=g5LU@22 zUSKc6@`!?CrxnppyL?oFz|8<&?|i|O(o?sAxC^x$OhW}=zvk4QWY^+bbJfFW+(a_M z1cTL#($}Q08phLsv{?=P=&pH!z^K;S;HMn27kZKBBxyD~mEJIrdZJVi=5>{Du|>g2 zm_)dm>(7OA8UdC`z)mSQf$SXD@P)#e)`O#gB*;2WScd`XBN4-hoxTuP)3437oA6x=x9x`HKEe|E^ef+;lSS?u~9;F`Qamhs2W zj+`Fuls8l3D4?v-;loVF65&+5#Ok9gJ^^a2 zFjY@Quqv`j^q#B_j4j_=Q7ppWeR5qliq(Vn zIwKGhnRsrB$155IF6=N=0v0}BF_Lfz6&SpcAEgH_(0RQp;SpM6kM`iO?N78|jbf$a zjK48<=!(ZFE>SQ<)eygOaxHL=DaMu1XJWpm4_?l+7$jJn#Z<-DNTi>YO|iu{ghE^y z;1tSTAa)GFM61qTI>k1G;AVs-M^<|U+M@o98(5F%s|b4y-S`2?XIrZ-dzONb1AvF|Q6MmGX}QyjTRi>Kp$ z_y{C-T!AV{(R7+zsku*ynG+WJZf{y9J#DAMEq9AMYLj9jXBmx&-FEISab>L+q%XLM z8672ae|MZl^CZ7R?`Em0D6V%Z*8}wbqGndGT+nyPySD~o9uszSdh)pK#oa^9l7Y6{ zxT&WD5!084m#pV?S37}}ut_ipUc^N?#u>xQQtt)F!e+w!gdlJwt)>HNoAuAm4*D>nR#}ZG&$Na zJy-I}qm?{#jTz7{L=0t&&BqKhLJ7MUaM3*agiWL(S;SGV>j!4w`3co0$Q*8N=Aur? zzgUemu1Hn5X#c0Fq9P>&%Yd%{z z3B2(bvXy?#_93_3#y1HZqR-SI#lACw?y_yw7_&x6o_PwkunV{nxSrULDZeUuoeJjN z;m={vaai{nE<-T4b19gob#_?EJ#(QlZ|b&_Xj8{#T0qF}57)khu_uwWO^K&^Rs~(0 z2QHk-71qXvKLxU(J10IS1I#=)@}l3U%7c&D+t}p|5DI zJpE42@59a0w~3Sc*Z%Z5X(uXUqFUow*x`|mml20x!xKp4pJi1UV5IcSxysExWgKe>0X8|1eUyajLS5fD{pFNT0YPdl&8p7Uf!}g$Oq8yZ1I2=?)ICyxOHJ`SRF_Wuvmc?l&yGd<*(*bbjpe*40JI1rVCP1n?m?4Rr8C&oD z+@1;}c4YAn@;(|f=EnIvIRMwfGBS6!MTAkp>8#3rOrw04&>lM)H8&Tj3J!_XR6FRfv28+3fEu={Ag-eMvtYXR*hOcv z26hDH(m|0Sfl$(UM(I3~;m@z}h}w28=W{t;a%;F`{t`!p!>R+V3AaCEGJk9VBCt*I z*`aB8)QB`J>gu5Uo{|pO*u97y3sGbLgsCkt{TbThWNN{$F9xgUMRwfQ8I1L}f`;8D z{=?BHG7Z>O0`9s?-T zoW})k3Do5eFH>!T)!qAhn@4cOurJ;AYi8CR%NATqy_ApSQg+5EmG@V3qOzaWEur{& zQJi7b>CndQYwy-D2t7m>0cJbugsg7Ix(GXVKfbelJwNH7o*9gk`qF@$o%V5FBo?TH zQH^rgn4Er!PWTnE(_l^Q6&)+X<8W`s#TGvBr~25v8J^Xfo{TGrU@bjhg5!xZ$h)t= z&WHGh3SQgo)#&>@^!y(U#8*MPm$-A-j}1%qaW=l9etGht+RHapkma0W8qIz(6@lI> ze!(MrSWMgYhxRDQk?*HXtOINgw?w9IR$u~BMqJ!5>pmST_U!z$kQ6rJFrKzPw4A-? zsouLD94iayg_xEJSuORGu2H8Wifyf3vwM7(=@oMDB@k0t zdwZ$vkscub22H1ht{OMNhw;DLsiPy-KVCy376%@~<~RQGZRcCq?RRcF@ZIH32!1Ccn;?5B zi3>&#P5|S|IjZNEK^L(pUez_xzGlPlQkfT*ajq0lPiKztgh^Mb z`~o<$=aeWMO504Hi|gc02lj>U*URH$g~5GL*okzb8}_ju795mAdN(I z+qM;6wK=ADEP(v*{mKI2z$Ocf0O&fz0#1q8BEji)lP5uhC=JucBLx|4o zVm(|860;BI2L&TpKook}w-Bz0P;>j}^Oh&Ai`WjdJTuRJqxAbPeQ;)~9F$4zFF3b+ zJEytSXjp;Z;yv>*y<*U9g0xHYD%fyovKU3inH=2ZL1Y(2NNYBIxj%UhF8mjLk79m1 zb_yY+MUOz-&S&yxTM9kx!kG-7$itMgLjE_3=2v2HP@V-(MbzMaT}1*plQENrjO%`9 zo97XruHd?Q?)iW5Yb`kHl5oW7tYF#+Ypt%}ncG$^DFr@!yOqOzo3t=W%lT(i4=xxo z6E(|YeYjOGzza$S=;X(Y=GQ=y@@)hB9WRXcx_P-dy?vb@?mwQis=noB=zTane?$0s z`Fn!|Kis~~A5I_P@q1Y--69N-P=XmjoiWhIF%1z!2j|OdI75o0$mN&UNBLWVg`#Ah zqhD_-iEO)AS2=o%k)RaW( zs7~`3beN}%3#dVICvEay_$RX^zm8bR5|!?Do=N}uOfh%Hg$ZRv6ud}>HZokd4Jl|r z+5_#0n9$7)rc1yDtOQVZLr!N3NR`R)>Vjo|KL-pqdldOXWx^l~@V_&&LQcYuH)phr-pO-qv?;GLU-N+e0(D#O%YTYFoWpV_~-^_z3lA}Rql}i z4Eg@mO{j&b%XlhrTs$<7UemE^g=3zF@h}m7FNbu4(2$5KZ~O5|igYz!piLFCoD57R zmrx)SIKei9SyMH9)*7g7c5$-&5qUwr4g+fTVN<#(=dOd+40Fo7L9z6+FdtS?uycC; zcl}iEL)44Mrj4E5EAJ_|0K8|9O~I`DbPUtpRH70~uYQVr286LHIUJ^G7E|=^8A4`s1Ey zGF+!K$KhJTk$V8T@B=oN1@e$}M}T`Pl7efhgW+Xwr=_yLb<6=Y^9vEt%t}cQJwpfZ znnVuDnUk6BCQV7Laiu;3`nB^}F<8;eh+4?n@nfLPYk7H-<4Sa!2&UX<8G}fE^{L{Q zr>lqi*T>t#8gncz71vCOCb1H~4Lh^%@(dfW{s#3sHkS)Ntr$DH!6R|T{4jtkj;}J^ zsp80^Nn-{tZl~4Ozg>JT2u6Ndrv>G@E(0w&^ku4cdz*E$r85dpP-r@{c;;; zsjLU#Zht&wo;?%j4PDl9cqjtJr|fa{{IxqWNsqFIKEN8VC)gFGR?YpiR~|eTC2&y3`rt%T6ppW>cms0 zv*H6ut}Fe0zh!6cl)gzoIp(lY+>`Tz4bOVDGE0$ObQ>27aAT(AShv113g~&3^lkAWnBYMqp zb!S88>^!^&I!e=YMGP0AKwGqEGX8m=pKL^q)hWv*H=~c8(I9fuBJfLNVFeO)Sef%)--=A~BIw-SImit*FA$caz>? zK8k_^zHbf$6W|dFE?yoq=R!S$8do-g;dN0}ZNeFJz4`&$YrbyoEY=bjVAM7Xj&l%F z6LehELMlXKmKLSm6dfrjf9g6nS5i3F$tXkNFF@OHh$$ge5QoY?QBUZ@`DxN#xXD35 zvj#?RG0sOOne#^`A>3@}jtwDB51^6k6NUIAf)Co4tTApR#y?m57=}Zi&+0jtL2$28 zVSXp1@hIDpd-mGM!FS&}Qt2xAl-NP=MhoVwQs{%Hn?)_`!P2 zxJH0p2GtpE%HB-CbgM)t1W0NYi94(x#H0su3={=%lUuCqBMK#wqK z;SJb;MReXRPA*l}qZ(M?;{99bVqqeqZuNO?lzi;oM_a(M2+ErUsO~+l3Wl63qEOm4 zZ>fs;-N&Pj`rhvMncmNX?G=7EzZyxaYsB9k#)#L7_iu)wvS_RU z%uBceX_g)z;h!$Iha>!51m!|dNJ2^?2ECb{ckq4x3%&;+8IvOPFsNM(Msz`F)pf~$ zN_={jMu?SVG*!~{7DE3+bZs$Q1fov*0S~>$zmiMZ|^KwdKBGR)Hf|qN}h# zXGK4dUwjB(bpmyO4PFtwUxj_5RYctuY8CnTfdhXI9(Y@}_pXxd!&{Hp|f70g+&J1lRhq*kq3 z>UnEPt(FKo$;MVb=@rnTlB+stFaWG3gn$i!<9kgt@>tc>LcY64RYGc3UI2#y=X)yn z<_137KJpZL+xQT6h%bbLU-?z=%Mi+7)%n90Tmt8ot9dFf0)?(C0rsy?(r)AOID66+ zx9eR~jzM-FE%vgzJ)d8u->!#KjeeY*+#fC&!&319D(PWOUSOYIKrw`|#W0I`2`K@D ze%C*TKhba2BUr}l)ld%zqsfrHSn7geTQ%MB`CGt??^HEk_TRYvn)|j>9tj(n;XNX; zAANE)Q%rlj!mNU%c4Z!z{s2?imZEvzo7v6sZ53umg;?X!SA1-|_7{;StFs$)m6o>> zi)LmXE%0k97~j*whStK8L+e$nRXMmJLL|GBKsE0N1j4B}5)@)vq(&^&WM34Y|5))AN<`JlhV+s^E z1H}2bPwEExzIRa1@wdAN?DlG^eQpK`H1DXu+lxwjr!gCPDop*r*6yiM5kIVkg4aL* zcX1or;-H#TZv$5)!V7ein`3V#jVWB`fF;h+JsV9#4YAo{eX9+ZPZSAWX6TJDeY8wv z=6+)_uXSvlh>hqQOAr{Q^jLy-A6FTVzvrq5+h{i(XgVyWmywy;S1=3bdO4UB?_MGs zBQNXNf<5W#@&`HtwjqV$&7CMjX67ASy%vEdB<11Uj-=MiK3>B^uwR+ttv9#{P z$5Q-L&nPr9U_}K4sc}A!u!ZykVVbWy&ppK6a$+;VRQv;Ax(w#9sqVxd6VZUxIUQ>v zXl%C@D<*`mlXEB-BxvJN>tSnv{k~yZn^0P|rnk8O=#EJ2Ej`n+V$yTL7R<%l%WWZ`Z_ZS=WPr3jsRPcVpxn8DAr?J)vbH&_fJ&1ca%C54ZN{tk#bd&6 zgkKorVr(+&A06HYF6887=m{+Aej}ON$w2NIn?1cClbsmXTAhgPtz0~NRF8g@R&j_M z9f-S0{0?!MQ_H3xas(6W&FX@Gm#0H$xFvjgku(uloHXEuop<&`2zTv8mF5ry6-#I! zK(V>yU*g7P*=CxW^OjF(8k+r*hx5GYSgDL&@WHk0y0o%AAfk|XI(*@v%FuR>wTeGrqyK{ zTLY;Cp(gHq%0qvLp>_C1md%kAEl0?`v$8KBBjhchAm-rBhrkHu$0C2R0%lyCSO$pO zGiWNYMm0%iku$F-Dt*+k{T^XkX3KQ0!Su8?>$+W2+{0xT96zx2&8tR5qP7Fw9_t5v zn`df$lv97?a>dEv;S(n$K^bMD%Oq%4CRdgy9w`55A7SKl+{vcJVvGh4-Xa?$o2ZhwYdt< z^fO!dBw|}y#cn@1cInE?ok@bBjkl|2eT2tBs;Iot$9C`(`+4lSTxKq4mJGw+7rhtDu?(Bgl5}Io8j8YHdcb{-b zX_h%roA88&>%W^I9_8B{ls-s>;-7&MK7m90oiUnU9HKULSccQ4byKGsd_h^D4dBUx z#iTxQZWiu54YB~jfWE2o8(+z!HyVa+E0*D4!ep0P64JduAxF+Krr4K{60|t9lIEoeoruS_IG| z>^KYUeZo&tL45JSRd%6C1~{@>I;#SVBF1?G6oaufc~8me)U5t=RymRC ziMOkXx}r`W9}-f!uX(s(S%F@C2Un!0uI~!Fj#{HX{x@`rz;K6-sny70L_MdHBXG@(+rkP6Mw0~bsy8;7eF zvnHrL43r_B5gLd+$lg1-NX=WrgLjE*b^`&hP(!=MbG$=+P$FQm{ns1PCw5D~0Nl2L2S^NnN8Sd#Kn$3zO6?(LEwo372dol_&ZGa;SnKGyJ zt%9Ih&}x-JJF8CSp}-o!Yb7FM#3pqfaUKB6#T;gHY=ecCWrHLR+E6&u*$|HbYzOQp zg*cjT(CH(UT~w#1!KYS-UZ;5mc)!`SWLqR|s}c1^>kyAJYMsBTL3C%CT|{iN&+P5Q zd>yF@Z!xnwjx-F2CDZn*zg$$@?9*19K!x3S(Kd#jjPSJgKbeS_*yE5_Z~%ZWwEt<_ zwJ~urFg9>9_{TwHt9!;CwjloM^2BbiiE z|DlrHu0h^%y@kO?vT#qd@^0H^Wxai$4vaYD_;A4RklYZA^VqvhSHR_u%eR9n02^5U zL%7^-b~J=yQ&wPt0{mwwVqc`9B72EMT_yg81x^TY;ythI0*tP3ETYd}h63-A94Sb7HX%N-pf!v(4v&{I}@NpJ|E^&Y^V2tS?u_Qo^h-HL=%4?6J z^!rp~WdnP%4#rhb<%#;+BnO_w423MdDw0a)`jQ1GFyEgT`I>{!GGb(yzGa6q*m9GZ zdIyRyH!-md$nxW^{O9z%0nqfJgm~gYSY=x*yX$c}w(veA zL$M;26A?pO#VMK%WdP5b&vL;}XZZahPk=Vd;qK0&s(sWmro{uc<4dg{?ZktYg(N3 zfg!PIwtRd11v>w?b4^(!mQ0%~#5a+-Cz|n_grNk^#;Gs?#}(tZ`I9(l1IEkMI{H4yJyMGBbJEJngZAC3uOa(3o&X3wS67GmE$$!9hVds2U!ekU z)b0j3v@8iNPQZb?CGJOmk)be>?7tY!yL^;+_-Ogrh(sUeW3c$4-|}(mZ`P*;RV~X- zdgl+8HYwSDnBwCPSh={w55NnNkqIk=b+sS)>zar&(-yWnF zl-xm57Tf`Xz5H&alaFGGM1qG|_hR#?cl{pI4}@gae!x`ErrNoCNFH~k5tfbGRX5Bte$aH3m zJ;rZdo2zYJ_>G-)`d0WnPSs?9!zbeSfB%GgQlf4?c?Xn6y ziOq&@!A3=yFB)U&+)r02<;b07ldWJkJV$SP6Cm`(ONmfB8Cf-@<0UTfVr`&uM_*Vw z#pxv&(}JkS+K+;m3$WjQV!9hFF5d|fa#~=#fQ73=$K9VO< z=z-`va(q!1ZBGeSv%mNTQKnJe+^dxJ>*cIiVq({(JmvM+lg>`9-S=E^-k zA=dePfgO65q8F6&mr>9GY8e2Gkgmmg_5HNEf9dS?iY@jdeXLfI@lo3;xFxRu^3Q-Q zA(Ho|Cb;jyv;W7}kuRh9_DxB865opt7T$vi+3tiErMt@}XevR=imPpOmoddIuBEGe z8G|v_@aUI!=WNH}#{$#r$m{G5>Og*ET2)3{*nPJ5EjKo_=7c;q_FR(|xT;AE1lt%3(ip|6HGO>ZGBd-K_JI*TtVhH^Oii_#1PZ zPR6@fD=){+rf?T}`8w_tpLmXLRJIFfw>B^11 zgw6qQj#F-LZYuKur`!%A<3TTD_qH zDU4BZwFR$$xjJwO-s{VF`2#yMFY*iepHl)}vL-72zf%>Pf2S(2|E*UzJ6Tx&n-G!} zCoKmV5V|g@(Lf`%kn}A7LdZH^D8;jg%!sUm*^CY7ktO9CN3PmL@28xfJ2lC#{Xovd`Qu8gq)0tyU=A9N~(h{g#>!{FYD@pX-5B zxq+zkVJQdtAOXSOWtsRw|K@|`gxI9Nf&C|vTZPE@`;Y(taCiU!NdJxGe;tneU(u&p z62Wnqix+b(2SWGZBqKPBj$+9NcuBz zGB2v4`J1xLJCR_pM~$Ym5vA2=%hpjBQne~Z&g-vWuZlQ1zKC*)@|nf1MtVckMtd)A zc1SI{osYYFz1wx-uKGI|Uqxym(J^zkX=4cs^g-0R3ckHQAqe_5r1LNnH}N5L?h;X= zqSWB=eqqoEPkuL{=H8rji}239p&coOY>fn-$~{&SIS*(X1U+Ik1@UBt2_)r(mn5i6 z2a&WC)r@4;yflV`Bi!2+f~vL{rgW1K2rO+umQ;g*Mu~|hAZMuwNkIfq5%ouOk#-z` zLBocT{zK$?=s)V3et*7`oqT^iV9G!h;js%u<0h()7-#f-+uPv-Ms)CG_WJWNw%0eo zGxxUl?!_L0pOrYLawG|JmY>w)3w+iyUj*s7zUzWk(P%yaR&0W%BFC4j_tqPV?efYl zFRFIApuaYRRmx8cl7+0qB4j7hI?0 zx3jvm1FHxi`v*@)%UYwiwp2^80tS2TI}*lt-d6SZ;}9ORZ{P`n`J&Yie0e*g-=(3s zfS(+ZMN9j^{%09}Eb4)^dK(y|kf>Kk0KG*bwG}IfMQJjB*QlgOIDJ3=-u52~C#w>G zFeoG^rCjpo(CTJqb%zK@OLUdot3lSPS+={Ng{vY`lYB%lX5K{tW zwl-uk(ht=}Ib~&bfKDSIkrm1W(pRCc0ZxX|rq0)yQUa3fSceTukSHp-BnZWYKWAtM zRKOT4P*LaPHB4|>D9+^K7ax2Qskyd&9g04j#)2GK?pMA0=yTrjCJ-87C)m52{}Nhi zB37}8#eq<9BTn#(ar9n98iXYH0U&HOn@U>mm%8h4!cBzEMjP|z2f+a#s_&FNx6kDW ziIej!-=JY($w|D-ET2vA#`ghLBa$$NTY)USO~d`$M71|Xkc}X4^M^w}AxINlvW1F> zXXqFsyham0l+6cBB@V9VH2*%zmP@I@r*|2z5YOFxN8_=^n z(O+_?eS#3IJ|{^^`*~kwORYlP>@flDMnSZ;HOwc{LL=UHff|(>8EJIh38o!yxf3s< zN6`T2zQPF)JBHCl8OJV&RB!}}$D&RE8UciAYo5o;gEgg+^ur#+R-q3^&a40mSK;r? zl1j(x(7o8-u6n4qJlviKQHskg@-c@#WKj7T_V!7-G_(l$smKV_X8@#6miMmhO=A}? zSdy$Kd-ECct3I;?=$HCY;J!n)O_@nX>o?X3%Z_097tDLZOBpYvQ*Xjcq)Fw$r*cCjbXaFafewW^P>0sj#5&ZajN_&K=Ns5SpioAQWl(&c({!-g^$gH3c7q_dN=bncU za1T3Y>*c3DI3x^N7S;Wz2sOlbtr=Syr`?BKcz<*Q_c{$gKy12Qso7EKJq4U(dbhWF zX0lm!iNks8J@2iKyz%a9TNjSf9B`v%;W$r*)y_VpL}{wnx$;H+Ldl8&$#&luDDxNS z4~An&ZuUPF%PkR4kYiD@dN@qtDv|xWx<2}Vl!13lW}D)T@CZE!(`&3v9UN1@0<$h z4L>E#A^v(fT=${RK&crOcMLJ=5rAA7lQ~eUEjKTMdy*Uof>WE9uq`F)0(1N`=z(A^ z%q0HX+YQp{xujSMP77VEJmVF`l;nJhFvCPTJFo3Facvpl@uk}VCijUPqD{oQApxFU z{u1e}iLVzHm=RdB5q-oVNrdiT9R+7^?cT@eWI)_J?Y*_9v#;W&T=x%NR1^(f2tMOb zwYN^<)(4T1CXnEs{jta>a@)MjxI4W85UhgK*R9|}X$>56bxPt>x>LB$O-6%pB(o4z zMjFH}e?L0TtpYk*S?*p|j;(!xCa|OgF+&qmA^qneJ`Q}mJ9|5N9*FQ;oFMi^{5+q& z7k!Ns&*3Eo4+ZY$RTs4XZS77t*Y%scKG_z8xWOW0>UL#9| z*V(3b8X*$+B8UiaY~i%Cyvg3VY#ixyK15v8s?XNuJ?P`(=kZ!{ep^4lUzyapTL6e< zWO5qy&4}22St`_O{2htTUSiZ(j;U(CXReC0b?&Kl`jiD$?H+7Lj$fff%)lbBftrYc z+6eG;#f7K{wzTY-&#vLN~fH{&N6q^4hkB^STd6N;Ca{-sN+V)Y5kjMi;xk~ zgu&)^UF4f)F0rRDbN{+^pU?T-c!C}i$8m~v73wvd*>%tEnA#WnW|6$xGO$U&)ouDn)W(EKb=R44>w@?=YNgXS3@VTj-UVlW&i9zsQ*0> z8d+QX&m;YAHEFvoHiVu_H3X0Vb1i9!TCf(s?KT@f5Qql>cpE}{(i=L&a3qO|nV-)Y z82p44&Cwnw_?9tuuPku0kgUh6+QKB-%Fi0)4ob~pQtWKYWbR~4aE}&t@6ifN{*L*v z3d0I)kHfUN4%dP5f99%4&l3g+BNOdcHlOvx=LjNcWk(}8Bl70YzgS1vjNRi6Rkp#+ zQE*~f@7u4Ma1hPyGZDB z@X)I3F5~)|E~}%=<#4XBP;SI=zFr74ofaBsCoO{ci)5+4;<0Pdv_N>W{tsdA6eLQt zZ3~ud+qP|2?Xqp}vTfV8ZQHhOW0$S2`(EqZj<}s4`IjqJMy{AK2jtHc-9?Z4F~VrT9SU9@&LJg^lAjOkN1M z=D66uvNVO@l~fKYgu)WAFQ;xMoSv+w=?!o-;>U>~Q?rY!tc^2P>UY%NFJYFmNSHRp zgL|^F+`hfPLj^4g;K%pV#?gpS|8@%_Qyr0p_8M%O^2G7_P~j(V)AfsjJwqqZ?A-h3 zl@`x^s`So1=PN>%#SKKRiwxc_51z=%1?5FQZxDa)O>fke<0c+lFxIm z0$<7Y%{BRg))qA2FLW)X<`cHhkiGkpTLY_>b;rTlHqahjf$X*wn4s-x8)s!PN+11O zB*@ffd{1;mU;;tVJ9E^JTupfS2}$kglk+`^5RORB1ba`2-AX7ts7yD3XaXk6vR`{B z*F)=_EzN=5X#3pKH6YIkaI!pH4ZDs=KY-S{{msqinY!+wJ;h*I`+!9JR8~F;jVaQe z3bDn+4b!IEx82UOg8;PJoXG_)SV`0uiSf0=zs&ozfyXyj^RT30ybYvOksJT|jST_m zjexvjsD}Yxj>$)qlwY+y46yM}xUAjJ1z1h_$*=PwhiM=|`)w(`p*{@gdDoY+KxlNt zk!v8zvLw^#qNq~}!qR6HT~!Z_DcTy9rvRhEpdl2|4iRQfOd@meIHwkEwrXu?BDW$< zw1n2b`7AxLv9&qT=iE;Wkmnu|=h76-&TzG|P@Lsw6j`s_P~Ls{-{SGg-L*GD&yCfZ zc1~NN>TC(aBm7feDeR|9IU^J}m$t*C^^)L9rw4MM>(?Ewjot=5;Z@Bu_J8p&yu~@N`pSy@2l&itY%TR=2+oj_)}~~SQ-w9 zVrgZl^{uq<#f>Dd0`gVg@IyXjoZ&4NmD|)wF(4p<{)6`|#61u(1zGH1nsiCV4`-0E zC4^$pq6xo$1Y;~U(_?`JLvSQTZOQqBU&co?g!DJII{Mz5m~|k%n|ZAns|oF>bFDuN z*b}e~^PP@$pWJ!I=o)*PriHqO2mXqDH*3qCI`I4{GYL4SeDyjU1IC%^N-C{lw%*N$ zi{BT`d=d4suf@}#1qUas>o)x)09jhjF4=$#vkw@7E31p3IOll3z)$iy9fQ5~rr=*{ z8sf>mnSb5b7ZNcZD&O3tzSCw-bRqNYX6?;Rua1ANX*+aZEPcFhTo5)2`2n;uLmcw0 zcveLr0&G7+#&AUQQgc8S*c(6$7d;`db;HXVzd*o%pXg?$<2h=Ts#B-f+`}PO34sTk zGDU)IiqZ{22{q37Nux?ui#hP}9Lq6r(4>I>InFMixIwUEh%SZv=`DoUGmJVO0nD?i zEX+(+iKqyTLbUHAjrsnO?LtWr`^OdPY>Wx*b>IJl z?IesvEATJ?0J|jr@6?C?s?tWbHZCR(PQN_Jwx)IL<_O|P4l&z7UJPMX*|LQvilm(+^+o?r_v}=Uo#e#<7-d93Ks>RlndzzZY4m#YlVH_q zXV0Zpvo5r^`9%wIL&b%xOGnL=;r@zcB74K;$ExZZdE*VY&U>pYZbmlU;6MIm@9lZX zR+H{KGjk7(wI`lktLF2S(Qd_J@URpflwJf*_=ga*9Psxy^Dm9fm21QK=F-aGzF5Ji zi}#Ys%LTW!-u*OHTMs1Hd>>ln#c=E$Lm)&tMv;b$telwx`lah@O|?#sZ7O?M+G9~~ z^FI9v;|&tYS3>Xi``TolGWv&6xm0Qo&st5N`pvGTEc1HKuZAzuACr$Yn|9uHG|vz{ zzT@NLMI7-T%tvO=gb81d=fhS0E{_!2l*r)%ieg4XTuSHd@l6&FJxxh`NR5;y1t(9^ z2*pY$wrWvWSX=C2qc&>q;rLR=hfkOCij}Lj+i2EI>WflGW_3K1LUY_F8gy!>1Fb7+Mp3?>v%cjU-bm!tQzMpC>LQ$xhBWp5g zSiskEN|VsgVm~thLr)+Spc1Z?PJevv;6=F`km@5xG6~v3T6#uYeuO%*QQDf3IY0ry zgvPyi_j^QFJFEi?UFY#m%67J3KWf0xSpvA*mX71qVs&rKdJ)&8E$j;id*WS*!9V^{ z&ZmJ_5xji`o{0~d7Z&Oq=3zxA)q*jqmW2hIIsXzeFS==-QXrVH1I)F~uhH#D8x zk{{TWWbC#$?^-#}7e7mkh4A9f-4AZ3Y(nTz-z;j!Xp${XqiQ|t8n$I#*dVZe;&OAYrEOUp3#u6)6lEh;= zB)oofAjDHibVZO-OX)C_zVEzQiP^`=>V`Zbc*X8yPX;U!Dx*JUmbM&4lU_fcwN-^3 zQ_2=G{l(LMYTniX{tZrNt_qTSWQQ~~__!$7O)~?=k>&s>SYkHD`|-TFJRNz;Pcmuc z<tl>eu@S4{C@B0zow{LK0alUeCG| zgKVL#Ia#1Ys2HFwC-~BaJ>|T`nJ$52ba+h!#d_@z9IbO|yfL;S0NPQYJ_%F&34}Yx ziTNhy=GYHUy3LHeG%V=sK+SD2sgOJG1$vkVgD#^7qR-$aFrK%BHj}hr^lRYyo1h`r z8hgbnonDTJn2WeUn6Jbur1g0K#lkS%gRBT9G-FWM;*?hd#F{!KB6U~HMT@&xOAAVd z5VQV=I;8n{d6|eJyG2c(ONa4h(rGyAG$-8u4D%nqEE~CH4qHQc<+koJc!jru02WoK zePpumH;wNZgIZbtKL-2K-AZjQ6;Z!nk*DDMl>F8lWX0Qhn;VhB4csP;9C-$Ult4#J zgHwG;oQss(KhwqZo+0b2-k4$jJeF29v(%#F<>k@y05g~eOfMU*&)5_QR8q&d6Fz%N z2N0^{aDl5<%7h>JR)7eKqJkSjEGh3UyBcQ4J^e3o{p`$I&%zPFcxU&yG-kP4Gge}- zAb5t#yQ*_TY{aA%ET;MTlym(q)_U@>NM#&5IUQ6e@g$N-o8!nh63cVUOjf)Y$mc~>_@_Xgqx?#FMgt)KwM(u?^NGmhq-i2U6=ub2%)t~G-E7LW?kBjBJ9s)x$nnhy(fmWGgZ6jE&Wc2 zsHad%OGja<*MzTmw4@kOlJpLF+Ool4^4!o z!{0nx{+4K@XZ?X;dIumCi-Sbe>D81Y8uf|oh!x9fJcdzS82hnKw=tFG<}|%K0+De9 zRouPc%u{VyyyHahtDl_{V@@DA!(??1p${w~ylj0<3OjCsOj#qKE#GNjFkF_dQ}e<_ zLPaU6XbKaND7f|PGrnkDrd{13-^D6m=$y`e2ZZ$dD9K7`9l>NvbpL(JJ@WF=bSB}4 zy=Dxdxd4gHQrmnKPH-VeHPHkFjcXGlB121jOD6Fj?THEm0*vGFvVLa)b!oVC8`#?f z^=n{dVm5JGe_}Sq>%f*yYU4vZ1NORA?46F=Fk;$dTHK0uhEZg<+9JZgCnQ6@F9_eU zI~H#4lSk>mI47n`zm`P#Br=d|#?!*^@$|wuoFBI81Xtnd3&rX55<@m8qhaZdNdLL6 zZ2aad5;m6WW--OZ2Vh0+v8_LhP_Fd7KNn9g?0V#8VBC&mA$OPBkn{fVEsKxD{jlcS zHS= zCVC(x-1|H@@1qIW_X<}cV}T09pSix@8e1(6$N`jA#fUHL72H%y!Abb@FXPu}XD#0? z?YJ&Rv)BbY{vB;8PIu4S!Y_l~v#;4U4yCf@xLJ+?6I*8TS1uO!AaB==$0(=}ueQ1? z;!XaCIt7v$hkpS<7bLik7te@h$zlMvWGYW|F%chCy>N14*zGulDkraj4WoYRY&oUfh!h}Xy zugD8CD3}4={E>WRqTrkJG*R`2oi+*&;2ZMHe`ycuV32@Lr1RQ%G%| zG)~E*o*)1K_`e|H|A|NZF92a@VD$IDR8-YJj+?&}&x2Z?6}+)EV{N9^`r?vHrp2W( z&g@ZM8;*pwl(}Rau%9YV%(tg2ARmC}y|jDW*f1ZzSEoREM#i!Gdb+G&4zogQG*i@{ zbqpI3VrP^mrn2_3hL-Z73;Q@$uD|W;Fu70#2{>HKtP=8DES9(L&+~rY)^YC@v}#yF z-SDa?oYmK0c@wJ17|ML{ds5~7(Mol_#bpHYonnG0Iz1n5slA=;KDoQP^_(Rc)fpYb z#UjJLaK~tVz+gusaBBcrGP5XkfQj{Fzkt|Q-gs>oT`HQOhPW< zwnCaf9>Zi1Y&TSe=@wQlm9ms^rizl_2-iO^SDIr3ma-gI!DQ~T4p3F`7|_%urAdp_ zK*~~L=I2#H$jY+M1q;Lz`J_zwl4h^(rHBqp9HM3x5jv_2xhWdc)0T;Q8P0C_Pju7- zTKuy~*^O#uM(V@CV7254OyAJ7E4uDwH7Ytlzk1VFmkviIC3}nwgcxRIEqTrFJ)mdi7ubHxgkr$PGyI$QBY_;pWhKD4X z+>0=a*T4F$#<}E=j77!nfgOIcki^^;xaH?nze3&i+Th_-T3;tI9J|L+c-DdDLMXph z)7m*1>z7(`zuH73e`sf#>s zf7&@4P|dQAJ#T_x=G?L+B$Za3=&`N6=EVU%q*|z|Zeck6X?`yh{9BfaDh(VE6B>u2 zin&U4u$86I>I`V{nLdB?aDm@rPC zf7rL~an=M?i^4IdqsVfltx)#Nvi`4uW)KsgA#LP8uSQF<+9T@|t_eQ)mD z08_^P;KetpwM-bp-DpfV>DHu+{OjWJ?wNVdikz{Ptcvn_CCC-&^gdN!u$sCLx~`~s zN5d|@Z_`-R6*$^eiUKKsbJ3c#&QBxj;QSC{k1H$d9}b)srMYd4Ru7kHxbV=P)K}E? zrBO=_nk9>PnrPOOZA%OayAQ%6^mf16DekcZSN|0Sf;e(eeWS*e^1%p89! zRkONv%pI?uhyIHggR})u+?jgSr2Xb$q5w()ys$w-Majz45{q-^vMblgD z%1*gtN0X3?8Eu;5eut{5LL(Eo(xO>JnW$3OWYSfWe5_irQKh<_Q`Sb4D3;9R`RKU- zWt~hFy(@33k~blpTs$TrIMABD>Iv@FLmLe2jMdI2n2eD{;;PmmfK;Qp|f0~tEcXbAD*W;Vu|I*_ArJ6n|;pPfu19+IwjQP z1{d)-4lwXS>?72DOuCqHLmZLZ8v1fIo`!s&i9d5SYe4n(NQgj}g$OK*vaxvKaJO=` zK@2*MZ~A&=%M_+)j=>S8tC|}oNz^tfp4gNfK%N*3CtPHMvDUmhi5Z>ePs+%I$0<^v zviA}d^2lDv5KMVzWq?Lsard>CeYA$s+g_m=md(qEmfVMFRMs5O9lpgNCrXi@t_JlK8{NiVK2uP@&`z0P!LQfHxA+#gUUH@!6>pdp$$!J#&T z;kf)Q0qdqJi8vkBT9K)UJqDUw^<}Kc{eu^Iz4}McO5Kddk_I{y`3{MuJ$H2y&Y%P~ zr6>^3@qpZ(Zy(POafGLU?6ESw&>HluC~VRekEt@m8eY^U8H0^ES**@$!M)wQBaoNfi$Ur!z4nOE+Ohlk45D#GSM{e?6o(2EH z0N1JsrH3rMKJB>xl5Q(FzY-6L_(Bgzz)~A`86k@(d(eE`SMq=x&p*hY&$Ag#?MzM!BB2z`+y}-R!vA@ph6255bXEUbFSa zObadSVCN+O>OeIbASqFK&_!S}8ut%@h@5lsrQ@&d?q4_DnEqLnw$QSV@H({PeD#V{ zsNKfgTet8@h{u9gWpZ{7w2MdeuEh~TAhEB9D&9Kg^ItDn^=;~&BR$`#q1!X=nmPeX zj##u*n%+w0Vpi#vK=)aAlDEe!Q0Cj%D^QgcYH18Lc|dP3uCR^+<<@01pcj0bC*Tih z?`$!stKKEBoP21Roye!H3D#z%F^!iPwEI&YREIhN)+;WZc8f%(Y_lL z<~dw$2{v?wN{7bVgTYD+YzXD#mfqOgmmd7dbR9l~?$Oa_xIKq>f(j9>*c|D$tXTXf zxbO_n2iW30Fi;th)T*;PFkS7mt3<3Z8LRaKIRoV|_FUQ7TtArYqpd39R45`92FuB^ z5c4CMg^(>2?dFkya;2B$Q?=|ByXfE8Ib3L`rgCj|zuwMU_kI*_#O_V5kEX$4BlIhv z6>=xXf-;w2_Z6H-AaXShHkb2L&piQC=flM8{Qqg0$$GVRi&UNS!<^vgeEqi$)H6J@ z6!F);_XP|9fczgqqqEKb*@NmR^~Yo~ApAyj6H}=Wn8RKa76*zR<>8znX3;=ox#1P%@YI<0D<3t$-q!fasWl0-0e!^l*| zLhpfl&yWj6!5UUNMNXb0gZIc2NG4#R!jz;g=SFc#-S%7OiQykmUlE{Mz`=XIn!=_) z&Pgpks!0Pv#%q%>;<@KHkmW%#T(kre&s?bs*WFGHtsbn8!PUagnmMh)Dsco+qPaJP z&JoyU@DSbtHfswH1;XK-4J*AkZa1XL%Hk#6VlWF29;cb0wGGHjdzX~5&5M*Z4^pYS z`bT}n31f3~Vr3nQ*_5`cZ^V9-l}}7hn(M(W43?2)y4AZN{b7rdl6Q}N@)z&7w|8(k zYjheZdZVq>wb&dsFvXe3SBJ2j+p$1qVucsUM5d%C&ov9FL?GG*$nZ#AsM+Xe?-Xa> z`@?I3`?)BlF&7ftFrb?Fhh5}K4e}E^t4av1KeKs;r+u);KNllUeAQw$@qY4mJo$lU z6Oh+aG#@&O$5+m+Vg}>9LM-&L)p4Hoya>~HQ#TKfuvao&b{WVO+$#MxS6qMod0*^7v7FvG7=uyL9?S#D&-xm9-c#MHOUp>(fr@#Zc5e`1*5Lj&-8fd1Eo^?U!{(bp~}HpaFN zCdU77y&Cwxg=R~$3FrNn+woW92EhFPKKZ|wpmTC_qLz@Jk(HjE`qSFX$~4cs2lU?- z`hE8Q-omdG^WV4dU+381zarEBr_C6`fYxmPLa~8gDE2?EVPa+af0{CrRX1%mIS{;8 z)#@%Hn^cZpH+Hx-XabokbaSR7Bx zIoN+AI1U?;2i$;2N)`QKa8FH#QO8Z514r{9NEL*ViCSAJ+yeqyTSaQi=#KBY3BZ8< z5m2${5FntJ6}e3yQ{mFFNQe;=a0y5Gy6YMS9VwFh6#*jhA2B8m9dXYfj|1Y1h@T~| zARtS}NK9U7$8zQRQ{&Wj*Q{(KCX=xJ;(KSPQu}G~WT}qJ$b34qKiOtj;2?0B(PwgQ`acTUK$Y&m8 zY}htPfxulCC>KE`wouXhq99vJxZ?d<-T1E$Ii-^5r>XXw}a zY9<7we{|&ZkJ=b11q_5IGl|i49ds*{3o}k%xUl<7N!p+jiW-!)=@l9ff z`0$As4Ka!==Ayx@Yhl_|ZpcOk@*uHv#f_kvQy1}EZC`3oSHz>tPWXOpU1TA}nmnIL z8$zoC{gAaHs%X0r;*WOLt0X%2pp|QJH&rVxyeSu;EET3&`LQpTmzaFq7toKX>hm5|gVvkRj!rOSSodh+6bqY_ZoKaT{NurT8 zrIQ&9bVj}rkWhdh<`I-L0OBPc)?$?}s{r)~K~y342fw|ugpZIe4PABu9bajlm%;P% zp{m72C3b>;p_0wFd$@V>0$@G(7!_YTT9163o!=<`hA%a`3dwfnBi&9t?>wA+)YQ~i zebii3+$h+qZp8x4UY{;LE#1WF51L(6H7r9Nhh1Tt56%y}EmEHvh;Z|YEL+d`PGr72o# zjp3)1#vId5sii@ny)BRjP_9pz?l?J8=87BC^^DOBV8@QM78_;4NuG~ujWjIP?ND^% zgS$qe3Dt_du`NnNaAzyDQO<$>Zpxm=d-tvrSoQF45(}*J;eoH2e|;GAHbl+F~FE{lOZt>42Aq9Wbo_tYz`P6Kx**9pC@7R zVMXfE13kxgyX2JU-dU(?#A9>%Y&NC|o@QYYt-|u32GL)co{%p05S5M@$Cz3&Cm-9~ zq-TE^AD+HjrRT`zQ|Y~wFPU*v8h{NdGKADU%=kJnvEow&^m@TUl-+3>jQCVF{D6&8 zEzopL^I3&R(Nr_P=;csLLp9;x2m{JshHi8zX~tvbO`D#In#138n_*e^sLKZM6vU>p z=yx@1B>SL>V?t#@J$xF*4pl>35;`Lcz&}{M-*W85T?)yAnvk}g`EOD;;CZnt&9BMd zi4*{U?0+n=)&@52Ru(qSZvRXCw)my5u)hE4`74-emQyb#)RN?M=&aT&%~G|H%Wht~ z@v?{rAzA8@_N{G5e}C}N1LPRS$o5u~B~v1}z3!k>AJR;-LN8Dt`KMSEv>3N1nco!= zbJDA!i6Wh`L|4#SI44KmOI246i|3FtX;f5;=8Q=YD zto(x)d8-pIYB*KHR1hw134$*^p9j`h5zsGcsEhNr2FUJBj7L69#4IyQpe6JRACgq`zR>^?u}5bm|l=(z-!Y8RSJULhDku@<%i} zX~|s@V?nsGDW(4M6(Y-AO5)tyQjTRa(2D}@dEi`Ou18V>c+ajW+*!i+o}IdAbS+f# ztWJuHJE;0}PVwYcYYArok|xp|ut5>gWkDxbBwykF5S(LEjJ5#a_ zXtpIMyx2yU?54R7gf}Fyo-TAmJ<3%Z5x;Qoy2E)71_$3hF$@*ibj_b)IX?Hy8<%Lu&JBD{P#Ki6T!0JyQr}KW}L)m8& z$BS^B6#^3YdZ6glqjXhrH!(4_9~kzhKx_db-yk{%pvt8P zF_xo7zTtBOs06R^n8pT>0`*IkUJ**-uOC9qM+u=!NdZ%&NOt3vtECqOD3NWTA|WQZ zB#Meac+n(CDJ20XA5$I!p5jpS;WwK2%d7F5lBQDo6JMVWsV_s6Fpw5UU|3*S=!qfv zAsE1ug;+;BGT6lG^p605)$aS#-5#+DIfk$R8Aj3^X{Lz>FDMi7=mC$sQdDmcU-6ni z*iPnVZ*@KB&C>aBJ?(01>vr=pkq%c&%d75P!m%w-Sn|Ar&=bkb4Tgqt zj02FKmiB74gg8SUEW{~{AOddYl7M_Hwr467+zcHI#`ZC6(QE8Em}^%^H5?hEIWu6S z^{%!@_0ujh#D|FrL4pp*WLrDwYvi9{i-ruoXzI^mxl#yxX-JGEH*zXB`c?N$u}npQ zDl1XkEL)xAzrLe!h3hjQQt}GzBUR&K!N65yot_5*NOHQVO_)F6QX;|oZ>0S7P|VlQ zWw2d9OE5$tx(522lSoFZj%E*=7^aaGB1+OFOQj>UKD=c-H>E7j?f=0YyYMzcq%J); zZfx7LrYBvJC|L(we4Ui0+v3BP$;ZEzBcpd^y+Sj)Hcdsl2j-X%f=p^Z0k#6{+I{}u zuXb$<{Z&@YudYUWyubH2;_LkuX_T8Y#|=$-9Ct~mgBs&esfF} zS%lZ}w+#7Lh_CVCnqT9gchZnkVO6y?SY|=RpiJ5v45QWGb;ML)F%^z5vYHXT3@4s_ z=*aIf9RG5O^4d{}V)dCK-EhqKkpa}hz#g^5#*{)UQH`^W zEW0lwtckB+h2U;zsO@JvJTQhbmD)RHT39c6TGP_Xj)Oq;0#nSbfhsk8B@B z6SZ*ANa=a=VhI7LG>|_ue%;{UT1wg6PF%TqGHVq0v_95Lrv#J)C{xJz-mea!!NSHJT5+Ls$ z)U0;+7Ts!i?<2xLzX8@E0^kQ6Vf(eUt2E-81p)x)SQ3&-A09p?<5*lINJ$=hCc>ef zp%N0({y?_&m2yjCMNH(CKq3~fAh7XUMXi}`2#}2y3p!2V&Wx5o3v^i6*gA!Rp)-2d zElwOBB0+En=b~o;(9akW(Xs*<4IVu-<5#w=04xp$8&rQAWYKcG;vfr~s2O?XyfM}k z@?=t=00glS<72Ut4w7&WGJ&A6&o%R*po0-RCNaEt#C0&8d7v#ZX3M z#FXPP9P#i!4FomKVa(UoTt$tvxiEV95ok#>oqQ`!%A=2FSxv#c(onvXO}kN;ZbKtR z-96FMr!BxX)SFd@?ojhZLZrKvSET{a^VLXQ<>zfme$3|NK9TxFn!ZlX_q?S0FYz2M zGnTVuHUpbaa8@W++?Vl$i(UCh?t6s{(_dDViRd!u{?^_UbP9yWwXW-MJum9((Go}X z&dWLoFGysvsKNvxu&?zN>z3(kHqDE~72FL=`+k)IQpP(uSHd5Bss4Pa9y?r=8(jG; z=$%R1>+6L_c4DnxO7xwwD8tR^=$xUMdSZ;dS1Iw)#5I>`Lk_;Zr6 z23UchVaU+`+$Aw8!Vuw$(kC=g3dswBX1VnH;d2Th9teb$gb9HK1esvkgT*;u_~Uq+U+m{@3W-sb4t7L_8i$pZ-v(xRt8J+;a;C_@?m21QKxGq5#Va? zl69dO9zULwz#7xC_?2*AN^Epb3ZJb-9Y*9X4dqj}0y~to+~{kq#a3Od!4>U8(^v?w z@UT7$mTO8l4b5Q8kA_TQHwFEtpK#|4H=^W5$(MzaO=yrDhYRJ+i78bHLH#sjtclf< z!zO)O=JnAm<-BgLPGjKt&YCukEZMSjMb1##n_p%xM_i%H$9j5SS#!fI8;%WBsd4Sj zi#O<&3H(zdRpP_BR9;+(CWY`&=^H#}WGs}EH5g+xUUjW@O{Z2bo z^}@WJ>Kbey?5GS5C>CPLA$h)`ySje@2#_SyO^+3DNYn>rSuiLzfpp*;wxO zkN-exFsI_mwqM`m-xXYo&_U`0Bdo#sqG(}fZ3#Ey@!;&hF{PTX%0biQI* z4L*kT->@_WZ(!1K{~+<%j)?jlq%(ph+xNFp2lyIG2I4h$tj*4)Nn~ zsRd~w3xVw=$fLuqk>X;Or$ve;ciYq_(U?IlK`85K*P$K*{E8Q_f6GIorVlR33|l+- z*e-VQJ~8^@%l&1lH>r@DtxlXKE0{gaH=R8}w(uLnY5Z@dNxoAyTJo+>ufZL5>^Zj0 z`gQL-DkfTs9=mxRi5kH^ya;8|!Ko*c+3xk9wNXSpg{tR;v3k07N&FMHZmvIeH+8H( zZ{uB_d&|(^@-&A;Hs}ozCBP*zb%o^)2|8z0b2dQ+|=SGIw zUB$3e1c7A?i#R}xgY5n_=bKt+7Rq^xHpB&hW79urHIarh628T#q&<7HcoH;^s=sS^IB1 zJ#)VMAqv2fKZRWwTGWohyxOv6DMHMq6a{p`h<;Kj(&w zuK-xL@?5X>bA>s%m*F-+ZMapJAV6y7%OuyiAJLx+gdMa(Wd*HU7p0p>`qIi1%1 zyH5q5xZ}-5YKcnFcmEA<@pFu1HUH944p9HUc+1(*>i@!9(W=(AzjlFNv$6dmpY(v0 z0c3|h!-)ZK1{`Pyn?RtCJxqFvEE{^nk_1(_!ej4EOp-CTbsZ)oNzc~D!xp8?V&)}w z+xr`o_OA2DJCUT#%YxwF3thso7NxavP9tDU5g4D|P0J@(+UR;;R!zuCiVqpzK z_<(45g@rLp_HpfQx4ZBpi6j+iR^UCo!-vu%7k9-JI^PEs4iT-S;>pe4fOy8kE#6U8wV|@z}k5yJH^JACHd~ zsrqV}yY6I%($M)n`dI@J>%?Z>t$tc)U|se|{}*?~%mPkBM-i>DWldwzbzL-P*3f0P zh5mu3RQiVxuvETP!dFHd9HX#3zsP2DTx#M2*#ITw$?bw~>|f#=c&n zgeqh%c}q2p_sc;=w|kwF1wQ_fkf(!+ui@Sr%!}Cc-guk4R`(Mwg zS&sW?(*Xxjp(&f%qJRggeufv+`+!5NTeeuW3N5GZ7%4|080 z`SccI3`JB78}dMXCUyc}L0Lub+;q^ztcpFQ@u~~j4nwJJ+H∈`x3J7tef@K?ZfD z5kg#;5zufb=MwTcqc$17xENDKp7*hOKXabMkQ!??!U2cId@2LtTvI<#1>?r>ydA3< z?~e;U;(!p5-TS3NwVf7}2B#yHy}ec@Q9)_TR`T>5d}bChX4M>|H}K}SeEnGmcipms zC`A@xU~LFry;dPEnwX=E_a_qOJa53;FOEaZf5Mps8#sH|z`br_L3J?OVV1+%;bUnw zcfV=!UZ%l225cVkL0_0t^soHM3>!Av^I|8?&urfJC2n3(;1^ea3J{GmBXM>z~AmS3GjS@5Eg zJkvS74glkECdRs{H0&WuZ^HQu+eOphDt!{|$yf~HFPCUK0>^gc`j=4d3;3UMLeMnJ zisx?>!e0si0OJ3VXtFbKa5QoFWtpZmwQM&z(0pEM_sZZ?6qoz;@zEGAT(Xz4u?{yd zH!aX1fRgLaMZ#z-C@3vlHomSM-Qp#rlIZ)L%wU`1MREomzh($YNA3Cc+`c3RdB_>1 z7^fjb$HA&4r4Crb)#NDtj8km|1(|lsnz@V5acP)v!X9chNm3BGIfn-LC4!NgD-Bih zNEZ>A2DGa}sP>C^o31r0#-7(f+byQLV8Khh5)+O5lL2 zL=byaV9cQXuC0@yJ+9sCznSYb@b(}Xt&AfEVgmhM+8q&9;z?u^^0^FlZQ)^)G$&8+ zr!>jaS|PRW7cf{cDet>}6XRLdEx0 zM2CTv#Fwv~PlLNKPsUpe%Q?ajdL5QXPBjth83_)QK#B{&aP*9>nMpa0pDDcqXvi=l z5^+pKCl>t@R0JDu{p)Cmr#NV4G)}q%T-|d_zZMra8G5O0JSvSnGx&~OCA}uxvO_df z-%77p@EFvJ~YN=zj8Mf4C$8SLUfFmi@iW{7ep*3^#KWp;hI|%h0BN}`5 zCYKK7sD~7hTeTy6f??I9Xvij^A1^{b{qDM$E;Ed&SUt@o z1=X+uSb2!L2#^WQnaTi^8txHki4tI8w4c;Z>>e$d$`IOX`&?5gs#qZ6D*(f)XCqt9 zKW$iaVs>N29et<=0`)8%16lniP)xvTrNAKmaZtAmIrGuu)GCSQm0{P#WJvY|%sR9V zK#9}Br<03~d+ZgtImg;Ung!a$sJBl#z;LcAS_DhzHnUY#h@2hgt@xxD{|j8$JJ4O< z*qjAYoK_QhV+u$46DhO#_Q!~`@SJ;)BhRZ6vj~UjF+;vT>HwNC*aOXe^ROmiJxLi_ zsCRHs9)RNrJXC%cifFYuUxRM&)-Igx6#2+U7Xvvc&$hc&g=E=xH%XT#W1FnK=)^h^ zFrcmm1q!CvhRUoVM)Y^Gix=$e&TAWqDYofYll^%(|g?ZEgg{S~K|!(~ck0Y8)5<4220pETgfqZbTWcXz zsydJO>VnPqPl6|OM`0VFKGDc6@sO+L@OrbWTQ!4joyQw?nyo3QCG+8UN!}85On9Q; zV+z{-&fLmsd|FX_F|%fg>rGC?c{Sm+`phf5>;Kmv6ci=w{+m0?EQts!g4z+-LrnGS*sbKsiYH>5DNhQ>m+2Y-k zl^L#!^eVuIvSK%>{p((C-@my3SV=g@?OBl~zJK zo_O|YLs;OITfzv?C_+0&Rn4Ndg|yn&!*QUf^~HOJ&q9u3gCx3NQwSG@Ni3Nc-mVTu z^OdyTz*u@)pZG>we6%y3ua!1yBq<58NotIUkN(73RG!UjAWo*ty(^x8+s<-V~`CTY02@ANtK-p-Aa^ zoRlwgcOu)+y*SK`&3|DR$`!sD@??UcKg@sJ^#&e$apdrzM`u)<-GG=3nE3q>Cds?c!c|~LAnGwTBMC% zd2v9J4K~gn)wax#XzZ6+77}|27xeDRTC;KG_2Fu1PIQ_D-8;q~+;E*H!?K~UOEpMm zmH!Y(E)mvV9bBGvK6%Hv|4x<`loffc^w-Sfq59D>^_AS;!j9z(4u!Ovk0ito43U zIhDQc1>yQoHs>SVpY4365sa4IQV2yY?rGeh*0?eTGpwaYn}Nr$Y@4{mWNx=X+D!ya z)L?PK0E1%$L#mXmkiZuRk31PeD?Xuuhqkixl-%cIBr|&=(96XEr(Csb8NJTiUxco0h>_MADSSQ^cYnxFLxAtPfSz%59e|<_2nd4(P<; zf0(Rcpfw$5J(_SZ%=__Lbe&X~?jGveqNNVBo8Zrt(Py&5H&#ocPX6H;U&nZd8~+_3 z;8tgY_ol|mc#Mwzweacy21RI!Q>O`Mz1p(0{Miz=zQ8xfqWL5`$(O@cx$G{1c`0QH zr;yrFq2Rut>?0Y}*o6)_BaZ_Lf@kR@_8c~AhG8VDVY^Hdt~(p6VJQOxhki(DJml1Z zg5DU}{l;L!jm(aXLitJKoGp*JGJ=4{e}8?He{GPu6>5Xws%6gVy@qBH zD{IPLY7Rf6BI)2ZQK6h1uKAC_do2QKMK06UC2JKJzx#P!?$!+8XQ$4W@14Fk_-9~E zTv36x{s-hgdpCx{`vm>pK(N$bqbT|R*toRiY)b@D`$Bu1ubzf%1ph+6aX8xgeRr(W!5z$Vaa|mA_})p? z2~v@Sg2q&yDkfGQlK&hI?b%jumoks@VhL^JK5{gwCMdI;G@jue)q_cG=gS`%Aa0S~ zHjUoumd<5&;p5e)m8FJlNEp=(sD+aww|2I;3nS8a1SS%?BAEaJ3DQ=y6)JS8ISbzk zJ4`=p%#*$}VT=)?NT%Gu15BFqXGdljMQfCxR67i(1kzR$c1o1!1R0qntNE1aPIt*u zCzuUq)08}Ty|jc-+@=)b4e(SnPlzr||6L`-0VtF3N)+PUu>5fHsJBSds#Rmg_VMm= zPjDKt7Y-5%nwcmu779Ta*=4h2yYVJ}BZ0UmN1+?e26< zkOg0J{==lcvAx;qY2zAempw#89GPZzMHo5YPOFU4Z%K_zQ7764l}<4o z1ua#r`jKZnVapVLgJHW2;l-iTh$hKQxXS?Zl`>=zrm979gn6bANnb9Jj$!LAhR;K2 zXy6GCO}1T0ot(` zwr$(CZ9D1MPCB;r#7;W4ZQHh!j%{;#ubD5i*P3~M!S(H4b=P@R6%&U+{YB9mfy#{b zWZmgL%0eowyu>x|?R{ytE6ugrwUqpCPH0rOZFT<#NA}gq6eqP(XQ4aG0G2xj&gHTjjp;u9Zhw=8j8oN$Y;%(KKn*a-P)%~b@C@*tOqPOejI>s47|2?JZEwF8*|NmI|Xwo zchewUhVdI-1;MEPJ@K=CX1*{DXY7s({yTQz_S}7T82_P^cV)elmVCyZVfHNzy~av~ zR=f!`Pi^OS!S>^|`Z>NDXFvCdi+<=;=;Xi!#UQ!kc{6Lsa{icnpTTQ$3Hubjtsr>n zuF&nLeh=1Mn1cGTy3&L_rPo`uBunmJ_Rh}k^9*5;_zsGcGl4<-@%uFyGaSuO$r_!w^LruiiW1tyot`D`01AY+RC%K4MvBSjrYAtoSyok=CRhZwhM-oV z^%Q(tEF&6O{2vZAZQQ@c;z}k@ZBxZ*wbEyWRv6G*H|oVm)x`ZMRO2q~Al_pLWgwL< za(asN#)6FdTwH5V#{C*Excg z$;PDSJ~nZ#66;sChTIXCUdFjUPRur7*%~j?H+GQ25zoaGM~i&o#Dj-)`YN|491Bx;qPr1hh^`O?R2MY!c`}VzWlIH_e zg!oJo2R!$!CH85GZ+c}ZoN#j#bIQc~OC9qC796iz#4a7%wR&x}J9|g6+bQP=*tu{a*mpNV}pi&2}(EV15?n zU>F?>)eIa=GCs{dyuo>{_u&jiFTXsquwSJP5KXuN?iwv{qH@NsXnqejM=HxULdFZ$ zxET)9`38-r`DFimbgar}8C`N+$wLBZoYtNAE{2CVJIarahWeEK#{%-U>~ zYfE4ag*yuy!NQMK6h9$Ow`M%7HM;Moq?F#ub2`Kv{!DxjjagR$PL)^waHOZ}+7kZ9 z&&gEwo>w(fww{Y*H*2FhNHy{I$mJfR!){hLuiyhnD88x+&L}RMQdLqW zZuxVc11Wm>QH@Q8QIS#Dy+#@UeAiW_KQWdXCV8+ELGhO?eARZSd-p0JCXv_p86Puk z2(cIAgWOYeGgZH-E-u%@E;X+qtoMc&RJY(N(dUvk>yyk85G$+gs%FI$bCSde_32l9 z9#y_7ah*EfX=bsz>wf)n({!cnp|&-lkA~>fVzNAQPpXzgLlCio<2elWW25)o&%(P4 zE9x+t{101|*{;uY;L6g?^P#P=n*8&?R8KZoXt0RONc^PnYzO+?*`J3_nDz2ILQk_QhiZWQyWjbaGX;?VFPP(~ojvwq~Pma%Q1tAQ#8iz;vd5q!1b& zctSP~qCW!h_CuuT8M^WHgt#}@gupkap4D#_pwEeYV5pxmua-Il2$vZ+Bx*QN%cg>G zy~I`yS*$c%Lt$mQ%S2LV2KT93e6Rs{QeCfUBA&p!S|eIG=L>YwN_Fkj3-=gdR!eL1KDQWEdr`d?vt6QieGO&Bzs1nn_M z6rZ+%GcWQio_+lzR<`#8PJn_H?};JKu%mC6g?OnYTx};}Eb};@iK&m7v)QI)BoVEw z%et^T+{n9rZf%?vg~4dkBa3ly`{ALxiuKPRK2pWfNyj3`F%J00x=QLA`J-Alau3c{ zqYT1J^Ivob&G&S8;9@4|G)zJkbUAvN@X%Pn=QDv!nCsA1XV(S!3}eK7igY3YLSht> zVHpR0DxA0pyQEgP!#e@`b~?yELzt?YcP+f8>2PS_o80Q+tG zM^QCaktc`k@h|{fVdhHAfC%2v{?19lUvq&>z4}NZ0+D9$Ew?~1hl@e!+!RGT<>4t= z*WheKHxsodZKFP+_n#oR3`vGW@S5dLPF0@1gaqC1u@4IyCHxGgKzxVPtoiX_>mKt$ zN;nGKafPdIAqx5_em~UuCn<{#DW)MYgIeSAEo_Ff0Z~R7JYs0#t}w3w%D5MEj#knY za{2ss>J`$qcWjHP{uMpGSTG8${8oI4`m8q_corl=??{y*3E}ier0DhI`Qb9R0^BGR z!a~qONW?HU<3`k4<(xRh-psRwV znD;Cg#sd7yuUV&dn}E)=!``q@o%4nrE{d=Edaxu#-M!ULIaiHA~=*rr_6kv z@OZnt?_-%~YoIH-oC4AQp8?Wpoxv1AWEVRivUwKr`wr{mj+(CS?&L}z5FUkob7{g= z#R9cqE6q~g@U$e)hYbQFN0_;5g~_H{t(shFoGWUPYeIi{JFGrDPz&34GubM?+p^r; zSr~M0MXD%O^3nlkf*xyTb#znOx3s#aAy2q zwdG&C2~uX*K?<&O-3xqi?)}@pa4!>Lmbu2dO}^B2r%~xT9Aa6OxkTDa%^lC*2AeK0 z+|we2&msm&zJYF{J(KRxEJ!hq3nwdA4KL!dG(_;Fvjp}!6j#_Y)`P8ut3k}i5N#R|B z&Epo=CuForC7jL{{5x^6t*zR^C-iPGYsU~qe$5q~j!!$EKsTRYEpf|5M+4lJ`^43F za>O-Ycs0Yds{^;1y8HeSgGBt%$+nZGe5H%gEl3P$%0NLj8ElyVGTkChn{jq$%h5$P zCout$YN;4nOYAq+x-V45pxgUeqmD3XO#y^lXeuz@?ZG=yc#G$4GHpC@b?j4%3tIQz z2Nk#O4MsI^1|2fKWCo?}hn52`7F=|0vhFQ)11#>On;JW9=hz##F$mo16t0?7X^l4% zge)Lc_v2_3biCpG5s)~%Yp{NQ&=D&j7|gC>!dRRu2CeU7dq2&na_K7Ch32ApKh?c7$RIPUEUZR9e3bU{^P;Vl3 zd|Q2P#ld<@b`)>#rR$>>ASa zOAmO-K4D76j45<0TA5KDTR|zLlDx%f0QKtNik8n64ZiZqS){(HY;|s$r#}wfz($VvpmZ)de^){%K2UBMUBW{2_vHcD49eA-61$>_`dEyi z!yctV%VrLv#H3PCqM^I$M7qFI;;Z}5fgBw`+g&gz_DOTQRROMs8tu zoEsTYz1FsaOSszjyt{xA`%d=~iTVqV@FtD#4E}GAsm&?xFXcaa#TE9S^!@)CfzD=5 zu2#-wc4qc2|Ixmo8r%QjSNvZLoRkP?;475ju$VmL8>_}e1L&X=TfCt_0_zo+H8WhK zX<{`G{de)p88^<4AtxojuHAPf3HwV^nv}+I0&~nrt;>#3Qif4*)C+)HMI0?xFkFs8 z7h|**4p{n}UxP+us?8;DA3lE}h@=oak8W#+XAI&O2zGV7dic4z@8GC?Pg!1paC#a# z7FN4#Tt?I`K%{zgm(LR?pN3Rj`EXQ5>j!Vz6?uWQ?^qt}sa-j@{Yy5LA3JtC9VV3h zX1UY}^{hu8Q4y++N`ndB#j$dXnFy>?!pm}-Q1BNg|JQftz_+=rzm9 zZ9b`^1&bL~X6DH|6z^xGyEAUf4Ejzcy`NheRk8P%X=W%@McJytA0fu9^iqbC-Gt^E z3ZC!pF3^^^g_Z*fy|YjIonb@g7_KFwk`*O*4V{j81cM9nZE=&nf#~ z=XeL+QO?!Fm6@ert|4!@E-}3VIam`(Wn4j(yF>Z-3RdXNco1kI(Ay}oPlrIKexGE% z;iQQXwI*#51Bc;Lrk_VCy(@ilsM=6xiVv2g|3YHuthjhb-GVL^d*ec@F0?y{g}#~; zT{5_X93L_*zo)&=GHhm6#B^Mxd+nW$h9oP|y!TmMbTHPE(FWRB5;{xGu-pCTb4|p5()ktWO zm&l*%TZ`J|GH6x3)OfzQTE}BD^9516YAvphWCN>65QDdXpnDX>%ka=`P3KbLu(GN{ zDhQ&rqu}3_!zC_N3S_&B3eCP;G${Q`%1GaZ_wiNIhBanuOyU>S84A^BizRM7W2Qs* zb?mJ+f!>aE(G$kXDb<>j9f^ad71T)`-Npul9Z?jr?@l&)J&%*S+QgA&@`%jPJ z!+Sj52B8dJ|G*bcVS^}y{kSxuU-%9F9(yEGw)H0Vd}knWAhds}8c%RgybH2Qsx;i-B9YeB$z8>Z!N+wZhSnVmSyKHQtVi$9m5hg&jh%jwc? zFPjmO%vNY_Uh%Dao_`r93-%)S$n_5)`*IPC`P%MAl}Euud&STC9KO6(d@Sf+{gV(+ zlrNM4O5~mu#YdeIW#lCUIdKjzwhsAWXnI!e{XeT%fk(E2qD}(ev_xD2IUYCLPyrE< zjK%X^_09nho;aI|Q$ROyCF~x)FIp%i%G)a!0m208hyV4lasEd3MF7}~!b z%}lJ!t^W6K_7%U)Z`{#@-8Whd!*i&yz0BmJ32@3e%1s#s&Dg^zw9Xn!2rVE{$HHNT zhE5_dz`aAS-;}q-UEbl4^b{gWd-_$z_1X2=8D?zZDyJ3tuhNW&y`OcO@dCQ+s4kR7 z%amC$>}5`p3i0-`g8j^Ij}Ez#+g(n+Dneh%2^#wQ!h=&~pOv^oABn*ppQmOVRLR;9exYd>6D03P0ImEX7gAN~k!hgIFclg)xea>4 zMan~5sx)+BOk^3->USEa01kEH`h zlE1@+QXEO#&DF()4Rv+iQYXT>qW*m12QooW)Ws{m4tFlBWK|g3aB__H{ywQb`6UzC zt#AtAioh1tY5_7rvGC{f782(7hD7DWWcEf5R*z(&N1@#LzI_wbUm^Xz-JnN3bS(Zz z9hRv&4bfHKYQnFIfCwtsI3(V^iYJivFW?gNSJuMg|lQ(jXjuS^*owZf;lmHcF-Vo-|U7ACZ_pOO9P2$eT`aO|#Oqa!`kPFIdfFWav?*9ApA|(U?=2Y zC*@!#?CZ-l$Tz{W7yPqI$-Y~k^IWLk7$IC%@7+oIqcqRpL*E3 zD*qzKLU8*lu9;@QhK)gCz2H5{sarw%i5wv$cMu#N*#qk}uSApkdmHihOz6yrr_QOp zmJN|TzRz}#uXotvi#mJP`|%kn!oY^oJ@e=IZcO7&t}ZcB#neUfrJE)?@&I+{O@3IQ zKUD#p4^}Oi{m$_odGZ`CukB0L#>-#(<4CO-`CVce79jPWsB2LW0fZIdF&1h5Z~5=F zJIkWY`O@6d!d{axC2f3gKOGgf)|!3yfdpkT$6pX^xXO9-j7mUce!$P}G^yDY7jfAPc3XizLC#djLR@52&+Vb*jhj0|3XED#VLfI`Fgh;buSZ6T6^Qu*e}) zU<#47X4)tJNo^;8Xd!ItEJ22a9SR5@PcV8i>i`Mu-VAa;@hS{ooO$FD3o9S+${0hd zOVQvu#3!J#cFY4#h{Z-}0(K4Rii{&xNoeyGK}~KJ{P>QDcw!U{2JOeQ2@`e44>{e% z(bMg{vr>`Zv8Rp_eR6>3?-C+nsJmap+v<)e4!z>}F-`1QQG7$3x!(SjxG`?2t_s^?Esg8@H=Y^`SwF{_QaJWtBJ6>K}!$E$H4~$|QVRABR6Dw%BC1baZwa{#wyq=v6_cyF=!V$_0G`v|~rJ z^7@0%*YRg3WpSsQcyT4X{yKDb1Jlk8)^-%vFk9xhQ}p_1*zIncSLmVUa$K@&iYfEz zBAF5NpxrRq1rJrzUY7!9j52NfEKWubE7@;B)Nz#FYfN|JOO2$1If;jSPy;_>p$T+Z zS+eap)4r$CC1)i?Z4}rNMN!H00UXcAsQD?WvaukwkhQMDKYjmwsGs57j}hGm#kT-X zHTjwh#!)Ff+Z}Ml(ch8VW*@vpQUbiPE6ewiFpHQveY@{Ad5DqErx%bD#7*nV{f<{U zNr;OEy(Wh0tOMjfS4~RCZD&4S6|v0kAoIfD;Ts&ros+;eT=P(4m4A5ZK#he`i*f{C zYkal|#D*g499|Au4FmHG+H=3Q43*n|wzfwP_GyMwZ~)3^7QETo~M; zlj5S|@C?C8d!M93gSWpnOnB`KU}$VGYKa-o?Ea1TjTjpw<%Y zx!>ow^K9UzDO>l{tnK&wE!BxDS;(engU>2-CFZA^Cso?m`q1Qqk+GPd4@RT~ok+?@#)z`n6Y|(#I2Yy{S6pDOjfQ zQf4>r_uaUH)n;IM0xfgwI-E}jl|ZZSCPomC7Pjl6*3dke@=IIJMT~mygAohatm{+| zjwT6pDd#UR8?~V5rv(pp`ldbeyyp(oDD9p>!JS+euds3tYjbm$F#{DUAkgKyx*AqY zdBVl7)=g!c{6#>*OX{|MuW#Mf)|{3N{pfrp&RaFIXrqZ=HJ+L$*ZSYQeAep-%Fgl& zGxpa<)wbNNDvB~A!C&_0e@K2@>b30!e$_NwE{5(IHe4;yWf@mm)@1DJ_JNUnB z{`h>py`6x`Np2w7cWXxpC(j7S519d%rJc#?S@EMpH@7pVn2Z~fo7Kk)rW_b3M8ndH zp`%_M+~QSbp&|jh+-cWQXlKs$!HK;@TwsH=STDesi&b<+MQOz(f#Eo;;l9c%clCb| zj%*3D0o1K{7w6>*@S9k~n)2tsJymFts9;kCey?$!!o3(|=ifI%nFTV_vPaO)`O-2E zNCn9g9w5yW%JMC~yTCo2HJvX)ubnI&OcZ16CWVX23g-b-I--T?2bq&pl*s+o(k7fV zfWB4-f(tKKROkZ!B&$eV#YDo+7<%_JvDc(oQr1U{3slZoEnf{G@^&9@4=IU$yK!^* zy0{+FpX%~)**f&>h*58o04m=?uLsazUge01CeBY>4O>N(cj-l zQ7U*`9f2~%8vr(9O3Oou88D;m<7SL-PnoaC=+=Fsl_qMudqpJ5h{pvhGeev-_X^ni z-kV-*C9nCF#WrURBgEw}!?CS#9~F(9eBF@RlGTCjYhPo(jU;sg*lbAiTz7oK$8l?n6$V`b9@! zRfD()Al~K+6LoyA3zh4(IoIh=Q%i~7p&%axH6ECU%4L8hR+n(t=SP}+ zV0?GOxoKhZrq_hr7?E^MjZSUT;>9v4qj*0RRTNsg`eyg2JLz16jVKct3ST*Y+a!2d zIHzj3YHB77x!y}uIn3qq2YLTfAlP2JePw$j293B?tHCy>Z|ZAik1iaI#>!OBQ~2!S zl_YI<){yK9EGrGRnZbp<$836< z5_hEapCEJS*HCElTtQazzhs#-bV_{**A;2`Ic8%{NbnO6veOK>9W?zNq!hzKZXlT- zZ!l~BhB3&uw7l^ODfdZ|XOq5fcd18je$42PT?&F(&1 zUza#KJ7Za*7Rs3vqWSkKvPm*M=jIp7+PY6(HZ*U-Qy+I3VRV&ZrLVHPc3U>cN*e_6 zj6&LYRSMDqyw)}vxgk9wMtoDd>8_P>>HrT1RtcWH$Hp%{Ij-ND;5gXG0f=;|PfVA( zAKP0BIrZp>Gi*fWVUzA)kdJst=W>r@qeAABzb709voA+XP>PdY+x{ic-vB0Z*QqoG zo|_v*D@Q=cr#gqXS*7QW(4u%cAx%}AcSB>XMT zaz}FbU!-%s=^f)ybbF2Oe za}&bs_ITsbDPUV^R_KfMk*qm-y(whsjBIfn1@hA*s8`R~hfE$CnvcU4cj4FN>Qkv# z9HRHe=ZB6879BC4>90+OQ@S_iBxe`BCL`j}ZvBPDk8WurkS#EuW(Ip_SGL>p|Wp!$K5{3U{od(0EB1XqsdW7enb?Ey~AY zhL}L!D4Vn9o32H2leM5Gb6J&1Ql!qBSXNsx0i?to!-FmaZiFDqb_u-L%`^I8( z9_2wGsVD%LHERh3S3EHiyLIA2ZME=mO|^nPFJyf$TP0%LuE<&|Zu%9&WEW|x6R7kN z0}ZD}iQN8fYwjv&_f*Wzfu@C21kq-wzhv5n-juCnBt5LU(7!NnoHUS7L$P|%j|lt| z9=5qnpJtE22LH#~+CfiLK4Z+GlH`YTStt?ghfd~nA2Tncsv;j>XDkTW>jH`0G|)KI z#Rx62Fl@#+i>4}^;Lbu9O*v3n{wj0K=QB=KeAYRm<_bm6e6rImPSc`d{KB?I)>55F zE)uZ#N3Udh(|O@!$f-1kwc9OXc+zq@=e<;YTiDep;<*K0qif$k;Cbb&ozSav04n*) zJF?!n=W+o->5{No^Jy-2pu)=mA8_;yy`>i+vgVAf`L~U(6%o1_+KNLOicM-R&(tN= z!oOO7RakqD#cwnQEg0AGSr(j)pp?61Ly4FS?K`q>(%o!}Ir0tzq3&zIunLN(^W|_& za}eP=_AdylLBfx1hMGF*(B|JUQW6VrGDaNWaS0bSXbH(sHpH;G{Hm~RRZh{0TZ*#v z!_N~BcioMcU=iJkB>U4qIz#44KYKXnaFc8ejBxg$*Uq};!s{S9dv~{VC=(uPcP!^- zQ1Pe8c&WH*Hg5;;>bEOD<4`_~tSP-jI8M8aDr$SNg8*30Vq*Dj5vBY`=n7=SW^OIc zjr;T(zJ3ip{f+>0i$T@wY?bCOUjyqKcqy2hoJd3?Ai8kKAgHkujlA%KVI+hF(3On$ z<}2Wr)~xP}3g+mWt!I58Kf4Wjdx6T_VhhH@P#7CaVfm>Hy$@!$qvK-SVHff8ON_di z!uS$D)A{wNf;pc-2}XWdLI|m542?n7_0S+5#Wl1$^%U^v$n~_mB6*s>TtZ`xFx$D8 zAI!sMsu|Qa*d~$i6Ri9ApP&lhijbG?p7 zL$L=b%1&JS<;&GFC&d$XPYN0c-7PsK`tLnFolg@_Y!zrR)C^@6rBoITu0`RY;X}Zb zY`bWJ z1U3Pv%<^@|Pyi7%5Hth&3G_?;lw%!3?+t&{CB?D%lzn~y?MI&F0mwVfcMNeRKSlwxa#{vgCT?BUSU$Dcd%~}(1Cwy z{$tzXLuvm;xS!a=Qz4}_-RIY9k-O6mdI(dH#Y_Kq$^1(6x@_dIuxe;@UF!Ht#sqUd zNBP(sD)-}Mzs=O>%ef~*wa51#b_`MXD&XwEN>iNU6lZuRUyy>k9d7UT-wfn(2_hie zJx3taZq`OBuBw2EeOgxguvy+R+&tuaSdf3++eHNEP4aBx5<63IO&Jd-ev(cE5Ei#>l2^~yHHYi~& zf)+!n^)s#F&I}&|u#QQPnKmQ~Q)Goln6S0XV5V%q3D#IjV zS1WyZuy~~xO_UgL(SiR|mkeY?8MZHG)D*t@Q`0cXLWVi6M2b(Fz0a4$6!5PQ;2r>Y!t&N2)0ptE=PT|RV80Sk-ROH&Iih;h3B1rW7GoUnX!a+DrpDOL~Eg^k8 zR9hMlQ;O%>#$S!Gu$^-}iLT#{F~Rb@7F>c`II8M+Lb4OyR7EEdhhbSZ=_9k#Qgc}y zL)VpW10knfvY2z(R_>+z7EKf04o8u3nr}+Xxn?QXBetMD z>2Wtj+c!TxCItkNW~nFfHwn=e=VT`9QpUK~P$tU-s$2PHMx<^N0_vwl&`+}JCno5h z#LZ@bhDJSBCp%e`2zj2~a^IT01#z3?Z*>}aut9H1;#)$pbYJ=`ii@pTVFo;#vU@`s z3Q&xdmes=D>o#r({^aN0oQR5Qy1dnR z=t!F_}dq-2!vp8{#VY$N=2h23}5||D{YQI9@e7-b(fIqzzZW0%V?Zt$p z!SyK}N?(#rP}%knZNSa=0=%DtK?-gxp~=(c9lNfUv0y;WbkEA>IUa3oD4r5o*4&I5r^WXfWx z9Wj#r!!)1Nos!2GZ4G@jg_;P|wxF6f8$7+*17=KmX(*O+(vPLS)Vz)_HB6}tg}-jr ze=6ML^}tVzi!;UMl@+cL05c17mnI;6u0^Bz>z3chynHPtW-r=>3;bRA#ah5%x?iOB zm33Pn^`sAaFL4)#^-O4Ri^X@^Pq*;*Z#~y7HBte;gNhTD zsBnIPrFf3v_W&@^BxUt{}vE zs)Qm{>ne*myvjXkmiM*wu33*^PAgQMNTNFrH1WHZ-YZ#Ym{@$KeHsF3m%74@iKWd6 zPoWfk=)5%~)xdB;CH-T&m9C57&B*_r*GutkLGe;v62W8~XqgqGj-l$I=XlaF+pqr2 zmRv~N+a8@ZTWrtRk4vf!aLYLd1LasOTv3o9FuG+!KRr;@L2{ZmE&< zmitm%0~!~;vAFg`ix0y0jr&$c*?W`|nQ(!d-TOIl!&r>;S=rt8qlU7Uq%6@m&5xD) zLg3YPl)urUyG1iPE!AYa@1J{2rTqEx}IQ{fV{(!}WscGmkD2aaOlYJ}Y}7{NGd ze&M;E|B_^XcRD`A5KPT&X{L|++Axy$jMllMxM2?uv31`p=~dFj}>)B@nR#CBI z4W1esM$J4Oa+O(eEm{OC#d*=F|1f$WVjmAomHBMj-T(?W@EB~taAn84#xcg2-x;*K zNowrQ-XtJ;zP?&*3OINj!hLX-UiQ zzKO9dcevS8wDvu4yZ^IWwmTXQQnttT zXDBux1w5*^N`0y&_R}NCRP}K;UkPsh^Jq0HwD?fp%%oFjyEMo)?&u2TLevmnNwcL^ zzs(|YeFxB;M&nF-#~fCC*s}gofLu@q$Adst!Ma+RbfUx1e9>v$reMg96_&pb@gZ}P zY>_<{P=RL^I9S>Ej*=3=^0%atj@nFICZt(ZVS@?c_?GvLNqdRb-u+YkG|WeJ+(F8X zY}#}NB5K&LW~1u{bo)M??m7cfaGBlO6 zT;694Buyz?rZRQv2LyQr;JP7D#M6s$+mLq5Mnx>(DjGg|s7X-fjJXZ>H%s?9OVVcB zz=UM$p^RoO%lR$b|I~iQ7?LS*oUH@+%2O6y7llY2>Wmo`Yq6Ir$u7u8+38=Qz_K*J z&Mt7E*~ddsHT+565TxM5E$^1AVxS=h-cRv8@#{Qv7S?BVKlNFJ zf~NwwLBF3>TjoUbmfn%gs)-$}$2V*$g-+TL#>Mq2&#=J-nqZp(a&-Y$?2lOmdUMLl zJS@gbP*=Cw-5kttwkk(m8(~v!q0oHB73nh7xuVdl>*>(BN4^r;p^feZ&%jv!OZoZO zwYwM_cOs8m&!5!|YELfr_PT4;zzIVEXX(@a^xe4dt&64FH0+1QAG47Ow#UXt>sD0P zCWfTqj20fa3u zJ|BG1X~H;U9Dil4c5tcF-p*nab4-RAE18b#uMa3!T7J9Sc91!nIW7J6mP8$2gUG4K zLl=4dOz_zWj5M%Oh3^JBm0hl@=jhWA5`}Qg`=J>GyL(Mm!zwysl$Mf=M@ZDh3KESpRV>67PZ ztj?i~+#sUlUGghijVqouG38)l9rAVtj!KS=nMZ!lir)P7nG?Rd*w(IUrp`y9e4a;~ z8snL)fH-=3UI(_J)zEI*H~xy}&g+yda>=$9P2Z1Ux34I{mL<-0fRgubdK*Fp2nu3SvVtm-(d+8?ekT@UiZJD$hu*^#Y7{)ogZ9$Vr`%Chef^!xqjE5qii;eb4%Y4G~KA4a~su|u9+Y; zVaSA3$wy0*YC2ggmG*9tBZdyAb+!w>s%fUa^f2*RVd0rD?Wvu~YxCipQw&>=M;B!~ ze7G$(6@*@;UDTyj>3w-hSX!5Oph!4)cH({$`{TrxB3j{!AZA77-?#UmF;$8DtDClA z*OTYy{Dhf_Ba?jV9b7i8cL$&%>^nib=a4*Rp!%&lz1WWPTj#{<^#m>N@Q81qelU=X zdTuGClIj;FNp)}RUIzne2LW7s644px^n&4d<%`QK;vk#O9TS)y<;6LRY)TWb7OFGF z!nw~aaVBS61WlKq@uQ>T=iTG!XcA%E-mc^OU@4*EU|cw1Y{|z^CM(h$RIv+!sDmZ2 zzCx1chsRM2)~509Z^Pbn$osc0;PjInDMS!MJmc#K5?Nd+qicWWmmZ+^{wqT!C`#|Y z7YB=TB$LGA*n#icU>TTZCF2q42GI-mww#w%FhjAAVGp;Y$4k>GzoTDzGt zq2E^|#04D{ZuDZ~-as59)YObl)A+RGi`Zj*B}l;O&Sehrb45A)h6T*2(|iUIjiGF5 z0_ZL@<4srdq6abf_8_@l-3Nd3(*m7MA{=6P2qDP!4+C=wX?(&=cY!H@EP4)T5=lL( zBU}2oLhK7ztddBt!R6?SOXCB(V?+M*(bVr_?4Pq#7^;oaGL4Z@XOzjfArBRNCid@t z{sq@{nj7>0s~|G`q|qVO0E0+Q+w&I&ox9nPGKxh}r!+g_lTmfyK%Oilm)z*j^QV{Y z?vwt3;RIGwODppx&#Rq_q>p5RE!NlQQ-BeW2-np_iDuO!vcxST0jFnbn6eE1YQ%!g( zSI0=`96*y=;mplb;qX-gnkpUsFbzB+_JbMzVX~|s3>q0?njCp`7#U!Rhg#F%_!u}r z&(Ac|Uf4r{O%X+gzPNwfeG?+wC2SJiv)tbcv1w#bq%Fviq0q-S@)H@qTioWmLwGO;vgm?kfvK*236Sab8Lbe{uppj0Dd6OI8J)r&_{ z3`P6`YleWQFCH3qmJ74Ls+@40GzmP@75o^K)T6XRox~3^I99V1Rxs)(2sR;Xu_-)T zBqIci9waov6+M*+#jF>_Fv`$LFs^j+hp1vd= zYLGC~SOA0wD~*U^{00H=7E3wShS>@wyL68YL;S~u*N&7#yIR)@&w?`TlYwx;8xMm` z;&Ql;Wb?{x&nHE(R5A3|2f>HY;6E#OK9{@n6}@+%lQv;@i63eG+kLq#PDJ1m0x&$# zg+>QYX^>ic4|~8#!#X>5p+RFkS7At3MPn{fg32J8hS4lwsenzznA`LntLHGh28@XZ z(Oe*b7m?}J1h8CY$MTY<6#mhT_Ma9Of!m&mn0^?)4k0kC(7X*`!S4-N*z4WzeeHJ_ zRm6E&pJ`X7jZUuQJ)Tno8BwpJYv6ft$A?#^^O5`B)kn!teo3|=pS9xF=NI!G~bMHb(c5UjG4Cl5{ef=E%!|s(mJPRHvXBSO@O(q-K{H{FL1b6kl=h~)xV@UcnBDbuenJr>Baf$`JJyDe z|Qpuo!jgHi7mB=N@E*@ zwSlW1Q7N0L5+uz1M#}lT|9kuOq3D3OT)%F@)Hkq=8zpHkVl01J&4;TSz9vnZR<6oC zYYbI@B+Y60b`m%0AkfVZQ#$(NxslY8I1 znVWZKzGC(2b69Kt&MxU*)w`?qKTVuD<60I{mM7{b80Zi-Y1}GyzQMN<3`C0EW4VcE zZsI1W=MWV)&-{b-c78T95-I`$y1l_h?7@mz)vBeaxYlj=eGZqgkMbhKf*D|?+@-r) z+G6p7pI~3PpXT9+@_~bq)(g){qkWKYjLSyi6dIp4O5`jb>MQ1;fJ-;Oyx9>VMf;1> zYgc{8>I@?7?eyTRW)n0}^Fbt-xA6AMv2TW-s`IqV=kt-U%kGkvkgx#OCkn$1Iku8}rCNlElzpHmxNQ!x{vJd5-Da#ir#aOb zJ8VN_Rc+q88~drLY!7_0(2(RBM#d{zlQ?rs(eLE zS#I2V(MDWmh?^tfycx1PYGCYGFuxJVgss@lq+Nms**-0$9PXT4#(N2V-03nqRIJPp zNag7>&rrw{@;-y)DdpO8WN_9+sjj=911MdJeAak+|*B1q(Sd9q7+(K$cGcI?+%$EYHn6bRtI}>0*MY^1m z#-@xt*XlMs1knNHb1iaFq6HA%u}&~Q>il-uB+fzAx?JM3`qZ&m zU}+IAvlLJiUHDhl`2u4gx~cS1A(NWUI;&ry1ru(f94RM+x;g4Pw-H)+mLNNo_ChRJ zsZ=kYk0EU#N_-2`9xpj{LFOSx*_H>k&($OGl2zH#>jK0%5(vv^?B?_7cfOJ&q^_Ni zVs&*}v>y5Z@dT}^r9bcMbxn~W8vq@1$LV1R%#?ea(qqLO-JVdE1e~Pz^>4}eaAIFlR(hk3h(i>%kKrW11$sG_37m;1`JQRE_81lWNSsog z&33l_bd+kIPzId_E-1ywj0Z0&QiKqDqfZL82m|C&(AAjYozKOjVcEH4P621<5-&O zT+h1G^UEl%_5jC|c+ANCBAGA2HwcLY3KDEh%#QUGdE=3HD{+EZR!GMtF{gmG+~xB! zPxey9rr*Cpcw*c^Eg`#ZH2CFr#9W1K->r45Yi`5@m;2;DLDqb z(FRQKcZd0JNu2;;C+3&LtU#D24*_!i=4YCAQ! zM$51x$9jS9fG4Kx6zvf!y@%m!QMj`Hm0$73H??Wpgfh;Ah+r%;v^-dH zPiFa4ue<|}GS9fKIYbLR?m>ZF=|Kda=3x`QZzzK8g;C!iTx|Bkh9gX~q@Sb;3dzpC-h>obO!SzHTrKt@U5Z(u`cZ1BbwQrW{BJudV=c&pmAI za=>MBqgi6swjdfs%2xjnkHj}JL5OlXK_648wRNn4LRiO6o^8I( zu#S^5j302pI#&l*gp8me;;e9FgBMypbSrIV*l84pD_9a`{E8J})6xijg$$vtB5$pD$7NwmMfD5v>~_uMV;9r#v&-uK zw|<|2amKZ#kW+I^{R3zR>!j)Z?1c5(v(52bu949z>#RHl+3vGbKFe((8lLbwYGkk& zWOjU8S}cqJ^GJX;3&nB6D%hu!wQdBmPp{P3qYsoK?F<{u86N3)FbaKVxJv*kLzlC>JU25M?AmrBX9(vO!8S2jf5CEX$~4abU| zxQ3h-rp8imK`QiG=Li;ad>!0_^^#TXzfT|l}J4LMm^H7yu%OTc#B;G8Y5{R+j zq3L9$Qdg=nQ-l^WJG=JPs-uuDPPYm<9w&WSZ!awpS(C#bp&BYfibaBFf>@J&XBpKL z&y7$FA#kOJqI3Z#?59gQ4@nuT;Z0$n?hIk?OVfN5G1y+k^qIuNjN)aGJuZGI1xt_o6J ziU>Dh9!uWU^^Yy?tDzMbGSQ+1o1q)gtGJJ&hHEU1UWakL?)sitl2W%_Z52y8O`W*3 zSjev}bJYHD!PfD~nH-{0%7Pf>tX5j?6eq*Ng>;XvkdJR32hwb2zrzQ3s`Mne4I8HN zY-+qX=9W5}%8WYQRdloV(}vHJVzy4#24o{>`b{!2s~pj|S3Se|(n~}C%*v~huWtd1 z_UQOXv@oglnB`Y2rAiE8N2`D>>SbgN{UnFFht(m7>bkh+QZ*)}J1PXq4r^H zkR`>NI-Zs&n>~PRFJA)R!T)oEa?Ekw;3hZ_Py!$c9^pSlcb%Qgt^SpjN6p-JRRr

mq=(7Uo`kWm`Sg6|TK@fE0`I%guyOz>KMC!@ z3OloH$TE3ygi@?}8gTbyK>_CQ4CU?zWQBbwJ0gtFB$tH*RzI^U~|1RQ{K5GE&eq6RRCPZ$$e2cQpy*)vb`k!Rtn zKg6>P#s=-m??z(k=B09r4LCuS7!cpBoujByKSu9JF;fZcY4cw5%~S*-9Ynqee%Zob zbiO;F1h?E3i_7!yP?m5G#A);)E$j*nQ>$T;MC?wLV3;-Us(6z=fEFhe+&519tfr|0 zvpxHPz%x*M%r(lmki}^u5n5fof-3|K-3sbx{ScRv@7diF6yw-o6V+c1YZI}XF6T5} zH4;SKz?w`+fe1R~xbA0O+--kYqa_vLKEoGmDb_D?i7X=9=G<#PN7FOz-*Ikz!`>=8 zmL?g_MIq)>-(g3A8z zyttAfz7%pDiEdnY{_MEGSCVDG!LPQRWf8&yb|_5q+ugf{uGL5iZHh&0Gf5l)%T+}8 z4_)#1kPI^o`I7dJuSQfy8uN7E?S#p>6xBT6V!{@`24QLfKSb)o*X_2LFjsqycBNEI zNeXk{Xg{)uKsV^G?&nLG_vTjRA!66#up6_wh&7@CRK?o0Ovayf_RgsTzNVY;Y$ zL1fIUf8QnL;1WnUZgLK$T2rF z9KUsDRg##eE7?7c8h$4~%UUNpp)XVX4Xs0<@H6Qiv{-ywbhvb-@>C2^dO@uwgi094 zy%ID}hEO!^fqZvS1T;`SY5*6pC%Ms7ExE8*#D{Cpm)#E9=bB=V)g{l({-uRnzKAo- zXIDY?*O8euLAQ2nUB}a?6@m)&@@-Xr>#}mL25%-GP8QaJJ%(kK*xMd{*%FB|eXG`Y zc{wr7GJZ7UV32n?q=s_wR^mhMm@PLF{aUxdT-)qQLM=Np_6_uu^3HPITV z>Qv)$P22awUt9b=@TkEO7Cqwmji{B|ftIz!zZGWD6{azVGKNylUod#=?VeIg6|$kL zbT!+bQi)+9n1LJ(ju~sMlVz)B$1_C3XDu7^OCAMZ^*)}?uPzevHn-iqgXX}6oVjs{ zvAPbN*yv0m3wR!HAi0k@i6N-;|U$WbqNC-m3?=fQ}0st-o6Pv9%%-eHk=*bIhm zwkJ&RR^`CC6IRSvg5geZQo~%dTtMtzU46ZH!EX9G(xmNj=lLvt<6<*fs5>9AXQu50 zTQ0B7hr^!#4hO1LX+6j{x){aQ6XC;^S3pym?#_4Z0rlCO@ziqSI} zy=`@_XBu3KyK%<*%f<|gW%W3=Xkd$g>{kWZrvc9Gs#MQ#!-rvVFO(gN&9`Ey76So` zrE9k-630fdT9DH`ZsV2$wc;WS5K9q;x*ll-#)cot!o`^A#Yxa@s8JDIsLrKIQJVQ! z4$g69@0>?c6nofct{$uvC`EWw5b*f@_7q2WV^BMVJV0P}R?|eOQX9N*S zoky_%TCU02zQfZM14#Tbrs8p>CEeqkC~Qgllny;&^TQT=TgqVg_zxk5$4C_(eQ}&{ zrLfS-88NY>NwCpiEsCV^!@=peFPB&R@lU7ZgvuF(?YK?&u-*f+p+i0#zP9EcTftP@&>P`DLS>(B`x zq28v@emTXJMdp+KJVU7)dT7wtcY&YG3AqVFZ zKtcUTlG$&b5$N=CX3*Y;*9SMb!!6&mm`$Kv&7SI~5D&$zm%Jf6m^~rOmnAb1O-C2~ zDbs`RX2{{yz-q9X{0LX}#}Hr3@g1KLp{yKUC}BIj_}S=3IN;1`FLY2~#12iG$pVg1 zr8k{5OmOK|b<3;{)ri=16F2-uOrb7Qa1SEetriz~#NTst!38|sxMoRb{&eN?itZ1R?Ti6M9^ z`0G(HlxnziWTX#fZXlyGxFdB&%rwn2r7%%5@=mM!+{dssd!)tWD1E`WLR8r}zjoPa z5;Nc6l1qp*)Ued7R&VC)K@JCyCReqAdNz2ITTEmHnHK+C*;HxZE`Ln^;;`OcL1KGX zR9KK2#|G}4B=!aS#1O%87I63uyQ z{V19u=eSzXEU_07nw*bk?0F62hcn8bjAm4cQsrs*S&}5MJ`uyogHqV!EW0jYN@KZD z_r-xT1?c>UJ=j_}QR^=qamNa`1h+JDvveDiDEIV0r9dtwC! zLqe0Ev3|n}j@~)ig~k(Zm%?N>taTYWu3{eQ7{XH-5biE+6M3T!jyuYDQzmmgWHN?T zIDHif-r0~No6)heJ|d=qW24!js8^ZYE^Zr5C z(6h}HS@%@ns?AhxG)))t7wlHsp%2fz#FLoKqwT{7TgR0*V?)^_UwG6fo``)j1bA;a zyo0{D);14L4qnQK1Xc4-{N*saEOB0ODLs}Ez+}5m4s>O5# zG+hEIR+TV83zPNb2Av?R#(mhay?B)M>2yk-IR!1whysD41w8Gk*T6IMFNHd zKuS}0a0JD5uKmwKw_O-YF7e1>S|8S(z>rCH!jV6c@l<^yQlLl#2kv>+iBPx5s*DSj z2Fqp7fdGXoi^Hg@E`;o|{^Z5;siT9e@o0KDrWlX{x|6%7xLv)+7IilR$BRLytIza? z1KHCxbZW4jDyb2%%~%_VX{BO1$O}5Zrx!<#7y(gF1E3kzP>L6f8f7Z`3i3Prq_rv6dw8 zcu3|zE7(2c`%y+=J+nv(rxx>%q7cOBFT6nm>xuSurRDVDH99Pb8)1M2b&d0+2-J)@ zP$^RIohg0K9*iYAH!MLa7Eql))s02n4L}XWsZ|7y*VGGvl3-nlu=;XgWKvXm^l{%T z0V>mzAlH8i%%q}zhxz{b4iN=LcCqz*rqH7T$NdK(YJ;Nup^StO|9nJYvCEQd3D#$Z z;$u<4VUy3Q9O_I>k+~LhZ8Mm+U0vMJX`i9!dfn9@G&Mlz!V19&$ z>b8xXvRC(LnaP8GNaOOg5{vCTZ3UY6g(WNxIUCUy8zB-`pri-!vn>&ma^ubX!3{k> zwnkP4V8Gp91Hn}@T7(`N+j(| z>zbO=kW+5J`7C*)(}{pHoQACYFjaB<^GxWmLj1YAhLV!JC;PEkoq>{MK*)v zvCxrQDT02)xTMbQ+Hr5+=ZlqU`SbXrMWqu%YHij0I0iW?M7LLhsuyoo>tSE4mJtI- zm3RaHfI^ZkrkBCa*2`dJ0YPqKK}OH2RdB<(KX*KVR-i%dkKbsdol(S=eBcPA0c3U# zEa0bWo+|^4*#n&wArW4E{xI!xX!i}ajQuW2ko;x&qDUyyo%AP$(knHCn{s|v=s24* z9mRZsL=ooDhwYFWdxXa%$+3xOeS#`o2%6-Sp}i_tRw%nNN5OH(#|-#y~{R)g|{V z8o?1Xtenkzj) ze$XeCFD!nske~lZx|&Yc-gI`cmmD{IG%-eY;%vY??h7{8iOQsl;TYaiYUrB|4V5cy z;aEZp?DoPju0QX@t`~CFmydC_n=dV%M=I*NY$DRwQ03w?UxVt;*33H&Go=6o%x$Lm ztdDvMQQcMJp?4?BxB9SL-fi;0y5wgH`R-hUDXNfQYeJ8gS)gAR4wT7n|4~u-u{<`^ zwHRoP;qBUCmy~~fgbw|M*&xv1{`A4!IRO$LJ20%YET{LWMgR0kj5)!uX4KW2?HG%HH84FuDtt6D~iM69*(&Pt#lJMC#@i zo@x3e<;B)f!@hg8tzYFlYcn0+5vE3}hF2gv6;_A%ipr2_%NG*svz%lvER7=MrS4uA zwZ+Szjs!Vs=DX)MB%78G63)}*xP#p5Svplk;y5ll25A{Wy_ZDtD9E20BhB}zE`tx+ zWZUl40h@-h^=x2tE?GFcRnF*nrGk=s%jqVi36;^%4n^MG#VmgAdFQ{Et$Vd$WOx6} z+a8A}q>oJbAsI>W%Um zJr9~O4^(!Z%iwl+(2_%p(E6gbrl!;oqwAMmoHun;K|uSha1|43xS<|>b>S>y*3{8L z>4ruHpQ3Ms zXr4)JN3h|D%(n+g%ulpylVRC(#WaufscTy9lLn=SD_0-t&r%L*=(Mpjs~G93tG_HR zrXZQ*!M`^s)S)%HE#^2Pmdc@MdhzH6`i z7aum4a`4RrX+uZ;bc?qmE{rmlT)cyIW!rS?4>Jj^Wk7-1_S6*0weFStGkb-Uc+NY@ zUO@cz6Sqh?_1_RDVV(CV>;|+tjHvB9bz&DZf*!DL%A~M>4TTG} z%ix|&)Xht>!j|7=95S^5GBtDPVhL`MLl?dt$N~R^Q;>*0Md~kr!(vPwCX)q4DXg4A z>ab=i&83H@oqer^fR66@9ajHqbEMm5S-kDa1Voay2VuJqVenJes>X(LG6AMdEa(`8 zkW2l}$O$V$u}$&g`H30k62Tw^%c+xF)xlTpImC3tLNPeOKvnMBGdG(O?-gtoX^KfU z%N8{C{xA-32luD^6z=&KL#fKyj88QkkTxIKphG+pGs_P?u9Y{O+J8BM#bOZ5IxY#DLVXrENA5A zTVAzzDnu$HYDWdQo!wTI_E>V`1SSC!nX1Yfjk3#m32wYWhuf^b2887Fu z*PErCFUonf|2S-8^?-G`jlg;6U#Ix!gZ((5@573uQKp75=X6^JV@*7f~`E|-)<{6=SLYD1(a@Q1z}WUqZVQzl`tiEPVZ;9`j@;>F>0DF7#RY~3BpM^Wp3sOMtIY0Cm1s5 z*M=5Sb8VC~1KG60i%KQ_QVcC1oQ~ z!N%sNLZ!16{z%=Kw}?sDZgS8s@1eN+Y`fxIu0`9o%0iy)LTMw?S;S7d&eq6Y z7EuQ;XHZ=W+j^}zKGm>!=yAMQ@V;_VQpta8p*5SI8zjTvbTYuhsCuL~GI-w)k6|=# zLRryoI%rz0ix3FCjK?iNQUMdo(a@7WUGho6t94TeMyIe3XLXTA%yNEXSTsI`^w!L$e*td051&0#<7tRMFFo3%mBy# z3LVih(6KPnG0+*AJ37&t+nCtW1G1Vrxa->4n*Yjf>g47WAu};aJuxXBMK41)LPJ3r zy+<`ECOuvY7Vk04nX1wfmg=shHBFh?Pg+Qt%GvKR{M6DC=1%#!P@JIK%?M76NJY$w zj_x;!0jy5!otO_mK#G8TtAH$s|6Zv6e|?b>7ZjFJ6#nIgwt^;LVJj*?FOy?LkviAv zWQRrCv5Rs6l3<5rG#`Y_jfkUXBJb@5m;0zyTA!yzqpE=$+c5r>DT zA?EV-Y8J)Fg2^LjE!xMAejRbbw%y^aV66F6R+nEIx54GQo>S|4CN5!Pmh1qhAvps) zIPXa>HI6B(`#R~=q(n%l8g5@v3~6cP4Z2c&|D7(`)SLF;M(*GUZ5Ppt%nV0p(YL?QSWJf*{si zVs<6C^x9LG3wYzhYahkbVlZ2BUVlvSPo;JZyA&Z5M>kC*_nNK6el}gB#inVMi_|9v z_4cY^=HQJ6?#?+sywlc3ZYLV$1U6sGhWI$ih)X1??9=Yv54K zqg7jffF-{KRkYB6?6)F+&MgwScG8pszEhb~?v3RWT#8G!^rx}vm1q%CsQ~uuhGF5P z4L^*sk#C25Ca64a-Gr|sKXZqniA4^RY`z%CyLm)sv=8!-@_LgrOUffadsO)WXrusLg^VN~I*azJQdXb7kpP(lU%3BvAR9Sl`cV$)6g_*@IH9!r23>SG+DIK zjw)*HM)~6agLnd*{!2zXj)M=uirP!8U>nRKNG+U?7{n2f5Act~csqBfYPIWQO~1jZ z#_p9B(W!J_emN3f1Ejc^2X@X#s_m&BD(8v`*xl%NuH>7tS@7HAh&eUkpK*S6e=0 z^D`OcqoEzAophr})Khvalj;hLQJ>{5P>Z%f9ex!6|3cczVc%+Q=_Q3^g}L|gCd_T8 zCZj}?U!Yh_z)L}6env3`o-}(x`JM0GzC|oJ1++vArwKiQEkC)4J#N1$V1|p1sFA`A;=#3&j$&mmUt8oPX__aiwBjOiWPT{g2=r^Dn)k+ zaF3m~ydNdFOlxX~%P!E?-^$E0;fhhH)Y5tkbh}8?3?Q{a-TEpx&%F%a6q1iX# zrm{0C;s-M9^{8qL*9(mNHiVs+eD@U!!j9i=0_aWvRW*R1VLL#^2lwx90t&){vI;_# z3oS|IV#q*potb_$gZDS-k%-z43u27?TEi3>4(q4uA>V@*T7^^*cYEJr6= z{fpjnTWCO|BtehsByp&uDj{iS<8NS(6ROK6bp^ATM|mvM+$z%zrbJzr#Z z_1yr|{mM!$a$vq}rS6KN!U;_>7FZ`$NmuW>BH3s;>Ed=r5JOOZDnccEzKdD_RVS%C z7+cOtbAtYWXi|S$Hy*yDaLs75-ky}{*&cYwKI0K~8q zv0Uu7iVsLN_HB4WNu3q)`Lf0Yq2)h5`@cT>h)d(JPFs8boW8`A_s^$G3!lDQNR0|? z&Ofiw(T!z>MUe!ZMt#AQRoZ)B(D(af62;jz@$rIFTHCzUgXd_jM$Eh?@iJHH$a(lQ z*MAbxUtW{eA|2vz@J!xZ{($nuGh?ho{3=Hxrep=Q*GbCx_+E`vwaSZO`8?n=jXxlP z7EHbaF;oUlDWh*{*ZFiV_9Qep^VE{9xD%q}hBm>;W9lRI{Uw6HqM{_D3 zt|d=oKMGVgeeX7A;Wh_f-|>z_d{}RjqbF?cvKWO%icw-l_P@&)I+EmlON`DcjKgma zYpNGXlBfu>i+rX$<+56!vMn<1O}|CV9`ftB#_PeO%Q7{wXdh^8S{>h?$7RZ^lt|-{ z1p}_FhI$Lj4mqZN7lIftuN@P>(?;p5@!#9^mNzkQflc)G|xHaSiNCpfxZ=9Esb2dZpIGS1npQLVJ z(=OD`kLId1-CR`*M_HZ1HOADyvQl4WcL~6IeQP}Jvaj*xaxlMP_nD6dN-n0O?29M# zvq&&u7b)6BdvFK)SK};Ll1VBHsv{2kcznQy#SOhCdFOr0w2TQ|$G3_fVA?Xt=*mEV zl*8Nptm~Sjv34Wr1BeOI^K(efe!AUDrltq3^%#e1+)d8E&|1BE)+;JkKeW!MEdhshWRr|Ke^R%#H0Z7EUSnn6Fyi7bCnKt&xf zoaolIp+9k{2wKLY_^b#GsqBu{=8zB zlLY>N3IbR(1_TKdmu{(c5w@kx1^C7dIFJCxAEm(m*C$sqV`D3NU0rhB#oSuo+~&WUD`)XLc>zq10gr#b=E{GR0{?WtfB)y#9gUrw z?VN0FtsHf&Y)wtgZA|}*BVReJq)jVB8q=EprMudKMe*F#)1U|SRkNH zlHdD4RQRt5Lt7ibx(lcOQVANT#6eF$7yutY`Oo1L3h38Aov_n4wEQpTpXH!5O96FA z0W`pWnlq{V)%<^Uy#GF!J;TzAkpM0K6EI>>{fQzw>k*$NV(LXh$6q{hquRa8H1rX*h!2gs2|B(O7`8V)aSmgKm{IdGP z9YB}-Zz=Gf{J)rARnq@;as88H2C!gN>wy000rby*5>a&jPX+%%G`2GNn_&lhROm_r z80t=>|JJ_v!^r}FCZm5MTI<`mTbbK9yZx78@QGs}qY>bx17O1YOCb7>QsAFw|Bp;( zN2|YM292{UcmggjOMn^>@gG3I8E~g^`6s5GzJuex^rwICoZqF!|6msQtNQ&D@IQpd ze~11qjr}L|)&HNMe-+C9o%y>0=$}mUuzz6wsR#Ny@ON#cKY@A){{Z|=Y3X!J zc>62xzr}?A1b#mHAK?Fu%Kjc9{wws?{NnfE?w`=HzlZ)M)caT7znuIny7nh8{q%30 z{8y~)chc{%u|G-VXMaojBU1J|?DuEvKVdXie+&Ef6Stfs7+@Vgpl*P_IKUl7^x@aH F{|5~6n|uHO literal 0 HcmV?d00001 From 1d95451be1f3080904c00cc4c4a6cc519efdf321 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 17:13:01 +0100 Subject: [PATCH 07/82] gh-63207: Use GetSystemTimePreciseAsFileTime() in time.time() (#116822) --- Doc/whatsnew/3.13.rst | 6 +++ ...4-03-14-17-21-25.gh-issue-63207.LV16SL.rst | 4 ++ Python/pytime.c | 38 +++++++++++-------- 3 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a48db949d8f401..b665e6f1c85915 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -570,6 +570,12 @@ time instead of the ``GetTickCount64()`` clock which has a resolution of 15.6 ms. (Contributed by Victor Stinner in :gh:`88494`.) +* On Windows, :func:`time.time()` now uses the + ``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better + than 1 us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a + resolution of 15.6 ms. + (Contributed by Victor Stinner in :gh:`63207`.) + tkinter ------- diff --git a/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst b/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst new file mode 100644 index 00000000000000..1f77555d5e7d31 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-17-21-25.gh-issue-63207.LV16SL.rst @@ -0,0 +1,4 @@ +On Windows, :func:`time.time()` now uses the +``GetSystemTimePreciseAsFileTime()`` clock to have a resolution better than 1 +us, instead of the ``GetSystemTimeAsFileTime()`` clock which has a resolution +of 15.6 ms. Patch by Victor Stinner. diff --git a/Python/pytime.c b/Python/pytime.c index 45be6a3dbd3341..d5b38047b6db31 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -55,6 +55,14 @@ #endif +#ifdef MS_WINDOWS +static _PyTimeFraction py_qpc_base = {0, 0}; + +// Forward declaration +static int py_win_perf_counter_frequency(_PyTimeFraction *base, int raise_exc); +#endif + + static PyTime_t _PyTime_GCD(PyTime_t x, PyTime_t y) { @@ -895,7 +903,7 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) FILETIME system_time; ULARGE_INTEGER large; - GetSystemTimeAsFileTime(&system_time); + GetSystemTimePreciseAsFileTime(&system_time); large.u.LowPart = system_time.dwLowDateTime; large.u.HighPart = system_time.dwHighDateTime; /* 11,644,473,600,000,000,000: number of nanoseconds between @@ -904,18 +912,17 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) PyTime_t ns = large.QuadPart * 100 - 11644473600000000000; *tp = ns; if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; + // GetSystemTimePreciseAsFileTime() is implemented using + // QueryPerformanceCounter() internally. + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { + return -1; + } + } - info->implementation = "GetSystemTimeAsFileTime()"; + info->implementation = "GetSystemTimePreciseAsFileTime()"; info->monotonic = 0; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->adjustable = 1; } @@ -1063,16 +1070,15 @@ py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); - static _PyTimeFraction base = {0, 0}; - if (base.denom == 0) { - if (py_win_perf_counter_frequency(&base, raise_exc) < 0) { + if (py_qpc_base.denom == 0) { + if (py_win_perf_counter_frequency(&py_qpc_base, raise_exc) < 0) { return -1; } } if (info) { info->implementation = "QueryPerformanceCounter()"; - info->resolution = _PyTimeFraction_Resolution(&base); + info->resolution = _PyTimeFraction_Resolution(&py_qpc_base); info->monotonic = 1; info->adjustable = 0; } @@ -1088,7 +1094,7 @@ py_get_win_perf_counter(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) "LONGLONG is larger than PyTime_t"); ticks = (PyTime_t)ticksll; - *tp = _PyTimeFraction_Mul(ticks, &base); + *tp = _PyTimeFraction_Mul(ticks, &py_qpc_base); return 0; } #endif // MS_WINDOWS From f139d840fb543f8357ce9fc8f845c4e945a0ce85 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 17:14:42 +0100 Subject: [PATCH 08/82] gh-116869: Fix redefinition of the _PyOptimizerObject type (#116963) Defining a type twice is a C11 feature and so makes the C API incompatible with C99. Fix the issue by only defining the type once. Example of warning (treated as an error): In file included from Include/Python.h:122: Include/cpython/optimizer.h:77:3: error: redefinition of typedef '_PyOptimizerObject' is a C11 feature [-Werror,-Wtypedef-redefinition] } _PyOptimizerObject; ^ build/Include/cpython/optimizer.h:60:35: note: previous definition is here typedef struct _PyOptimizerObject _PyOptimizerObject; ^ --- Include/cpython/optimizer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 6d7b8bc3c1433a..df83e6d16a429d 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -65,7 +65,7 @@ typedef int (*optimize_func)( _Py_CODEUNIT *instr, _PyExecutorObject **exec_ptr, int curr_stackentries); -typedef struct _PyOptimizerObject { +struct _PyOptimizerObject { PyObject_HEAD optimize_func optimize; /* These thresholds are treated as signed so do not exceed INT16_MAX @@ -74,7 +74,7 @@ typedef struct _PyOptimizerObject { uint16_t side_threshold; uint16_t backedge_threshold; /* Data needed by the optimizer goes here, but is opaque to the VM */ -} _PyOptimizerObject; +}; /** Test support **/ typedef struct { From 7707b14489644073ab0153f5751c6ddbf3fc6f91 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 18 Mar 2024 17:15:29 +0100 Subject: [PATCH 09/82] gh-115258: Fix hanging tests for threading queue shutdown (#115940) This reinstates `test_shutdown_immediate_all_methods_in_many_threads` and improves `test_shutdown_all_methods_in_many_threads`. --- Lib/test/test_queue.py | 132 ++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index ad31ba1af03b6f..c4d10110132393 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -317,97 +317,107 @@ def test_shutdown_all_methods_in_one_thread(self): def test_shutdown_immediate_all_methods_in_one_thread(self): return self._shutdown_all_methods_in_one_thread(True) - def _write_msg_thread(self, q, n, results, delay, - i_when_exec_shutdown, - event_start, event_end): - event_start.wait() - for i in range(1, n+1): + def _write_msg_thread(self, q, n, results, + i_when_exec_shutdown, event_shutdown, + barrier_start): + # All `write_msg_threads` + # put several items into the queue. + for i in range(0, i_when_exec_shutdown//2): + q.put((i, 'LOYD')) + # Wait for the barrier to be complete. + barrier_start.wait() + + for i in range(i_when_exec_shutdown//2, n): try: q.put((i, "YDLO")) - results.append(True) except self.queue.ShutDown: results.append(False) - # triggers shutdown of queue - if i == i_when_exec_shutdown: - event_end.set() - time.sleep(delay) - # end of all puts - q.join() + break - def _read_msg_thread(self, q, nb, results, delay, event_start): - event_start.wait() - block = True - while nb: - time.sleep(delay) + # Trigger queue shutdown. + if i == i_when_exec_shutdown: + # Only one thread should call shutdown(). + if not event_shutdown.is_set(): + event_shutdown.set() + results.append(True) + + def _read_msg_thread(self, q, results, barrier_start): + # Get at least one item. + q.get(True) + q.task_done() + # Wait for the barrier to be complete. + barrier_start.wait() + while True: try: - # Get at least one message - q.get(block) - block = False + q.get(False) q.task_done() - results.append(True) - nb -= 1 except self.queue.ShutDown: - results.append(False) - nb -= 1 + results.append(True) + break except self.queue.Empty: pass - q.join() - def _shutdown_thread(self, q, event_end, immediate): + def _shutdown_thread(self, q, results, event_end, immediate): event_end.wait() q.shutdown(immediate) - q.join() + results.append(q.qsize() == 0) - def _join_thread(self, q, delay, event_start): - event_start.wait() - time.sleep(delay) + def _join_thread(self, q, barrier_start): + # Wait for the barrier to be complete. + barrier_start.wait() q.join() def _shutdown_all_methods_in_many_threads(self, immediate): + # Run a 'multi-producers/consumers queue' use case, + # with enough items into the queue. + # When shutdown, all running threads will be joined. q = self.type2test() ps = [] - ev_start = threading.Event() - ev_exec_shutdown = threading.Event() res_puts = [] res_gets = [] - delay = 1e-4 - read_process = 4 - nb_msgs = read_process * 16 - nb_msgs_r = nb_msgs // read_process - when_exec_shutdown = nb_msgs // 2 - lprocs = ( - (self._write_msg_thread, 1, (q, nb_msgs, res_puts, delay, - when_exec_shutdown, - ev_start, ev_exec_shutdown)), - (self._read_msg_thread, read_process, (q, nb_msgs_r, - res_gets, delay*2, - ev_start)), - (self._join_thread, 2, (q, delay*2, ev_start)), - (self._shutdown_thread, 1, (q, ev_exec_shutdown, immediate)), - ) - # start all threds + res_shutdown = [] + write_threads = 4 + read_threads = 6 + join_threads = 2 + nb_msgs = 1024*64 + nb_msgs_w = nb_msgs // write_threads + when_exec_shutdown = nb_msgs_w // 2 + # Use of a Barrier to ensure that + # - all write threads put all their items into the queue, + # - all read thread get at least one item from the queue, + # and keep on running until shutdown. + # The join thread is started only when shutdown is immediate. + nparties = write_threads + read_threads + if immediate: + nparties += join_threads + barrier_start = threading.Barrier(nparties) + ev_exec_shutdown = threading.Event() + lprocs = [ + (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, + when_exec_shutdown, ev_exec_shutdown, + barrier_start)), + (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), + (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), + ] + if immediate: + lprocs.append((self._join_thread, join_threads, (q, barrier_start))) + # start all threads. for func, n, args in lprocs: for i in range(n): ps.append(threading.Thread(target=func, args=args)) ps[-1].start() - # set event in order to run q.shutdown() - ev_start.set() - - if not immediate: - assert(len(res_gets) == len(res_puts)) - assert(res_gets.count(True) == res_puts.count(True)) - else: - assert(len(res_gets) <= len(res_puts)) - assert(res_gets.count(True) <= res_puts.count(True)) - - for thread in ps[1:]: + for thread in ps: thread.join() - @unittest.skip("test times out (gh-115258)") + self.assertTrue(True in res_puts) + self.assertEqual(res_gets.count(True), read_threads) + if immediate: + self.assertListEqual(res_shutdown, [True]) + self.assertTrue(q.empty()) + def test_shutdown_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(False) - @unittest.skip("test times out (gh-115258)") def test_shutdown_immediate_all_methods_in_many_threads(self): return self._shutdown_all_methods_in_many_threads(True) From 849e0716d378d6f9f724d1b3c386f6613d52a49d Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 18 Mar 2024 12:07:25 -0500 Subject: [PATCH 10/82] gh-115119: Switch Windows build to mpdecimal external (GH-115182) This includes adding what should be a relatively temporary `Modules/_decimal/windows/mpdecimal.h` shim to choose between `mpdecimal32vc.h` or `mpdecimal64vc.h` based on which of `CONFIG_64` or `CONFIG_32` is defined. --- ...-02-08-14-48-15.gh-issue-115119.qMt32O.rst | 3 + Misc/externals.spdx.json | 22 ++++++ Modules/_decimal/windows/mpdecimal.h | 17 +++++ PCbuild/_decimal.vcxproj | 68 ++++++++++--------- PCbuild/_decimal.vcxproj.filters | 68 ++++++++++--------- PCbuild/get_externals.bat | 1 + PCbuild/python.props | 1 + 7 files changed, 117 insertions(+), 63 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst create mode 100644 Modules/_decimal/windows/mpdecimal.h diff --git a/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst new file mode 100644 index 00000000000000..f95fed1084cf4f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-02-08-14-48-15.gh-issue-115119.qMt32O.rst @@ -0,0 +1,3 @@ +Switched from vendored ``libmpdecimal`` code to a separately-hosted external +package in the ``cpython-source-deps`` repository when building the +``_decimal`` module. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 2acfccbb004d6b..6df6401835c6f1 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -43,6 +43,28 @@ "primaryPackagePurpose": "SOURCE", "versionInfo": "3.4.4" }, + { + "SPDXID": "SPDXRef-PACKAGE-mpdecimal", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "93118043651ffa33dcaaab445bae4f8929fca25d2d749079b78e97f220c3d8b1" + } + ], + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-2.5.1.tar.gz", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceType": "cpe23Type" + } + ], + "licenseConcluded": "NOASSERTION", + "name": "mpdecimal", + "originator": "Organization: bytereef.org", + "primaryPackagePurpose": "SOURCE", + "versionInfo": "2.5.1" + }, { "SPDXID": "SPDXRef-PACKAGE-openssl", "checksums": [ diff --git a/Modules/_decimal/windows/mpdecimal.h b/Modules/_decimal/windows/mpdecimal.h new file mode 100644 index 00000000000000..77bc6229fbc119 --- /dev/null +++ b/Modules/_decimal/windows/mpdecimal.h @@ -0,0 +1,17 @@ +/* Windows mpdecimal.h shim + * + * Generally, the mpdecimal library build will copy the correct header into + * place named "mpdecimal.h", but since we're building it ourselves directly + * into _decimal.pyd, we need to pick the right one. + * + * */ + +#if defined(_MSC_VER) + #if defined(CONFIG_64) + #include + #elif defined(CONFIG_32) + #include + #else + #error "Unknown configuration!" + #endif +#endif diff --git a/PCbuild/_decimal.vcxproj b/PCbuild/_decimal.vcxproj index 490d7df87eb1c6..ee7421484b5312 100644 --- a/PCbuild/_decimal.vcxproj +++ b/PCbuild/_decimal.vcxproj @@ -93,51 +93,55 @@ - _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + BUILD_LIBMPDEC;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) CONFIG_32;PPRO;MASM;%(PreprocessorDefinitions) CONFIG_32;ANSI;%(PreprocessorDefinitions) CONFIG_64;ANSI;%(PreprocessorDefinitions) CONFIG_64;MASM;%(PreprocessorDefinitions) - ..\Modules\_decimal;..\Modules\_decimal\libmpdec;%(AdditionalIncludeDirectories) + ..\Modules\_decimal;..\Modules\_decimal\windows;$(mpdecimalDir)\libmpdec;%(AdditionalIncludeDirectories) - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + true true true diff --git a/PCbuild/_decimal.vcxproj.filters b/PCbuild/_decimal.vcxproj.filters index 0cbd3d0736c241..e4bdb64ec1fb9f 100644 --- a/PCbuild/_decimal.vcxproj.filters +++ b/PCbuild/_decimal.vcxproj.filters @@ -21,49 +21,55 @@ Header Files - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + Header Files\libmpdec - + + Header Files\libmpdec + + + Header Files\libmpdec + + Header Files\libmpdec @@ -71,46 +77,46 @@ Source Files - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec - + Source Files\libmpdec @@ -120,8 +126,8 @@ - + Source Files\libmpdec - \ No newline at end of file + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 60ce12b725e233..f5b7e114c98ded 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,6 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 +set libraries=%libraries% mpdecimal-2.5.1 set libraries=%libraries% sqlite-3.45.1.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 diff --git a/PCbuild/python.props b/PCbuild/python.props index e21f1f60464bc8..a8d08073fbd11e 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,6 +74,7 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include + $(ExternalsDir)\mpdecimal-2.5.1\ $(ExternalsDir)openssl-3.0.13\ $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include From 76d086890790f1bfbe05d12e02cadb539db5b0b1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:08:43 -0700 Subject: [PATCH 11/82] Cleanup tier2 debug output (#116920) Various tweaks, including a slight refactor of the special cases for `_PUSH_FRAME`/`_POP_FRAME` to show the actual operand emitted. --- Python/ceval.c | 9 ++++++-- Python/optimizer.c | 43 ++++++++++++++++++++++-------------- Python/optimizer_analysis.c | 18 +++++++++++---- Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 1 + 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index f817f288903694..9dbcd3da3fec27 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,7 +1017,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uopcode = next_uop->opcode; #ifdef Py_DEBUG if (lltrace >= 3) { - printf("%4d uop: ", (int)(next_uop - (current_executor == NULL ? next_uop : current_executor->trace))); + if (next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT) { + printf("%4d uop: ", 0); + } + else { + printf("%4d uop: ", (int)(next_uop - current_executor->trace)); + } _PyUOpPrint(next_uop); printf(" stack_level=%d\n", (int)(stack_pointer - _PyFrame_Stackbase(frame))); @@ -1113,7 +1118,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyUOpPrint(&next_uop[-1]); printf(", exit %u, temp %d, target %d -> %s]\n", exit_index, exit->temperature, exit->target, - _PyOpcode_OpName[frame->instr_ptr->op.code]); + _PyOpcode_OpName[_PyCode_CODE(_PyFrame_GetCode(frame))[exit->target].op.code]); } #endif Py_INCREF(exit->executor); diff --git a/Python/optimizer.c b/Python/optimizer.c index 88c45f2e73c682..a6d6ffe5378636 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -313,7 +313,7 @@ _PyUOpPrint(const _PyUOpInstruction *uop) else { printf("%s", name); } - printf(" (%d, target=%d, operand=%" PRIx64 ")", + printf(" (%d, target=%d, operand=%#" PRIx64 ")", uop->oparg, uop->target, (uint64_t)uop->operand); @@ -528,7 +528,7 @@ translate_bytecode_to_trace( } #endif - DPRINTF(4, + DPRINTF(2, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), @@ -546,7 +546,7 @@ translate_bytecode_to_trace( uint32_t oparg = instr->op.arg; uint32_t extended = 0; - DPRINTF(3, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); if (opcode == ENTER_EXECUTOR) { assert(oparg < 256); @@ -606,7 +606,7 @@ translate_bytecode_to_trace( confidence = confidence * (18 - bitcount) / 20; } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", + DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", target, _PyOpcode_OpName[opcode], oparg, counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); if (confidence < CONFIDENCE_CUTOFF) { @@ -617,7 +617,7 @@ translate_bytecode_to_trace( _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *target_instr = next_instr + oparg; if (jump_likely) { - DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n", + DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); instr = target_instr; ADD_TO_TRACE(uopcode, max_length, 0, INSTR_IP(next_instr, code)); @@ -716,12 +716,12 @@ translate_bytecode_to_trace( expansion->uops[i].offset); Py_FatalError("garbled expansion"); } - ADD_TO_TRACE(uop, oparg, operand, target); + if (uop == _POP_FRAME) { TRACE_STACK_POP(); /* Set the operand to the function object returned to, * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)func; + ADD_TO_TRACE(uop, oparg, (uintptr_t)func, target); DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -730,6 +730,7 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + if (uop == _PUSH_FRAME) { assert(i + 1 == nuops); int func_version_offset = @@ -738,7 +739,7 @@ translate_bytecode_to_trace( + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); - DPRINTF(3, "Function object: %p\n", func); + DPRINTF(2, "Function: version=%#x; object=%p\n", (int)func_version, new_func); if (new_func != NULL) { PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); if (new_code == code) { @@ -748,6 +749,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(new_code->co_filename), new_code->co_firstlineno); OPT_STAT_INC(recursive_call); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -756,6 +758,7 @@ translate_bytecode_to_trace( // Perhaps it may happen again, so don't bother tracing. // TODO: Reason about this -- is it better to bail or not? DPRINTF(2, "Bailing because co_version != func_version\n"); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -763,9 +766,9 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's code object, - * to assist optimization passes */ - trace[trace_length-1].operand = (uintptr_t)new_func; + /* Set the operand to the callee's function object, + * to assist optimization passes */ + ADD_TO_TRACE(uop, oparg, (uintptr_t)new_func, target); code = new_code; func = new_func; instr = _PyCode_CODE(code); @@ -777,9 +780,14 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } + DPRINTF(2, "Bail, new_func == NULL\n"); + ADD_TO_TRACE(uop, oparg, operand, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } + + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } break; } @@ -803,17 +811,18 @@ translate_bytecode_to_trace( // Skip short traces like _SET_IP, LOAD_FAST, _SET_IP, _EXIT_TRACE if (progress_needed || trace_length < 5) { OPT_STAT_INC(trace_too_short); - DPRINTF(4, - "No trace for %s (%s:%d) at byte offset %d\n", + DPRINTF(2, + "No trace for %s (%s:%d) at byte offset %d (%s)\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + 2 * INSTR_IP(initial_instr, code), + progress_needed ? "no progress" : "too short"); return 0; } ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); DPRINTF(1, - "Created a trace for %s (%s:%d) at byte offset %d -- length %d\n", + "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, @@ -938,6 +947,8 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende assert(next_exit == -1); assert(dest == executor->trace); dest->opcode = _START_EXECUTOR; + dest->oparg = 0; + dest->target = 0; dest->operand = (uintptr_t)executor; _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG @@ -947,7 +958,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } if (lltrace >= 2) { - printf("Optimized executor (length %d):\n", length); + printf("Optimized trace (length %d):\n", length); for (int i = 0; i < length; i++) { printf("%4d OPTIMIZED: ", i); _PyUOpPrint(&executor->trace[i]); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9fd4b1967ecc3b..0c95616848a85b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -35,6 +35,7 @@ #ifdef Py_DEBUG extern const char *_PyUOpName(int index); + extern void _PyUOpPrint(const _PyUOpInstruction *uop); static const char *const DEBUG_ENV = "PYTHON_OPT_DEBUG"; static inline int get_lltrace(void) { char *uop_debug = Py_GETENV(DEBUG_ENV); @@ -377,14 +378,20 @@ optimize_uops( _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; - DPRINTF(3, "Abstract interpreting %s:%d ", - _PyUOpName(opcode), - oparg); +#ifdef Py_DEBUG + if (get_lltrace() >= 3) { + printf("%4d abs: ", (int)(this_instr - trace)); + _PyUOpPrint(this_instr); + printf(" "); + } +#endif + switch (opcode) { + #include "optimizer_cases.c.h" default: - DPRINTF(1, "Unknown opcode in abstract interpreter\n"); + DPRINTF(1, "\nUnknown opcode in abstract interpreter\n"); Py_UNREACHABLE(); } assert(ctx->frame != NULL); @@ -397,11 +404,13 @@ optimize_uops( return 1; out_of_space: + DPRINTF(3, "\n"); DPRINTF(1, "Out of space in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; error: + DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; @@ -411,6 +420,7 @@ optimize_uops( // This means that the abstract interpreter has hit unreachable code. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's // simpler to just admit failure and not create the executor. + DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 54abbcd74d7934..ef08c0d8897c9f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -544,6 +544,7 @@ dummy_func(void) { (void)callable; PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + DPRINTF(3, "func: %p ", func); if (func == NULL) { goto error; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index cf36f1ba792775..610d1b1aede9cc 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1597,6 +1597,7 @@ int argcount = oparg; (void)callable; PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; + DPRINTF(3, "func: %p ", func); if (func == NULL) { goto error; } From 7e1f38f2de8f93de362433203faa5605a0c47f0e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:11:10 -0700 Subject: [PATCH 12/82] gh-116916: Remove separate next_func_version counter (#116918) Somehow we ended up with two separate counter variables tracking "the next function version". Most likely this was a historical accident where an old branch was updated incorrectly. This PR merges the two counters into a single one: `interp->func_state.next_version`. --- Include/internal/pycore_interp.h | 1 - Objects/codeobject.c | 6 +++--- Objects/funcobject.c | 8 ++++---- Python/pystate.c | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d79fd3b6039ef5..942f47340b3966 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -245,7 +245,6 @@ struct _is { uint16_t optimizer_side_threshold; - uint32_t next_func_version; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 30336fa86111a7..3df733eb4ee578 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -415,9 +415,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; PyInterpreterState *interp = _PyInterpreterState_GET(); - co->co_version = interp->next_func_version; - if (interp->next_func_version != 0) { - interp->next_func_version++; + co->co_version = interp->func_state.next_version; + if (interp->func_state.next_version != 0) { + interp->func_state.next_version++; } co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 08b2823d8cf024..a506166916de48 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -236,8 +236,9 @@ How does a function's `func_version` field get initialized? - A new version is allocated by `_PyFunction_GetVersionForCurrentState` when the specializer needs a version and the version is 0. -The latter allocates versions using a counter in the interpreter state; -when the counter wraps around to 0, no more versions are allocated. +The latter allocates versions using a counter in the interpreter state, +`interp->func_state.next_version`. +When the counter wraps around to 0, no more versions are allocated. There is one other special case: functions with a non-standard `vectorcall` field are not given a version. @@ -247,8 +248,7 @@ Code object versions -------------------- So where to code objects get their `co_version`? -There is a per-interpreter counter, `next_func_version`. -This is initialized to 1 when the interpreter is created. +They share the same counter, `interp->func_state.next_version`. Code objects get a new `co_version` allocated from this counter upon creation. Since code objects are nominally immutable, `co_version` can diff --git a/Python/pystate.c b/Python/pystate.c index 9f142223aff340..eedcb920cd1cf2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -630,7 +630,6 @@ init_interpreter(PyInterpreterState *interp, interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; (void)_Py_SetOptimizer(interp, NULL); - interp->next_func_version = 1; interp->executor_list_head = NULL; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ From 9c7b3688e6ff071456e0ee623b82870755ea7808 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Mar 2024 11:13:11 -0700 Subject: [PATCH 13/82] gh-108716: Cleanup remaining deepfreeze infrastructure (#116919) Keep Tools/build/deepfreeze.py around (we may repurpose it for deepfreezing non-code objects), and keep basic "clean" targets that remove the output of former deep-freeze activities, to keep the build directories of current devs clean. --- .gitignore | 1 - Makefile.pre.in | 43 ++---------------- PCbuild/_freeze_module.vcxproj | 19 -------- PCbuild/pythoncore.vcxproj | 5 --- Python/deepfreeze/README.txt | 6 --- Python/frozen.c | 28 ------------ Tools/build/freeze_modules.py | 79 +--------------------------------- configure | 3 +- configure.ac | 3 +- 9 files changed, 7 insertions(+), 180 deletions(-) delete mode 100644 Python/deepfreeze/README.txt diff --git a/.gitignore b/.gitignore index 3e1213ef925305..2d380a441d2394 100644 --- a/.gitignore +++ b/.gitignore @@ -142,7 +142,6 @@ Tools/unicode/data/ /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config diff --git a/Makefile.pre.in b/Makefile.pre.in index 3cf4de08a0c842..5958b6bdb05a57 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1400,7 +1400,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ -# "Bootstrap Python" used to run deepfreeze.py +# "Bootstrap Python" used to run Programs/_freeze_module.py BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ @@ -1419,7 +1419,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # # Freezing is a multi step process. It works differently for standard builds # and cross builds. Standard builds use Programs/_freeze_module and -# _bootstrap_python for freezing and deepfreezing, so users can build Python +# _bootstrap_python for freezing, so users can build Python # without an existing Python installation. Cross builds cannot execute # compiled binaries and therefore rely on an external build Python # interpreter. The build interpreter must have same version and same bytecode @@ -1433,12 +1433,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu # 5) create remaining frozen module headers with # ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python # script is used to test the cross compile code path. -# 6) deepfreeze modules with _bootstrap_python # # Cross compile process: # 1) create all frozen module headers with external build Python and # Programs/_freeze_module.py script. -# 2) deepfreeze modules with external build Python. # # FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py. @@ -1584,41 +1582,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules @echo "The Makefile was updated, you may need to re-run make." -############################################################################ -# Deepfreeze targets - -DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) - -# BEGIN: deepfreeze modules -$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \ - Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ - Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ - Python/frozen_modules/zipimport.h:zipimport \ - Python/frozen_modules/abc.h:abc \ - Python/frozen_modules/codecs.h:codecs \ - Python/frozen_modules/io.h:io \ - Python/frozen_modules/_collections_abc.h:_collections_abc \ - Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ - Python/frozen_modules/genericpath.h:genericpath \ - Python/frozen_modules/ntpath.h:ntpath \ - Python/frozen_modules/posixpath.h:posixpath \ - Python/frozen_modules/os.h:os \ - Python/frozen_modules/site.h:site \ - Python/frozen_modules/stat.h:stat \ - Python/frozen_modules/importlib.util.h:importlib.util \ - Python/frozen_modules/importlib.machinery.h:importlib.machinery \ - Python/frozen_modules/runpy.h:runpy \ - Python/frozen_modules/__hello__.h:__hello__ \ - Python/frozen_modules/__phello__.h:__phello__ \ - Python/frozen_modules/__phello__.ham.h:__phello__.ham \ - Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ - Python/frozen_modules/__phello__.spam.h:__phello__.spam \ - Python/frozen_modules/frozen_only.h:frozen_only \ - -o Python/deepfreeze/deepfreeze.c -# END: deepfreeze modules - # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen @@ -2933,7 +2896,7 @@ clean-retain-profile: pycremoval -rm -f python.html python*.js python.data python*.symbols python*.map -rm -f $(WASM_STDLIB) -rm -f Programs/_testembed Programs/_freeze_module - -rm -f Python/deepfreeze/*.[co] + -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST -rm -f jit_stencils.h diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 3a8a417a6bf47a..bce92c91f1ca0d 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -476,25 +476,6 @@ - - - - -$(IntDir)\deepfreeze_mappings.txt - - - - - - - - diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 88a4a7c9564309..9131ce87db6c84 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -632,11 +632,6 @@ - - - - - diff --git a/Python/deepfreeze/README.txt b/Python/deepfreeze/README.txt deleted file mode 100644 index 276ab51143ab33..00000000000000 --- a/Python/deepfreeze/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -This directory contains the generated .c files for all the deep-frozen -modules. Python/frozen.c depends on these files. - -None of these files are committed into the repo. - -See Tools/build/freeze_modules.py for more info. diff --git a/Python/frozen.c b/Python/frozen.c index 77f51a7f750965..627f2ff9413562 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -66,34 +66,6 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#define GET_CODE(name) _Py_get_##name##_toplevel - -/* Start extern declarations */ -extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); -extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void); -extern PyObject *_Py_get_zipimport_toplevel(void); -extern PyObject *_Py_get_abc_toplevel(void); -extern PyObject *_Py_get_codecs_toplevel(void); -extern PyObject *_Py_get_io_toplevel(void); -extern PyObject *_Py_get__collections_abc_toplevel(void); -extern PyObject *_Py_get__sitebuiltins_toplevel(void); -extern PyObject *_Py_get_genericpath_toplevel(void); -extern PyObject *_Py_get_ntpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_os_toplevel(void); -extern PyObject *_Py_get_site_toplevel(void); -extern PyObject *_Py_get_stat_toplevel(void); -extern PyObject *_Py_get_importlib_util_toplevel(void); -extern PyObject *_Py_get_importlib_machinery_toplevel(void); -extern PyObject *_Py_get_runpy_toplevel(void); -extern PyObject *_Py_get___hello___toplevel(void); -extern PyObject *_Py_get___phello___toplevel(void); -extern PyObject *_Py_get___phello___ham_toplevel(void); -extern PyObject *_Py_get___phello___ham_eggs_toplevel(void); -extern PyObject *_Py_get___phello___spam_toplevel(void); -extern PyObject *_Py_get_frozen_only_toplevel(void); -/* End extern declarations */ - static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index a541b4b33c519b..eef2d0af046f51 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -20,8 +20,6 @@ # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the # .gitattributes and .gitignore files needs to be updated. FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') -DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze') -DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt' FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') @@ -233,7 +231,7 @@ def iter_subs(): ####################################### # frozen source files -class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')): +class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')): @classmethod def from_id(cls, frozenid, pyfile=None): @@ -241,8 +239,7 @@ def from_id(cls, frozenid, pyfile=None): pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py' #assert os.path.exists(pyfile), (frozenid, pyfile) frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR) - deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR) - return cls(frozenid, pyfile, frozenfile, deepfreezefile) + return cls(frozenid, pyfile, frozenfile) @property def frozenid(self): @@ -508,13 +505,6 @@ def regen_frozen(modules): lines.append(f'/* {mod.section} */') lastsection = mod.section - # Also add a extern declaration for the corresponding - # deepfreeze-generated function. - orig_name = mod.source.id - code_name = orig_name.replace(".", "_") - get_code_name = "_Py_get_%s_toplevel" % code_name - externlines.append("extern PyObject *%s(void);" % get_code_name) - pkg = 'true' if mod.ispkg else 'false' size = f"(int)sizeof({mod.symbol})" line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' @@ -549,13 +539,6 @@ def regen_frozen(modules): headerlines, FROZEN_FILE, ) - lines = replace_block( - lines, - "/* Start extern declarations */", - "/* End extern declarations */", - externlines, - FROZEN_FILE, - ) lines = replace_block( lines, "static const struct _frozen bootstrap_modules[] =", @@ -591,8 +574,6 @@ def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] - deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)", - "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') @@ -614,8 +595,6 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\") - deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -643,13 +622,6 @@ def regen_makefile(modules): rules, MAKEFILE, ) - lines = replace_block( - lines, - "# BEGIN: deepfreeze modules", - "# END: deepfreeze modules", - deepfreezerules, - MAKEFILE, - ) outfile.writelines(lines) @@ -657,9 +629,6 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}' - deepfreezerules = [f' '] - deepfreezemappings = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) @@ -673,9 +642,6 @@ def regen_pcbuild(modules): filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezemappings.append(f' \n') - - corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): @@ -688,36 +654,6 @@ def regen_pcbuild(modules): PCBUILD_PROJECT, ) outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezemappings, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - [deepfreezemappingsfile, ], - PCBUILD_PROJECT, - ) - outfile.writelines(lines) - with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - deepfreezerules, - PCBUILD_PROJECT, - ) - outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() @@ -729,17 +665,6 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) - print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') - with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): - lines = infile.readlines() - lines = replace_block( - lines, - '', - '', - corelines, - PCBUILD_FILTERS, - ) - outfile.writelines(lines) ####################################### diff --git a/configure b/configure index 9b40c48979b1be..07dce38c92724f 100755 --- a/configure +++ b/configure @@ -27318,8 +27318,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for build directories" >&5 printf %s "checking for build directories... " >&6; } for dir in $SRCDIRS; do diff --git a/configure.ac b/configure.ac index 02913cef06a5f9..3e676c56693a3c 100644 --- a/configure.ac +++ b/configure.ac @@ -6826,8 +6826,7 @@ SRCDIRS="\ Parser/lexer \ Programs \ Python \ - Python/frozen_modules \ - Python/deepfreeze" + Python/frozen_modules" AC_MSG_CHECKING([for build directories]) for dir in $SRCDIRS; do if test ! -d $dir; then From 590a26010d5d7f27890f89820645580bb8f28547 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 20:15:20 +0100 Subject: [PATCH 14/82] gh-116869: Add test_cext test: build a C extension (#116954) --- Lib/test/test_cext/__init__.py | 92 +++++++++++++++++++ Lib/test/test_cext/extension.c | 80 ++++++++++++++++ Lib/test/test_cext/setup.py | 47 ++++++++++ Makefile.pre.in | 1 + ...-03-18-10-58-47.gh-issue-116869.lN0GBl.rst | 2 + 5 files changed, 222 insertions(+) create mode 100644 Lib/test/test_cext/__init__.py create mode 100644 Lib/test/test_cext/extension.c create mode 100644 Lib/test/test_cext/setup.py create mode 100644 Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py new file mode 100644 index 00000000000000..302ea3d3a82300 --- /dev/null +++ b/Lib/test/test_cext/__init__.py @@ -0,0 +1,92 @@ +# gh-116869: Build a basic C test extension to check that the Python C API +# does not emit C compiler warnings. + +import os.path +import shutil +import subprocess +import sysconfig +import unittest +from test import support + + +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') + + +# gh-110119: pip does not currently support 't' in the ABI flag use by +# --disable-gil builds. Once it does, we can remove this skip. +@unittest.skipIf(support.Py_GIL_DISABLED, + 'test does not work with --disable-gil') +@support.requires_subprocess() +@support.requires_resource('cpu') +class TestExt(unittest.TestCase): + def test_build_c99(self): + self.check_build('c99', '_test_c99_ext') + + def test_build_c11(self): + self.check_build('c11', '_test_c11_ext') + + # With MSVC, the linker fails with: cannot open file 'python311.lib' + # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 + @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') + # Building and running an extension in clang sanitizing mode is not + # straightforward + @unittest.skipIf( + '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), + 'test does not work with analyzing builds') + # the test uses venv+pip: skip if it's not available + @support.requires_venv_with_pip() + def check_build(self, clang_std, extension_name): + venv_dir = 'env' + with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: + self._check_build(clang_std, extension_name, python_exe) + + def _check_build(self, clang_std, extension_name, python_exe): + pkg_dir = 'pkg' + os.mkdir(pkg_dir) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + + def run_cmd(operation, cmd): + env = os.environ.copy() + env['CPYTHON_TEST_STD'] = clang_std + env['CPYTHON_TEST_EXT_NAME'] = extension_name + if support.verbose: + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True, env=env) + else: + proc = subprocess.run(cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + print(proc.stdout, end='') + self.fail( + f"{operation} failed with exit code {proc.returncode}") + + # Build and install the C extension + cmd = [python_exe, '-X', 'dev', + '-m', 'pip', 'install', '--no-build-isolation', + os.path.abspath(pkg_dir)] + run_cmd('Install', cmd) + + # Do a reference run. Until we test that running python + # doesn't leak references (gh-94755), run it so one can manually check + # -X showrefcount results against this baseline. + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', 'pass'] + run_cmd('Reference run', cmd) + + # Import the C extension + cmd = [python_exe, + '-X', 'dev', + '-X', 'showrefcount', + '-c', f"import {extension_name}"] + run_cmd('Import', cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c new file mode 100644 index 00000000000000..cfecad39f8af14 --- /dev/null +++ b/Lib/test/test_cext/extension.c @@ -0,0 +1,80 @@ +// gh-116869: Basic C test extension to check that the Python C API +// does not emit C compiler warnings. + +// Always enable assertions +#undef NDEBUG + +#include "Python.h" + +#if defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L +# define NAME _test_c2x_ext +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define NAME _test_c11_ext +#else +# define NAME _test_c99_ext +#endif + +#define _STR(NAME) #NAME +#define STR(NAME) _STR(NAME) + +PyDoc_STRVAR(_testcext_add_doc, +"add(x, y)\n" +"\n" +"Return the sum of two integers: x + y."); + +static PyObject * +_testcext_add(PyObject *Py_UNUSED(module), PyObject *args) +{ + long i, j; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { + return NULL; + } + long res = i + j; + return PyLong_FromLong(res); +} + + +static PyMethodDef _testcext_methods[] = { + {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {NULL, NULL, 0, NULL} // sentinel +}; + + +static int +_testcext_exec(PyObject *module) +{ + if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot _testcext_slots[] = { + {Py_mod_exec, _testcext_exec}, + {0, NULL} +}; + + +PyDoc_STRVAR(_testcext_doc, "C test extension."); + +static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, // m_base + STR(NAME), // m_name + _testcext_doc, // m_doc + 0, // m_size + _testcext_methods, // m_methods + _testcext_slots, // m_slots + NULL, // m_traverse + NULL, // m_clear + NULL, // m_free +}; + + +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + +PyMODINIT_FUNC +FUNC_NAME(NAME)(void) +{ + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py new file mode 100644 index 00000000000000..dd57a5fb8e1e1b --- /dev/null +++ b/Lib/test/test_cext/setup.py @@ -0,0 +1,47 @@ +# gh-91321: Build a basic C test extension to check that the Python C API is +# compatible with C and does not emit C compiler warnings. +import os +import shlex +import sys +import sysconfig +from test import support + +from setuptools import setup, Extension + + +SOURCE = 'extension.c' +if not support.MS_WINDOWS: + # C compiler flags for GCC and clang + CFLAGS = [ + # The purpose of test_cext extension is to check that building a C + # extension using the Python C API does not emit C compiler warnings. + '-Werror', + ] +else: + # Don't pass any compiler flag to MSVC + CFLAGS = [] + + +def main(): + std = os.environ["CPYTHON_TEST_STD"] + name = os.environ["CPYTHON_TEST_EXT_NAME"] + cflags = [*CFLAGS, f'-std={std}'] + + # Remove existing -std options to only test ours + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + ext = Extension( + name, + sources=[SOURCE], + extra_compile_args=cflags) + setup(name='internal' + name, version='0.0', ext_modules=[ext]) + + +if __name__ == "__main__": + main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 5958b6bdb05a57..404e7ee3e42054 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2330,6 +2330,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/interpreters \ test/test_asyncio \ test/test_capi \ + test/test_cext \ test/test_concurrent_futures \ test/test_cppext \ test/test_ctypes \ diff --git a/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst b/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst new file mode 100644 index 00000000000000..71044b4930355a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-18-10-58-47.gh-issue-116869.lN0GBl.rst @@ -0,0 +1,2 @@ +Add ``test_cext`` test: build a C extension to check if the Python C API +emits C compiler warnings. Patch by Victor Stinner. From a9c304cf020e2fa3ae78fd88359dfc808c9dd639 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 20:16:58 +0100 Subject: [PATCH 15/82] gh-116869: Make C API compatible with ISO C90 (#116950) Make the C API compatible with -Werror=declaration-after-statement compiler flag again. --- Include/cpython/longintrepr.h | 3 ++- Include/object.h | 3 +-- .../next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index f037c7bb90deda..3246908ba982e2 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -129,9 +129,10 @@ _PyLong_IsCompact(const PyLongObject* op) { static inline Py_ssize_t _PyLong_CompactValue(const PyLongObject *op) { + Py_ssize_t sign; assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); - Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } diff --git a/Include/object.h b/Include/object.h index 34141af7b7f7ef..b0c0dba06ca139 100644 --- a/Include/object.h +++ b/Include/object.h @@ -343,8 +343,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; static inline Py_ssize_t Py_SIZE(PyObject *ob) { assert(ob->ob_type != &PyLong_Type); assert(ob->ob_type != &PyBool_Type); - PyVarObject *var_ob = _PyVarObject_CAST(ob); - return var_ob->ob_size; + return _PyVarObject_CAST(ob)->ob_size; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) diff --git a/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst b/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst new file mode 100644 index 00000000000000..9b9d943f2e6d19 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-18-09-58-46.gh-issue-116869.LFDVKM.rst @@ -0,0 +1,2 @@ +Make the C API compatible with ``-Werror=declaration-after-statement`` +compiler flag again. Patch by Victor Stinner. From 415964417771946dcb7a163951913adf84644b6d Mon Sep 17 00:00:00 2001 From: "Pierre Ossman (ThinLinc team)" Date: Mon, 18 Mar 2024 21:15:53 +0100 Subject: [PATCH 16/82] gh-113538: Add asycio.Server.{close,abort}_clients (redo) (#116784) These give applications the option of more forcefully terminating client connections for asyncio servers. Useful when terminating a service and there is limited time to wait for clients to finish up their work. This is a do-over with a test fix for gh-114432, which was reverted. --- Doc/library/asyncio-eventloop.rst | 25 +++++ Doc/whatsnew/3.13.rst | 5 + Lib/asyncio/base_events.py | 25 +++-- Lib/asyncio/events.py | 8 ++ Lib/asyncio/proactor_events.py | 4 +- Lib/asyncio/selector_events.py | 6 +- Lib/test/test_asyncio/test_server.py | 96 +++++++++++++++++-- ...-01-22-15-50-58.gh-issue-113538.v2wrwg.rst | 3 + 8 files changed, 152 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 06c5c877ccc173..d6ed817b13676f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1641,6 +1641,31 @@ Do not instantiate the :class:`Server` class directly. coroutine to wait until the server is closed (and no more connections are active). + .. method:: close_clients() + + Close all existing incoming client connections. + + Calls :meth:`~asyncio.BaseTransport.close` on all associated + transports. + + :meth:`close` should be called before :meth:`close_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + + .. method:: abort_clients() + + Close all existing incoming client connections immediately, + without waiting for pending operations to complete. + + Calls :meth:`~asyncio.WriteTransport.abort` on all associated + transports. + + :meth:`close` should be called before :meth:`abort_clients` when + closing the server to avoid races with new clients connecting. + + .. versionadded:: 3.13 + .. method:: get_loop() Return the event loop associated with the server object. diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index b665e6f1c85915..0553cc97c5c75a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -270,6 +270,11 @@ asyncio the buffer size. (Contributed by Jamie Phan in :gh:`115199`.) +* Add :meth:`asyncio.Server.close_clients` and + :meth:`asyncio.Server.abort_clients` methods which allow to more + forcefully close an asyncio server. + (Contributed by Pierre Ossman in :gh:`113538`.) + base64 --- diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 6c5cf28e7c59d4..f0e690b61a73dd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -279,7 +279,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, ssl_handshake_timeout, ssl_shutdown_timeout=None): self._loop = loop self._sockets = sockets - self._active_count = 0 + # Weak references so we don't break Transport's ability to + # detect abandoned transports + self._clients = weakref.WeakSet() self._waiters = [] self._protocol_factory = protocol_factory self._backlog = backlog @@ -292,14 +294,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' - def _attach(self): + def _attach(self, transport): assert self._sockets is not None - self._active_count += 1 + self._clients.add(transport) - def _detach(self): - assert self._active_count > 0 - self._active_count -= 1 - if self._active_count == 0 and self._sockets is None: + def _detach(self, transport): + self._clients.discard(transport) + if len(self._clients) == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -348,9 +349,17 @@ def close(self): self._serving_forever_fut.cancel() self._serving_forever_fut = None - if self._active_count == 0: + if len(self._clients) == 0: self._wakeup() + def close_clients(self): + for transport in self._clients.copy(): + transport.close() + + def abort_clients(self): + for transport in self._clients.copy(): + transport.abort() + async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 680749325025db..be495469a0558b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -175,6 +175,14 @@ def close(self): """Stop serving. This leaves existing connections open.""" raise NotImplementedError + def close_clients(self): + """Close all active connections.""" + raise NotImplementedError + + def abort_clients(self): + """Close all active connections immediately.""" + raise NotImplementedError + def get_loop(self): """Get the event loop the Server object is attached to.""" raise NotImplementedError diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index a512db6367b20a..397a8cda757895 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._called_connection_lost = False self._eof_written = False if self._server is not None: - self._server._attach() + self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) if waiter is not None: # only wake up the waiter when connection_made() has been called @@ -167,7 +167,7 @@ def _call_connection_lost(self, exc): self._sock = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None self._called_connection_lost = True diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 8e888d26ea0737..f94bf10b4225e7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -791,7 +791,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._paused = False # Set when pause_reading() called if self._server is not None: - self._server._attach() + self._server._attach(self) loop._transports[self._sock_fd] = self def __repr__(self): @@ -868,6 +868,8 @@ def __del__(self, _warn=warnings.warn): if self._sock is not None: _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() + if self._server is not None: + self._server._detach(self) def _fatal_error(self, exc, message='Fatal error on transport'): # Should be called from exception handler only. @@ -906,7 +908,7 @@ def _call_connection_lost(self, exc): self._loop = None server = self._server if server is not None: - server._detach() + server._detach(self) self._server = None def get_write_buffer_size(self): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 918faac909b9bf..4ca8a166a0f1a1 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -125,8 +125,12 @@ async def main(srv): class TestServer2(unittest.IsolatedAsyncioTestCase): async def test_wait_closed_basic(self): - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -137,7 +141,8 @@ async def serve(*args): self.assertFalse(task1.done()) # active count != 0, not closed: should block - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) task2 = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task1.done()) @@ -152,7 +157,8 @@ async def serve(*args): self.assertFalse(task2.done()) self.assertFalse(task3.done()) - srv._detach() + wr.close() + await wr.wait_closed() # active count == 0, closed: should unblock await task1 await task2 @@ -161,8 +167,12 @@ async def serve(*args): async def test_wait_closed_race(self): # Test a regression in 3.12.0, should be fixed in 3.12.1 - async def serve(*args): - pass + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) self.addCleanup(srv.close) @@ -170,13 +180,83 @@ async def serve(*args): task = asyncio.create_task(srv.wait_closed()) await asyncio.sleep(0) self.assertFalse(task.done()) - srv._attach() + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) loop = asyncio.get_running_loop() loop.call_soon(srv.close) - loop.call_soon(srv._detach) + loop.call_soon(wr.close) await srv.wait_closed() + async def test_close_clients(self): + async def serve(rd, wr): + try: + await rd.read() + finally: + wr.close() + await wr.wait_closed() + + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (rd, wr) = await asyncio.open_connection(addr[0], addr[1]) + self.addCleanup(wr.close) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + + srv.close() + srv.close_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) + + async def test_abort_clients(self): + async def serve(rd, wr): + fut.set_result((rd, wr)) + await wr.wait_closed() + + fut = asyncio.Future() + srv = await asyncio.start_server(serve, socket_helper.HOSTv4, 0) + self.addCleanup(srv.close) + + addr = srv.sockets[0].getsockname() + (c_rd, c_wr) = await asyncio.open_connection(addr[0], addr[1], limit=4096) + self.addCleanup(c_wr.close) + + (s_rd, s_wr) = await fut + + # Limit the socket buffers so we can reliably overfill them + s_sock = s_wr.get_extra_info('socket') + s_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65536) + c_sock = c_wr.get_extra_info('socket') + c_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536) + + # Get the reader in to a paused state by sending more than twice + # the configured limit + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + s_wr.write(b'a' * 4096) + while c_wr.transport.is_reading(): + await asyncio.sleep(0) + + # Get the writer in a waiting state by sending data until the + # socket buffers are full on both server and client sockets and + # the kernel stops accepting more data + s_wr.write(b'a' * c_sock.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + s_wr.write(b'a' * s_sock.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF)) + self.assertNotEqual(s_wr.transport.get_write_buffer_size(), 0) + + task = asyncio.create_task(srv.wait_closed()) + await asyncio.sleep(0) + self.assertFalse(task.done()) + srv.close() + srv.abort_clients() + await asyncio.sleep(0) + await asyncio.sleep(0) + self.assertTrue(task.done()) # Test the various corner cases of Unix server socket removal diff --git a/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst new file mode 100644 index 00000000000000..5c59af98e136bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-22-15-50-58.gh-issue-113538.v2wrwg.rst @@ -0,0 +1,3 @@ +Add :meth:`asyncio.Server.close_clients` and +:meth:`asyncio.Server.abort_clients` methods which allow to more forcefully +close an asyncio server. From ecb4a2b711d62f1395ddbff52576d0cca8a1b43e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 22:03:55 +0100 Subject: [PATCH 17/82] gh-116417: Move limited C API list.c tests to _testlimitedcapi (#116602) Split list.c and set.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. --- Lib/test/test_capi/test_list.py | 29 ++-- Lib/test/test_capi/test_set.py | 32 ++-- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/list.c | 150 +----------------- Modules/_testcapi/set.c | 172 --------------------- Modules/_testlimitedcapi.c | 6 + Modules/_testlimitedcapi/list.c | 169 ++++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 2 + Modules/_testlimitedcapi/set.c | 189 +++++++++++++++++++++++ PCbuild/_testlimitedcapi.vcxproj | 2 + PCbuild/_testlimitedcapi.vcxproj.filters | 2 + 11 files changed, 404 insertions(+), 351 deletions(-) create mode 100644 Modules/_testlimitedcapi/list.c create mode 100644 Modules/_testlimitedcapi/set.c diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index dceb4fce3c077b..0896a971f5c727 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -4,6 +4,7 @@ from test.support import import_helper from collections import UserList _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN @@ -25,7 +26,7 @@ def __del__(self): class CAPITest(unittest.TestCase): def test_check(self): # Test PyList_Check() - check = _testcapi.list_check + check = _testlimitedcapi.list_check self.assertTrue(check([1, 2])) self.assertTrue(check([])) self.assertTrue(check(ListSubclass([1, 2]))) @@ -39,7 +40,7 @@ def test_check(self): def test_list_check_exact(self): # Test PyList_CheckExact() - check = _testcapi.list_check_exact + check = _testlimitedcapi.list_check_exact self.assertTrue(check([1])) self.assertTrue(check([])) self.assertFalse(check(ListSubclass([1]))) @@ -51,7 +52,7 @@ def test_list_check_exact(self): def test_list_new(self): # Test PyList_New() - list_new = _testcapi.list_new + list_new = _testlimitedcapi.list_new lst = list_new(0) self.assertEqual(lst, []) self.assertIs(type(lst), list) @@ -62,7 +63,7 @@ def test_list_new(self): def test_list_size(self): # Test PyList_Size() - size = _testcapi.list_size + size = _testlimitedcapi.list_size self.assertEqual(size([1, 2]), 2) self.assertEqual(size(ListSubclass([1, 2])), 2) self.assertRaises(SystemError, size, UserList()) @@ -98,11 +99,11 @@ def check_list_get_item(self, getitem, exctype): def test_list_getitem(self): # Test PyList_GetItem() - self.check_list_get_item(_testcapi.list_getitem, SystemError) + self.check_list_get_item(_testlimitedcapi.list_getitem, SystemError) def test_list_get_item_ref(self): # Test PyList_GetItemRef() - self.check_list_get_item(_testcapi.list_get_item_ref, TypeError) + self.check_list_get_item(_testlimitedcapi.list_get_item_ref, TypeError) def test_list_get_item(self): # Test PyList_GET_ITEM() @@ -119,7 +120,7 @@ def test_list_get_item(self): def test_list_setitem(self): # Test PyList_SetItem() - setitem = _testcapi.list_setitem + setitem = _testlimitedcapi.list_setitem lst = [1, 2, 3] setitem(lst, 0, 10) self.assertEqual(lst, [10, 2, 3]) @@ -151,7 +152,7 @@ def test_list_set_item(self): def test_list_insert(self): # Test PyList_Insert() - insert = _testcapi.list_insert + insert = _testlimitedcapi.list_insert lst = [1, 2, 3] insert(lst, 0, 23) self.assertEqual(lst, [23, 1, 2, 3]) @@ -173,7 +174,7 @@ def test_list_insert(self): def test_list_append(self): # Test PyList_Append() - append = _testcapi.list_append + append = _testlimitedcapi.list_append lst = [1, 2, 3] append(lst, 10) self.assertEqual(lst, [1, 2, 3, 10]) @@ -186,7 +187,7 @@ def test_list_append(self): def test_list_getslice(self): # Test PyList_GetSlice() - getslice = _testcapi.list_getslice + getslice = _testlimitedcapi.list_getslice lst = [1, 2, 3] # empty @@ -210,7 +211,7 @@ def test_list_getslice(self): def test_list_setslice(self): # Test PyList_SetSlice() - list_setslice = _testcapi.list_setslice + list_setslice = _testlimitedcapi.list_setslice def set_slice(lst, low, high, value): lst = lst.copy() self.assertEqual(list_setslice(lst, low, high, value), 0) @@ -265,7 +266,7 @@ def set_slice(lst, low, high, value): def test_list_sort(self): # Test PyList_Sort() - sort = _testcapi.list_sort + sort = _testlimitedcapi.list_sort lst = [4, 6, 7, 3, 1, 5, 9, 2, 0, 8] sort(lst) self.assertEqual(lst, list(range(10))) @@ -281,7 +282,7 @@ def test_list_sort(self): def test_list_reverse(self): # Test PyList_Reverse() - reverse = _testcapi.list_reverse + reverse = _testlimitedcapi.list_reverse def list_reverse(lst): self.assertEqual(reverse(lst), 0) return lst @@ -295,7 +296,7 @@ def list_reverse(lst): def test_list_astuple(self): # Test PyList_AsTuple() - astuple = _testcapi.list_astuple + astuple = _testlimitedcapi.list_astuple self.assertEqual(astuple([]), ()) self.assertEqual(astuple([2, 5, 10]), (2, 5, 10)) diff --git a/Lib/test/test_capi/test_set.py b/Lib/test/test_capi/test_set.py index 5235f81543e0b6..499a5148d782ab 100644 --- a/Lib/test/test_capi/test_set.py +++ b/Lib/test/test_capi/test_set.py @@ -2,8 +2,10 @@ from test.support import import_helper -# Skip this test if the _testcapi or _testinternalcapi modules aren't available. +# Skip this test if the _testcapi, _testlimitedcapi or _testinternalcapi +# modules aren't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') class set_subclass(set): @@ -23,7 +25,7 @@ def assertImmutable(self, action, *args): class TestSetCAPI(BaseSetTests, unittest.TestCase): def test_set_check(self): - check = _testcapi.set_check + check = _testlimitedcapi.set_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -33,7 +35,7 @@ def test_set_check(self): # CRASHES: check(NULL) def test_set_check_exact(self): - check = _testcapi.set_checkexact + check = _testlimitedcapi.set_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertFalse(check(frozenset())) @@ -43,7 +45,7 @@ def test_set_check_exact(self): # CRASHES: check(NULL) def test_frozenset_check(self): - check = _testcapi.frozenset_check + check = _testlimitedcapi.frozenset_check self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -53,7 +55,7 @@ def test_frozenset_check(self): # CRASHES: check(NULL) def test_frozenset_check_exact(self): - check = _testcapi.frozenset_checkexact + check = _testlimitedcapi.frozenset_checkexact self.assertFalse(check(set())) self.assertTrue(check(frozenset())) self.assertTrue(check(frozenset({1, 2}))) @@ -63,7 +65,7 @@ def test_frozenset_check_exact(self): # CRASHES: check(NULL) def test_anyset_check(self): - check = _testcapi.anyset_check + check = _testlimitedcapi.anyset_check self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -74,7 +76,7 @@ def test_anyset_check(self): # CRASHES: check(NULL) def test_anyset_check_exact(self): - check = _testcapi.anyset_checkexact + check = _testlimitedcapi.anyset_checkexact self.assertTrue(check(set())) self.assertTrue(check({1, 2})) self.assertTrue(check(frozenset())) @@ -85,7 +87,7 @@ def test_anyset_check_exact(self): # CRASHES: check(NULL) def test_set_new(self): - set_new = _testcapi.set_new + set_new = _testlimitedcapi.set_new self.assertEqual(set_new().__class__, set) self.assertEqual(set_new(), set()) self.assertEqual(set_new((1, 1, 2)), {1, 2}) @@ -98,7 +100,7 @@ def test_set_new(self): set_new((1, {})) def test_frozenset_new(self): - frozenset_new = _testcapi.frozenset_new + frozenset_new = _testlimitedcapi.frozenset_new self.assertEqual(frozenset_new().__class__, frozenset) self.assertEqual(frozenset_new(), frozenset()) self.assertEqual(frozenset_new((1, 1, 2)), frozenset({1, 2})) @@ -111,7 +113,7 @@ def test_frozenset_new(self): frozenset_new((1, {})) def test_set_size(self): - get_size = _testcapi.set_size + get_size = _testlimitedcapi.set_size self.assertEqual(get_size(set()), 0) self.assertEqual(get_size(frozenset()), 0) self.assertEqual(get_size({1, 1, 2}), 2) @@ -134,7 +136,7 @@ def test_set_get_size(self): # CRASHES: get_size(object()) def test_set_contains(self): - contains = _testcapi.set_contains + contains = _testlimitedcapi.set_contains for cls in (set, frozenset, set_subclass, frozenset_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -147,7 +149,7 @@ def test_set_contains(self): # CRASHES: contains(NULL, NULL) def test_add(self): - add = _testcapi.set_add + add = _testlimitedcapi.set_add for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -165,7 +167,7 @@ def test_add(self): # CRASHES: add(NULL, NULL) def test_discard(self): - discard = _testcapi.set_discard + discard = _testlimitedcapi.set_discard for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) @@ -187,7 +189,7 @@ def test_discard(self): # CRASHES: discard(NULL, NULL) def test_pop(self): - pop = _testcapi.set_pop + pop = _testlimitedcapi.set_pop orig = (1, 2) for cls in (set, set_subclass): with self.subTest(cls=cls): @@ -204,7 +206,7 @@ def test_pop(self): # CRASHES: pop(NULL) def test_clear(self): - clear = _testcapi.set_clear + clear = _testlimitedcapi.set_clear for cls in (set, set_subclass): with self.subTest(cls=cls): instance = cls((1, 2)) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index deada66cf1a807..d4743d31b465a9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/pyos.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 2cb6499e28336d..09cec4c30c8c36 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -1,32 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -list_check(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_Check(obj)); -} - -static PyObject * -list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyList_CheckExact(obj)); -} - -static PyObject * -list_new(PyObject* Py_UNUSED(module), PyObject *obj) -{ - return PyList_New(PyLong_AsSsize_t(obj)); -} - -static PyObject * -list_size(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyList_Size(obj)); -} static PyObject * list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) @@ -35,17 +9,6 @@ list_get_size(PyObject *Py_UNUSED(module), PyObject *obj) RETURN_SIZE(PyList_GET_SIZE(obj)); } -static PyObject * -list_getitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return Py_XNewRef(PyList_GetItem(obj, i)); -} static PyObject * list_get_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -59,31 +22,6 @@ list_get_item(PyObject *Py_UNUSED(module), PyObject *args) return Py_XNewRef(PyList_GET_ITEM(obj, i)); } -static PyObject * -list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &obj, &i)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetItemRef(obj, i); -} - -static PyObject * -list_setitem(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); - -} static PyObject * list_set_item(PyObject *Py_UNUSED(module), PyObject *args) @@ -100,79 +38,6 @@ list_set_item(PyObject *Py_UNUSED(module), PyObject *args) } -static PyObject * -list_insert(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t where; - if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); - -} - -static PyObject * -list_append(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_Append(obj, value)); -} - -static PyObject * -list_getslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { - return NULL; - } - NULLABLE(obj); - return PyList_GetSlice(obj, ilow, ihigh); - -} - -static PyObject * -list_setslice(PyObject *Py_UNUSED(module), PyObject *args) -{ - PyObject *obj, *value; - Py_ssize_t ilow, ihigh; - if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); -} - -static PyObject * -list_sort(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Sort(obj)); -} - -static PyObject * -list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyList_Reverse(obj)); -} - -static PyObject * -list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyList_AsTuple(obj); -} - static PyObject * list_clear(PyObject* Py_UNUSED(module), PyObject *obj) @@ -196,25 +61,12 @@ list_extend(PyObject* Py_UNUSED(module), PyObject *args) static PyMethodDef test_methods[] = { - {"list_check", list_check, METH_O}, - {"list_check_exact", list_check_exact, METH_O}, - {"list_new", list_new, METH_O}, - {"list_size", list_size, METH_O}, {"list_get_size", list_get_size, METH_O}, - {"list_getitem", list_getitem, METH_VARARGS}, {"list_get_item", list_get_item, METH_VARARGS}, - {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, - {"list_setitem", list_setitem, METH_VARARGS}, {"list_set_item", list_set_item, METH_VARARGS}, - {"list_insert", list_insert, METH_VARARGS}, - {"list_append", list_append, METH_VARARGS}, - {"list_getslice", list_getslice, METH_VARARGS}, - {"list_setslice", list_setslice, METH_VARARGS}, - {"list_sort", list_sort, METH_O}, - {"list_reverse", list_reverse, METH_O}, - {"list_astuple", list_astuple, METH_O}, {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, + {NULL}, }; diff --git a/Modules/_testcapi/set.c b/Modules/_testcapi/set.c index 2fbd0aeffcd9f9..31b52cee5e9623 100644 --- a/Modules/_testcapi/set.c +++ b/Modules/_testcapi/set.c @@ -1,75 +1,6 @@ #include "parts.h" #include "util.h" -static PyObject * -set_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Check(obj)); -} - -static PyObject * -set_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_CheckExact(obj)); -} - -static PyObject * -frozenset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_Check(obj)); -} - -static PyObject * -frozenset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyFrozenSet_CheckExact(obj)); -} - -static PyObject * -anyset_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_Check(obj)); -} - -static PyObject * -anyset_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PyAnySet_CheckExact(obj)); -} - -static PyObject * -set_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PySet_New(iterable); -} - -static PyObject * -frozenset_new(PyObject *self, PyObject *args) -{ - PyObject *iterable = NULL; - if (!PyArg_ParseTuple(args, "|O", &iterable)) { - return NULL; - } - return PyFrozenSet_New(iterable); -} - -static PyObject * -set_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySet_Size(obj)); -} - static PyObject * set_get_size(PyObject *self, PyObject *obj) { @@ -77,111 +8,8 @@ set_get_size(PyObject *self, PyObject *obj) RETURN_SIZE(PySet_GET_SIZE(obj)); } -static PyObject * -set_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Contains(obj, item)); -} - -static PyObject * -set_add(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Add(obj, item)); -} - -static PyObject * -set_discard(PyObject *self, PyObject *args) -{ - PyObject *obj, *item; - if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(item); - RETURN_INT(PySet_Discard(obj, item)); -} - -static PyObject * -set_pop(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySet_Pop(obj); -} - -static PyObject * -set_clear(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_INT(PySet_Clear(obj)); -} - -static PyObject * -test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) -{ - // Test that `frozenset` can be used with `PySet_Add`, - // when frozenset is just created in CAPI. - PyObject *fs = PyFrozenSet_New(NULL); - if (fs == NULL) { - return NULL; - } - PyObject *num = PyLong_FromLong(1); - if (num == NULL) { - goto error; - } - if (PySet_Add(fs, num) < 0) { - goto error; - } - int contains = PySet_Contains(fs, num); - if (contains < 0) { - goto error; - } - else if (contains == 0) { - goto unexpected; - } - Py_DECREF(fs); - Py_DECREF(num); - Py_RETURN_NONE; - -unexpected: - PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); -error: - Py_DECREF(fs); - Py_XDECREF(num); - return NULL; -} - static PyMethodDef test_methods[] = { - {"set_check", set_check, METH_O}, - {"set_checkexact", set_checkexact, METH_O}, - {"frozenset_check", frozenset_check, METH_O}, - {"frozenset_checkexact", frozenset_checkexact, METH_O}, - {"anyset_check", anyset_check, METH_O}, - {"anyset_checkexact", anyset_checkexact, METH_O}, - - {"set_new", set_new, METH_VARARGS}, - {"frozenset_new", frozenset_new, METH_VARARGS}, - - {"set_size", set_size, METH_O}, {"set_get_size", set_get_size, METH_O}, - {"set_contains", set_contains, METH_VARARGS}, - {"set_add", set_add, METH_VARARGS}, - {"set_discard", set_discard, METH_VARARGS}, - {"set_pop", set_pop, METH_O}, - {"set_clear", set_clear, METH_O}, - - {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 49bf6a3ea39613..a25b0aa2de7c99 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -35,9 +35,15 @@ PyInit__testlimitedcapi(void) if (_PyTestCapi_Init_HeaptypeRelative(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_List(mod) < 0) { + return NULL; + } if (_PyTestCapi_Init_PyOS(mod) < 0) { return NULL; } + if (_PyTestCapi_Init_Set(mod) < 0) { + return NULL; + } if (_PyTestCapi_Init_Sys(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c new file mode 100644 index 00000000000000..0917900da390ce --- /dev/null +++ b/Modules/_testlimitedcapi/list.c @@ -0,0 +1,169 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +list_check(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_Check(obj)); +} + +static PyObject * +list_check_exact(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyList_CheckExact(obj)); +} + +static PyObject * +list_new(PyObject* Py_UNUSED(module), PyObject *obj) +{ + return PyList_New(PyLong_AsSsize_t(obj)); +} + +static PyObject * +list_size(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyList_Size(obj)); +} + +static PyObject * +list_getitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return Py_XNewRef(PyList_GetItem(obj, i)); +} + +static PyObject * +list_get_item_ref(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &obj, &i)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetItemRef(obj, i); +} + +static PyObject * +list_setitem(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "OnO", &obj, &i, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetItem(obj, i, Py_XNewRef(value))); + +} + +static PyObject * +list_insert(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t where; + if (!PyArg_ParseTuple(args, "OnO", &obj, &where, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Insert(obj, where, Py_XNewRef(value))); + +} + +static PyObject * +list_append(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_Append(obj, value)); +} + +static PyObject * +list_getslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "Onn", &obj, &ilow, &ihigh)) { + return NULL; + } + NULLABLE(obj); + return PyList_GetSlice(obj, ilow, ihigh); + +} + +static PyObject * +list_setslice(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *obj, *value; + Py_ssize_t ilow, ihigh; + if (!PyArg_ParseTuple(args, "OnnO", &obj, &ilow, &ihigh, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyList_SetSlice(obj, ilow, ihigh, value)); +} + +static PyObject * +list_sort(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Sort(obj)); +} + +static PyObject * +list_reverse(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyList_Reverse(obj)); +} + +static PyObject * +list_astuple(PyObject* Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyList_AsTuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"list_check", list_check, METH_O}, + {"list_check_exact", list_check_exact, METH_O}, + {"list_new", list_new, METH_O}, + {"list_size", list_size, METH_O}, + {"list_getitem", list_getitem, METH_VARARGS}, + {"list_get_item_ref", list_get_item_ref, METH_VARARGS}, + {"list_setitem", list_setitem, METH_VARARGS}, + {"list_insert", list_insert, METH_VARARGS}, + {"list_append", list_append, METH_VARARGS}, + {"list_getslice", list_getslice, METH_VARARGS}, + {"list_setslice", list_setslice, METH_VARARGS}, + {"list_sort", list_sort, METH_O}, + {"list_reverse", list_reverse, METH_O}, + {"list_astuple", list_astuple, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_List(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 9bc52413382eb5..bb867eccbfe5c3 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,7 +25,9 @@ int _PyTestCapi_Init_ByteArray(PyObject *module); int _PyTestCapi_Init_Bytes(PyObject *module); int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); +int _PyTestCapi_Init_List(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Set(PyObject *module); int _PyTestCapi_Init_Sys(PyObject *module); int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c new file mode 100644 index 00000000000000..471b515f874473 --- /dev/null +++ b/Modules/_testlimitedcapi/set.c @@ -0,0 +1,189 @@ +#include "parts.h" +#include "util.h" + +static PyObject * +set_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Check(obj)); +} + +static PyObject * +set_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_CheckExact(obj)); +} + +static PyObject * +frozenset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_Check(obj)); +} + +static PyObject * +frozenset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyFrozenSet_CheckExact(obj)); +} + +static PyObject * +anyset_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_Check(obj)); +} + +static PyObject * +anyset_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PyAnySet_CheckExact(obj)); +} + +static PyObject * +set_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PySet_New(iterable); +} + +static PyObject * +frozenset_new(PyObject *self, PyObject *args) +{ + PyObject *iterable = NULL; + if (!PyArg_ParseTuple(args, "|O", &iterable)) { + return NULL; + } + return PyFrozenSet_New(iterable); +} + +static PyObject * +set_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySet_Size(obj)); +} + +static PyObject * +set_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Contains(obj, item)); +} + +static PyObject * +set_add(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Add(obj, item)); +} + +static PyObject * +set_discard(PyObject *self, PyObject *args) +{ + PyObject *obj, *item; + if (!PyArg_ParseTuple(args, "OO", &obj, &item)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(item); + RETURN_INT(PySet_Discard(obj, item)); +} + +static PyObject * +set_pop(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySet_Pop(obj); +} + +static PyObject * +set_clear(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_INT(PySet_Clear(obj)); +} + +static PyObject * +test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // Test that `frozenset` can be used with `PySet_Add`, + // when frozenset is just created in CAPI. + PyObject *fs = PyFrozenSet_New(NULL); + if (fs == NULL) { + return NULL; + } + PyObject *num = PyLong_FromLong(1); + if (num == NULL) { + goto error; + } + if (PySet_Add(fs, num) < 0) { + goto error; + } + int contains = PySet_Contains(fs, num); + if (contains < 0) { + goto error; + } + else if (contains == 0) { + goto unexpected; + } + Py_DECREF(fs); + Py_DECREF(num); + Py_RETURN_NONE; + +unexpected: + PyErr_SetString(PyExc_ValueError, "set does not contain expected value"); +error: + Py_DECREF(fs); + Py_XDECREF(num); + return NULL; +} + +static PyMethodDef test_methods[] = { + {"set_check", set_check, METH_O}, + {"set_checkexact", set_checkexact, METH_O}, + {"frozenset_check", frozenset_check, METH_O}, + {"frozenset_checkexact", frozenset_checkexact, METH_O}, + {"anyset_check", anyset_check, METH_O}, + {"anyset_checkexact", anyset_checkexact, METH_O}, + + {"set_new", set_new, METH_VARARGS}, + {"frozenset_new", frozenset_new, METH_VARARGS}, + + {"set_size", set_size, METH_O}, + {"set_contains", set_contains, METH_VARARGS}, + {"set_add", set_add, METH_VARARGS}, + {"set_discard", set_discard, METH_VARARGS}, + {"set_pop", set_pop, METH_O}, + {"set_clear", set_clear, METH_O}, + + {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Set(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 1afeacaa93396e..dc2ccca6bad963 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,7 +97,9 @@ + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index b3eeb86185c372..33616be32c09b2 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,7 +12,9 @@ + + From 165cb4578c3cbd4d21faf1050193c297662fd031 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 18 Mar 2024 17:05:43 -0400 Subject: [PATCH 18/82] gh-116941: Fix pyatomic_std.h syntax errors (#116967) --- Include/cpython/pyatomic_std.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index f3970a45df24f9..ef34bb0b77dfe5 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -916,6 +916,7 @@ _Py_atomic_load_uint32_acquire(const uint32_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(uint32_t)*)obj, + memory_order_acquire); } static inline Py_ssize_t @@ -923,6 +924,7 @@ _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) { _Py_USING_STD; return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj, + memory_order_acquire); } From dc2d0f46540159aa22774777791105140e4d74c1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Mar 2024 23:06:52 +0100 Subject: [PATCH 19/82] gh-116417: Fix WASI build of _testlimitedcapi (#116974) Use different function names between _testcapi and _testlimitedcapi to not confuse the WASI linker. --- Modules/_testlimitedcapi.c | 16 ++++++++-------- Modules/_testlimitedcapi/bytearray.c | 2 +- Modules/_testlimitedcapi/bytes.c | 2 +- Modules/_testlimitedcapi/heaptype_relative.c | 3 ++- Modules/_testlimitedcapi/list.c | 2 +- Modules/_testlimitedcapi/parts.h | 16 ++++++++-------- Modules/_testlimitedcapi/pyos.c | 2 +- Modules/_testlimitedcapi/set.c | 2 +- Modules/_testlimitedcapi/sys.c | 2 +- Modules/_testlimitedcapi/vectorcall_limited.c | 3 ++- 10 files changed, 26 insertions(+), 24 deletions(-) diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index a25b0aa2de7c99..9edc4fe3a9e970 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -26,28 +26,28 @@ PyInit__testlimitedcapi(void) return NULL; } - if (_PyTestCapi_Init_ByteArray(mod) < 0) { + if (_PyTestLimitedCAPI_Init_ByteArray(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_Bytes(mod) < 0) { + if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_HeaptypeRelative(mod) < 0) { + if (_PyTestLimitedCAPI_Init_HeaptypeRelative(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_List(mod) < 0) { + if (_PyTestLimitedCAPI_Init_List(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_PyOS(mod) < 0) { + if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_Set(mod) < 0) { + if (_PyTestLimitedCAPI_Init_Set(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_Sys(mod) < 0) { + if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { return NULL; } - if (_PyTestCapi_Init_VectorcallLimited(mod) < 0) { + if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { return NULL; } return mod; diff --git a/Modules/_testlimitedcapi/bytearray.c b/Modules/_testlimitedcapi/bytearray.c index dc47ed2c306f40..68b029e25af677 100644 --- a/Modules/_testlimitedcapi/bytearray.c +++ b/Modules/_testlimitedcapi/bytearray.c @@ -113,7 +113,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_ByteArray(PyObject *m) +_PyTestLimitedCAPI_Init_ByteArray(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/bytes.c b/Modules/_testlimitedcapi/bytes.c index a14c4f9d4d30a8..157d4089954af5 100644 --- a/Modules/_testlimitedcapi/bytes.c +++ b/Modules/_testlimitedcapi/bytes.c @@ -245,7 +245,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_Bytes(PyObject *m) +_PyTestLimitedCAPI_Init_Bytes(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index d0316dd4fc63b4..7c508c6182bc8a 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -331,7 +331,8 @@ static PyMethodDef TestMethods[] = { }; int -_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { +_PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } diff --git a/Modules/_testlimitedcapi/list.c b/Modules/_testlimitedcapi/list.c index 0917900da390ce..3022cbf9191b2e 100644 --- a/Modules/_testlimitedcapi/list.c +++ b/Modules/_testlimitedcapi/list.c @@ -159,7 +159,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_List(PyObject *m) +_PyTestLimitedCAPI_Init_List(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index bb867eccbfe5c3..ee67f10e4b4830 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -22,13 +22,13 @@ # error "Py_BUILD_CORE macro must not be defined" #endif -int _PyTestCapi_Init_ByteArray(PyObject *module); -int _PyTestCapi_Init_Bytes(PyObject *module); -int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); -int _PyTestCapi_Init_List(PyObject *module); -int _PyTestCapi_Init_PyOS(PyObject *module); -int _PyTestCapi_Init_Set(PyObject *module); -int _PyTestCapi_Init_Sys(PyObject *module); -int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); +int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); +int _PyTestLimitedCAPI_Init_List(PyObject *module); +int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); +int _PyTestLimitedCAPI_Init_Set(PyObject *module); +int _PyTestLimitedCAPI_Init_Sys(PyObject *module); +int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/pyos.c b/Modules/_testlimitedcapi/pyos.c index 63140e914875db..0f61801eae322a 100644 --- a/Modules/_testlimitedcapi/pyos.c +++ b/Modules/_testlimitedcapi/pyos.c @@ -50,7 +50,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_PyOS(PyObject *mod) +_PyTestLimitedCAPI_Init_PyOS(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 471b515f874473..35da5fa5f008e1 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -179,7 +179,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_Set(PyObject *m) +_PyTestLimitedCAPI_Init_Set(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/sys.c b/Modules/_testlimitedcapi/sys.c index aa40e3cd5b9b29..7d8b7a8569e515 100644 --- a/Modules/_testlimitedcapi/sys.c +++ b/Modules/_testlimitedcapi/sys.c @@ -46,7 +46,7 @@ static PyMethodDef test_methods[] = { }; int -_PyTestCapi_Init_Sys(PyObject *m) +_PyTestLimitedCAPI_Init_Sys(PyObject *m) { if (PyModule_AddFunctions(m, test_methods) < 0) { return -1; diff --git a/Modules/_testlimitedcapi/vectorcall_limited.c b/Modules/_testlimitedcapi/vectorcall_limited.c index fc1a89c9098e1b..784126c17fccc1 100644 --- a/Modules/_testlimitedcapi/vectorcall_limited.c +++ b/Modules/_testlimitedcapi/vectorcall_limited.c @@ -182,7 +182,8 @@ static PyMethodDef TestMethods[] = { }; int -_PyTestCapi_Init_VectorcallLimited(PyObject *m) { +_PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *m) +{ if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } From 1d82a41235ac5619d36ac7e289fcbb686c1d9350 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 00:14:29 +0100 Subject: [PATCH 20/82] gh-116869: Enable test_cext and test_cppext on Free Threading build (#116973) Remove the "if Py_GIL_DISABLED" skip and move all "skip" decorators to the class. Use support.check_sanitizer() --- Lib/test/test_cext/__init__.py | 24 +++++++++--------------- Lib/test/test_cppext/__init__.py | 27 ++++++++++----------------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 302ea3d3a82300..730b91abfb2715 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -4,7 +4,6 @@ import os.path import shutil import subprocess -import sysconfig import unittest from test import support @@ -13,10 +12,15 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# gh-110119: pip does not currently support 't' in the ABI flag use by -# --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(support.Py_GIL_DISABLED, - 'test does not work with --disable-gil') +# With MSVC, the linker fails with: cannot open file 'python311.lib' +# https://github.com/python/cpython/pull/32175#issuecomment-1111175897 +@unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') class TestExt(unittest.TestCase): @@ -26,16 +30,6 @@ def test_build_c99(self): def test_build_c11(self): self.check_build('c11', '_test_c11_ext') - # With MSVC, the linker fails with: cannot open file 'python311.lib' - # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') - # Building and running an extension in clang sanitizing mode is not - # straightforward - @unittest.skipIf( - '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), - 'test does not work with analyzing builds') - # the test uses venv+pip: skip if it's not available - @support.requires_venv_with_pip() def check_build(self, clang_std, extension_name): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index c6039bd17b0662..66ead9a3d78c66 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -4,7 +4,6 @@ import shutil import unittest import subprocess -import sysconfig from test import support @@ -12,30 +11,24 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# gh-110119: pip does not currently support 't' in the ABI flag use by -# --disable-gil builds. Once it does, we can remove this skip. -@unittest.skipIf(support.Py_GIL_DISABLED, - 'test does not work with --disable-gil') +# With MSVC, the linker fails with: cannot open file 'python311.lib' +# https://github.com/python/cpython/pull/32175#issuecomment-1111175897 +@unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') +# Building and running an extension in clang sanitizing mode is not +# straightforward +@support.skip_if_sanitizer('test does not work with analyzing builds', + address=True, memory=True, ub=True, thread=True) +# the test uses venv+pip: skip if it's not available +@support.requires_venv_with_pip() @support.requires_subprocess() +@support.requires_resource('cpu') class TestCPPExt(unittest.TestCase): - @support.requires_resource('cpu') def test_build_cpp11(self): self.check_build(False, '_testcpp11ext') - @support.requires_resource('cpu') def test_build_cpp03(self): self.check_build(True, '_testcpp03ext') - # With MSVC, the linker fails with: cannot open file 'python311.lib' - # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 - @unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') - # Building and running an extension in clang sanitizing mode is not - # straightforward - @unittest.skipIf( - '-fsanitize' in (sysconfig.get_config_var('PY_CFLAGS') or ''), - 'test does not work with analyzing builds') - # the test uses venv+pip: skip if it's not available - @support.requires_venv_with_pip() def check_build(self, std_cpp03, extension_name): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: From a22d05f04c074dbb4f71e7837f54c0bb693db75d Mon Sep 17 00:00:00 2001 From: Aviel Boag Date: Tue, 19 Mar 2024 02:53:14 +0200 Subject: [PATCH 21/82] gh-105866: fix dataclass with slots=True, weakref_slot=True (#105870) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Kirill Podoprigora Co-authored-by: Carl Meyer --- Lib/dataclasses.py | 4 +++- Lib/test/test_dataclasses/__init__.py | 11 +++++++++++ .../2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 45ce5a98b51ae0..e511eff4125038 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1159,8 +1159,10 @@ def _dataclass_setstate(self, state): def _get_slots(cls): match cls.__dict__.get('__slots__'): + # A class which does not define __slots__ at all is equivalent + # to a class defining __slots__ = ('__dict__', '__weakref__') case None: - return + yield from ('__dict__', '__weakref__') case str(slot): yield slot # Slots may be any iterable, but we cannot handle an iterator diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ede74b0dd15ccf..e27abac5111394 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3498,6 +3498,17 @@ class A(Base): self.assertIs(a.__weakref__, a_ref) + def test_dataclass_derived_weakref_slot(self): + class A: + pass + + @dataclass(slots=True, weakref_slot=True) + class B(A): + pass + + B() + + class TestDescriptors(unittest.TestCase): def test_set_name(self): # See bpo-33141. diff --git a/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst b/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst new file mode 100644 index 00000000000000..28eae1232742f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-16-19-17-06.gh-issue-105866.0NBveV.rst @@ -0,0 +1 @@ +Fixed ``_get_slots`` bug which caused error when defining dataclasses with slots and a weakref_slot. From a3cf0fada09b74b1a6981cc06c4dd0bb1091b092 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 19 Mar 2024 12:18:07 +0900 Subject: [PATCH 22/82] gh-116621: Specialize list.extend for dict items (gh-116888) --- Objects/listobject.c | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index 096043bb3d3c51..fc20a9bff3af47 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1289,7 +1289,7 @@ list_extend_set(PyListObject *self, PySetObject *other) PyObject **dest = self->ob_item + m; while (_PySet_NextEntry((PyObject *)other, &setpos, &key, &hash)) { Py_INCREF(key); - *dest = key; + FT_ATOMIC_STORE_PTR_RELEASE(*dest, key); dest++; } Py_SET_SIZE(self, m + n); @@ -1312,7 +1312,7 @@ list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) while (_PyDict_Next((PyObject *)dict, &pos, &keyvalue[0], &keyvalue[1], NULL)) { PyObject *obj = keyvalue[which_item]; Py_INCREF(obj); - *dest = obj; + FT_ATOMIC_STORE_PTR_RELEASE(*dest, obj); dest++; } @@ -1320,12 +1320,39 @@ list_extend_dict(PyListObject *self, PyDictObject *dict, int which_item) return 0; } +static int +list_extend_dictitems(PyListObject *self, PyDictObject *dict) +{ + Py_ssize_t m = Py_SIZE(self); + Py_ssize_t n = PyDict_GET_SIZE(dict); + if (list_resize(self, m + n) < 0) { + return -1; + } + + PyObject **dest = self->ob_item + m; + Py_ssize_t pos = 0; + Py_ssize_t i = 0; + PyObject *key, *value; + while (_PyDict_Next((PyObject *)dict, &pos, &key, &value, NULL)) { + PyObject *item = PyTuple_Pack(2, key, value); + if (item == NULL) { + Py_SET_SIZE(self, m + i); + return -1; + } + FT_ATOMIC_STORE_PTR_RELEASE(*dest, item); + dest++; + i++; + } + + Py_SET_SIZE(self, m + n); + return 0; +} + static int _list_extend(PyListObject *self, PyObject *iterable) { // Special case: // lists and tuples which can use PySequence_Fast ops - // TODO(@corona10): Add more special cases for other types. int res = -1; if ((PyObject *)self == iterable) { Py_BEGIN_CRITICAL_SECTION(self); @@ -1359,6 +1386,12 @@ _list_extend(PyListObject *self, PyObject *iterable) res = list_extend_dict(self, dict, 1 /*values*/); Py_END_CRITICAL_SECTION2(); } + else if (Py_IS_TYPE(iterable, &PyDictItems_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION2(self, dict); + res = list_extend_dictitems(self, dict); + Py_END_CRITICAL_SECTION2(); + } else { Py_BEGIN_CRITICAL_SECTION(self); res = list_extend_iter_lock_held(self, iterable); From a8e93d3dca086896e668b88b6c5450eaf644c0e7 Mon Sep 17 00:00:00 2001 From: Bogdan Romanyuk <65823030+wrongnull@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:20:38 +0300 Subject: [PATCH 23/82] gh-115756: make PyCode_GetFirstFree an unstable API (GH-115781) --- Doc/c-api/code.rst | 8 +++++++- Doc/whatsnew/3.13.rst | 4 ++++ Include/cpython/code.h | 6 +++++- .../C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst | 3 +++ Objects/frameobject.c | 2 +- Objects/typeobject.c | 2 +- Python/ceval.c | 2 +- Python/compile.c | 2 +- 8 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index f6fdd7574323c7..968c472219c643 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -34,10 +34,16 @@ bound into a function. Return the number of free variables in a code object. -.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co) +.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) Return the position of the first free variable in a code object. + .. versionchanged:: 3.13 + + Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. + .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Return a new code object. If you need a dummy code object to create a frame, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0553cc97c5c75a..0e04dcd196d306 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1471,6 +1471,10 @@ Changes in the Python API than directories only. Users may add a trailing slash to match only directories. +* :c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed + to :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`) + Build Changes ============= diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 4e7e212fd099db..d5dac1765638f9 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -226,11 +226,15 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -static inline int PyCode_GetFirstFree(PyCodeObject *op) { +static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) { assert(PyCode_Check(op)); return op->co_nlocalsplus - op->co_nfreevars; } +Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) { + return PyUnstable_Code_GetFirstFree(op); +} + #define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) diff --git a/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst b/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst new file mode 100644 index 00000000000000..6960395fe229a3 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-19-09-49-04.gh-issue-115756.4Ls_Tl.rst @@ -0,0 +1,3 @@ +:c:func:`!PyCode_GetFirstFree` is an ustable API now and has been renamed to +:c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in +:gh:`115781`) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a914c61aac2fd5..d55c246d80dd6a 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1140,7 +1140,7 @@ frame_init_get_vars(_PyInterpreterFrame *frame) /* Free vars have not been initialized -- Do that */ PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; - int offset = PyCode_GetFirstFree(co); + int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = Py_NewRef(o); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 24f31492985164..06c2fc8e6ca072 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -10884,7 +10884,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyCodeObject *co, // Look for __class__ in the free vars. PyTypeObject *type = NULL; - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; i++) { assert((_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_FREE) != 0); PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); diff --git a/Python/ceval.c b/Python/ceval.c index 9dbcd3da3fec27..b35a321c943123 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2903,7 +2903,7 @@ _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg) if (_PyErr_Occurred(tstate)) return; name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg); - if (oparg < PyCode_GetFirstFree(co)) { + if (oparg < PyUnstable_Code_GetFirstFree(co)) { _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, name); } else { diff --git a/Python/compile.c b/Python/compile.c index 6b17f3bcaf2264..3291d31a5cc8ed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1830,7 +1830,7 @@ compiler_make_closure(struct compiler *c, location loc, PyCodeObject *co, Py_ssize_t flags) { if (co->co_nfreevars) { - int i = PyCode_GetFirstFree(co); + int i = PyUnstable_Code_GetFirstFree(co); for (; i < co->co_nlocalsplus; ++i) { /* Bypass com_addop_varname because it will generate LOAD_DEREF but LOAD_CLOSURE is needed. From b1bc37597f0d36084c4dcb15977fe6d4b9322cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20R=C3=B6thlisberger?= Date: Tue, 19 Mar 2024 09:59:08 +0000 Subject: [PATCH 24/82] gh-116957: configparser: Do post-process values after DuplicateOptionError (GH-116958) If you catch DuplicateOptionError / DuplicateSectionError when reading a config file (the intention is to skip invalid config files) and then attempt to use the ConfigParser instance, any values it *had* read successfully so far, were stored as a list instead of string! Later `get` calls would raise "AttributeError: 'list' object has no attribute 'find'" from somewhere deep in the interpolation code. --- Lib/configparser.py | 184 +++++++++--------- Lib/test/test_configparser.py | 15 ++ ...-03-18-14-36-50.gh-issue-116957.dTCs4f.rst | 3 + 3 files changed, 111 insertions(+), 91 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst diff --git a/Lib/configparser.py b/Lib/configparser.py index 241f10aee93ec4..8f182eec306b8b 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -961,102 +961,104 @@ def _read(self, fp, fpname): lineno = 0 indent_level = 0 e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: - if self._empty_lines_in_values: - # add empty line to the value, but only if there was no - # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join - else: - # empty line marks end of value - indent_level = sys.maxsize - continue - # continuation line? - first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - if cursect[optname] is None: - raise MultilineContinuationError(fpname, lineno, line) - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults + try: + for lineno, line in enumerate(fp, start=1): + comment_start = sys.maxsize + # strip inline comments + inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} + while comment_start == sys.maxsize and inline_prefixes: + next_prefixes = {} + for prefix, index in inline_prefixes.items(): + index = line.find(prefix, index+1) + if index == -1: + continue + next_prefixes[prefix] = index + if index == 0 or (index > 0 and line[index-1].isspace()): + comment_start = min(comment_start, index) + inline_prefixes = next_prefixes + # strip full line comments + for prefix in self._comment_prefixes: + if line.strip().startswith(prefix): + comment_start = 0 + break + if comment_start == sys.maxsize: + comment_start = None + value = line[:comment_start].strip() + if not value: + if self._empty_lines_in_values: + # add empty line to the value, but only if there was no + # comment on the line + if (comment_start is None and + cursect is not None and + optname and + cursect[optname] is not None): + cursect[optname].append('') # newlines added at join else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? + # empty line marks end of value + indent_level = sys.maxsize + continue + # continuation line? + first_nonspace = self.NONSPACECRE.search(line) + cur_indent_level = first_nonspace.start() if first_nonspace else 0 + if (cursect is not None and optname and + cur_indent_level > indent_level): + if cursect[optname] is None: + raise MultilineContinuationError(fpname, lineno, line) + cursect[optname].append(value) + # a section header or option header? else: - mo = self._optcre.match(value) + indent_level = cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(value) if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] + sectname = mo.group('header') + if sectname in self._sections: + if self._strict and sectname in elements_added: + raise DuplicateSectionError(sectname, fpname, + lineno) + cursect = self._sections[sectname] + elements_added.add(sectname) + elif sectname == self.default_section: + cursect = self._defaults else: - # valueless option handling - cursect[optname] = None + cursect = self._dict() + self._sections[sectname] = cursect + self._proxies[sectname] = SectionProxy(self, sectname) + elements_added.add(sectname) + # So sections can't start with a continuation line + optname = None + # no section header in the file? + elif cursect is None: + raise MissingSectionHeaderError(fpname, lineno, line) + # an option line? else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() + mo = self._optcre.match(value) + if mo: + optname, vi, optval = mo.group('option', 'vi', 'value') + if not optname: + e = self._handle_error(e, fpname, lineno, line) + optname = self.optionxform(optname.rstrip()) + if (self._strict and + (sectname, optname) in elements_added): + raise DuplicateOptionError(sectname, optname, + fpname, lineno) + elements_added.add((sectname, optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + cursect[optname] = [optval] + else: + # valueless option handling + cursect[optname] = None + else: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + e = self._handle_error(e, fpname, lineno, line) + finally: + self._join_multiline_values() # if any parsing errors occurred, raise an exception if e: raise e diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 5d58e34740adaf..6340e378c4f21a 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -646,6 +646,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" diff --git a/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst b/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst new file mode 100644 index 00000000000000..51fe04957e26bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-18-14-36-50.gh-issue-116957.dTCs4f.rst @@ -0,0 +1,3 @@ +configparser: Don't leave ConfigParser values in an invalid state (stored as +a list instead of a str) after an earlier read raised DuplicateSectionError +or DuplicateOptionError. From 039d20ae5428dfe3d70404d8a5297c70d41e2e2d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 11:44:13 +0100 Subject: [PATCH 25/82] gh-116417: Move limited C API abstract.c tests to _testlimitedcapi (#116986) Split abstract.c and float.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. Update test_bytes and test_class. --- Lib/test/test_bytes.py | 16 +- Lib/test/test_capi/test_abstract.py | 113 ++--- Lib/test/test_capi/test_float.py | 17 +- Lib/test/test_class.py | 6 +- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/abstract.c | 583 +---------------------- Modules/_testcapi/float.c | 73 --- Modules/_testlimitedcapi.c | 6 + Modules/_testlimitedcapi/abstract.c | 582 ++++++++++++++++++++++ Modules/_testlimitedcapi/float.c | 92 ++++ Modules/_testlimitedcapi/parts.h | 2 + PCbuild/_testlimitedcapi.vcxproj | 2 + PCbuild/_testlimitedcapi.vcxproj.filters | 2 + 13 files changed, 774 insertions(+), 722 deletions(-) create mode 100644 Modules/_testlimitedcapi/abstract.c create mode 100644 Modules/_testlimitedcapi/float.c diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 71bb1e75c6affd..9e1985bb3a7639 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -991,13 +991,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1256,7 +1256,7 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') def test_getitem_error(self): b = bytearray(b'python') @@ -1354,7 +1354,7 @@ def setitem_as_mapping(b, i, val): b[i] = val def setitem_as_sequence(b, i, val): - self._testcapi.sequence_setitem(b, i, val) + self._testlimitedcapi.sequence_setitem(b, i, val) def do_tests(setitem): b = bytearray([1, 2, 3]) @@ -1401,7 +1401,7 @@ def del_as_mapping(b, i): del b[i] def del_as_sequence(b, i): - self._testcapi.sequence_delitem(b, i) + self._testlimitedcapi.sequence_delitem(b, i) def do_tests(delete): b = bytearray(range(10)) @@ -1810,7 +1810,7 @@ def __index__(self): with self.subTest("tp_as_sequence"): b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - self._testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) class AssortedBytesTest(unittest.TestCase): diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 97ed939928c360..7e6cc9a2d0154b 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -4,6 +4,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX NULL = None @@ -74,7 +75,7 @@ def assertTypedEqual(self, actual, expected): def test_object_str(self): # Test PyObject_Str() - object_str = _testcapi.object_str + object_str = _testlimitedcapi.object_str self.assertTypedEqual(object_str(''), '') self.assertTypedEqual(object_str('abc'), 'abc') self.assertTypedEqual(object_str('\U0001f40d'), '\U0001f40d') @@ -87,7 +88,7 @@ def test_object_str(self): def test_object_repr(self): # Test PyObject_Repr() - object_repr = _testcapi.object_repr + object_repr = _testlimitedcapi.object_repr self.assertTypedEqual(object_repr(''), "''") self.assertTypedEqual(object_repr('abc'), "'abc'") self.assertTypedEqual(object_repr('\U0001f40d'), "'\U0001f40d'") @@ -100,7 +101,7 @@ def test_object_repr(self): def test_object_ascii(self): # Test PyObject_ASCII() - object_ascii = _testcapi.object_ascii + object_ascii = _testlimitedcapi.object_ascii self.assertTypedEqual(object_ascii(''), "''") self.assertTypedEqual(object_ascii('abc'), "'abc'") self.assertTypedEqual(object_ascii('\U0001f40d'), r"'\U0001f40d'") @@ -113,7 +114,7 @@ def test_object_ascii(self): def test_object_bytes(self): # Test PyObject_Bytes() - object_bytes = _testcapi.object_bytes + object_bytes = _testlimitedcapi.object_bytes self.assertTypedEqual(object_bytes(b''), b'') self.assertTypedEqual(object_bytes(b'abc'), b'abc') self.assertTypedEqual(object_bytes(BytesSubclass(b'abc')), b'abc') @@ -132,7 +133,7 @@ def test_object_bytes(self): self.assertTypedEqual(object_bytes(NULL), b'') def test_object_getattr(self): - xgetattr = _testcapi.object_getattr + xgetattr = _testlimitedcapi.object_getattr obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -146,7 +147,7 @@ def test_object_getattr(self): # CRASHES xgetattr(NULL, 'a') def test_object_getattrstring(self): - getattrstring = _testcapi.object_getattrstring + getattrstring = _testlimitedcapi.object_getattrstring obj = TestObject() obj.a = 11 setattr(obj, '\U0001f40d', 22) @@ -188,7 +189,7 @@ def test_object_getoptionalattrstring(self): # CRASHES getoptionalattrstring(NULL, b'a') def test_object_hasattr(self): - xhasattr = _testcapi.object_hasattr + xhasattr = _testlimitedcapi.object_hasattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -212,7 +213,7 @@ def test_object_hasattr(self): # CRASHES xhasattr(NULL, 'a') def test_object_hasattrstring(self): - hasattrstring = _testcapi.object_hasattrstring + hasattrstring = _testlimitedcapi.object_hasattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -264,7 +265,7 @@ def test_object_hasattrstringwitherror(self): # CRASHES hasattrstring(NULL, b'a') def test_object_setattr(self): - xsetattr = _testcapi.object_setattr + xsetattr = _testlimitedcapi.object_setattr obj = TestObject() xsetattr(obj, 'a', 5) self.assertEqual(obj.a, 5) @@ -284,7 +285,7 @@ def test_object_setattr(self): # CRASHES xsetattr(NULL, 'a', 5) def test_object_setattrstring(self): - setattrstring = _testcapi.object_setattrstring + setattrstring = _testlimitedcapi.object_setattrstring obj = TestObject() setattrstring(obj, b'a', 5) self.assertEqual(obj.a, 5) @@ -305,7 +306,7 @@ def test_object_setattrstring(self): # CRASHES setattrstring(NULL, b'a', 5) def test_object_delattr(self): - xdelattr = _testcapi.object_delattr + xdelattr = _testlimitedcapi.object_delattr obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -322,7 +323,7 @@ def test_object_delattr(self): # CRASHES xdelattr(NULL, 'a') def test_object_delattrstring(self): - delattrstring = _testcapi.object_delattrstring + delattrstring = _testlimitedcapi.object_delattrstring obj = TestObject() obj.a = 1 setattr(obj, '\U0001f40d', 2) @@ -340,7 +341,7 @@ def test_object_delattrstring(self): def test_mapping_check(self): - check = _testcapi.mapping_check + check = _testlimitedcapi.mapping_check self.assertTrue(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -351,7 +352,7 @@ def test_mapping_check(self): self.assertFalse(check(NULL)) def test_mapping_size(self): - for size in _testcapi.mapping_size, _testcapi.mapping_length: + for size in _testlimitedcapi.mapping_size, _testlimitedcapi.mapping_length: self.assertEqual(size({1: 2}), 1) self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) @@ -363,7 +364,7 @@ def test_mapping_size(self): self.assertRaises(SystemError, size, NULL) def test_object_getitem(self): - getitem = _testcapi.object_getitem + getitem = _testlimitedcapi.object_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertRaises(KeyError, getitem, dct, 'b') @@ -383,7 +384,7 @@ def test_object_getitem(self): self.assertRaises(SystemError, getitem, NULL, 'a') def test_mapping_getitemstring(self): - getitemstring = _testcapi.mapping_getitemstring + getitemstring = _testlimitedcapi.mapping_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertRaises(KeyError, getitemstring, dct, b'b') @@ -437,7 +438,7 @@ def test_mapping_getoptionalitemstring(self): # CRASHES getitemstring(NULL, b'a') def test_mapping_haskey(self): - haskey = _testcapi.mapping_haskey + haskey = _testlimitedcapi.mapping_haskey dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -486,7 +487,7 @@ def test_mapping_haskey(self): 'null argument to internal routine') def test_mapping_haskeystring(self): - haskeystring = _testcapi.mapping_haskeystring + haskeystring = _testlimitedcapi.mapping_haskeystring dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -527,7 +528,7 @@ def test_mapping_haskeystring(self): "null argument to internal routine") def test_mapping_haskeywitherror(self): - haskey = _testcapi.mapping_haskeywitherror + haskey = _testlimitedcapi.mapping_haskeywitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskey(dct, 'a')) self.assertFalse(haskey(dct, 'b')) @@ -548,7 +549,7 @@ def test_mapping_haskeywitherror(self): # CRASHES haskey(NULL, 'a')) def test_mapping_haskeystringwitherror(self): - haskeystring = _testcapi.mapping_haskeystringwitherror + haskeystring = _testlimitedcapi.mapping_haskeystringwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(haskeystring(dct, b'a')) self.assertFalse(haskeystring(dct, b'b')) @@ -565,7 +566,7 @@ def test_mapping_haskeystringwitherror(self): # CRASHES haskeystring(NULL, b'a') def test_object_setitem(self): - setitem = _testcapi.object_setitem + setitem = _testlimitedcapi.object_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -591,7 +592,7 @@ def test_object_setitem(self): self.assertRaises(SystemError, setitem, NULL, 'a', 5) def test_mapping_setitemstring(self): - setitemstring = _testcapi.mapping_setitemstring + setitemstring = _testlimitedcapi.mapping_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -611,7 +612,7 @@ def test_mapping_setitemstring(self): self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) def test_object_delitem(self): - for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + for delitem in _testlimitedcapi.object_delitem, _testlimitedcapi.mapping_delitem: dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -637,7 +638,7 @@ def test_object_delitem(self): self.assertRaises(SystemError, delitem, NULL, 'a') def test_mapping_delitemstring(self): - delitemstring = _testcapi.mapping_delitemstring + delitemstring = _testlimitedcapi.mapping_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -677,23 +678,23 @@ def items(self): for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), dict_obj, OrderedDict(dict_obj), Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.mapping_keys(mapping), + self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), list(mapping.keys())) - self.assertListEqual(_testcapi.mapping_values(mapping), + self.assertListEqual(_testlimitedcapi.mapping_values(mapping), list(mapping.values())) - self.assertListEqual(_testcapi.mapping_items(mapping), + self.assertListEqual(_testlimitedcapi.mapping_items(mapping), list(mapping.items())) def test_mapping_keys_valuesitems_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) - self.assertRaises(AttributeError, _testcapi.mapping_values, object()) - self.assertRaises(AttributeError, _testcapi.mapping_items, object()) - self.assertRaises(AttributeError, _testcapi.mapping_keys, []) - self.assertRaises(AttributeError, _testcapi.mapping_values, []) - self.assertRaises(AttributeError, _testcapi.mapping_items, []) - self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) - self.assertRaises(SystemError, _testcapi.mapping_values, NULL) - self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_values, []) + self.assertRaises(AttributeError, _testlimitedcapi.mapping_items, []) + self.assertRaises(SystemError, _testlimitedcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testlimitedcapi.mapping_items, NULL) class BadMapping: def keys(self): @@ -703,12 +704,12 @@ def values(self): def items(self): return None bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testlimitedcapi.mapping_items, bad_mapping) def test_sequence_check(self): - check = _testcapi.sequence_check + check = _testlimitedcapi.sequence_check self.assertFalse(check({1: 2})) self.assertTrue(check([1, 2])) self.assertTrue(check((1, 2))) @@ -719,7 +720,7 @@ def test_sequence_check(self): # CRASHES check(NULL) def test_sequence_size(self): - for size in _testcapi.sequence_size, _testcapi.sequence_length: + for size in _testlimitedcapi.sequence_size, _testlimitedcapi.sequence_length: self.assertEqual(size([1, 2]), 2) self.assertEqual(size((1, 2)), 2) self.assertEqual(size('abc'), 3) @@ -731,7 +732,7 @@ def test_sequence_size(self): self.assertRaises(SystemError, size, NULL) def test_sequence_getitem(self): - getitem = _testcapi.sequence_getitem + getitem = _testlimitedcapi.sequence_getitem lst = ['a', 'b', 'c'] self.assertEqual(getitem(lst, 1), 'b') self.assertEqual(getitem(lst, -1), 'c') @@ -744,7 +745,7 @@ def test_sequence_getitem(self): self.assertRaises(SystemError, getitem, NULL, 1) def test_sequence_concat(self): - concat = _testcapi.sequence_concat + concat = _testlimitedcapi.sequence_concat self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) @@ -757,7 +758,7 @@ def test_sequence_concat(self): self.assertRaises(SystemError, concat, NULL, []) def test_sequence_repeat(self): - repeat = _testcapi.sequence_repeat + repeat = _testlimitedcapi.sequence_repeat self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) self.assertEqual(repeat(['a', 'b'], 0), []) @@ -771,7 +772,7 @@ def test_sequence_repeat(self): self.assertRaises(SystemError, repeat, NULL, 2) def test_sequence_inplaceconcat(self): - inplaceconcat = _testcapi.sequence_inplaceconcat + inplaceconcat = _testlimitedcapi.sequence_inplaceconcat lst = ['a', 'b'] res = inplaceconcat(lst, [1, 2]) self.assertEqual(res, ['a', 'b', 1, 2]) @@ -790,7 +791,7 @@ def test_sequence_inplaceconcat(self): self.assertRaises(SystemError, inplaceconcat, NULL, []) def test_sequence_inplacerepeat(self): - inplacerepeat = _testcapi.sequence_inplacerepeat + inplacerepeat = _testlimitedcapi.sequence_inplacerepeat lst = ['a', 'b'] res = inplacerepeat(lst, 2) self.assertEqual(res, ['a', 'b', 'a', 'b']) @@ -807,7 +808,7 @@ def test_sequence_inplacerepeat(self): self.assertRaises(SystemError, inplacerepeat, NULL, 2) def test_sequence_setitem(self): - setitem = _testcapi.sequence_setitem + setitem = _testlimitedcapi.sequence_setitem lst = ['a', 'b', 'c'] setitem(lst, 1, 'x') self.assertEqual(lst, ['a', 'x', 'c']) @@ -825,7 +826,7 @@ def test_sequence_setitem(self): self.assertRaises(SystemError, setitem, NULL, 1, 'x') def test_sequence_delitem(self): - delitem = _testcapi.sequence_delitem + delitem = _testlimitedcapi.sequence_delitem lst = ['a', 'b', 'c'] delitem(lst, 1) self.assertEqual(lst, ['a', 'c']) @@ -840,7 +841,7 @@ def test_sequence_delitem(self): self.assertRaises(SystemError, delitem, NULL, 1) def test_sequence_setslice(self): - setslice = _testcapi.sequence_setslice + setslice = _testlimitedcapi.sequence_setslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -882,7 +883,7 @@ def __setitem__(self, index, value): self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') def test_sequence_delslice(self): - delslice = _testcapi.sequence_delslice + delslice = _testlimitedcapi.sequence_delslice # Correct case: for start in [*range(-6, 7), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: @@ -920,7 +921,7 @@ def __delitem__(self, index): self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) def test_sequence_count(self): - count = _testcapi.sequence_count + count = _testlimitedcapi.sequence_count lst = ['a', 'b', 'a'] self.assertEqual(count(lst, 'a'), 2) @@ -935,7 +936,7 @@ def test_sequence_count(self): self.assertRaises(SystemError, count, NULL, 'a') def test_sequence_contains(self): - contains = _testcapi.sequence_contains + contains = _testlimitedcapi.sequence_contains lst = ['a', 'b', 'a'] self.assertEqual(contains(lst, 'a'), 1) @@ -954,7 +955,7 @@ def test_sequence_contains(self): # CRASHES contains(NULL, 'a') def test_sequence_index(self): - index = _testcapi.sequence_index + index = _testlimitedcapi.sequence_index lst = ['a', 'b', 'a'] self.assertEqual(index(lst, 'a'), 0) @@ -974,7 +975,7 @@ def test_sequence_index(self): self.assertRaises(SystemError, index, NULL, 'a') def test_sequence_list(self): - xlist = _testcapi.sequence_list + xlist = _testlimitedcapi.sequence_list self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) @@ -984,7 +985,7 @@ def test_sequence_list(self): self.assertRaises(SystemError, xlist, NULL) def test_sequence_tuple(self): - xtuple = _testcapi.sequence_tuple + xtuple = _testlimitedcapi.sequence_tuple self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) @@ -994,7 +995,7 @@ def test_sequence_tuple(self): self.assertRaises(SystemError, xtuple, NULL) def test_number_check(self): - number_check = _testcapi.number_check + number_check = _testlimitedcapi.number_check self.assertTrue(number_check(1 + 1j)) self.assertTrue(number_check(1)) self.assertTrue(number_check(0.5)) diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index cb94d562645916..92c987794142c9 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -9,6 +9,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -29,7 +30,7 @@ class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() - check = _testcapi.float_check + check = _testlimitedcapi.float_check self.assertTrue(check(4.25)) self.assertTrue(check(FloatSubclass(4.25))) @@ -41,7 +42,7 @@ def test_check(self): def test_checkexact(self): # Test PyFloat_CheckExact() - checkexact = _testcapi.float_checkexact + checkexact = _testlimitedcapi.float_checkexact self.assertTrue(checkexact(4.25)) self.assertFalse(checkexact(FloatSubclass(4.25))) @@ -53,7 +54,7 @@ def test_checkexact(self): def test_fromstring(self): # Test PyFloat_FromString() - fromstring = _testcapi.float_fromstring + fromstring = _testlimitedcapi.float_fromstring self.assertEqual(fromstring("4.25"), 4.25) self.assertEqual(fromstring(b"4.25"), 4.25) @@ -72,13 +73,13 @@ def test_fromstring(self): def test_fromdouble(self): # Test PyFloat_FromDouble() - fromdouble = _testcapi.float_fromdouble + fromdouble = _testlimitedcapi.float_fromdouble self.assertEqual(fromdouble(4.25), 4.25) def test_asdouble(self): # Test PyFloat_AsDouble() - asdouble = _testcapi.float_asdouble + asdouble = _testlimitedcapi.float_asdouble class BadFloat3: def __float__(self): @@ -109,19 +110,19 @@ def __float__(self): def test_getinfo(self): # Test PyFloat_GetInfo() - getinfo = _testcapi.float_getinfo + getinfo = _testlimitedcapi.float_getinfo self.assertEqual(getinfo(), sys.float_info) def test_getmax(self): # Test PyFloat_GetMax() - getmax = _testcapi.float_getmax + getmax = _testlimitedcapi.float_getmax self.assertEqual(getmax(), sys.float_info.max) def test_getmin(self): # Test PyFloat_GetMax() - getmin = _testcapi.float_getmin + getmin = _testlimitedcapi.float_getmin self.assertEqual(getmin(), sys.float_info.min) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index d59271435e9eb0..ad89a22c625dfd 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -448,15 +448,15 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys from test.support import import_helper - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) - self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testlimitedcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d4743d31b465a9..d6c322e715efe9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c index a8ba009eb6a54b..b126aee5b9777b 100644 --- a/Modules/_testcapi/abstract.c +++ b/Modules/_testcapi/abstract.c @@ -2,59 +2,6 @@ #include "util.h" -static PyObject * -object_repr(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Repr(arg); -} - -static PyObject * -object_ascii(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_ASCII(arg); -} - -static PyObject * -object_str(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Str(arg); -} - -static PyObject * -object_bytes(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyObject_Bytes(arg); -} - -static PyObject * -object_getattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyObject_GetAttr(obj, attr_name); -} - -static PyObject * -object_getattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyObject_GetAttrString(obj, attr_name); -} - static PyObject * object_getoptionalattr(PyObject *self, PyObject *args) { @@ -106,31 +53,6 @@ object_getoptionalattrstring(PyObject *self, PyObject *args) } } -static PyObject * -object_hasattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); -} - -static PyObject * -object_hasattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { - return NULL; - } - NULLABLE(obj); - return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); -} - static PyObject * object_hasattrwitherror(PyObject *self, PyObject *args) { @@ -157,121 +79,17 @@ object_hasattrstringwitherror(PyObject *self, PyObject *args) } static PyObject * -object_setattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value; - if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - NULLABLE(value); - RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); -} - -static PyObject * -object_setattrstring(PyObject *self, PyObject *args) -{ - PyObject *obj, *value; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(value); - RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); -} - -static PyObject * -object_delattr(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name; -if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - RETURN_INT(PyObject_DelAttr(obj, attr_name)); -} - -static PyObject * -object_delattrstring(PyObject *self, PyObject *args) +mapping_getoptionalitemstring(PyObject *self, PyObject *args) { - PyObject *obj; + PyObject *obj, *value = UNINITIALIZED_PTR; const char *attr_name; Py_ssize_t size; if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { return NULL; } NULLABLE(obj); - RETURN_INT(PyObject_DelAttrString(obj, attr_name)); -} - -static PyObject * -number_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyBool_FromLong(PyNumber_Check(obj)); -} - -static PyObject * -mapping_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyMapping_Check(obj)); -} - -static PyObject * -mapping_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Size(obj)); -} - -static PyObject * -mapping_length(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyMapping_Length(obj)); -} - -static PyObject * -object_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyObject_GetItem(mapping, key); -} - -static PyObject * -mapping_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyMapping_GetItemString(mapping, key); -} - -static PyObject * -mapping_getoptionalitem(PyObject *self, PyObject *args) -{ - PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(attr_name); - switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -281,23 +99,22 @@ mapping_getoptionalitem(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); Py_UNREACHABLE(); } } static PyObject * -mapping_getoptionalitemstring(PyObject *self, PyObject *args) +mapping_getoptionalitem(PyObject *self, PyObject *args) { - PyObject *obj, *value = UNINITIALIZED_PTR; - const char *attr_name; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + PyObject *obj, *attr_name, *value = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { return NULL; } NULLABLE(obj); + NULLABLE(attr_name); - switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { case -1: assert(value == NULL); return NULL; @@ -307,399 +124,19 @@ mapping_getoptionalitemstring(PyObject *self, PyObject *args) case 1: return value; default: - Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); Py_UNREACHABLE(); } } -static PyObject * -mapping_haskey(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - return PyLong_FromLong(PyMapping_HasKey(mapping, key)); -} - -static PyObject * -mapping_haskeystring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); -} - -static PyObject * -mapping_haskeywitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); -} - -static PyObject * -mapping_haskeystringwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); -} - -static PyObject * -object_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyObject_SetItem(mapping, key, value)); -} - -static PyObject * -mapping_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyMapping_SetItemString(mapping, key, value)); -} - -static PyObject * -object_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyObject_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyMapping_DelItem(mapping, key)); -} - -static PyObject * -mapping_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyMapping_DelItemString(mapping, key)); -} - -static PyObject * -mapping_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Keys(obj); -} - -static PyObject * -mapping_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Values(obj); -} - -static PyObject * -mapping_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyMapping_Items(obj); -} - - -static PyObject * -sequence_check(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PySequence_Check(obj)); -} - -static PyObject * -sequence_size(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Size(obj)); -} - -static PyObject * -sequence_length(PyObject* self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PySequence_Length(obj)); -} - -static PyObject * -sequence_concat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_Concat(seq1, seq2); -} - -static PyObject * -sequence_repeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_Repeat(seq, count); -} - -static PyObject * -sequence_inplaceconcat(PyObject *self, PyObject *args) -{ - PyObject *seq1, *seq2; - if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { - return NULL; - } - NULLABLE(seq1); - NULLABLE(seq2); - - return PySequence_InPlaceConcat(seq1, seq2); -} - -static PyObject * -sequence_inplacerepeat(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t count; - if (!PyArg_ParseTuple(args, "On", &seq, &count)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_InPlaceRepeat(seq, count); -} - -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - return PySequence_GetItem(seq, i); -} - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(val); - - RETURN_INT(PySequence_SetItem(seq, i, val)); -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - NULLABLE(seq); - - RETURN_INT(PySequence_DelItem(seq, i)); -} - -static PyObject * -sequence_setslice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - NULLABLE(sequence); - NULLABLE(obj); - - RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); -} - -static PyObject * -sequence_delslice(PyObject *self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - NULLABLE(sequence); - - RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); -} - -static PyObject * -sequence_count(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Count(seq, value)); -} - -static PyObject * -sequence_contains(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_INT(PySequence_Contains(seq, value)); -} - -static PyObject * -sequence_index(PyObject *self, PyObject *args) -{ - PyObject *seq, *value; - if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { - return NULL; - } - NULLABLE(seq); - NULLABLE(value); - - RETURN_SIZE(PySequence_Index(seq, value)); -} - -static PyObject * -sequence_list(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_List(obj); -} - -static PyObject * -sequence_tuple(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PySequence_Tuple(obj); -} - static PyMethodDef test_methods[] = { - {"object_repr", object_repr, METH_O}, - {"object_ascii", object_ascii, METH_O}, - {"object_str", object_str, METH_O}, - {"object_bytes", object_bytes, METH_O}, - - {"object_getattr", object_getattr, METH_VARARGS}, - {"object_getattrstring", object_getattrstring, METH_VARARGS}, {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, - {"object_hasattr", object_hasattr, METH_VARARGS}, - {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, {"object_hasattrwitherror", object_hasattrwitherror, METH_VARARGS}, {"object_hasattrstringwitherror", object_hasattrstringwitherror, METH_VARARGS}, - {"object_setattr", object_setattr, METH_VARARGS}, - {"object_setattrstring", object_setattrstring, METH_VARARGS}, - {"object_delattr", object_delattr, METH_VARARGS}, - {"object_delattrstring", object_delattrstring, METH_VARARGS}, - - {"number_check", number_check, METH_O}, - {"mapping_check", mapping_check, METH_O}, - {"mapping_size", mapping_size, METH_O}, - {"mapping_length", mapping_length, METH_O}, - {"object_getitem", object_getitem, METH_VARARGS}, - {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, - {"mapping_haskey", mapping_haskey, METH_VARARGS}, - {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, - {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, - {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, - {"object_setitem", object_setitem, METH_VARARGS}, - {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, - {"object_delitem", object_delitem, METH_VARARGS}, - {"mapping_delitem", mapping_delitem, METH_VARARGS}, - {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, - {"mapping_keys", mapping_keys, METH_O}, - {"mapping_values", mapping_values, METH_O}, - {"mapping_items", mapping_items, METH_O}, - - {"sequence_check", sequence_check, METH_O}, - {"sequence_size", sequence_size, METH_O}, - {"sequence_length", sequence_length, METH_O}, - {"sequence_concat", sequence_concat, METH_VARARGS}, - {"sequence_repeat", sequence_repeat, METH_VARARGS}, - {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, - {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"sequence_setslice", sequence_setslice, METH_VARARGS}, - {"sequence_delslice", sequence_delslice, METH_VARARGS}, - {"sequence_count", sequence_count, METH_VARARGS}, - {"sequence_contains", sequence_contains, METH_VARARGS}, - {"sequence_index", sequence_index, METH_VARARGS}, - {"sequence_list", sequence_list, METH_O}, - {"sequence_tuple", sequence_tuple, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index 4fcbaf3bb2aa1e..15ea97ec4520b7 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -6,71 +6,6 @@ #include "clinic/float.c.h" -static PyObject * -float_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_Check(obj)); -} - -static PyObject * -float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyFloat_CheckExact(obj)); -} - -static PyObject * -float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyFloat_FromString(obj); -} - -static PyObject * -float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - if (!PyArg_Parse(obj, "d", &d)) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double d; - - NULLABLE(obj); - d = PyFloat_AsDouble(obj); - if (d == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(d); -} - -static PyObject * -float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_GetInfo(); -} - -static PyObject * -float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMax()); -} - -static PyObject * -float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) -{ - return PyFloat_FromDouble(PyFloat_GetMin()); -} - /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -165,14 +100,6 @@ _testcapi_float_unpack_impl(PyObject *module, const char *data, } static PyMethodDef test_methods[] = { - {"float_check", float_check, METH_O}, - {"float_checkexact", float_checkexact, METH_O}, - {"float_fromstring", float_fromstring, METH_O}, - {"float_fromdouble", float_fromdouble, METH_O}, - {"float_asdouble", float_asdouble, METH_O}, - {"float_getinfo", float_getinfo, METH_NOARGS}, - {"float_getmax", float_getmax, METH_NOARGS}, - {"float_getmin", float_getmin, METH_NOARGS}, _TESTCAPI_FLOAT_PACK_METHODDEF _TESTCAPI_FLOAT_UNPACK_METHODDEF {NULL}, diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 9edc4fe3a9e970..756e2fb252f2e2 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -26,12 +26,18 @@ PyInit__testlimitedcapi(void) return NULL; } + if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_ByteArray(mod) < 0) { return NULL; } if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_HeaptypeRelative(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/abstract.c b/Modules/_testlimitedcapi/abstract.c new file mode 100644 index 00000000000000..6056dd100d6069 --- /dev/null +++ b/Modules/_testlimitedcapi/abstract.c @@ -0,0 +1,582 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +object_repr(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Repr(arg); +} + +static PyObject * +object_ascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_ASCII(arg); +} + +static PyObject * +object_str(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Str(arg); +} + +static PyObject * +object_bytes(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyObject_Bytes(arg); +} + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + +static PyObject * +number_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyBool_FromLong(PyNumber_Check(obj)); +} + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +mapping_haskeywitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_HasKeyWithError(mapping, key)); +} + +static PyObject * +mapping_haskeystringwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_HasKeyStringWithError(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_repr", object_repr, METH_O}, + {"object_ascii", object_ascii, METH_O}, + {"object_str", object_str, METH_O}, + {"object_bytes", object_bytes, METH_O}, + + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"number_check", number_check, METH_O}, + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"mapping_haskeywitherror", mapping_haskeywitherror, METH_VARARGS}, + {"mapping_haskeystringwitherror", mapping_haskeystringwitherror, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/float.c b/Modules/_testlimitedcapi/float.c new file mode 100644 index 00000000000000..f9865e4be2c4a8 --- /dev/null +++ b/Modules/_testlimitedcapi/float.c @@ -0,0 +1,92 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +float_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_Check(obj)); +} + +static PyObject * +float_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyFloat_CheckExact(obj)); +} + +static PyObject * +float_fromstring(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyFloat_FromString(obj); +} + +static PyObject * +float_fromdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + if (!PyArg_Parse(obj, "d", &d)) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_asdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double d; + + NULLABLE(obj); + d = PyFloat_AsDouble(obj); + if (d == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(d); +} + +static PyObject * +float_getinfo(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_GetInfo(); +} + +static PyObject * +float_getmax(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMax()); +} + +static PyObject * +float_getmin(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(arg)) +{ + return PyFloat_FromDouble(PyFloat_GetMin()); +} + + +static PyMethodDef test_methods[] = { + {"float_check", float_check, METH_O}, + {"float_checkexact", float_checkexact, METH_O}, + {"float_fromstring", float_fromstring, METH_O}, + {"float_fromdouble", float_fromdouble, METH_O}, + {"float_asdouble", float_asdouble, METH_O}, + {"float_getinfo", float_getinfo, METH_NOARGS}, + {"float_getmax", float_getmax, METH_NOARGS}, + {"float_getmin", float_getmin, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Float(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} + diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index ee67f10e4b4830..461fe7aa2cc337 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -22,8 +22,10 @@ # error "Py_BUILD_CORE macro must not be defined" #endif +int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); int _PyTestLimitedCAPI_Init_List(PyObject *module); int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index dc2ccca6bad963..43136697c5dac0 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -94,8 +94,10 @@ + + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 33616be32c09b2..d3cdc47957eed5 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -9,8 +9,10 @@ + + From 5405e9e5b51f3bd883aee5c1a52a39a56e2fb2b4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 12:05:22 +0100 Subject: [PATCH 26/82] gh-116869: Build test_cext with -Werror=declaration-after-statement (#116990) Make sure that the C API does not emit compiler warnings when built with -Werror=declaration-after-statement. --- Lib/test/test_cext/__init__.py | 2 ++ Lib/test/test_cext/setup.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 730b91abfb2715..77929495336638 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -1,5 +1,7 @@ # gh-116869: Build a basic C test extension to check that the Python C API # does not emit C compiler warnings. +# +# Python C API must build with -Werror=declaration-after-statement. import os.path import shutil diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index dd57a5fb8e1e1b..aed7e6d46dd8d0 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -16,6 +16,10 @@ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + + # gh-116869: The Python C API must build with + # -Werror=declaration-after-statement. + '-Werror=declaration-after-statement', ] else: # Don't pass any compiler flag to MSVC From 0f278012e88fa9607d85bc6c7265fd394f0ac163 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 19 Mar 2024 07:06:43 -0400 Subject: [PATCH 27/82] gh-116808: Fix optimized trace length histogram (GH-116827) --- Python/optimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index a6d6ffe5378636..bb00e0d2575784 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -913,6 +913,8 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende if (executor == NULL) { return NULL; } + OPT_HIST(length, optimized_trace_length_hist); + /* Initialize exits */ for (int i = 0; i < exit_count; i++) { executor->exits[i].executor = &COLD_EXITS[i]; @@ -1051,7 +1053,6 @@ uop_optimize( if (executor == NULL) { return -1; } - OPT_HIST(Py_SIZE(executor), optimized_trace_length_hist); *exec_ptr = executor; return 1; } From 1312094b398e6153da2b8068bbfeca0360012964 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 13:19:45 +0100 Subject: [PATCH 28/82] gh-116869: Test the limited C API in test_cext (#116995) --- Lib/test/test_cext/__init__.py | 22 +++++++++++----- Lib/test/test_cext/extension.c | 16 +++++------- Lib/test/test_cext/setup.py | 47 ++++++++++++++++++++++++---------- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 77929495336638..992939241c591e 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -27,17 +27,24 @@ @support.requires_resource('cpu') class TestExt(unittest.TestCase): def test_build_c99(self): - self.check_build('c99', '_test_c99_ext') + self.check_build('_test_c99_ext', std='c99') def test_build_c11(self): - self.check_build('c11', '_test_c11_ext') + self.check_build('_test_c11_ext', std='c11') - def check_build(self, clang_std, extension_name): + def test_build_limited(self): + self.check_build('_test_limited_ext', limited=True) + + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_ext', limited=True, std='c11') + + def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(clang_std, extension_name, python_exe) + self._check_build(extension_name, python_exe, + std=std, limited=limited) - def _check_build(self, clang_std, extension_name, python_exe): + def _check_build(self, extension_name, python_exe, std, limited): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -45,7 +52,10 @@ def _check_build(self, clang_std, extension_name, python_exe): def run_cmd(operation, cmd): env = os.environ.copy() - env['CPYTHON_TEST_STD'] = clang_std + if std: + env['CPYTHON_TEST_STD'] = std + if limited: + env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: print('Run:', ' '.join(cmd)) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index cfecad39f8af14..a2433f2663c667 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -6,12 +6,8 @@ #include "Python.h" -#if defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L -# define NAME _test_c2x_ext -#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -# define NAME _test_c11_ext -#else -# define NAME _test_c99_ext +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" #endif #define _STR(NAME) #NAME @@ -25,11 +21,11 @@ PyDoc_STRVAR(_testcext_add_doc, static PyObject * _testcext_add(PyObject *Py_UNUSED(module), PyObject *args) { - long i, j; + long i, j, res; if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { return NULL; } - long res = i + j; + res = i + j; return PyLong_FromLong(res); } @@ -59,7 +55,7 @@ PyDoc_STRVAR(_testcext_doc, "C test extension."); static struct PyModuleDef _testcext_module = { PyModuleDef_HEAD_INIT, // m_base - STR(NAME), // m_name + STR(MODULE_NAME), // m_name _testcext_doc, // m_doc 0, // m_size _testcext_methods, // m_methods @@ -74,7 +70,7 @@ static struct PyModuleDef _testcext_module = { #define FUNC_NAME(NAME) _FUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(NAME)(void) +FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcext_module); } diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index aed7e6d46dd8d0..5ca080a790a5bd 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -27,24 +27,43 @@ def main(): - std = os.environ["CPYTHON_TEST_STD"] - name = os.environ["CPYTHON_TEST_EXT_NAME"] - cflags = [*CFLAGS, f'-std={std}'] - - # Remove existing -std options to only test ours - cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: - cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] - cmd = shlex.join(cmd) - # CC env var overrides sysconfig CC variable in setuptools - os.environ['CC'] = cmd + std = os.environ.get("CPYTHON_TEST_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + + cflags = list(CFLAGS) + cflags.append(f'-DMODULE_NAME={module_name}') + + if std: + cflags.append(f'-std={std}') + + # Remove existing -std options to only test ours + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + if limited: + version = sys.hexversion + cflags.append(f'-DPy_LIMITED_API={version:#x}') + + for env_name in ('CC', 'CFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cflags!r}") ext = Extension( - name, + module_name, sources=[SOURCE], extra_compile_args=cflags) - setup(name='internal' + name, version='0.0', ext_modules=[ext]) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) if __name__ == "__main__": From a5574789876987b2b9aa19294c735fe795a5b5c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 13:30:39 +0100 Subject: [PATCH 29/82] gh-116417: Move limited C API unicode.c tests to _testlimitedcapi (#116993) Split unicode.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. Update test_codecs. --- Lib/test/test_capi/test_codecs.py | 46 +- Lib/test/test_capi/test_misc.py | 2 +- Lib/test/test_capi/test_unicode.py | 206 +-- Lib/test/test_codecs.py | 14 +- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/unicode.c | 1899 +-------------------- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/float.c | 1 - Modules/_testlimitedcapi/parts.h | 1 + Modules/_testlimitedcapi/unicode.c | 1938 ++++++++++++++++++++++ PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 12 files changed, 2082 insertions(+), 2032 deletions(-) create mode 100644 Modules/_testlimitedcapi/unicode.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 682c56979c6dfa..bd521a509d07ec 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -2,7 +2,7 @@ import sys from test.support import import_helper -_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -27,7 +27,7 @@ class CAPITest(unittest.TestCase): def test_fromencodedobject(self): """Test PyUnicode_FromEncodedObject()""" - fromencodedobject = _testcapi.unicode_fromencodedobject + fromencodedobject = _testlimitedcapi.unicode_fromencodedobject self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') @@ -52,7 +52,7 @@ def test_fromencodedobject(self): def test_decode(self): """Test PyUnicode_Decode()""" - decode = _testcapi.unicode_decode + decode = _testlimitedcapi.unicode_decode self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') @@ -70,7 +70,7 @@ def test_decode(self): def test_asencodedstring(self): """Test PyUnicode_AsEncodedString()""" - asencodedstring = _testcapi.unicode_asencodedstring + asencodedstring = _testlimitedcapi.unicode_asencodedstring self.assertEqual(asencodedstring('abc', NULL), b'abc') self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') @@ -93,7 +93,7 @@ def test_asencodedstring(self): def test_decodeutf8(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf8 = _testcapi.unicode_decodeutf8 + decodeutf8 = _testlimitedcapi.unicode_decodeutf8 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -113,7 +113,7 @@ def test_decodeutf8(self): def test_decodeutf8stateful(self): """Test PyUnicode_DecodeUTF8Stateful()""" - decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: b = s.encode('utf-8') @@ -136,7 +136,7 @@ def test_decodeutf8stateful(self): def test_asutf8string(self): """Test PyUnicode_AsUTF8String()""" - asutf8string = _testcapi.unicode_asutf8string + asutf8string = _testlimitedcapi.unicode_asutf8string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf8string(s), s.encode('utf-8')) @@ -148,7 +148,7 @@ def test_asutf8string(self): def test_decodeutf16(self): """Test PyUnicode_DecodeUTF16()""" - decodeutf16 = _testcapi.unicode_decodeutf16 + decodeutf16 = _testlimitedcapi.unicode_decodeutf16 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -192,7 +192,7 @@ def test_decodeutf16(self): def test_decodeutf16stateful(self): """Test PyUnicode_DecodeUTF16Stateful()""" - decodeutf16stateful = _testcapi.unicode_decodeutf16stateful + decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -238,7 +238,7 @@ def test_decodeutf16stateful(self): def test_asutf16string(self): """Test PyUnicode_AsUTF16String()""" - asutf16string = _testcapi.unicode_asutf16string + asutf16string = _testlimitedcapi.unicode_asutf16string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf16string(s), s.encode('utf-16')) @@ -250,7 +250,7 @@ def test_asutf16string(self): def test_decodeutf32(self): """Test PyUnicode_DecodeUTF8()""" - decodeutf32 = _testcapi.unicode_decodeutf32 + decodeutf32 = _testlimitedcapi.unicode_decodeutf32 naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -290,7 +290,7 @@ def test_decodeutf32(self): def test_decodeutf32stateful(self): """Test PyUnicode_DecodeUTF32Stateful()""" - decodeutf32stateful = _testcapi.unicode_decodeutf32stateful + decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful naturalbyteorder = -1 if sys.byteorder == 'little' else 1 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: @@ -342,7 +342,7 @@ def test_decodeutf32stateful(self): def test_asutf32string(self): """Test PyUnicode_AsUTF32String()""" - asutf32string = _testcapi.unicode_asutf32string + asutf32string = _testlimitedcapi.unicode_asutf32string for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: self.assertEqual(asutf32string(s), s.encode('utf-32')) @@ -354,7 +354,7 @@ def test_asutf32string(self): def test_decodelatin1(self): """Test PyUnicode_DecodeLatin1()""" - decodelatin1 = _testcapi.unicode_decodelatin1 + decodelatin1 = _testlimitedcapi.unicode_decodelatin1 self.assertEqual(decodelatin1(b'abc'), 'abc') self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') @@ -365,7 +365,7 @@ def test_decodelatin1(self): def test_aslatin1string(self): """Test PyUnicode_AsLatin1String()""" - aslatin1string = _testcapi.unicode_aslatin1string + aslatin1string = _testlimitedcapi.unicode_aslatin1string self.assertEqual(aslatin1string('abc'), b'abc') self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') @@ -377,7 +377,7 @@ def test_aslatin1string(self): def test_decodeascii(self): """Test PyUnicode_DecodeASCII()""" - decodeascii = _testcapi.unicode_decodeascii + decodeascii = _testlimitedcapi.unicode_decodeascii self.assertEqual(decodeascii(b'abc'), 'abc') self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') @@ -392,7 +392,7 @@ def test_decodeascii(self): def test_asasciistring(self): """Test PyUnicode_AsASCIIString()""" - asasciistring = _testcapi.unicode_asasciistring + asasciistring = _testlimitedcapi.unicode_asasciistring self.assertEqual(asasciistring('abc'), b'abc') @@ -403,7 +403,7 @@ def test_asasciistring(self): def test_decodecharmap(self): """Test PyUnicode_DecodeCharmap()""" - decodecharmap = _testcapi.unicode_decodecharmap + decodecharmap = _testlimitedcapi.unicode_decodecharmap self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') @@ -426,7 +426,7 @@ def test_decodecharmap(self): def test_ascharmapstring(self): """Test PyUnicode_AsCharmapString()""" - ascharmapstring = _testcapi.unicode_ascharmapstring + ascharmapstring = _testlimitedcapi.unicode_ascharmapstring self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') @@ -443,7 +443,7 @@ def test_ascharmapstring(self): def test_decodeunicodeescape(self): """Test PyUnicode_DecodeUnicodeEscape()""" - decodeunicodeescape = _testcapi.unicode_decodeunicodeescape + decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape self.assertEqual(decodeunicodeescape(b'abc'), 'abc') self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') @@ -467,7 +467,7 @@ def test_decodeunicodeescape(self): def test_asunicodeescapestring(self): """Test PyUnicode_AsUnicodeEscapeString()""" - asunicodeescapestring = _testcapi.unicode_asunicodeescapestring + asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring self.assertEqual(asunicodeescapestring('abc'), b'abc') self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') @@ -481,7 +481,7 @@ def test_asunicodeescapestring(self): def test_decoderawunicodeescape(self): """Test PyUnicode_DecodeRawUnicodeEscape()""" - decoderawunicodeescape = _testcapi.unicode_decoderawunicodeescape + decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') @@ -503,7 +503,7 @@ def test_decoderawunicodeescape(self): def test_asrawunicodeescapestring(self): """Test PyUnicode_AsRawUnicodeEscapeString()""" - asrawunicodeescapestring = _testcapi.unicode_asrawunicodeescapestring + asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring self.assertEqual(asrawunicodeescapestring('abc'), b'abc') self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index eb0bc13911701a..d3fcd0b59dfa49 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2338,7 +2338,7 @@ class Test_testcapi(unittest.TestCase): # Suppress warning from PyUnicode_FromUnicode(). @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_widechar(self): - _testcapi.test_widechar() + _testlimitedcapi.test_widechar() def test_version_api_data(self): self.assertEqual(_testcapi.Py_Version, sys.hexversion) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 91c425e483f0ff..a64c75c415c3fe 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -8,6 +8,10 @@ from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -84,10 +88,10 @@ def test_fill(self): # TODO: Test PyUnicode_Fill() with non-modifiable unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_writechar(self): """Test PyUnicode_WriteChar()""" - from _testcapi import unicode_writechar as writechar + from _testlimitedcapi import unicode_writechar as writechar strings = [ # one string for every kind @@ -115,10 +119,10 @@ def test_writechar(self): # unicode. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_resize(self): """Test PyUnicode_Resize()""" - from _testcapi import unicode_resize as resize + from _testlimitedcapi import unicode_resize as resize strings = [ # all strings have exactly 3 characters @@ -141,10 +145,10 @@ def test_resize(self): # and with NULL as the address. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_append(self): """Test PyUnicode_Append()""" - from _testcapi import unicode_append as append + from _testlimitedcapi import unicode_append as append strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -169,10 +173,10 @@ def test_append(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_appendanddel(self): """Test PyUnicode_AppendAndDel()""" - from _testcapi import unicode_appendanddel as appendanddel + from _testlimitedcapi import unicode_appendanddel as appendanddel strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -196,10 +200,10 @@ def test_appendanddel(self): # TODO: Check reference counts. @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstringandsize(self): """Test PyUnicode_FromStringAndSize()""" - from _testcapi import unicode_fromstringandsize as fromstringandsize + from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize self.assertEqual(fromstringandsize(b'abc'), 'abc') self.assertEqual(fromstringandsize(b'abc', 2), 'ab') @@ -221,10 +225,10 @@ def test_fromstringandsize(self): self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromstring(self): """Test PyUnicode_FromString()""" - from _testcapi import unicode_fromstring as fromstring + from _testlimitedcapi import unicode_fromstring as fromstring self.assertEqual(fromstring(b'abc'), 'abc') self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') @@ -273,10 +277,10 @@ def test_fromkindanddata(self): # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_substring(self): """Test PyUnicode_Substring()""" - from _testcapi import unicode_substring as substring + from _testlimitedcapi import unicode_substring as substring strings = [ 'ab', 'ab\xa1\xa2', @@ -297,10 +301,10 @@ def test_substring(self): # CRASHES substring(NULL, 0, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getlength(self): """Test PyUnicode_GetLength()""" - from _testcapi import unicode_getlength as getlength + from _testlimitedcapi import unicode_getlength as getlength for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -311,10 +315,10 @@ def test_getlength(self): # CRASHES getlength(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_readchar(self): """Test PyUnicode_ReadChar()""" - from _testcapi import unicode_readchar as readchar + from _testlimitedcapi import unicode_readchar as readchar for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -330,10 +334,10 @@ def test_readchar(self): # CRASHES readchar(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromobject(self): """Test PyUnicode_FromObject()""" - from _testcapi import unicode_fromobject as fromobject + from _testlimitedcapi import unicode_fromobject as fromobject for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -352,7 +356,7 @@ def test_from_format(self): """Test PyUnicode_FromFormat()""" # Length modifiers "j" and "t" are not tested here because ctypes does # not expose types for intmax_t and ptrdiff_t. - # _testcapi.test_string_from_format() has a wider coverage of all + # _testlimitedcapi.test_string_from_format() has a wider coverage of all # formats. import_helper.import_module('ctypes') from ctypes import ( @@ -609,40 +613,6 @@ def check_format(expected, format, *args): check_format('xyz', b'%V', None, b'xyz') - # test %T - check_format('type: str', - b'type: %T', py_object("abc")) - check_format(f'type: st', - b'type: %.2T', py_object("abc")) - check_format(f'type: str', - b'type: %10T', py_object("abc")) - - class LocalType: - pass - obj = LocalType() - fullname = f'{__name__}.{LocalType.__qualname__}' - check_format(f'type: {fullname}', - b'type: %T', py_object(obj)) - fullname_alt = f'{__name__}:{LocalType.__qualname__}' - check_format(f'type: {fullname_alt}', - b'type: %T#', py_object(obj)) - - # test %N - check_format('type: str', - b'type: %N', py_object(str)) - check_format(f'type: st', - b'type: %.2N', py_object(str)) - check_format(f'type: str', - b'type: %10N', py_object(str)) - - check_format(f'type: {fullname}', - b'type: %N', py_object(type(obj))) - check_format(f'type: {fullname_alt}', - b'type: %N#', py_object(type(obj))) - with self.assertRaisesRegex(TypeError, "%N argument must be a type"): - check_format('type: str', - b'type: %N', py_object("abc")) - # test %ls check_format('abc', b'%ls', c_wchar_p('abc')) check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) @@ -741,10 +711,10 @@ class LocalType: PyUnicode_FromFormat, b'%+i', c_int(10)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_interninplace(self): """Test PyUnicode_InternInPlace()""" - from _testcapi import unicode_interninplace as interninplace + from _testlimitedcapi import unicode_interninplace as interninplace s = b'abc'.decode() r = interninplace(s) @@ -754,10 +724,10 @@ def test_interninplace(self): # CRASHES interninplace(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_internfromstring(self): """Test PyUnicode_InternFromString()""" - from _testcapi import unicode_internfromstring as internfromstring + from _testlimitedcapi import unicode_internfromstring as internfromstring self.assertEqual(internfromstring(b'abc'), 'abc') self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') @@ -768,10 +738,10 @@ def test_internfromstring(self): # CRASHES internfromstring(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromwidechar(self): """Test PyUnicode_FromWideChar()""" - from _testcapi import unicode_fromwidechar as fromwidechar + from _testlimitedcapi import unicode_fromwidechar as fromwidechar from _testcapi import SIZEOF_WCHAR_T if SIZEOF_WCHAR_T == 2: @@ -803,11 +773,11 @@ def test_fromwidechar(self): self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" - from _testcapi import unicode_aswidechar - from _testcapi import unicode_aswidechar_null + from _testlimitedcapi import unicode_aswidechar + from _testlimitedcapi import unicode_aswidechar_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) @@ -851,11 +821,11 @@ def test_aswidechar(self): self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" - from _testcapi import unicode_aswidecharstring - from _testcapi import unicode_aswidecharstring_null + from _testlimitedcapi import unicode_aswidecharstring + from _testlimitedcapi import unicode_aswidecharstring_null from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') @@ -927,10 +897,10 @@ def test_asucs4copy(self): # CRASHES asucs4copy(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_fromordinal(self): """Test PyUnicode_FromOrdinal()""" - from _testcapi import unicode_fromordinal as fromordinal + from _testlimitedcapi import unicode_fromordinal as fromordinal self.assertEqual(fromordinal(0x61), 'a') self.assertEqual(fromordinal(0x20ac), '\u20ac') @@ -956,11 +926,11 @@ def test_asutf8(self): # CRASHES unicode_asutf8(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" - from _testcapi import unicode_asutf8andsize - from _testcapi import unicode_asutf8andsize_null + from _testlimitedcapi import unicode_asutf8andsize + from _testlimitedcapi import unicode_asutf8andsize_null self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) @@ -979,10 +949,10 @@ def test_asutf8andsize(self): # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_getdefaultencoding(self): """Test PyUnicode_GetDefaultEncoding()""" - from _testcapi import unicode_getdefaultencoding as getdefaultencoding + from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding self.assertEqual(getdefaultencoding(), b'utf-8') @@ -1007,10 +977,10 @@ def test_transform_decimal_and_space(self): # CRASHES transform_decimal(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_concat(self): """Test PyUnicode_Concat()""" - from _testcapi import unicode_concat as concat + from _testlimitedcapi import unicode_concat as concat self.assertEqual(concat('abc', 'def'), 'abcdef') self.assertEqual(concat('abc', 'где'), 'abcгде') @@ -1028,10 +998,10 @@ def test_concat(self): # CRASHES concat('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_split(self): """Test PyUnicode_Split()""" - from _testcapi import unicode_split as split + from _testlimitedcapi import unicode_split as split self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) @@ -1056,10 +1026,10 @@ def test_split(self): # CRASHES split(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rsplit(self): """Test PyUnicode_RSplit()""" - from _testcapi import unicode_rsplit as rsplit + from _testlimitedcapi import unicode_rsplit as rsplit self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) @@ -1085,10 +1055,10 @@ def test_rsplit(self): # CRASHES rsplit(NULL, '|') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_partition(self): """Test PyUnicode_Partition()""" - from _testcapi import unicode_partition as partition + from _testlimitedcapi import unicode_partition as partition self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) @@ -1105,10 +1075,10 @@ def test_partition(self): # CRASHES partition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_rpartition(self): """Test PyUnicode_RPartition()""" - from _testcapi import unicode_rpartition as rpartition + from _testlimitedcapi import unicode_rpartition as rpartition self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) @@ -1125,10 +1095,10 @@ def test_rpartition(self): # CRASHES rpartition('a|b|c', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_splitlines(self): """Test PyUnicode_SplitLines()""" - from _testcapi import unicode_splitlines as splitlines + from _testlimitedcapi import unicode_splitlines as splitlines self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) self.assertEqual(splitlines('a\nb\rc\r\nd', True), @@ -1143,10 +1113,10 @@ def test_splitlines(self): # CRASHES splitlines(NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_translate(self): """Test PyUnicode_Translate()""" - from _testcapi import unicode_translate as translate + from _testlimitedcapi import unicode_translate as translate self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') @@ -1168,10 +1138,10 @@ def test_translate(self): # CRASHES translate(NULL, []) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_join(self): """Test PyUnicode_Join()""" - from _testcapi import unicode_join as join + from _testlimitedcapi import unicode_join as join self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') self.assertEqual(join('|', ['a', '', 'c']), 'a||c') self.assertEqual(join('', ['a', 'b', 'c']), 'abc') @@ -1186,10 +1156,10 @@ def test_join(self): self.assertRaises(SystemError, join, '|', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_count(self): """Test PyUnicode_Count()""" - from _testcapi import unicode_count + from _testlimitedcapi import unicode_count for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1217,10 +1187,10 @@ def test_count(self): # CRASHES unicode_count(str, NULL, 0, len(str)) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_tailmatch(self): """Test PyUnicode_Tailmatch()""" - from _testcapi import unicode_tailmatch as tailmatch + from _testlimitedcapi import unicode_tailmatch as tailmatch str = 'ababahalamaha' self.assertEqual(tailmatch(str, 'aba', 0, len(str), -1), 1) @@ -1252,10 +1222,10 @@ def test_tailmatch(self): # CRASHES tailmatch(str, NULL, 0, len(str), -1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_find(self): """Test PyUnicode_Find()""" - from _testcapi import unicode_find as find + from _testlimitedcapi import unicode_find as find for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1293,10 +1263,10 @@ def test_find(self): # CRASHES find(str, NULL, 0, len(str), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_findchar(self): """Test PyUnicode_FindChar()""" - from _testcapi import unicode_findchar + from _testlimitedcapi import unicode_findchar for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": for i, ch in enumerate(str): @@ -1329,10 +1299,10 @@ def test_findchar(self): # CRASHES unicode_findchar(NULL, ord('!'), 0, len(str), 1), 1) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_replace(self): """Test PyUnicode_Replace()""" - from _testcapi import unicode_replace as replace + from _testlimitedcapi import unicode_replace as replace str = 'abracadabra' self.assertEqual(replace(str, 'a', '='), '=br=c=d=br=') @@ -1360,10 +1330,10 @@ def test_replace(self): # CRASHES replace(NULL, 'a', '=') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_compare(self): """Test PyUnicode_Compare()""" - from _testcapi import unicode_compare as compare + from _testlimitedcapi import unicode_compare as compare self.assertEqual(compare('abc', 'abc'), 0) self.assertEqual(compare('abc', 'def'), -1) @@ -1382,10 +1352,10 @@ def test_compare(self): # CRASHES compare('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_comparewithasciistring(self): """Test PyUnicode_CompareWithASCIIString()""" - from _testcapi import unicode_comparewithasciistring as comparewithasciistring + from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring self.assertEqual(comparewithasciistring('abc', b'abc'), 0) self.assertEqual(comparewithasciistring('abc', b'def'), -1) @@ -1399,11 +1369,11 @@ def test_comparewithasciistring(self): # CRASHES comparewithasciistring(NULL, b'abc') @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8(self): # Test PyUnicode_EqualToUTF8() - from _testcapi import unicode_equaltoutf8 as equaltoutf8 - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1445,11 +1415,11 @@ def test_equaltoutf8(self): '\ud801'.encode("utf8", "surrogatepass")), 0) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_equaltoutf8andsize(self): # Test PyUnicode_EqualToUTF8AndSize() - from _testcapi import unicode_equaltoutf8andsize as equaltoutf8andsize - from _testcapi import unicode_asutf8andsize as asutf8andsize + from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize + from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize strings = [ 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', @@ -1514,10 +1484,10 @@ def check_not_equal_encoding(text, encoding): # CRASHES equaltoutf8andsize('abc', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_richcompare(self): """Test PyUnicode_RichCompare()""" - from _testcapi import unicode_richcompare as richcompare + from _testlimitedcapi import unicode_richcompare as richcompare LT, LE, EQ, NE, GT, GE = range(6) strings = ('abc', 'абв', '\U0001f600', 'abc\0') @@ -1542,10 +1512,10 @@ def test_richcompare(self): # CRASHES richcompare('abc', NULL, op) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_format(self): """Test PyUnicode_Format()""" - from _testcapi import unicode_format as format + from _testlimitedcapi import unicode_format as format self.assertEqual(format('x=%d!', 42), 'x=42!') self.assertEqual(format('x=%d!', (42,)), 'x=42!') @@ -1555,10 +1525,10 @@ def test_format(self): self.assertRaises(SystemError, format, NULL, 42) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_contains(self): """Test PyUnicode_Contains()""" - from _testcapi import unicode_contains as contains + from _testlimitedcapi import unicode_contains as contains self.assertEqual(contains('abcd', ''), 1) self.assertEqual(contains('abcd', 'b'), 1) @@ -1577,10 +1547,10 @@ def test_contains(self): # CRASHES contains('abcd', NULL) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_isidentifier(self): """Test PyUnicode_IsIdentifier()""" - from _testcapi import unicode_isidentifier as isidentifier + from _testlimitedcapi import unicode_isidentifier as isidentifier self.assertEqual(isidentifier("a"), 1) self.assertEqual(isidentifier("b0"), 1) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index fe3776d6dd9337..e05b95c2d60bad 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -13,9 +13,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -2224,14 +2224,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2240,7 +2240,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2251,12 +2251,12 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d6c322e715efe9..99aef6f39cfd1e 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d0954bbe36ff9d..015db9017139d0 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -3,101 +3,29 @@ #include "parts.h" #include "util.h" +/* Test PyUnicode_New() */ static PyObject * -codec_incrementalencoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalEncoder(encoding, errors); -} - -static PyObject * -codec_incrementaldecoder(PyObject *self, PyObject *args) -{ - const char *encoding, *errors = NULL; - if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", - &encoding, &errors)) - return NULL; - return PyCodec_IncrementalDecoder(encoding, errors); -} - -static PyObject * -test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); - int result; - if (py_s == NULL) - return NULL; - result = PyUnicode_CompareWithASCIIString(py_s, "str"); - Py_DECREF(py_s); - if (!result) { - PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " - "should not compare equal to c string."); - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +unicode_new(PyObject *self, PyObject *args) { -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; - size_t wtextlen = 1; - const wchar_t invalid[1] = {(wchar_t)0x110000u}; -#else - const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; - size_t wtextlen = 2; -#endif - PyObject *wide, *utf8; - - wide = PyUnicode_FromWideChar(wtext, wtextlen); - if (wide == NULL) - return NULL; + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; - utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); - if (utf8 == NULL) { - Py_DECREF(wide); + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { return NULL; } - if (PyUnicode_GET_LENGTH(wide) != PyUnicode_GET_LENGTH(utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "have different length"); - return NULL; - } - if (PyUnicode_Compare(wide, utf8)) { - Py_DECREF(wide); - Py_DECREF(utf8); - if (PyErr_Occurred()) - return NULL; - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "wide string and utf8 string " - "are different"); + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { return NULL; } - - Py_DECREF(wide); - Py_DECREF(utf8); - -#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) - wide = PyUnicode_FromWideChar(invalid, 1); - if (wide == NULL) - PyErr_Clear(); - else { - PyErr_SetString(PyExc_AssertionError, - "test_widechar: " - "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); return NULL; } -#endif - Py_RETURN_NONE; + return result; } @@ -128,30 +56,6 @@ unicode_copy(PyObject *unicode) return copy; } -/* Test PyUnicode_New() */ -static PyObject * -unicode_new(PyObject *self, PyObject *args) -{ - Py_ssize_t size; - unsigned int maxchar; - PyObject *result; - - if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { - return NULL; - } - - result = PyUnicode_New(size, (Py_UCS4)maxchar); - if (!result) { - return NULL; - } - if (size > 0 && maxchar <= 0x10ffff && - PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) - { - Py_DECREF(result); - return NULL; - } - return result; -} /* Test PyUnicode_Fill() */ static PyObject * @@ -178,129 +82,6 @@ unicode_fill(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, filled); } -/* Test PyUnicode_WriteChar() */ -static PyObject * -unicode_writechar(PyObject *self, PyObject *args) -{ - PyObject *to, *to_copy; - Py_ssize_t index; - unsigned int character; - int result; - - if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { - return NULL; - } - - NULLABLE(to); - if (!(to_copy = unicode_copy(to)) && to) { - return NULL; - } - - result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); - if (result == -1 && PyErr_Occurred()) { - Py_DECREF(to_copy); - return NULL; - } - return Py_BuildValue("(Ni)", to_copy, result); -} - -/* Test PyUnicode_Resize() */ -static PyObject * -unicode_resize(PyObject *self, PyObject *args) -{ - PyObject *obj, *copy; - Py_ssize_t length; - int result; - - if (!PyArg_ParseTuple(args, "On", &obj, &length)) { - return NULL; - } - - NULLABLE(obj); - if (!(copy = unicode_copy(obj)) && obj) { - return NULL; - } - result = PyUnicode_Resize(©, length); - if (result == -1 && PyErr_Occurred()) { - Py_XDECREF(copy); - return NULL; - } - if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { - if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { - Py_DECREF(copy); - return NULL; - } - } - return Py_BuildValue("(Ni)", copy, result); -} - -/* Test PyUnicode_Append() */ -static PyObject * -unicode_append(PyObject *self, PyObject *args) -{ - PyObject *left, *right, *left_copy; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { - return NULL; - } - PyUnicode_Append(&left_copy, right); - return left_copy; -} - -/* Test PyUnicode_AppendAndDel() */ -static PyObject * -unicode_appendanddel(PyObject *self, PyObject *args) -{ - PyObject *left, *right, *left_copy; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - if (!(left_copy = unicode_copy(left)) && left) { - return NULL; - } - Py_XINCREF(right); - PyUnicode_AppendAndDel(&left_copy, right); - return left_copy; -} - -/* Test PyUnicode_FromStringAndSize() */ -static PyObject * -unicode_fromstringandsize(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { - return NULL; - } - - if (size == -100) { - size = bsize; - } - return PyUnicode_FromStringAndSize(s, size); -} - -/* Test PyUnicode_FromString() */ -static PyObject * -unicode_fromstring(PyObject *self, PyObject *arg) -{ - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { - return NULL; - } - return PyUnicode_FromString(s); -} /* Test PyUnicode_FromKindAndData() */ static PyObject * @@ -326,211 +107,9 @@ unicode_fromkindanddata(PyObject *self, PyObject *args) return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); } -/* Test PyUnicode_Substring() */ -static PyObject * -unicode_substring(PyObject *self, PyObject *args) -{ - PyObject *str; - Py_ssize_t start, end; - - if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { - return NULL; - } - - NULLABLE(str); - return PyUnicode_Substring(str, start, end); -} - -/* Test PyUnicode_GetLength() */ -static PyObject * -unicode_getlength(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - RETURN_SIZE(PyUnicode_GetLength(arg)); -} - -/* Test PyUnicode_ReadChar() */ -static PyObject * -unicode_readchar(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t index; - Py_UCS4 result; - - if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { - return NULL; - } - - NULLABLE(unicode); - result = PyUnicode_ReadChar(unicode, index); - if (result == (Py_UCS4)-1) - return NULL; - return PyLong_FromUnsignedLong(result); -} - -/* Test PyUnicode_FromEncodedObject() */ -static PyObject * -unicode_fromencodedobject(PyObject *self, PyObject *args) -{ - PyObject *obj; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { - return NULL; - } - - NULLABLE(obj); - return PyUnicode_FromEncodedObject(obj, encoding, errors); -} - -/* Test PyUnicode_FromObject() */ -static PyObject * -unicode_fromobject(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_FromObject(arg); -} - -/* Test PyUnicode_InternInPlace() */ -static PyObject * -unicode_interninplace(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - Py_XINCREF(arg); - PyUnicode_InternInPlace(&arg); - return arg; -} - -/* Test PyUnicode_InternFromString() */ -static PyObject * -unicode_internfromstring(PyObject *self, PyObject *arg) -{ - const char *s; - Py_ssize_t size; - - if (!PyArg_Parse(arg, "z#", &s, &size)) { - return NULL; - } - return PyUnicode_InternFromString(s); -} - -/* Test PyUnicode_FromWideChar() */ -static PyObject * -unicode_fromwidechar(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t bsize; - Py_ssize_t size = -100; - - if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { - return NULL; - } - if (size == -100) { - if (bsize % SIZEOF_WCHAR_T) { - PyErr_SetString(PyExc_AssertionError, - "invalid size in unicode_fromwidechar()"); - return NULL; - } - size = bsize / SIZEOF_WCHAR_T; - } - return PyUnicode_FromWideChar((const wchar_t *)s, size); -} - -/* Test PyUnicode_AsWideChar() */ -static PyObject * -unicode_aswidechar(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t buflen, size; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - buffer = PyMem_New(wchar_t, buflen); - if (buffer == NULL) - return PyErr_NoMemory(); - - size = PyUnicode_AsWideChar(unicode, buffer, buflen); - if (size == -1) { - PyMem_Free(buffer); - return NULL; - } - - if (size < buflen) - buflen = size + 1; - else - buflen = size; - result = PyUnicode_FromWideChar(buffer, buflen); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as buffer */ -static PyObject * -unicode_aswidechar_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - NULLABLE(unicode); - RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); -} - -/* Test PyUnicode_AsWideCharString() */ -static PyObject * -unicode_aswidecharstring(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - Py_ssize_t size = UNINITIALIZED_SIZE; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, &size); - if (buffer == NULL) { - assert(size == UNINITIALIZED_SIZE); - return NULL; - } - - result = PyUnicode_FromWideChar(buffer, size + 1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return Py_BuildValue("(Nn)", result, size); -} - -/* Test PyUnicode_AsWideCharString() with NULL as the size address */ -static PyObject * -unicode_aswidecharstring_null(PyObject *self, PyObject *args) -{ - PyObject *unicode, *result; - wchar_t *buffer; - - if (!PyArg_ParseTuple(args, "O", &unicode)) - return NULL; - - NULLABLE(unicode); - buffer = PyUnicode_AsWideCharString(unicode, NULL); - if (buffer == NULL) - return NULL; - - result = PyUnicode_FromWideChar(buffer, -1); - PyMem_Free(buffer); - if (result == NULL) - return NULL; - return result; -} -/* Test PyUnicode_AsUCS4() */ +// Test PyUnicode_AsUCS4(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * unicode_asucs4(PyObject *self, PyObject *args) { @@ -562,7 +141,9 @@ unicode_asucs4(PyObject *self, PyObject *args) return result; } -/* Test PyUnicode_AsUCS4Copy() */ + +// Test PyUnicode_AsUCS4Copy(). +// Part of the limited C API, but the test needs PyUnicode_FromKindAndData(). static PyObject * unicode_asucs4copy(PyObject *self, PyObject *args) { @@ -586,17 +167,6 @@ unicode_asucs4copy(PyObject *self, PyObject *args) return result; } -/* Test PyUnicode_FromOrdinal() */ -static PyObject * -unicode_fromordinal(PyObject *self, PyObject *args) -{ - int ordinal; - - if (!PyArg_ParseTuple(args, "i", &ordinal)) - return NULL; - - return PyUnicode_FromOrdinal(ordinal); -} /* Test PyUnicode_AsUTF8() */ static PyObject * @@ -617,911 +187,6 @@ unicode_asutf8(PyObject *self, PyObject *args) return PyBytes_FromStringAndSize(s, buflen); } -/* Test PyUnicode_AsUTF8AndSize() */ -static PyObject * -unicode_asutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - Py_ssize_t size = UNINITIALIZED_SIZE; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, &size); - if (s == NULL) { - assert(size == -1); - return NULL; - } - - return Py_BuildValue("(y#n)", s, buflen, size); -} - -/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ -static PyObject * -unicode_asutf8andsize_null(PyObject *self, PyObject *args) -{ - PyObject *unicode; - Py_ssize_t buflen; - const char *s; - - if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) - return NULL; - - NULLABLE(unicode); - s = PyUnicode_AsUTF8AndSize(unicode, NULL); - if (s == NULL) - return NULL; - - return PyBytes_FromStringAndSize(s, buflen); -} - -/* Test PyUnicode_GetDefaultEncoding() */ -static PyObject * -unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - const char *s = PyUnicode_GetDefaultEncoding(); - if (s == NULL) - return NULL; - - return PyBytes_FromString(s); -} - -/* Test PyUnicode_Decode() */ -static PyObject * -unicode_decode(PyObject *self, PyObject *args) -{ - const char *s; - Py_ssize_t size; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) - return NULL; - - return PyUnicode_Decode(s, size, encoding, errors); -} - -/* Test PyUnicode_AsEncodedString() */ -static PyObject * -unicode_asencodedstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *encoding; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_AsEncodedString(unicode, encoding, errors); -} - -/* Test PyUnicode_BuildEncodingMap() */ -static PyObject * -unicode_buildencodingmap(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_BuildEncodingMap(arg); -} - -/* Test PyUnicode_DecodeUTF7() */ -static PyObject * -unicode_decodeutf7(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF7(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF7Stateful() */ -static PyObject * -unicode_decodeutf7stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeUTF8() */ -static PyObject * -unicode_decodeutf8(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUTF8(data, size, errors); -} - -/* Test PyUnicode_DecodeUTF8Stateful() */ -static PyObject * -unicode_decodeutf8stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsUTF8String() */ -static PyObject * -unicode_asutf8string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF8String(arg); -} - -/* Test PyUnicode_DecodeUTF32() */ -static PyObject * -unicode_decodeutf32(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF32Stateful() */ -static PyObject * -unicode_decodeutf32stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF32String() */ -static PyObject * -unicode_asutf32string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF32String(arg); -} - -/* Test PyUnicode_DecodeUTF16() */ -static PyObject * -unicode_decodeutf16(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); - if (!result) { - return NULL; - } - return Py_BuildValue("(iN)", byteorder, result); -} - -/* Test PyUnicode_DecodeUTF16Stateful() */ -static PyObject * -unicode_decodeutf16stateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - int byteorder = UNINITIALIZED_INT; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(iNn)", byteorder, result, consumed); -} - -/* Test PyUnicode_AsUTF16String() */ -static PyObject * -unicode_asutf16string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUTF16String(arg); -} - -/* Test PyUnicode_DecodeUnicodeEscape() */ -static PyObject * -unicode_decodeunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsUnicodeEscapeString() */ -static PyObject * -unicode_asunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decoderawunicodeescape(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); -} - -/* Test PyUnicode_AsRawUnicodeEscapeString() */ -static PyObject * -unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsRawUnicodeEscapeString(arg); -} - -static PyObject * -unicode_decodelatin1(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLatin1(data, size, errors); -} - -/* Test PyUnicode_AsLatin1String() */ -static PyObject * -unicode_aslatin1string(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsLatin1String(arg); -} - -/* Test PyUnicode_DecodeASCII() */ -static PyObject * -unicode_decodeascii(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeASCII(data, size, errors); -} - -/* Test PyUnicode_AsASCIIString() */ -static PyObject * -unicode_asasciistring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsASCIIString(arg); -} - -/* Test PyUnicode_DecodeCharmap() */ -static PyObject * -unicode_decodecharmap(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - PyObject *mapping; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) - return NULL; - - NULLABLE(mapping); - return PyUnicode_DecodeCharmap(data, size, mapping, errors); -} - -/* Test PyUnicode_AsCharmapString() */ -static PyObject * -unicode_ascharmapstring(PyObject *self, PyObject *args) -{ - PyObject *unicode; - PyObject *mapping; - - if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) - return NULL; - - NULLABLE(unicode); - NULLABLE(mapping); - return PyUnicode_AsCharmapString(unicode, mapping); -} - -#ifdef MS_WINDOWS - -/* Test PyUnicode_DecodeMBCS() */ -static PyObject * -unicode_decodembcs(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeMBCS(data, size, errors); -} - -/* Test PyUnicode_DecodeMBCSStateful() */ -static PyObject * -unicode_decodembcsstateful(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_DecodeCodePageStateful() */ -static PyObject * -unicode_decodecodepagestateful(PyObject *self, PyObject *args) -{ - int code_page; - const char *data; - Py_ssize_t size; - const char *errors = NULL; - Py_ssize_t consumed = UNINITIALIZED_SIZE; - PyObject *result; - - if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) - return NULL; - - result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); - if (!result) { - assert(consumed == UNINITIALIZED_SIZE); - return NULL; - } - return Py_BuildValue("(Nn)", result, consumed); -} - -/* Test PyUnicode_AsMBCSString() */ -static PyObject * -unicode_asmbcsstring(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_AsMBCSString(arg); -} - -/* Test PyUnicode_EncodeCodePage() */ -static PyObject * -unicode_encodecodepage(PyObject *self, PyObject *args) -{ - int code_page; - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeCodePage(code_page, unicode, errors); -} - -#endif /* MS_WINDOWS */ - -/* Test PyUnicode_DecodeLocaleAndSize() */ -static PyObject * -unicode_decodelocaleandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocaleAndSize(data, size, errors); -} - -/* Test PyUnicode_DecodeLocale() */ -static PyObject * -unicode_decodelocale(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - const char *errors; - - if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) - return NULL; - - return PyUnicode_DecodeLocale(data, errors); -} - -/* Test PyUnicode_EncodeLocale() */ -static PyObject * -unicode_encodelocale(PyObject *self, PyObject *args) -{ - PyObject *unicode; - const char *errors; - - if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) - return NULL; - - NULLABLE(unicode); - return PyUnicode_EncodeLocale(unicode, errors); -} - -/* Test PyUnicode_DecodeFSDefault() */ -static PyObject * -unicode_decodefsdefault(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#", &data, &size)) - return NULL; - - return PyUnicode_DecodeFSDefault(data); -} - -/* Test PyUnicode_DecodeFSDefaultAndSize() */ -static PyObject * -unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) -{ - const char *data; - Py_ssize_t size; - - if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) - return NULL; - - return PyUnicode_DecodeFSDefaultAndSize(data, size); -} - -/* Test PyUnicode_EncodeFSDefault() */ -static PyObject * -unicode_encodefsdefault(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - return PyUnicode_EncodeFSDefault(arg); -} - -/* Test PyUnicode_Concat() */ -static PyObject * -unicode_concat(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_Concat(left, right); -} - -/* Test PyUnicode_Split() */ -static PyObject * -unicode_split(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Split(s, sep, maxsplit); -} - -/* Test PyUnicode_RSplit() */ -static PyObject * -unicode_rsplit(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - Py_ssize_t maxsplit = -1; - - if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RSplit(s, sep, maxsplit); -} - -/* Test PyUnicode_Splitlines() */ -static PyObject * -unicode_splitlines(PyObject *self, PyObject *args) -{ - PyObject *s; - int keepends = 0; - - if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) - return NULL; - - NULLABLE(s); - return PyUnicode_Splitlines(s, keepends); -} - -/* Test PyUnicode_Partition() */ -static PyObject * -unicode_partition(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_Partition(s, sep); -} - -/* Test PyUnicode_RPartition() */ -static PyObject * -unicode_rpartition(PyObject *self, PyObject *args) -{ - PyObject *s; - PyObject *sep; - - if (!PyArg_ParseTuple(args, "OO", &s, &sep)) - return NULL; - - NULLABLE(s); - NULLABLE(sep); - return PyUnicode_RPartition(s, sep); -} - -/* Test PyUnicode_Translate() */ -static PyObject * -unicode_translate(PyObject *self, PyObject *args) -{ - PyObject *obj; - PyObject *table; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) - return NULL; - - NULLABLE(obj); - NULLABLE(table); - return PyUnicode_Translate(obj, table, errors); -} - -/* Test PyUnicode_Join() */ -static PyObject * -unicode_join(PyObject *self, PyObject *args) -{ - PyObject *sep; - PyObject *seq; - - if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) - return NULL; - - NULLABLE(sep); - NULLABLE(seq); - return PyUnicode_Join(sep, seq); -} - -/* Test PyUnicode_Count() */ -static PyObject * -unicode_count(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - - if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); -} - -/* Test PyUnicode_Find() */ -static PyObject * -unicode_find(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - Py_ssize_t result; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - result = PyUnicode_Find(str, substr, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); - return NULL; - } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Tailmatch() */ -static PyObject * -unicode_tailmatch(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - Py_ssize_t start; - Py_ssize_t end; - int direction; - - if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); -} - -/* Test PyUnicode_FindChar() */ -static PyObject * -unicode_findchar(PyObject *self, PyObject *args) -{ - PyObject *str; - int direction; - unsigned int ch; - Py_ssize_t result; - Py_ssize_t start, end; - - if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, - &start, &end, &direction)) { - return NULL; - } - NULLABLE(str); - result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); - if (result == -2) { - assert(PyErr_Occurred()); - return NULL; - } - assert(!PyErr_Occurred()); - return PyLong_FromSsize_t(result); -} - -/* Test PyUnicode_Replace() */ -static PyObject * -unicode_replace(PyObject *self, PyObject *args) -{ - PyObject *str; - PyObject *substr; - PyObject *replstr; - Py_ssize_t maxcount = -1; - - if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) - return NULL; - - NULLABLE(str); - NULLABLE(substr); - NULLABLE(replstr); - return PyUnicode_Replace(str, substr, replstr, maxcount); -} - -/* Test PyUnicode_Compare() */ -static PyObject * -unicode_compare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int result; - - if (!PyArg_ParseTuple(args, "OO", &left, &right)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - result = PyUnicode_Compare(left, right); - if (result == -1 && PyErr_Occurred()) { - return NULL; - } - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_CompareWithASCIIString() */ -static PyObject * -unicode_comparewithasciistring(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - - if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) - return NULL; - - NULLABLE(left); - result = PyUnicode_CompareWithASCIIString(left, right); - if (result == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8() */ -static PyObject * -unicode_equaltoutf8(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - int result; - - if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { - return NULL; - } - - NULLABLE(left); - result = PyUnicode_EqualToUTF8(left, right); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_EqualToUTF8AndSize() */ -static PyObject * -unicode_equaltoutf8andsize(PyObject *self, PyObject *args) -{ - PyObject *left; - const char *right = NULL; - Py_ssize_t right_len; - Py_ssize_t size = -100; - int result; - - if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { - return NULL; - } - - NULLABLE(left); - if (size == -100) { - size = right_len; - } - result = PyUnicode_EqualToUTF8AndSize(left, right, size); - assert(!PyErr_Occurred()); - return PyLong_FromLong(result); -} - -/* Test PyUnicode_RichCompare() */ -static PyObject * -unicode_richcompare(PyObject *self, PyObject *args) -{ - PyObject *left; - PyObject *right; - int op; - - if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) - return NULL; - - NULLABLE(left); - NULLABLE(right); - return PyUnicode_RichCompare(left, right, op); -} - -/* Test PyUnicode_Format() */ -static PyObject * -unicode_format(PyObject *self, PyObject *args) -{ - PyObject *format; - PyObject *fargs; - - if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) - return NULL; - - NULLABLE(format); - NULLABLE(fargs); - return PyUnicode_Format(format, fargs); -} - -/* Test PyUnicode_Contains() */ -static PyObject * -unicode_contains(PyObject *self, PyObject *args) -{ - PyObject *container; - PyObject *element; - - if (!PyArg_ParseTuple(args, "OO", &container, &element)) - return NULL; - - NULLABLE(container); - NULLABLE(element); - RETURN_INT(PyUnicode_Contains(container, element)); -} - -/* Test PyUnicode_IsIdentifier() */ -static PyObject * -unicode_isidentifier(PyObject *self, PyObject *arg) -{ - NULLABLE(arg); - RETURN_INT(PyUnicode_IsIdentifier(arg)); -} /* Test PyUnicode_CopyCharacters() */ static PyObject * @@ -1555,541 +220,14 @@ unicode_copycharacters(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", to_copy, copied); } -static int -check_raised_systemerror(PyObject *result, char* msg) -{ - if (result) { - // no exception - PyErr_Format(PyExc_AssertionError, - "SystemError not raised: %s", - msg); - return 0; - } - if (PyErr_ExceptionMatches(PyExc_SystemError)) { - // expected exception - PyErr_Clear(); - return 1; - } - // unexpected exception - return 0; -} - -static PyObject * -test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *result; - PyObject *unicode = PyUnicode_FromString("None"); - -#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ - result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ - if (EXPECTED == NULL) { \ - if (!check_raised_systemerror(result, FORMAT)) { \ - goto Fail; \ - } \ - } \ - else if (result == NULL) \ - return NULL; \ - else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ - PyErr_Format(PyExc_AssertionError, \ - "test_string_from_format: failed at \"%s\" " \ - "expected \"%s\" got \"%s\"", \ - FORMAT, EXPECTED, PyUnicode_AsUTF8(result)); \ - goto Fail; \ - } \ - Py_XDECREF(result) - -#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) - -#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ - CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) - - // Unrecognized - CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); - - // "%%" (options are rejected) - CHECK_FORMAT_0( "%%", "%"); - CHECK_FORMAT_0( "%0%", NULL); - CHECK_FORMAT_0("%00%", NULL); - CHECK_FORMAT_0( "%2%", NULL); - CHECK_FORMAT_0("%02%", NULL); - CHECK_FORMAT_0("%.0%", NULL); - CHECK_FORMAT_0("%.2%", NULL); - - // "%c" - CHECK_FORMAT_1( "%c", "c", 'c'); - CHECK_FORMAT_1( "%0c", "c", 'c'); - CHECK_FORMAT_1("%00c", "c", 'c'); - CHECK_FORMAT_1( "%2c", NULL, 'c'); - CHECK_FORMAT_1("%02c", NULL, 'c'); - CHECK_FORMAT_1("%.0c", NULL, 'c'); - CHECK_FORMAT_1("%.2c", NULL, 'c'); - - // Integers - CHECK_FORMAT_1("%d", "123", (int)123); - CHECK_FORMAT_1("%i", "123", (int)123); - CHECK_FORMAT_1("%u", "123", (unsigned int)123); - CHECK_FORMAT_1("%x", "7b", (unsigned int)123); - CHECK_FORMAT_1("%X", "7B", (unsigned int)123); - CHECK_FORMAT_1("%o", "173", (unsigned int)123); - CHECK_FORMAT_1("%ld", "123", (long)123); - CHECK_FORMAT_1("%li", "123", (long)123); - CHECK_FORMAT_1("%lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); - CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); - CHECK_FORMAT_1("%lo", "173", (unsigned long)123); - CHECK_FORMAT_1("%lld", "123", (long long)123); - CHECK_FORMAT_1("%lli", "123", (long long)123); - CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); - CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); - CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); - CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%zu", "123", (size_t)123); - CHECK_FORMAT_1("%zx", "7b", (size_t)123); - CHECK_FORMAT_1("%zX", "7B", (size_t)123); - CHECK_FORMAT_1("%zo", "173", (size_t)123); - CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); - CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); - CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); - CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); - CHECK_FORMAT_1("%jd", "123", (intmax_t)123); - CHECK_FORMAT_1("%ji", "123", (intmax_t)123); - CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); - CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); - CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); - CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); - - CHECK_FORMAT_1("%d", "-123", (int)-123); - CHECK_FORMAT_1("%i", "-123", (int)-123); - CHECK_FORMAT_1("%ld", "-123", (long)-123); - CHECK_FORMAT_1("%li", "-123", (long)-123); - CHECK_FORMAT_1("%lld", "-123", (long long)-123); - CHECK_FORMAT_1("%lli", "-123", (long long)-123); - CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); - CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); - CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); - - // Integers: width < length - CHECK_FORMAT_1("%1d", "123", (int)123); - CHECK_FORMAT_1("%1i", "123", (int)123); - CHECK_FORMAT_1("%1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%1ld", "123", (long)123); - CHECK_FORMAT_1("%1li", "123", (long)123); - CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%1lld", "123", (long long)123); - CHECK_FORMAT_1("%1lli", "123", (long long)123); - CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%1zu", "123", (size_t)123); - CHECK_FORMAT_1("%1x", "7b", (int)123); - - CHECK_FORMAT_1("%1d", "-123", (int)-123); - CHECK_FORMAT_1("%1i", "-123", (int)-123); - CHECK_FORMAT_1("%1ld", "-123", (long)-123); - CHECK_FORMAT_1("%1li", "-123", (long)-123); - CHECK_FORMAT_1("%1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); - - // Integers: width > length - CHECK_FORMAT_1("%5d", " 123", (int)123); - CHECK_FORMAT_1("%5i", " 123", (int)123); - CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); - CHECK_FORMAT_1("%5ld", " 123", (long)123); - CHECK_FORMAT_1("%5li", " 123", (long)123); - CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); - CHECK_FORMAT_1("%5lld", " 123", (long long)123); - CHECK_FORMAT_1("%5lli", " 123", (long long)123); - CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); - CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5zu", " 123", (size_t)123); - CHECK_FORMAT_1("%5x", " 7b", (int)123); - - CHECK_FORMAT_1("%5d", " -123", (int)-123); - CHECK_FORMAT_1("%5i", " -123", (int)-123); - CHECK_FORMAT_1("%5ld", " -123", (long)-123); - CHECK_FORMAT_1("%5li", " -123", (long)-123); - CHECK_FORMAT_1("%5lld", " -123", (long long)-123); - CHECK_FORMAT_1("%5lli", " -123", (long long)-123); - CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); - - // Integers: width > length, 0-flag - CHECK_FORMAT_1("%05d", "00123", (int)123); - CHECK_FORMAT_1("%05i", "00123", (int)123); - CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%05ld", "00123", (long)123); - CHECK_FORMAT_1("%05li", "00123", (long)123); - CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%05lld", "00123", (long long)123); - CHECK_FORMAT_1("%05lli", "00123", (long long)123); - CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05zu", "00123", (size_t)123); - CHECK_FORMAT_1("%05x", "0007b", (int)123); - - CHECK_FORMAT_1("%05d", "-0123", (int)-123); - CHECK_FORMAT_1("%05i", "-0123", (int)-123); - CHECK_FORMAT_1("%05ld", "-0123", (long)-123); - CHECK_FORMAT_1("%05li", "-0123", (long)-123); - CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); - CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); - CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); - - // Integers: precision < length - CHECK_FORMAT_1("%.1d", "123", (int)123); - CHECK_FORMAT_1("%.1i", "123", (int)123); - CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); - CHECK_FORMAT_1("%.1ld", "123", (long)123); - CHECK_FORMAT_1("%.1li", "123", (long)123); - CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); - CHECK_FORMAT_1("%.1lld", "123", (long long)123); - CHECK_FORMAT_1("%.1lli", "123", (long long)123); - CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); - CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.1zu", "123", (size_t)123); - CHECK_FORMAT_1("%.1x", "7b", (int)123); - - CHECK_FORMAT_1("%.1d", "-123", (int)-123); - CHECK_FORMAT_1("%.1i", "-123", (int)-123); - CHECK_FORMAT_1("%.1ld", "-123", (long)-123); - CHECK_FORMAT_1("%.1li", "-123", (long)-123); - CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); - CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); - CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); - - // Integers: precision > length - CHECK_FORMAT_1("%.5d", "00123", (int)123); - CHECK_FORMAT_1("%.5i", "00123", (int)123); - CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); - CHECK_FORMAT_1("%.5ld", "00123", (long)123); - CHECK_FORMAT_1("%.5li", "00123", (long)123); - CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); - CHECK_FORMAT_1("%.5lld", "00123", (long long)123); - CHECK_FORMAT_1("%.5lli", "00123", (long long)123); - CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); - CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); - CHECK_FORMAT_1("%.5x", "0007b", (int)123); - - CHECK_FORMAT_1("%.5d", "-00123", (int)-123); - CHECK_FORMAT_1("%.5i", "-00123", (int)-123); - CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); - CHECK_FORMAT_1("%.5li", "-00123", (long)-123); - CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); - CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); - - // Integers: width > precision > length - CHECK_FORMAT_1("%7.5d", " 00123", (int)123); - CHECK_FORMAT_1("%7.5i", " 00123", (int)123); - CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); - CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); - CHECK_FORMAT_1("%7.5li", " 00123", (long)123); - CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); - CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); - CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); - CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); - CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); - CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); - - CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); - CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); - CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); - CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); - - // Integers: width > precision > length, 0-flag - CHECK_FORMAT_1("%07.5d", "0000123", (int)123); - CHECK_FORMAT_1("%07.5i", "0000123", (int)123); - CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); - CHECK_FORMAT_1("%07.5li", "0000123", (long)123); - CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); - CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%07.5x", "000007b", (int)123); - - CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); - CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); - CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); - CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); - - // Integers: precision > width > length - CHECK_FORMAT_1("%5.7d", "0000123", (int)123); - CHECK_FORMAT_1("%5.7i", "0000123", (int)123); - CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%5.7li", "0000123", (long)123); - CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%5.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision > width > length, 0-flag - CHECK_FORMAT_1("%05.7d", "0000123", (int)123); - CHECK_FORMAT_1("%05.7i", "0000123", (int)123); - CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); - CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); - CHECK_FORMAT_1("%05.7li", "0000123", (long)123); - CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); - CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); - CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); - CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); - CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); - CHECK_FORMAT_1("%05.7x", "000007b", (int)123); - - CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); - CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); - CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); - CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); - CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); - - // Integers: precision = 0, arg = 0 (empty string in C) - CHECK_FORMAT_1("%.0d", "0", (int)0); - CHECK_FORMAT_1("%.0i", "0", (int)0); - CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); - CHECK_FORMAT_1("%.0ld", "0", (long)0); - CHECK_FORMAT_1("%.0li", "0", (long)0); - CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); - CHECK_FORMAT_1("%.0lld", "0", (long long)0); - CHECK_FORMAT_1("%.0lli", "0", (long long)0); - CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); - CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); - CHECK_FORMAT_1("%.0zu", "0", (size_t)0); - CHECK_FORMAT_1("%.0x", "0", (int)0); - - // Strings - CHECK_FORMAT_1("%s", "None", "None"); - CHECK_FORMAT_1("%ls", "None", L"None"); - CHECK_FORMAT_1("%U", "None", unicode); - CHECK_FORMAT_1("%A", "None", Py_None); - CHECK_FORMAT_1("%S", "None", Py_None); - CHECK_FORMAT_1("%R", "None", Py_None); - CHECK_FORMAT_2("%V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%V", "None", NULL, "None"); - CHECK_FORMAT_2("%lV", "None", NULL, L"None"); - - // Strings: width < length - CHECK_FORMAT_1("%1s", "None", "None"); - CHECK_FORMAT_1("%1ls", "None", L"None"); - CHECK_FORMAT_1("%1U", "None", unicode); - CHECK_FORMAT_1("%1A", "None", Py_None); - CHECK_FORMAT_1("%1S", "None", Py_None); - CHECK_FORMAT_1("%1R", "None", Py_None); - CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1V", "None", NULL, "None"); - CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); - - // Strings: width > length - CHECK_FORMAT_1("%5s", " None", "None"); - CHECK_FORMAT_1("%5ls", " None", L"None"); - CHECK_FORMAT_1("%5U", " None", unicode); - CHECK_FORMAT_1("%5A", " None", Py_None); - CHECK_FORMAT_1("%5S", " None", Py_None); - CHECK_FORMAT_1("%5R", " None", Py_None); - CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); - CHECK_FORMAT_2("%5V", " None", NULL, "None"); - CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); - - // Strings: precision < length - CHECK_FORMAT_1("%.1s", "N", "None"); - CHECK_FORMAT_1("%.1ls", "N", L"None"); - CHECK_FORMAT_1("%.1U", "N", unicode); - CHECK_FORMAT_1("%.1A", "N", Py_None); - CHECK_FORMAT_1("%.1S", "N", Py_None); - CHECK_FORMAT_1("%.1R", "N", Py_None); - CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); - CHECK_FORMAT_2("%.1V", "N", NULL, "None"); - CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); - - // Strings: precision > length - CHECK_FORMAT_1("%.5s", "None", "None"); - CHECK_FORMAT_1("%.5ls", "None", L"None"); - CHECK_FORMAT_1("%.5U", "None", unicode); - CHECK_FORMAT_1("%.5A", "None", Py_None); - CHECK_FORMAT_1("%.5S", "None", Py_None); - CHECK_FORMAT_1("%.5R", "None", Py_None); - CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); - - // Strings: precision < length, width > length - CHECK_FORMAT_1("%5.1s", " N", "None"); - CHECK_FORMAT_1("%5.1ls"," N", L"None"); - CHECK_FORMAT_1("%5.1U", " N", unicode); - CHECK_FORMAT_1("%5.1A", " N", Py_None); - CHECK_FORMAT_1("%5.1S", " N", Py_None); - CHECK_FORMAT_1("%5.1R", " N", Py_None); - CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); - CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); - CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); - - // Strings: width < length, precision > length - CHECK_FORMAT_1("%1.5s", "None", "None"); - CHECK_FORMAT_1("%1.5ls", "None", L"None"); - CHECK_FORMAT_1("%1.5U", "None", unicode); - CHECK_FORMAT_1("%1.5A", "None", Py_None); - CHECK_FORMAT_1("%1.5S", "None", Py_None); - CHECK_FORMAT_1("%1.5R", "None", Py_None); - CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); - CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); - CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); - - Py_XDECREF(unicode); - Py_RETURN_NONE; - - Fail: - Py_XDECREF(result); - Py_XDECREF(unicode); - return NULL; - -#undef CHECK_FORMAT_2 -#undef CHECK_FORMAT_1 -#undef CHECK_FORMAT_0 -} static PyMethodDef TestMethods[] = { - {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, - {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, - {"test_unicode_compare_with_ascii", - test_unicode_compare_with_ascii, METH_NOARGS}, - {"test_string_from_format", test_string_from_format, METH_NOARGS}, - {"test_widechar", test_widechar, METH_NOARGS}, {"unicode_new", unicode_new, METH_VARARGS}, {"unicode_fill", unicode_fill, METH_VARARGS}, - {"unicode_writechar", unicode_writechar, METH_VARARGS}, - {"unicode_resize", unicode_resize, METH_VARARGS}, - {"unicode_append", unicode_append, METH_VARARGS}, - {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, - {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, - {"unicode_fromstring", unicode_fromstring, METH_O}, {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, - {"unicode_substring", unicode_substring, METH_VARARGS}, - {"unicode_getlength", unicode_getlength, METH_O}, - {"unicode_readchar", unicode_readchar, METH_VARARGS}, - {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, - {"unicode_fromobject", unicode_fromobject, METH_O}, - {"unicode_interninplace", unicode_interninplace, METH_O}, - {"unicode_internfromstring", unicode_internfromstring, METH_O}, - {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, - {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, - {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, - {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, - {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, - {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, - {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, - {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, - {"unicode_decode", unicode_decode, METH_VARARGS}, - {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, - {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, - {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, - {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, - {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, - {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, - {"unicode_asutf8string", unicode_asutf8string, METH_O}, - {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, - {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, - {"unicode_asutf16string", unicode_asutf16string, METH_O}, - {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, - {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, - {"unicode_asutf32string", unicode_asutf32string, METH_O}, - {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, - {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, - {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, - {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, - {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, - {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, - {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, - {"unicode_asasciistring", unicode_asasciistring, METH_O}, - {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, - {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, -#ifdef MS_WINDOWS - {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, - {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, - {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, - {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, - {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, -#endif /* MS_WINDOWS */ - {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, - {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, - {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, - {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, - {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, - {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, - {"unicode_concat", unicode_concat, METH_VARARGS}, - {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, - {"unicode_split", unicode_split, METH_VARARGS}, - {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, - {"unicode_partition", unicode_partition, METH_VARARGS}, - {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, - {"unicode_translate", unicode_translate, METH_VARARGS}, - {"unicode_join", unicode_join, METH_VARARGS}, - {"unicode_count", unicode_count, METH_VARARGS}, - {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, - {"unicode_find", unicode_find, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, - {"unicode_replace", unicode_replace, METH_VARARGS}, - {"unicode_compare", unicode_compare, METH_VARARGS}, - {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, - {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, - {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, - {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, - {"unicode_format", unicode_format, METH_VARARGS}, - {"unicode_contains", unicode_contains, METH_VARARGS}, - {"unicode_isidentifier", unicode_isidentifier, METH_O}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {NULL}, }; @@ -2099,6 +237,5 @@ _PyTestCapi_Init_Unicode(PyObject *m) { if (PyModule_AddFunctions(m, TestMethods) < 0) { return -1; } - return 0; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 756e2fb252f2e2..c038f7a9e2d1a5 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -53,6 +53,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Unicode(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_VectorcallLimited(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/float.c b/Modules/_testlimitedcapi/float.c index f9865e4be2c4a8..88dc91f682ff39 100644 --- a/Modules/_testlimitedcapi/float.c +++ b/Modules/_testlimitedcapi/float.c @@ -89,4 +89,3 @@ _PyTestLimitedCAPI_Init_Float(PyObject *mod) return 0; } - diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 461fe7aa2cc337..f8b84e9388184e 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -31,6 +31,7 @@ int _PyTestLimitedCAPI_Init_List(PyObject *module); int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); int _PyTestLimitedCAPI_Init_Set(PyObject *module); int _PyTestLimitedCAPI_Init_Sys(PyObject *module); +int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/unicode.c b/Modules/_testlimitedcapi/unicode.c new file mode 100644 index 00000000000000..2b70d09108a333 --- /dev/null +++ b/Modules/_testlimitedcapi/unicode.c @@ -0,0 +1,1938 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.13 to test PyUnicode_EqualToUTF8() +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + +#include // ptrdiff_t +#include // memset() + + +static PyObject * +codec_incrementalencoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementalencoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalEncoder(encoding, errors); +} + +static PyObject * +codec_incrementaldecoder(PyObject *self, PyObject *args) +{ + const char *encoding, *errors = NULL; + if (!PyArg_ParseTuple(args, "s|s:test_incrementaldecoder", + &encoding, &errors)) + return NULL; + return PyCodec_IncrementalDecoder(encoding, errors); +} + +static PyObject * +test_unicode_compare_with_ascii(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *py_s = PyUnicode_FromStringAndSize("str\0", 4); + int result; + if (py_s == NULL) + return NULL; + result = PyUnicode_CompareWithASCIIString(py_s, "str"); + Py_DECREF(py_s); + if (!result) { + PyErr_SetString(PyExc_AssertionError, "Python string ending in NULL " + "should not compare equal to c string."); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + const wchar_t wtext[2] = {(wchar_t)0x10ABCDu}; + size_t wtextlen = 1; + const wchar_t invalid[1] = {(wchar_t)0x110000u}; +#else + const wchar_t wtext[3] = {(wchar_t)0xDBEAu, (wchar_t)0xDFCDu}; + size_t wtextlen = 2; +#endif + PyObject *wide, *utf8; + + wide = PyUnicode_FromWideChar(wtext, wtextlen); + if (wide == NULL) + return NULL; + + utf8 = PyUnicode_FromString("\xf4\x8a\xaf\x8d"); + if (utf8 == NULL) { + Py_DECREF(wide); + return NULL; + } + + if (PyUnicode_GetLength(wide) != PyUnicode_GetLength(utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "have different length"); + return NULL; + } + if (PyUnicode_Compare(wide, utf8)) { + Py_DECREF(wide); + Py_DECREF(utf8); + if (PyErr_Occurred()) + return NULL; + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "wide string and utf8 string " + "are different"); + return NULL; + } + + Py_DECREF(wide); + Py_DECREF(utf8); + +#if defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) + wide = PyUnicode_FromWideChar(invalid, 1); + if (wide == NULL) + PyErr_Clear(); + else { + PyErr_SetString(PyExc_AssertionError, + "test_widechar: " + "PyUnicode_FromWideChar(L\"\\U00110000\", 1) didn't fail"); + return NULL; + } +#endif + Py_RETURN_NONE; +} + + +static PyObject * +unicode_copy(PyObject *unicode) +{ + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + // Create a new string by encoding to UTF-8 and then decode from UTF-8 + PyObject *utf8 = PyUnicode_AsUTF8String(unicode); + if (!utf8) { + return NULL; + } + + PyObject *copy = PyUnicode_DecodeUTF8( + PyBytes_AsString(utf8), + PyBytes_Size(utf8), + NULL); + Py_DECREF(utf8); + + return copy; +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +static void +unicode_fill(PyObject *str, Py_ssize_t start, Py_ssize_t end, Py_UCS4 ch) +{ + assert(0 <= start); + assert(end <= PyUnicode_GetLength(str)); + for (Py_ssize_t i = start; i < end; i++) { + int res = PyUnicode_WriteChar(str, i, ch); + assert(res == 0); + } +} + + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GetLength(obj)) { + unicode_fill(copy, PyUnicode_GetLength(obj), length, 0U); + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_SIZE(PyUnicode_GetLength(arg)); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + +/* Test PyUnicode_FromEncodedObject() */ +static PyObject * +unicode_fromencodedobject(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &obj, &encoding, &errors)) { + return NULL; + } + + NULLABLE(obj); + return PyUnicode_FromEncodedObject(obj, encoding, errors); +} + +/* Test PyUnicode_FromObject() */ +static PyObject * +unicode_fromobject(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_FromObject(arg); +} + +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + +/* Test PyUnicode_AsWideChar() */ +static PyObject * +unicode_aswidechar(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t buflen, size; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + buffer = PyMem_New(wchar_t, buflen); + if (buffer == NULL) + return PyErr_NoMemory(); + + size = PyUnicode_AsWideChar(unicode, buffer, buflen); + if (size == -1) { + PyMem_Free(buffer); + return NULL; + } + + if (size < buflen) + buflen = size + 1; + else + buflen = size; + result = PyUnicode_FromWideChar(buffer, buflen); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + RETURN_SIZE(PyUnicode_AsWideChar(unicode, NULL, buflen)); +} + +/* Test PyUnicode_AsWideCharString() */ +static PyObject * +unicode_aswidecharstring(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + Py_ssize_t size = UNINITIALIZED_SIZE; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, &size); + if (buffer == NULL) { + assert(size == UNINITIALIZED_SIZE); + return NULL; + } + + result = PyUnicode_FromWideChar(buffer, size + 1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return Py_BuildValue("(Nn)", result, size); +} + +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; + + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8AndSize() */ +static PyObject * +unicode_asutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = UNINITIALIZED_SIZE; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) { + assert(size == -1); + return NULL; + } + + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test PyUnicode_Decode() */ +static PyObject * +unicode_decode(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t size; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#z|z", &s, &size, &encoding, &errors)) + return NULL; + + return PyUnicode_Decode(s, size, encoding, errors); +} + +/* Test PyUnicode_AsEncodedString() */ +static PyObject * +unicode_asencodedstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *encoding; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "Oz|z", &unicode, &encoding, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_AsEncodedString(unicode, encoding, errors); +} + +/* Test PyUnicode_BuildEncodingMap() */ +static PyObject * +unicode_buildencodingmap(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_BuildEncodingMap(arg); +} + +/* Test PyUnicode_DecodeUTF7() */ +static PyObject * +unicode_decodeutf7(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF7(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF7Stateful() */ +static PyObject * +unicode_decodeutf7stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF7Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsUTF8String() */ +static PyObject * +unicode_asutf8string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF8String(arg); +} + +/* Test PyUnicode_DecodeUTF32() */ +static PyObject * +unicode_decodeutf32(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF32Stateful() */ +static PyObject * +unicode_decodeutf32stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF32Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF32String() */ +static PyObject * +unicode_asutf32string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF32String(arg); +} + +/* Test PyUnicode_DecodeUTF16() */ +static PyObject * +unicode_decodeutf16(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16(data, size, errors, &byteorder); + if (!result) { + return NULL; + } + return Py_BuildValue("(iN)", byteorder, result); +} + +/* Test PyUnicode_DecodeUTF16Stateful() */ +static PyObject * +unicode_decodeutf16stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + int byteorder = UNINITIALIZED_INT; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &byteorder, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF16Stateful(data, size, errors, &byteorder, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(iNn)", byteorder, result, consumed); +} + +/* Test PyUnicode_AsUTF16String() */ +static PyObject * +unicode_asutf16string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUTF16String(arg); +} + +/* Test PyUnicode_DecodeUnicodeEscape() */ +static PyObject * +unicode_decodeunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsUnicodeEscapeString() */ +static PyObject * +unicode_asunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decoderawunicodeescape(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeRawUnicodeEscape(data, size, errors); +} + +/* Test PyUnicode_AsRawUnicodeEscapeString() */ +static PyObject * +unicode_asrawunicodeescapestring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsRawUnicodeEscapeString(arg); +} + +static PyObject * +unicode_decodelatin1(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLatin1(data, size, errors); +} + +/* Test PyUnicode_AsLatin1String() */ +static PyObject * +unicode_aslatin1string(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsLatin1String(arg); +} + +/* Test PyUnicode_DecodeASCII() */ +static PyObject * +unicode_decodeascii(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeASCII(data, size, errors); +} + +/* Test PyUnicode_AsASCIIString() */ +static PyObject * +unicode_asasciistring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsASCIIString(arg); +} + +/* Test PyUnicode_DecodeCharmap() */ +static PyObject * +unicode_decodecharmap(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + PyObject *mapping; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#O|z", &data, &size, &mapping, &errors)) + return NULL; + + NULLABLE(mapping); + return PyUnicode_DecodeCharmap(data, size, mapping, errors); +} + +/* Test PyUnicode_AsCharmapString() */ +static PyObject * +unicode_ascharmapstring(PyObject *self, PyObject *args) +{ + PyObject *unicode; + PyObject *mapping; + + if (!PyArg_ParseTuple(args, "OO", &unicode, &mapping)) + return NULL; + + NULLABLE(unicode); + NULLABLE(mapping); + return PyUnicode_AsCharmapString(unicode, mapping); +} + +#ifdef MS_WINDOWS + +/* Test PyUnicode_DecodeMBCS() */ +static PyObject * +unicode_decodembcs(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeMBCS(data, size, errors); +} + +/* Test PyUnicode_DecodeMBCSStateful() */ +static PyObject * +unicode_decodembcsstateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeMBCSStateful(data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_DecodeCodePageStateful() */ +static PyObject * +unicode_decodecodepagestateful(PyObject *self, PyObject *args) +{ + int code_page; + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = UNINITIALIZED_SIZE; + PyObject *result; + + if (!PyArg_ParseTuple(args, "iy#|z", &code_page, &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeCodePageStateful(code_page, data, size, errors, &consumed); + if (!result) { + assert(consumed == UNINITIALIZED_SIZE); + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_AsMBCSString() */ +static PyObject * +unicode_asmbcsstring(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_AsMBCSString(arg); +} + +/* Test PyUnicode_EncodeCodePage() */ +static PyObject * +unicode_encodecodepage(PyObject *self, PyObject *args) +{ + int code_page; + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "iO|z", &code_page, &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeCodePage(code_page, unicode, errors); +} + +#endif /* MS_WINDOWS */ + +/* Test PyUnicode_DecodeLocaleAndSize() */ +static PyObject * +unicode_decodelocaleandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocaleAndSize(data, size, errors); +} + +/* Test PyUnicode_DecodeLocale() */ +static PyObject * +unicode_decodelocale(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeLocale(data, errors); +} + +/* Test PyUnicode_EncodeLocale() */ +static PyObject * +unicode_encodelocale(PyObject *self, PyObject *args) +{ + PyObject *unicode; + const char *errors; + + if (!PyArg_ParseTuple(args, "O|z", &unicode, &errors)) + return NULL; + + NULLABLE(unicode); + return PyUnicode_EncodeLocale(unicode, errors); +} + +/* Test PyUnicode_DecodeFSDefault() */ +static PyObject * +unicode_decodefsdefault(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#", &data, &size)) + return NULL; + + return PyUnicode_DecodeFSDefault(data); +} + +/* Test PyUnicode_DecodeFSDefaultAndSize() */ +static PyObject * +unicode_decodefsdefaultandsize(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "y#|n", &data, &size, &size)) + return NULL; + + return PyUnicode_DecodeFSDefaultAndSize(data, size); +} + +/* Test PyUnicode_EncodeFSDefault() */ +static PyObject * +unicode_encodefsdefault(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_EncodeFSDefault(arg); +} + +/* Test PyUnicode_Concat() */ +static PyObject * +unicode_concat(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_Concat(left, right); +} + +/* Test PyUnicode_Split() */ +static PyObject * +unicode_split(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Split(s, sep, maxsplit); +} + +/* Test PyUnicode_RSplit() */ +static PyObject * +unicode_rsplit(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RSplit(s, sep, maxsplit); +} + +/* Test PyUnicode_Splitlines() */ +static PyObject * +unicode_splitlines(PyObject *self, PyObject *args) +{ + PyObject *s; + int keepends = 0; + + if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + return NULL; + + NULLABLE(s); + return PyUnicode_Splitlines(s, keepends); +} + +/* Test PyUnicode_Partition() */ +static PyObject * +unicode_partition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Partition(s, sep); +} + +/* Test PyUnicode_RPartition() */ +static PyObject * +unicode_rpartition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RPartition(s, sep); +} + +/* Test PyUnicode_Translate() */ +static PyObject * +unicode_translate(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyObject *table; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + return NULL; + + NULLABLE(obj); + NULLABLE(table); + return PyUnicode_Translate(obj, table, errors); +} + +/* Test PyUnicode_Join() */ +static PyObject * +unicode_join(PyObject *self, PyObject *args) +{ + PyObject *sep; + PyObject *seq; + + if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + return NULL; + + NULLABLE(sep); + NULLABLE(seq); + return PyUnicode_Join(sep, seq); +} + +/* Test PyUnicode_Count() */ +static PyObject * +unicode_count(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + + if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Count(str, substr, start, end)); +} + +/* Test PyUnicode_Find() */ +static PyObject * +unicode_find(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Find(str, substr, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Tailmatch() */ +static PyObject * +unicode_tailmatch(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + RETURN_SIZE(PyUnicode_Tailmatch(str, substr, start, end, direction)); +} + +/* Test PyUnicode_FindChar() */ +static PyObject * +unicode_findchar(PyObject *self, PyObject *args) +{ + PyObject *str; + int direction; + unsigned int ch; + Py_ssize_t result; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, + &start, &end, &direction)) { + return NULL; + } + NULLABLE(str); + result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); + if (result == -2) { + assert(PyErr_Occurred()); + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Replace() */ +static PyObject * +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + PyObject *replstr; + Py_ssize_t maxcount = -1; + + if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + NULLABLE(replstr); + return PyUnicode_Replace(str, substr, replstr, maxcount); +} + +/* Test PyUnicode_Compare() */ +static PyObject * +unicode_compare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int result; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CompareWithASCIIString() */ +static PyObject * +unicode_comparewithasciistring(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) + return NULL; + + NULLABLE(left); + result = PyUnicode_CompareWithASCIIString(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8() */ +static PyObject * +unicode_equaltoutf8(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "Oz#", &left, &right, &right_len)) { + return NULL; + } + + NULLABLE(left); + result = PyUnicode_EqualToUTF8(left, right); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_EqualToUTF8AndSize() */ +static PyObject * +unicode_equaltoutf8andsize(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + Py_ssize_t size = -100; + int result; + + if (!PyArg_ParseTuple(args, "Oz#|n", &left, &right, &right_len, &size)) { + return NULL; + } + + NULLABLE(left); + if (size == -100) { + size = right_len; + } + result = PyUnicode_EqualToUTF8AndSize(left, right, size); + assert(!PyErr_Occurred()); + return PyLong_FromLong(result); +} + +/* Test PyUnicode_RichCompare() */ +static PyObject * +unicode_richcompare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int op; + + if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_RichCompare(left, right, op); +} + +/* Test PyUnicode_Format() */ +static PyObject * +unicode_format(PyObject *self, PyObject *args) +{ + PyObject *format; + PyObject *fargs; + + if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) + return NULL; + + NULLABLE(format); + NULLABLE(fargs); + return PyUnicode_Format(format, fargs); +} + +/* Test PyUnicode_Contains() */ +static PyObject * +unicode_contains(PyObject *self, PyObject *args) +{ + PyObject *container; + PyObject *element; + + if (!PyArg_ParseTuple(args, "OO", &container, &element)) + return NULL; + + NULLABLE(container); + NULLABLE(element); + RETURN_INT(PyUnicode_Contains(container, element)); +} + +/* Test PyUnicode_IsIdentifier() */ +static PyObject * +unicode_isidentifier(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + RETURN_INT(PyUnicode_IsIdentifier(arg)); +} + + +static int +check_raised_systemerror(PyObject *result, char* msg) +{ + if (result) { + // no exception + PyErr_Format(PyExc_AssertionError, + "SystemError not raised: %s", + msg); + return 0; + } + if (PyErr_ExceptionMatches(PyExc_SystemError)) { + // expected exception + PyErr_Clear(); + return 1; + } + // unexpected exception + return 0; +} + +static PyObject * +test_string_from_format(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *result; + PyObject *unicode = PyUnicode_FromString("None"); + +#define CHECK_FORMAT_2(FORMAT, EXPECTED, ARG1, ARG2) \ + result = PyUnicode_FromFormat(FORMAT, ARG1, ARG2); \ + if (EXPECTED == NULL) { \ + if (!check_raised_systemerror(result, FORMAT)) { \ + goto Fail; \ + } \ + } \ + else if (result == NULL) \ + return NULL; \ + else if (PyUnicode_CompareWithASCIIString(result, EXPECTED) != 0) { \ + PyObject *utf8 = PyUnicode_AsUTF8String(result); \ + PyErr_Format(PyExc_AssertionError, \ + "test_string_from_format: failed at \"%s\" " \ + "expected \"%s\" got \"%s\"", \ + FORMAT, EXPECTED, utf8); \ + Py_XDECREF(utf8); \ + goto Fail; \ + } \ + Py_XDECREF(result) + +#define CHECK_FORMAT_1(FORMAT, EXPECTED, ARG) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, ARG, 0) + +#define CHECK_FORMAT_0(FORMAT, EXPECTED) \ + CHECK_FORMAT_2(FORMAT, EXPECTED, 0, 0) + + // Unrecognized + CHECK_FORMAT_2("%u %? %u", NULL, 1, 2); + + // "%%" (options are rejected) + CHECK_FORMAT_0( "%%", "%"); + CHECK_FORMAT_0( "%0%", NULL); + CHECK_FORMAT_0("%00%", NULL); + CHECK_FORMAT_0( "%2%", NULL); + CHECK_FORMAT_0("%02%", NULL); + CHECK_FORMAT_0("%.0%", NULL); + CHECK_FORMAT_0("%.2%", NULL); + + // "%c" + CHECK_FORMAT_1( "%c", "c", 'c'); + CHECK_FORMAT_1( "%0c", "c", 'c'); + CHECK_FORMAT_1("%00c", "c", 'c'); + CHECK_FORMAT_1( "%2c", NULL, 'c'); + CHECK_FORMAT_1("%02c", NULL, 'c'); + CHECK_FORMAT_1("%.0c", NULL, 'c'); + CHECK_FORMAT_1("%.2c", NULL, 'c'); + + // Integers + CHECK_FORMAT_1("%d", "123", (int)123); + CHECK_FORMAT_1("%i", "123", (int)123); + CHECK_FORMAT_1("%u", "123", (unsigned int)123); + CHECK_FORMAT_1("%x", "7b", (unsigned int)123); + CHECK_FORMAT_1("%X", "7B", (unsigned int)123); + CHECK_FORMAT_1("%o", "173", (unsigned int)123); + CHECK_FORMAT_1("%ld", "123", (long)123); + CHECK_FORMAT_1("%li", "123", (long)123); + CHECK_FORMAT_1("%lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%lx", "7b", (unsigned long)123); + CHECK_FORMAT_1("%lX", "7B", (unsigned long)123); + CHECK_FORMAT_1("%lo", "173", (unsigned long)123); + CHECK_FORMAT_1("%lld", "123", (long long)123); + CHECK_FORMAT_1("%lli", "123", (long long)123); + CHECK_FORMAT_1("%llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%llx", "7b", (unsigned long long)123); + CHECK_FORMAT_1("%llX", "7B", (unsigned long long)123); + CHECK_FORMAT_1("%llo", "173", (unsigned long long)123); + CHECK_FORMAT_1("%zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%zu", "123", (size_t)123); + CHECK_FORMAT_1("%zx", "7b", (size_t)123); + CHECK_FORMAT_1("%zX", "7B", (size_t)123); + CHECK_FORMAT_1("%zo", "173", (size_t)123); + CHECK_FORMAT_1("%td", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%ti", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tu", "123", (ptrdiff_t)123); + CHECK_FORMAT_1("%tx", "7b", (ptrdiff_t)123); + CHECK_FORMAT_1("%tX", "7B", (ptrdiff_t)123); + CHECK_FORMAT_1("%to", "173", (ptrdiff_t)123); + CHECK_FORMAT_1("%jd", "123", (intmax_t)123); + CHECK_FORMAT_1("%ji", "123", (intmax_t)123); + CHECK_FORMAT_1("%ju", "123", (uintmax_t)123); + CHECK_FORMAT_1("%jx", "7b", (uintmax_t)123); + CHECK_FORMAT_1("%jX", "7B", (uintmax_t)123); + CHECK_FORMAT_1("%jo", "173", (uintmax_t)123); + + CHECK_FORMAT_1("%d", "-123", (int)-123); + CHECK_FORMAT_1("%i", "-123", (int)-123); + CHECK_FORMAT_1("%ld", "-123", (long)-123); + CHECK_FORMAT_1("%li", "-123", (long)-123); + CHECK_FORMAT_1("%lld", "-123", (long long)-123); + CHECK_FORMAT_1("%lli", "-123", (long long)-123); + CHECK_FORMAT_1("%zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%zi", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%td", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%ti", "-123", (ptrdiff_t)-123); + CHECK_FORMAT_1("%jd", "-123", (intmax_t)-123); + CHECK_FORMAT_1("%ji", "-123", (intmax_t)-123); + + // Integers: width < length + CHECK_FORMAT_1("%1d", "123", (int)123); + CHECK_FORMAT_1("%1i", "123", (int)123); + CHECK_FORMAT_1("%1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%1ld", "123", (long)123); + CHECK_FORMAT_1("%1li", "123", (long)123); + CHECK_FORMAT_1("%1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%1lld", "123", (long long)123); + CHECK_FORMAT_1("%1lli", "123", (long long)123); + CHECK_FORMAT_1("%1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%1zu", "123", (size_t)123); + CHECK_FORMAT_1("%1x", "7b", (int)123); + + CHECK_FORMAT_1("%1d", "-123", (int)-123); + CHECK_FORMAT_1("%1i", "-123", (int)-123); + CHECK_FORMAT_1("%1ld", "-123", (long)-123); + CHECK_FORMAT_1("%1li", "-123", (long)-123); + CHECK_FORMAT_1("%1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%1zi", "-123", (Py_ssize_t)-123); + + // Integers: width > length + CHECK_FORMAT_1("%5d", " 123", (int)123); + CHECK_FORMAT_1("%5i", " 123", (int)123); + CHECK_FORMAT_1("%5u", " 123", (unsigned int)123); + CHECK_FORMAT_1("%5ld", " 123", (long)123); + CHECK_FORMAT_1("%5li", " 123", (long)123); + CHECK_FORMAT_1("%5lu", " 123", (unsigned long)123); + CHECK_FORMAT_1("%5lld", " 123", (long long)123); + CHECK_FORMAT_1("%5lli", " 123", (long long)123); + CHECK_FORMAT_1("%5llu", " 123", (unsigned long long)123); + CHECK_FORMAT_1("%5zd", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zi", " 123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5zu", " 123", (size_t)123); + CHECK_FORMAT_1("%5x", " 7b", (int)123); + + CHECK_FORMAT_1("%5d", " -123", (int)-123); + CHECK_FORMAT_1("%5i", " -123", (int)-123); + CHECK_FORMAT_1("%5ld", " -123", (long)-123); + CHECK_FORMAT_1("%5li", " -123", (long)-123); + CHECK_FORMAT_1("%5lld", " -123", (long long)-123); + CHECK_FORMAT_1("%5lli", " -123", (long long)-123); + CHECK_FORMAT_1("%5zd", " -123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5zi", " -123", (Py_ssize_t)-123); + + // Integers: width > length, 0-flag + CHECK_FORMAT_1("%05d", "00123", (int)123); + CHECK_FORMAT_1("%05i", "00123", (int)123); + CHECK_FORMAT_1("%05u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%05ld", "00123", (long)123); + CHECK_FORMAT_1("%05li", "00123", (long)123); + CHECK_FORMAT_1("%05lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%05lld", "00123", (long long)123); + CHECK_FORMAT_1("%05lli", "00123", (long long)123); + CHECK_FORMAT_1("%05llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%05zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05zu", "00123", (size_t)123); + CHECK_FORMAT_1("%05x", "0007b", (int)123); + + CHECK_FORMAT_1("%05d", "-0123", (int)-123); + CHECK_FORMAT_1("%05i", "-0123", (int)-123); + CHECK_FORMAT_1("%05ld", "-0123", (long)-123); + CHECK_FORMAT_1("%05li", "-0123", (long)-123); + CHECK_FORMAT_1("%05lld", "-0123", (long long)-123); + CHECK_FORMAT_1("%05lli", "-0123", (long long)-123); + CHECK_FORMAT_1("%05zd", "-0123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05zi", "-0123", (Py_ssize_t)-123); + + // Integers: precision < length + CHECK_FORMAT_1("%.1d", "123", (int)123); + CHECK_FORMAT_1("%.1i", "123", (int)123); + CHECK_FORMAT_1("%.1u", "123", (unsigned int)123); + CHECK_FORMAT_1("%.1ld", "123", (long)123); + CHECK_FORMAT_1("%.1li", "123", (long)123); + CHECK_FORMAT_1("%.1lu", "123", (unsigned long)123); + CHECK_FORMAT_1("%.1lld", "123", (long long)123); + CHECK_FORMAT_1("%.1lli", "123", (long long)123); + CHECK_FORMAT_1("%.1llu", "123", (unsigned long long)123); + CHECK_FORMAT_1("%.1zd", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zi", "123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.1zu", "123", (size_t)123); + CHECK_FORMAT_1("%.1x", "7b", (int)123); + + CHECK_FORMAT_1("%.1d", "-123", (int)-123); + CHECK_FORMAT_1("%.1i", "-123", (int)-123); + CHECK_FORMAT_1("%.1ld", "-123", (long)-123); + CHECK_FORMAT_1("%.1li", "-123", (long)-123); + CHECK_FORMAT_1("%.1lld", "-123", (long long)-123); + CHECK_FORMAT_1("%.1lli", "-123", (long long)-123); + CHECK_FORMAT_1("%.1zd", "-123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.1zi", "-123", (Py_ssize_t)-123); + + // Integers: precision > length + CHECK_FORMAT_1("%.5d", "00123", (int)123); + CHECK_FORMAT_1("%.5i", "00123", (int)123); + CHECK_FORMAT_1("%.5u", "00123", (unsigned int)123); + CHECK_FORMAT_1("%.5ld", "00123", (long)123); + CHECK_FORMAT_1("%.5li", "00123", (long)123); + CHECK_FORMAT_1("%.5lu", "00123", (unsigned long)123); + CHECK_FORMAT_1("%.5lld", "00123", (long long)123); + CHECK_FORMAT_1("%.5lli", "00123", (long long)123); + CHECK_FORMAT_1("%.5llu", "00123", (unsigned long long)123); + CHECK_FORMAT_1("%.5zd", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zi", "00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%.5zu", "00123", (size_t)123); + CHECK_FORMAT_1("%.5x", "0007b", (int)123); + + CHECK_FORMAT_1("%.5d", "-00123", (int)-123); + CHECK_FORMAT_1("%.5i", "-00123", (int)-123); + CHECK_FORMAT_1("%.5ld", "-00123", (long)-123); + CHECK_FORMAT_1("%.5li", "-00123", (long)-123); + CHECK_FORMAT_1("%.5lld", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5lli", "-00123", (long long)-123); + CHECK_FORMAT_1("%.5zd", "-00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%.5zi", "-00123", (Py_ssize_t)-123); + + // Integers: width > precision > length + CHECK_FORMAT_1("%7.5d", " 00123", (int)123); + CHECK_FORMAT_1("%7.5i", " 00123", (int)123); + CHECK_FORMAT_1("%7.5u", " 00123", (unsigned int)123); + CHECK_FORMAT_1("%7.5ld", " 00123", (long)123); + CHECK_FORMAT_1("%7.5li", " 00123", (long)123); + CHECK_FORMAT_1("%7.5lu", " 00123", (unsigned long)123); + CHECK_FORMAT_1("%7.5lld", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5lli", " 00123", (long long)123); + CHECK_FORMAT_1("%7.5llu", " 00123", (unsigned long long)123); + CHECK_FORMAT_1("%7.5zd", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zi", " 00123", (Py_ssize_t)123); + CHECK_FORMAT_1("%7.5zu", " 00123", (size_t)123); + CHECK_FORMAT_1("%7.5x", " 0007b", (int)123); + + CHECK_FORMAT_1("%7.5d", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5i", " -00123", (int)-123); + CHECK_FORMAT_1("%7.5ld", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5li", " -00123", (long)-123); + CHECK_FORMAT_1("%7.5lld", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5lli", " -00123", (long long)-123); + CHECK_FORMAT_1("%7.5zd", " -00123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%7.5zi", " -00123", (Py_ssize_t)-123); + + // Integers: width > precision > length, 0-flag + CHECK_FORMAT_1("%07.5d", "0000123", (int)123); + CHECK_FORMAT_1("%07.5i", "0000123", (int)123); + CHECK_FORMAT_1("%07.5u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%07.5ld", "0000123", (long)123); + CHECK_FORMAT_1("%07.5li", "0000123", (long)123); + CHECK_FORMAT_1("%07.5lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%07.5lld", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5lli", "0000123", (long long)123); + CHECK_FORMAT_1("%07.5llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%07.5zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%07.5zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%07.5x", "000007b", (int)123); + + CHECK_FORMAT_1("%07.5d", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5i", "-000123", (int)-123); + CHECK_FORMAT_1("%07.5ld", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5li", "-000123", (long)-123); + CHECK_FORMAT_1("%07.5lld", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5lli", "-000123", (long long)-123); + CHECK_FORMAT_1("%07.5zd", "-000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%07.5zi", "-000123", (Py_ssize_t)-123); + + // Integers: precision > width > length + CHECK_FORMAT_1("%5.7d", "0000123", (int)123); + CHECK_FORMAT_1("%5.7i", "0000123", (int)123); + CHECK_FORMAT_1("%5.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%5.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%5.7li", "0000123", (long)123); + CHECK_FORMAT_1("%5.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%5.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%5.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%5.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%5.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%5.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%5.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%5.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%5.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%5.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%5.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision > width > length, 0-flag + CHECK_FORMAT_1("%05.7d", "0000123", (int)123); + CHECK_FORMAT_1("%05.7i", "0000123", (int)123); + CHECK_FORMAT_1("%05.7u", "0000123", (unsigned int)123); + CHECK_FORMAT_1("%05.7ld", "0000123", (long)123); + CHECK_FORMAT_1("%05.7li", "0000123", (long)123); + CHECK_FORMAT_1("%05.7lu", "0000123", (unsigned long)123); + CHECK_FORMAT_1("%05.7lld", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7lli", "0000123", (long long)123); + CHECK_FORMAT_1("%05.7llu", "0000123", (unsigned long long)123); + CHECK_FORMAT_1("%05.7zd", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zi", "0000123", (Py_ssize_t)123); + CHECK_FORMAT_1("%05.7zu", "0000123", (size_t)123); + CHECK_FORMAT_1("%05.7x", "000007b", (int)123); + + CHECK_FORMAT_1("%05.7d", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7i", "-0000123", (int)-123); + CHECK_FORMAT_1("%05.7ld", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7li", "-0000123", (long)-123); + CHECK_FORMAT_1("%05.7lld", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7lli", "-0000123", (long long)-123); + CHECK_FORMAT_1("%05.7zd", "-0000123", (Py_ssize_t)-123); + CHECK_FORMAT_1("%05.7zi", "-0000123", (Py_ssize_t)-123); + + // Integers: precision = 0, arg = 0 (empty string in C) + CHECK_FORMAT_1("%.0d", "0", (int)0); + CHECK_FORMAT_1("%.0i", "0", (int)0); + CHECK_FORMAT_1("%.0u", "0", (unsigned int)0); + CHECK_FORMAT_1("%.0ld", "0", (long)0); + CHECK_FORMAT_1("%.0li", "0", (long)0); + CHECK_FORMAT_1("%.0lu", "0", (unsigned long)0); + CHECK_FORMAT_1("%.0lld", "0", (long long)0); + CHECK_FORMAT_1("%.0lli", "0", (long long)0); + CHECK_FORMAT_1("%.0llu", "0", (unsigned long long)0); + CHECK_FORMAT_1("%.0zd", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zi", "0", (Py_ssize_t)0); + CHECK_FORMAT_1("%.0zu", "0", (size_t)0); + CHECK_FORMAT_1("%.0x", "0", (int)0); + + // Strings + CHECK_FORMAT_1("%s", "None", "None"); + CHECK_FORMAT_1("%ls", "None", L"None"); + CHECK_FORMAT_1("%U", "None", unicode); + CHECK_FORMAT_1("%A", "None", Py_None); + CHECK_FORMAT_1("%S", "None", Py_None); + CHECK_FORMAT_1("%R", "None", Py_None); + CHECK_FORMAT_2("%V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%V", "None", NULL, "None"); + CHECK_FORMAT_2("%lV", "None", NULL, L"None"); + + // Strings: width < length + CHECK_FORMAT_1("%1s", "None", "None"); + CHECK_FORMAT_1("%1ls", "None", L"None"); + CHECK_FORMAT_1("%1U", "None", unicode); + CHECK_FORMAT_1("%1A", "None", Py_None); + CHECK_FORMAT_1("%1S", "None", Py_None); + CHECK_FORMAT_1("%1R", "None", Py_None); + CHECK_FORMAT_2("%1V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1V", "None", NULL, "None"); + CHECK_FORMAT_2("%1lV", "None", NULL, L"None"); + + // Strings: width > length + CHECK_FORMAT_1("%5s", " None", "None"); + CHECK_FORMAT_1("%5ls", " None", L"None"); + CHECK_FORMAT_1("%5U", " None", unicode); + CHECK_FORMAT_1("%5A", " None", Py_None); + CHECK_FORMAT_1("%5S", " None", Py_None); + CHECK_FORMAT_1("%5R", " None", Py_None); + CHECK_FORMAT_2("%5V", " None", unicode, "ignored"); + CHECK_FORMAT_2("%5V", " None", NULL, "None"); + CHECK_FORMAT_2("%5lV", " None", NULL, L"None"); + + // Strings: precision < length + CHECK_FORMAT_1("%.1s", "N", "None"); + CHECK_FORMAT_1("%.1ls", "N", L"None"); + CHECK_FORMAT_1("%.1U", "N", unicode); + CHECK_FORMAT_1("%.1A", "N", Py_None); + CHECK_FORMAT_1("%.1S", "N", Py_None); + CHECK_FORMAT_1("%.1R", "N", Py_None); + CHECK_FORMAT_2("%.1V", "N", unicode, "ignored"); + CHECK_FORMAT_2("%.1V", "N", NULL, "None"); + CHECK_FORMAT_2("%.1lV", "N", NULL, L"None"); + + // Strings: precision > length + CHECK_FORMAT_1("%.5s", "None", "None"); + CHECK_FORMAT_1("%.5ls", "None", L"None"); + CHECK_FORMAT_1("%.5U", "None", unicode); + CHECK_FORMAT_1("%.5A", "None", Py_None); + CHECK_FORMAT_1("%.5S", "None", Py_None); + CHECK_FORMAT_1("%.5R", "None", Py_None); + CHECK_FORMAT_2("%.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%.5lV", "None", NULL, L"None"); + + // Strings: precision < length, width > length + CHECK_FORMAT_1("%5.1s", " N", "None"); + CHECK_FORMAT_1("%5.1ls"," N", L"None"); + CHECK_FORMAT_1("%5.1U", " N", unicode); + CHECK_FORMAT_1("%5.1A", " N", Py_None); + CHECK_FORMAT_1("%5.1S", " N", Py_None); + CHECK_FORMAT_1("%5.1R", " N", Py_None); + CHECK_FORMAT_2("%5.1V", " N", unicode, "ignored"); + CHECK_FORMAT_2("%5.1V", " N", NULL, "None"); + CHECK_FORMAT_2("%5.1lV"," N", NULL, L"None"); + + // Strings: width < length, precision > length + CHECK_FORMAT_1("%1.5s", "None", "None"); + CHECK_FORMAT_1("%1.5ls", "None", L"None"); + CHECK_FORMAT_1("%1.5U", "None", unicode); + CHECK_FORMAT_1("%1.5A", "None", Py_None); + CHECK_FORMAT_1("%1.5S", "None", Py_None); + CHECK_FORMAT_1("%1.5R", "None", Py_None); + CHECK_FORMAT_2("%1.5V", "None", unicode, "ignored"); + CHECK_FORMAT_2("%1.5V", "None", NULL, "None"); + CHECK_FORMAT_2("%1.5lV", "None", NULL, L"None"); + + Py_XDECREF(unicode); + Py_RETURN_NONE; + + Fail: + Py_XDECREF(result); + Py_XDECREF(unicode); + return NULL; + +#undef CHECK_FORMAT_2 +#undef CHECK_FORMAT_1 +#undef CHECK_FORMAT_0 +} + +static PyMethodDef TestMethods[] = { + {"codec_incrementalencoder", codec_incrementalencoder, METH_VARARGS}, + {"codec_incrementaldecoder", codec_incrementaldecoder, METH_VARARGS}, + {"test_unicode_compare_with_ascii", + test_unicode_compare_with_ascii, METH_NOARGS}, + {"test_string_from_format", test_string_from_format, METH_NOARGS}, + {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, + {"unicode_fromencodedobject",unicode_fromencodedobject, METH_VARARGS}, + {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, + {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, + {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, + {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_decode", unicode_decode, METH_VARARGS}, + {"unicode_asencodedstring", unicode_asencodedstring, METH_VARARGS}, + {"unicode_buildencodingmap", unicode_buildencodingmap, METH_O}, + {"unicode_decodeutf7", unicode_decodeutf7, METH_VARARGS}, + {"unicode_decodeutf7stateful",unicode_decodeutf7stateful, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_asutf8string", unicode_asutf8string, METH_O}, + {"unicode_decodeutf16", unicode_decodeutf16, METH_VARARGS}, + {"unicode_decodeutf16stateful",unicode_decodeutf16stateful, METH_VARARGS}, + {"unicode_asutf16string", unicode_asutf16string, METH_O}, + {"unicode_decodeutf32", unicode_decodeutf32, METH_VARARGS}, + {"unicode_decodeutf32stateful",unicode_decodeutf32stateful, METH_VARARGS}, + {"unicode_asutf32string", unicode_asutf32string, METH_O}, + {"unicode_decodeunicodeescape",unicode_decodeunicodeescape, METH_VARARGS}, + {"unicode_asunicodeescapestring",unicode_asunicodeescapestring,METH_O}, + {"unicode_decoderawunicodeescape",unicode_decoderawunicodeescape,METH_VARARGS}, + {"unicode_asrawunicodeescapestring",unicode_asrawunicodeescapestring,METH_O}, + {"unicode_decodelatin1", unicode_decodelatin1, METH_VARARGS}, + {"unicode_aslatin1string", unicode_aslatin1string, METH_O}, + {"unicode_decodeascii", unicode_decodeascii, METH_VARARGS}, + {"unicode_asasciistring", unicode_asasciistring, METH_O}, + {"unicode_decodecharmap", unicode_decodecharmap, METH_VARARGS}, + {"unicode_ascharmapstring", unicode_ascharmapstring, METH_VARARGS}, +#ifdef MS_WINDOWS + {"unicode_decodembcs", unicode_decodembcs, METH_VARARGS}, + {"unicode_decodembcsstateful",unicode_decodembcsstateful, METH_VARARGS}, + {"unicode_decodecodepagestateful",unicode_decodecodepagestateful,METH_VARARGS}, + {"unicode_asmbcsstring", unicode_asmbcsstring, METH_O}, + {"unicode_encodecodepage", unicode_encodecodepage, METH_VARARGS}, +#endif /* MS_WINDOWS */ + {"unicode_decodelocaleandsize",unicode_decodelocaleandsize, METH_VARARGS}, + {"unicode_decodelocale", unicode_decodelocale, METH_VARARGS}, + {"unicode_encodelocale", unicode_encodelocale, METH_VARARGS}, + {"unicode_decodefsdefault", unicode_decodefsdefault, METH_VARARGS}, + {"unicode_decodefsdefaultandsize",unicode_decodefsdefaultandsize,METH_VARARGS}, + {"unicode_encodefsdefault", unicode_encodefsdefault, METH_O}, + {"unicode_concat", unicode_concat, METH_VARARGS}, + {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, + {"unicode_split", unicode_split, METH_VARARGS}, + {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, + {"unicode_partition", unicode_partition, METH_VARARGS}, + {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, + {"unicode_translate", unicode_translate, METH_VARARGS}, + {"unicode_join", unicode_join, METH_VARARGS}, + {"unicode_count", unicode_count, METH_VARARGS}, + {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, + {"unicode_find", unicode_find, METH_VARARGS}, + {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_replace", unicode_replace, METH_VARARGS}, + {"unicode_compare", unicode_compare, METH_VARARGS}, + {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, + {"unicode_equaltoutf8", unicode_equaltoutf8, METH_VARARGS}, + {"unicode_equaltoutf8andsize",unicode_equaltoutf8andsize, METH_VARARGS}, + {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, + {"unicode_format", unicode_format, METH_VARARGS}, + {"unicode_contains", unicode_contains, METH_VARARGS}, + {"unicode_isidentifier", unicode_isidentifier, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Unicode(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 43136697c5dac0..3715d82f1e35ab 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -103,6 +103,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index d3cdc47957eed5..29177dfcc44067 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -18,6 +18,7 @@ + From 408e127159e54d87bb3464fd8bd60219dc527fac Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 19 Mar 2024 20:36:19 +0800 Subject: [PATCH 30/82] gh-114099 - Add iOS framework loading machinery. (GH-116454) Co-authored-by: Malcolm Smith Co-authored-by: Eric Snow --- .gitignore | 2 +- Doc/library/importlib.rst | 63 ++++++++++++++ Doc/tools/extensions/pyspecific.py | 2 +- Lib/ctypes/__init__.py | 11 +++ Lib/ctypes/util.py | 2 +- Lib/importlib/_bootstrap_external.py | 53 +++++++++++- Lib/importlib/abc.py | 6 +- Lib/importlib/machinery.py | 1 + Lib/inspect.py | 7 +- Lib/modulefinder.py | 7 +- Lib/test/test_capi/test_misc.py | 16 +++- Lib/test/test_import/__init__.py | 86 ++++++++++++++----- .../test_importlib/extension/test_finder.py | 25 +++++- .../test_importlib/extension/test_loader.py | 56 ++++++++---- Lib/test/test_importlib/test_util.py | 11 ++- Lib/test/test_importlib/util.py | 6 ++ ...-03-07-16-12-39.gh-issue-114099.ujdjn2.rst | 2 + configure | 1 - configure.ac | 1 - iOS/Resources/Info.plist.in | 2 +- .../iOSTestbed.xcodeproj/project.pbxproj | 2 +- iOS/testbed/iOSTestbedTests/iOSTestbedTests.m | 2 +- 22 files changed, 302 insertions(+), 62 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst diff --git a/.gitignore b/.gitignore index 2d380a441d2394..8872e9d5508ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,7 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre -iOSTestbed.* +/iOSTestbed.* iOS/Frameworks/ iOS/Resources/Info.plist iOS/testbed/build diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d92bb2f8e5cf83..b58ef359378e4f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1241,6 +1241,69 @@ find and load modules. and how the module's :attr:`__file__` is populated. +.. class:: AppleFrameworkLoader(name, path) + + A specialization of :class:`importlib.machinery.ExtensionFileLoader` that + is able to load extension modules in Framework format. + + For compatibility with the iOS App Store, *all* binary modules in an iOS app + must be dynamic libraries, contained in a framework with appropriate + metadata, stored in the ``Frameworks`` folder of the packaged app. There can + be only a single binary per framework, and there can be no executable binary + material outside the Frameworks folder. + + To accomodate this requirement, when running on iOS, extension module + binaries are *not* packaged as ``.so`` files on ``sys.path``, but as + individual standalone frameworks. To discover those frameworks, this loader + is be registered against the ``.fwork`` file extension, with a ``.fwork`` + file acting as a placeholder in the original location of the binary on + ``sys.path``. The ``.fwork`` file contains the path of the actual binary in + the ``Frameworks`` folder, relative to the app bundle. To allow for + resolving a framework-packaged binary back to the original location, the + framework is expected to contain a ``.origin`` file that contains the + location of the ``.fwork`` file, relative to the app bundle. + + For example, consider the case of an import ``from foo.bar import _whiz``, + where ``_whiz`` is implemented with the binary module + ``sources/foo/bar/_whiz.abi3.so``, with ``sources`` being the location + registered on ``sys.path``, relative to the application bundle. This module + *must* be distributed as + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz`` (creating the framework + name from the full import path of the module), with an ``Info.plist`` file + in the ``.framework`` directory identifying the binary as a framework. The + ``foo.bar._whiz`` module would be represented in the original location with + a ``sources/foo/bar/_whiz.abi3.fwork`` marker file, containing the path + ``Frameworks/foo.bar._whiz/foo.bar._whiz``. The framework would also contain + ``Frameworks/foo.bar._whiz.framework/foo.bar._whiz.origin``, containing the + path to the ``.fwork`` file. + + When a module is loaded with this loader, the ``__file__`` for the module + will report as the location of the ``.fwork`` file. This allows code to use + the ``__file__`` of a module as an anchor for file system traveral. + However, the spec origin will reference the location of the *actual* binary + in the ``.framework`` folder. + + The Xcode project building the app is responsible for converting any ``.so`` + files from wherever they exist in the ``PYTHONPATH`` into frameworks in the + ``Frameworks`` folder (including stripping extensions from the module file, + the addition of framework metadata, and signing the resulting framework), + and creating the ``.fwork`` and ``.origin`` files. This will usually be done + with a build step in the Xcode project; see the iOS documentation for + details on how to construct this build step. + + .. versionadded:: 3.13 + + .. availability:: iOS. + + .. attribute:: name + + Name of the module the loader supports. + + .. attribute:: path + + Path to the ``.fwork`` file for the extension module. + + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index cd441836f62bde..9709c4f4dc54aa 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -133,7 +133,7 @@ class Availability(SphinxDirective): known_platforms = frozenset({ "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", "GNU/kFreeBSD", "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", - "Unix", "VxWorks", "WASI", "Windows", "macOS", + "Unix", "VxWorks", "WASI", "Windows", "macOS", "iOS", # libc "BSD libc", "glibc", "musl", # POSIX platforms with pthreads diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index f63e31a3fb0107..36b2af7f2a0d66 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -348,6 +348,17 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, winmode=None): if name: name = _os.fspath(name) + + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + self._name = name flags = self._func_flags_ if use_errno: diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index c550883e7c7d4b..12d7428fe9a776 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b26be8583d0f81..4749a627c50c42 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -52,7 +52,7 @@ # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', -_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin' +_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY + _CASE_INSENSITIVE_PLATFORMS_STR_KEY) @@ -1714,6 +1714,46 @@ def __repr__(self): return f'FileFinder({self.path!r})' +class AppleFrameworkLoader(ExtensionFileLoader): + """A loader for modules that have been packaged as frameworks for + compatibility with Apple's iOS App Store policies. + """ + def create_module(self, spec): + # If the ModuleSpec has been created by the FileFinder, it will have + # been created with an origin pointing to the .fwork file. We need to + # redirect this to the location in the Frameworks folder, using the + # content of the .fwork file. + if spec.origin.endswith(".fwork"): + with _io.FileIO(spec.origin, 'r') as file: + framework_binary = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + spec.origin = _path_join(bundle_path, framework_binary) + + # If the loader is created based on the spec for a loaded module, the + # path will be pointing at the Framework location. If this occurs, + # get the original .fwork location to use as the module's __file__. + if self.path.endswith(".fwork"): + path = self.path + else: + with _io.FileIO(self.path + ".origin", 'r') as file: + origin = file.read().decode().strip() + bundle_path = _path_split(sys.executable)[0] + path = _path_join(bundle_path, origin) + + module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) + + _bootstrap._verbose_message( + "Apple framework extension module {!r} loaded from {!r} (path {!r})", + spec.name, + spec.origin, + path, + ) + + # Ensure that the __file__ points at the .fwork location + module.__file__ = path + + return module + # Import setup ############################################################### def _fix_up_module(ns, name, pathname, cpathname=None): @@ -1746,10 +1786,17 @@ def _get_supported_file_loaders(): Each item is a tuple (loader, suffixes). """ - extensions = ExtensionFileLoader, _imp.extension_suffixes() + if sys.platform in {"ios", "tvos", "watchos"}: + extension_loaders = [(AppleFrameworkLoader, [ + suffix.replace(".so", ".fwork") + for suffix in _imp.extension_suffixes() + ])] + else: + extension_loaders = [] + extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) source = SourceFileLoader, SOURCE_SUFFIXES bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES - return [extensions, source, bytecode] + return extension_loaders + [source, bytecode] def _set_bootstrap_module(_bootstrap_module): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index b56fa94eb9c135..37fef357fe2c0c 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -180,7 +180,11 @@ def get_code(self, fullname): else: return self.source_to_code(source, path) -_register(ExecutionLoader, machinery.ExtensionFileLoader) +_register( + ExecutionLoader, + machinery.ExtensionFileLoader, + machinery.AppleFrameworkLoader, +) class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index d9a19a13f7b275..fbd30b159fb752 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -12,6 +12,7 @@ from ._bootstrap_external import SourceFileLoader from ._bootstrap_external import SourcelessFileLoader from ._bootstrap_external import ExtensionFileLoader +from ._bootstrap_external import AppleFrameworkLoader from ._bootstrap_external import NamespaceLoader diff --git a/Lib/inspect.py b/Lib/inspect.py index 8a2b2c96e993b5..7336cea0dc3fdc 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -954,6 +954,10 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + # return a filename found in the linecache even if it doesn't exist on disk if filename in linecache.cache: return filename @@ -984,6 +988,7 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) @@ -1119,7 +1124,7 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index a0a020f9eeb9b4..ac478ee7f51722 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -72,7 +72,12 @@ def _find_module(name, path=None): if isinstance(spec.loader, importlib.machinery.SourceFileLoader): kind = _PY_SOURCE - elif isinstance(spec.loader, importlib.machinery.ExtensionFileLoader): + elif isinstance( + spec.loader, ( + importlib.machinery.ExtensionFileLoader, + importlib.machinery.AppleFrameworkLoader, + ) + ): kind = _C_EXTENSION elif isinstance(spec.loader, importlib.machinery.SourcelessFileLoader): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index d3fcd0b59dfa49..7365ead1c4749c 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2056,6 +2056,13 @@ def test_module_state_shared_in_global(self): self.addCleanup(os.close, r) self.addCleanup(os.close, w) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + script = textwrap.dedent(f""" import importlib.machinery import importlib.util @@ -2063,7 +2070,7 @@ def test_module_state_shared_in_global(self): fullname = '_test_module_state_shared' origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + loader = importlib.machinery.{loader}(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) attr_id = str(id(module.Error)).encode() @@ -2371,7 +2378,12 @@ class Test_ModuleStateAccess(unittest.TestCase): def setUp(self): fullname = '_testmultiphase_meth_state_access' # XXX origin = importlib.util.find_spec('_testmultiphase').origin - loader = importlib.machinery.ExtensionFileLoader(fullname, origin) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = importlib.machinery.AppleFrameworkLoader(fullname, origin) + else: + loader = importlib.machinery.ExtensionFileLoader(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7b0126226c4aba..4deed7f3ba2522 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -5,7 +5,11 @@ import importlib.util from importlib._bootstrap_external import _get_sourcefile from importlib.machinery import ( - BuiltinImporter, ExtensionFileLoader, FrozenImporter, SourceFileLoader, + AppleFrameworkLoader, + BuiltinImporter, + ExtensionFileLoader, + FrozenImporter, + SourceFileLoader, ) import marshal import os @@ -25,7 +29,7 @@ from test.support import os_helper from test.support import ( - STDLIB_DIR, swap_attr, swap_item, cpython_only, is_emscripten, + STDLIB_DIR, swap_attr, swap_item, cpython_only, is_apple_mobile, is_emscripten, is_wasi, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -66,6 +70,7 @@ def _require_loader(module, loader, skip): MODULE_KINDS = { BuiltinImporter: 'built-in', ExtensionFileLoader: 'extension', + AppleFrameworkLoader: 'framework extension', FrozenImporter: 'frozen', SourceFileLoader: 'pure Python', } @@ -91,7 +96,12 @@ def require_builtin(module, *, skip=False): assert module.__spec__.origin == 'built-in', module.__spec__ def require_extension(module, *, skip=False): - _require_loader(module, ExtensionFileLoader, skip) + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + _require_loader(module, AppleFrameworkLoader, skip) + else: + _require_loader(module, ExtensionFileLoader, skip) def require_frozen(module, *, skip=True): module = _require_loader(module, FrozenImporter, skip) @@ -134,7 +144,8 @@ def restore__testsinglephase(*, _orig=_testsinglephase): # it to its nominal state. sys.modules.pop('_testsinglephase', None) _orig._clear_globals() - _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) + origin = _orig.__spec__.origin + _testinternalcapi.clear_extension('_testsinglephase', origin) import _testsinglephase @@ -360,7 +371,7 @@ def test_from_import_missing_attr_has_name_and_so_path(self): self.assertEqual(cm.exception.path, _testcapi.__file__) self.assertRegex( str(cm.exception), - r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|fwork|pyd)\)" ) else: self.assertEqual( @@ -1689,6 +1700,14 @@ def pipe(self): os.set_blocking(r, False) return (r, w) + def create_extension_loader(self, modname, filename): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + return AppleFrameworkLoader(modname, filename) + else: + return ExtensionFileLoader(modname, filename) + def import_script(self, name, fd, filename=None, check_override=None): override_text = '' if check_override is not None: @@ -1697,12 +1716,19 @@ def import_script(self, name, fd, filename=None, check_override=None): _imp._override_multi_interp_extensions_check({check_override}) ''' if filename: + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + return textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} import os, sys {override_text} - loader = ExtensionFileLoader({name!r}, {filename!r}) + loader = {loader}({name!r}, {filename!r}) spec = spec_from_loader({name!r}, loader) try: module = module_from_spec(spec) @@ -1883,7 +1909,7 @@ def test_multi_init_extension_compat(self): def test_multi_init_extension_non_isolated_compat(self): modname = '_test_non_isolated' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -1901,7 +1927,7 @@ def test_multi_init_extension_non_isolated_compat(self): def test_multi_init_extension_per_interpreter_gil_compat(self): modname = '_test_shared_gil_only' filename = _testmultiphase.__file__ - loader = ExtensionFileLoader(modname, filename) + loader = self.create_extension_loader(modname, filename) spec = importlib.util.spec_from_loader(modname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -2034,10 +2060,25 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.find_spec(cls.NAME) - from importlib.machinery import ExtensionFileLoader - cls.FILE = spec.origin cls.LOADER = type(spec.loader) - assert cls.LOADER is ExtensionFileLoader + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader, and we need to differentiate between the + # spec.origin and the original file location. + if is_apple_mobile: + assert cls.LOADER is AppleFrameworkLoader + + cls.ORIGIN = spec.origin + with open(spec.origin + ".origin", "r") as f: + cls.FILE = os.path.join( + os.path.dirname(sys.executable), + f.read().strip() + ) + else: + assert cls.LOADER is ExtensionFileLoader + + cls.ORIGIN = spec.origin + cls.FILE = spec.origin # Start fresh. cls.clean_up() @@ -2053,14 +2094,15 @@ def tearDown(self): @classmethod def clean_up(cls): name = cls.NAME - filename = cls.FILE if name in sys.modules: if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == filename + assert sys.modules[name].__file__ == cls.FILE, \ + f"{sys.modules[name].__file__} != {cls.FILE}" + sys.modules[name]._clear_globals() del sys.modules[name] # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, filename) + _testinternalcapi.clear_extension(name, cls.ORIGIN) ######################### # helpers @@ -2068,7 +2110,7 @@ def clean_up(cls): def add_module_cleanup(self, name): def clean_up(): # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, self.FILE) + _testinternalcapi.clear_extension(name, self.ORIGIN) self.addCleanup(clean_up) def _load_dynamic(self, name, path): @@ -2091,7 +2133,7 @@ def load(self, name): except AttributeError: already_loaded = self.already_loaded = {} assert name not in already_loaded - mod = self._load_dynamic(name, self.FILE) + mod = self._load_dynamic(name, self.ORIGIN) self.assertNotIn(mod, already_loaded.values()) already_loaded[name] = mod return types.SimpleNamespace( @@ -2103,7 +2145,7 @@ def load(self, name): def re_load(self, name, mod): assert sys.modules[name] is mod assert mod.__dict__ == mod.__dict__ - reloaded = self._load_dynamic(name, self.FILE) + reloaded = self._load_dynamic(name, self.ORIGIN) return types.SimpleNamespace( name=name, module=reloaded, @@ -2129,7 +2171,7 @@ def clean_up(): name = {self.NAME!r} if name in sys.modules: sys.modules.pop(name)._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''')) _interpreters.destroy(interpid) self.addCleanup(clean_up) @@ -2146,7 +2188,7 @@ def import_in_subinterp(self, interpid=None, *, postcleanup = f''' {import_} mod._clear_globals() - _testinternalcapi.clear_extension(name, {self.FILE!r}) + _testinternalcapi.clear_extension(name, {self.ORIGIN!r}) ''' try: @@ -2184,7 +2226,7 @@ def check_common(self, loaded): # mod.__name__ might not match, but the spec will. self.assertEqual(mod.__spec__.name, loaded.name) self.assertEqual(mod.__file__, self.FILE) - self.assertEqual(mod.__spec__.origin, self.FILE) + self.assertEqual(mod.__spec__.origin, self.ORIGIN) if not isolated: self.assertTrue(issubclass(mod.error, Exception)) self.assertEqual(mod.int_const, 1969) @@ -2578,7 +2620,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # First, load in the main interpreter but then completely clear it. loaded_main = self.load(self.NAME) loaded_main.module._clear_globals() - _testinternalcapi.clear_extension(self.NAME, self.FILE) + _testinternalcapi.clear_extension(self.NAME, self.ORIGIN) # At this point: # * alive in 0 interpreters diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 3de120958fd27d..cdc8884d668a66 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -19,9 +20,27 @@ def setUp(self): ) def find_spec(self, fullname): - importer = self.machinery.FileFinder(util.EXTENSIONS.path, - (self.machinery.ExtensionFileLoader, - self.machinery.EXTENSION_SUFFIXES)) + if is_apple_mobile: + # Apple mobile platforms require a specialist loader that uses + # .fwork files as placeholders for the true `.so` files. + loaders = [ + ( + self.machinery.AppleFrameworkLoader, + [ + ext.replace(".so", ".fwork") + for ext in self.machinery.EXTENSION_SUFFIXES + ] + ) + ] + else: + loaders = [ + ( + self.machinery.ExtensionFileLoader, + self.machinery.EXTENSION_SUFFIXES + ) + ] + + importer = self.machinery.FileFinder(util.EXTENSIONS.path, *loaders) return importer.find_spec(fullname) diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index f4879e75847d8d..7607f0e0857595 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -1,3 +1,4 @@ +from test.support import is_apple_mobile from test.test_importlib import abc, util machinery = util.import_importlib('importlib.machinery') @@ -23,8 +24,15 @@ def setUp(self): raise unittest.SkipTest( f"{util.EXTENSIONS.name} is a builtin module" ) - self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + + self.loader = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) def load_module(self, fullname): with warnings.catch_warnings(): @@ -32,13 +40,11 @@ def load_module(self, fullname): return self.loader.load_module(fullname) def test_equality(self): - other = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass(util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertEqual(self.loader, other) def test_inequality(self): - other = self.machinery.ExtensionFileLoader('_' + util.EXTENSIONS.name, - util.EXTENSIONS.file_path) + other = self.LoaderClass('_' + util.EXTENSIONS.name, util.EXTENSIONS.file_path) self.assertNotEqual(self.loader, other) def test_load_module_API(self): @@ -58,8 +64,7 @@ def test_module(self): ('__package__', '')]: self.assertEqual(getattr(module, attr), value) self.assertIn(util.EXTENSIONS.name, sys.modules) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -86,7 +91,7 @@ def test_is_package(self): self.assertFalse(self.loader.is_package(util.EXTENSIONS.name)) for suffix in self.machinery.EXTENSION_SUFFIXES: path = os.path.join('some', 'path', 'pkg', '__init__' + suffix) - loader = self.machinery.ExtensionFileLoader('pkg', path) + loader = self.LoaderClass('pkg', path) self.assertTrue(loader.is_package('pkg')) @@ -101,6 +106,14 @@ class SinglePhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testsinglephase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -109,8 +122,8 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): with warnings.catch_warnings(): @@ -120,7 +133,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -137,8 +150,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) # No extension module as __init__ available for testing. test_package = None @@ -182,6 +194,14 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): def setUp(self): if not self.machinery.EXTENSION_SUFFIXES or not util.EXTENSIONS: raise unittest.SkipTest("Requires dynamic loading support.") + + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if is_apple_mobile: + self.LoaderClass = self.machinery.AppleFrameworkLoader + else: + self.LoaderClass = self.machinery.ExtensionFileLoader + self.name = '_testmultiphase' if self.name in sys.builtin_module_names: raise unittest.SkipTest( @@ -190,8 +210,7 @@ def setUp(self): finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) assert self.spec - self.loader = self.machinery.ExtensionFileLoader( - self.name, self.spec.origin) + self.loader = self.LoaderClass(self.name, self.spec.origin) def load_module(self): # Load the module from the test extension. @@ -202,7 +221,7 @@ def load_module(self): def load_module_by_name(self, fullname): # Load a module from the test extension by name. origin = self.spec.origin - loader = self.machinery.ExtensionFileLoader(fullname, origin) + loader = self.LoaderClass(fullname, origin) spec = importlib.util.spec_from_loader(fullname, loader) module = importlib.util.module_from_spec(spec) loader.exec_module(module) @@ -228,8 +247,7 @@ def test_module(self): with self.assertRaises(AttributeError): module.__path__ self.assertIs(module, sys.modules[self.name]) - self.assertIsInstance(module.__loader__, - self.machinery.ExtensionFileLoader) + self.assertIsInstance(module.__loader__, self.LoaderClass) def test_functionality(self): # Test basic functionality of stuff defined in an extension module. diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index a09286806e5152..a6a76e589761e0 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -707,13 +707,20 @@ def test_single_phase_init_module(self): @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module") def test_incomplete_multi_phase_init_module(self): + # Apple extensions must be distributed as frameworks. This requires + # a specialist loader. + if support.is_apple_mobile: + loader = "AppleFrameworkLoader" + else: + loader = "ExtensionFileLoader" + prescript = textwrap.dedent(f''' from importlib.util import spec_from_loader, module_from_spec - from importlib.machinery import ExtensionFileLoader + from importlib.machinery import {loader} name = '_test_shared_gil_only' filename = {_testmultiphase.__file__!r} - loader = ExtensionFileLoader(name, filename) + loader = {loader}(name, filename) spec = spec_from_loader(name, loader) ''') diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index a900cc1dddf425..89272484009c56 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -8,6 +8,7 @@ import os.path from test import support from test.support import import_helper +from test.support import is_apple_mobile from test.support import os_helper import unittest import sys @@ -43,6 +44,11 @@ def _extension_details(): global EXTENSIONS for path in sys.path: for ext in machinery.EXTENSION_SUFFIXES: + # Apple mobile platforms mechanically load .so files, + # but the findable files are labelled .fwork + if is_apple_mobile: + ext = ext.replace(".so", ".fwork") + filename = EXTENSIONS.name + ext file_path = os.path.join(path, filename) if os.path.exists(file_path): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst new file mode 100644 index 00000000000000..5405a3bdc36f9e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-07-16-12-39.gh-issue-114099.ujdjn2.rst @@ -0,0 +1,2 @@ +Added a Loader that can discover extension modules in an iOS-style Frameworks +folder. diff --git a/configure b/configure index 07dce38c92724f..229f0d32d322dd 100755 --- a/configure +++ b/configure @@ -12740,7 +12740,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/configure.ac b/configure.ac index 3e676c56693a3c..cd17977738482d 100644 --- a/configure.ac +++ b/configure.ac @@ -3285,7 +3285,6 @@ if test -z "$SHLIB_SUFFIX"; then esac ;; CYGWIN*) SHLIB_SUFFIX=.dll;; - iOS) SHLIB_SUFFIX=.dylib;; *) SHLIB_SUFFIX=.so;; esac fi diff --git a/iOS/Resources/Info.plist.in b/iOS/Resources/Info.plist.in index 3ecdc894f0a285..52c0a6e7fd7a55 100644 --- a/iOS/Resources/Info.plist.in +++ b/iOS/Resources/Info.plist.in @@ -17,7 +17,7 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION% + @VERSION@ CFBundleLongVersionString %VERSION%, (c) 2001-2024 Python Software Foundation. CFBundleSignature diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 4f138a4e7ccefd..4389c08ac1960d 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -273,7 +273,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_DYLIB=$2\n\n # The name of the .dylib file\n DYLIB=$(basename \"$FULL_DYLIB\")\n # The name of the .dylib file, relative to the install base\n RELATIVE_DYLIB=${FULL_DYLIB#$CODESIGNING_FOLDER_PATH/$INSTALL_BASE/}\n # The full dotted name of the binary module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $RELATIVE_DYLIB | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_DYLIB\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\" \n fi\n \n echo \"Installing binary for $RELATIVE_DYLIB\" \n mv \"$FULL_DYLIB\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library dylibs...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.dylib\" | while read FULL_DYLIB; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload \"$FULL_DYLIB\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index 53ea107db4a2de..e6a919c304ec8d 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -15,7 +15,7 @@ - (void)testPython { const char *argv[] = { "iOSTestbed", // argv[0] is the process that is running. "-uall", // Enable all resources - "-v", // run in verbose mode so we get test failure information + "-W", // Display test output on failure // To run a subset of tests, add the test names below; e.g., // "test_os", // "test_sys", From 27cf3ed00cfe942f4277c273a3dda8ee2ba61fc8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 14:42:44 +0100 Subject: [PATCH 31/82] gh-90872: Fix subprocess.Popen.wait() for negative timeout (#116989) On Windows, subprocess.Popen.wait() no longer calls WaitForSingleObject() with a negative timeout: pass 0 ms if the timeout is negative. --- Lib/subprocess.py | 2 ++ Lib/test/test_subprocess.py | 16 ++++++++++++++++ ...2024-03-19-11-08-26.gh-issue-90872.ghys95.rst | 3 +++ 3 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 1437bf8148282c..dbe15277866c99 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -1586,6 +1586,8 @@ def _wait(self, timeout): """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE + elif timeout <= 0: + timeout_millis = 0 else: timeout_millis = int(timeout * 1000) if self.returncode is None: diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index c44a778d5bbefe..d20b987961ea6f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1607,6 +1607,22 @@ def test_class_getitems(self): self.assertIsInstance(subprocess.Popen[bytes], types.GenericAlias) self.assertIsInstance(subprocess.CompletedProcess[str], types.GenericAlias) + @unittest.skipUnless(hasattr(subprocess, '_winapi'), + 'need subprocess._winapi') + def test_wait_negative_timeout(self): + proc = subprocess.Popen(ZERO_RETURN_CMD) + with proc: + patch = mock.patch.object( + subprocess._winapi, + 'WaitForSingleObject', + return_value=subprocess._winapi.WAIT_OBJECT_0) + with patch as mock_wait: + proc.wait(-1) # negative timeout + mock_wait.assert_called_once_with(proc._handle, 0) + proc.returncode = None + + self.assertEqual(proc.wait(), 0) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst b/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst new file mode 100644 index 00000000000000..ead68caa9fe88b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-19-11-08-26.gh-issue-90872.ghys95.rst @@ -0,0 +1,3 @@ +On Windows, :meth:`subprocess.Popen.wait` no longer calls +``WaitForSingleObject()`` with a negative timeout: pass ``0`` ms if the +timeout is negative. Patch by Victor Stinner. From a114d08a8912a50530ab3f19842c6ba73b0d1017 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 15:03:27 +0100 Subject: [PATCH 32/82] gh-92906: Enable test_cext and test_cppext on Windows (#117000) On Windows in release mode, the test_cext and test_cppext can now build C and C++ extensions. * test_cext now also builds the C extension without options. * test_cppext now also builds the C++ extension without options. * Add C++14 test to test_cppext; C++11 is not supported by MSVC. * Make setup_venv_with_pip_setuptools_wheel() quiet when support.verbose is false. Only show stdout and stderr on failure. --- Lib/test/support/__init__.py | 22 +++++---- Lib/test/test_cext/__init__.py | 29 ++++++++---- Lib/test/test_cext/extension.c | 2 + Lib/test/test_cext/setup.py | 35 ++++++++++++-- Lib/test/test_cppext/__init__.py | 35 +++++++++----- Lib/test/test_cppext/extension.cpp | 12 ++--- Lib/test/test_cppext/setup.py | 76 +++++++++++++++++++++--------- 7 files changed, 148 insertions(+), 63 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ce693e51aab31c..a1c7987fa0db47 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2251,16 +2251,25 @@ def _findwheel(pkgname): # and returns the path to the venv directory and the path to the python executable @contextlib.contextmanager def setup_venv_with_pip_setuptools_wheel(venv_dir): + import shlex import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2275,10 +2284,7 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): '-m', 'pip', 'install', _findwheel('setuptools'), _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) yield python diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 992939241c591e..7bf2381d844d1c 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -1,9 +1,11 @@ # gh-116869: Build a basic C test extension to check that the Python C API # does not emit C compiler warnings. # -# Python C API must build with -Werror=declaration-after-statement. +# The Python C API must be compatible with building +# with the -Werror=declaration-after-statement compiler flag. import os.path +import shlex import shutil import subprocess import unittest @@ -14,9 +16,10 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# With MSVC, the linker fails with: cannot open file 'python311.lib' -# https://github.com/python/cpython/pull/32175#issuecomment-1111175897 -@unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') # Building and running an extension in clang sanitizing mode is not # straightforward @support.skip_if_sanitizer('test does not work with analyzing builds', @@ -26,17 +29,22 @@ @support.requires_subprocess() @support.requires_resource('cpu') class TestExt(unittest.TestCase): - def test_build_c99(self): - self.check_build('_test_c99_ext', std='c99') + # Default build with no options + def test_build(self): + self.check_build('_test_cext') def test_build_c11(self): - self.check_build('_test_c11_ext', std='c11') + self.check_build('_test_c11_cext', std='c11') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + self.check_build('_test_c99_cext', std='c99') def test_build_limited(self): - self.check_build('_test_limited_ext', limited=True) + self.check_build('_test_limited_cext', limited=True) def test_build_limited_c11(self): - self.check_build('_test_limited_c11_ext', limited=True, std='c11') + self.check_build('_test_limited_c11_cext', limited=True, std='c11') def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' @@ -58,7 +66,7 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: - print('Run:', ' '.join(cmd)) + print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) else: proc = subprocess.run(cmd, @@ -67,6 +75,7 @@ def run_cmd(operation, cmd): stderr=subprocess.STDOUT, text=True) if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) print(proc.stdout, end='') self.fail( f"{operation} failed with exit code {proc.returncode}") diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index a2433f2663c667..662abd4005e8d2 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -39,9 +39,11 @@ static PyMethodDef _testcext_methods[] = { static int _testcext_exec(PyObject *module) { +#ifdef __STDC_VERSION__ if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { return -1; } +#endif return 0; } diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 5ca080a790a5bd..d6c4410fa5f1e3 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -1,6 +1,7 @@ # gh-91321: Build a basic C test extension to check that the Python C API is # compatible with C and does not emit C compiler warnings. import os +import platform import shlex import sys import sysconfig @@ -17,8 +18,8 @@ # extension using the Python C API does not emit C compiler warnings. '-Werror', - # gh-116869: The Python C API must build with - # -Werror=declaration-after-statement. + # gh-116869: The Python C API must be compatible with building + # with the -Werror=declaration-after-statement compiler flag. '-Werror=declaration-after-statement', ] else: @@ -34,22 +35,44 @@ def main(): cflags = list(CFLAGS) cflags.append(f'-DMODULE_NAME={module_name}') + # Add -std=STD or /std:STD (MSVC) compiler flag if std: - cflags.append(f'-std={std}') + if support.MS_WINDOWS: + cflags.append(f'/std:{std}') + std_prefix = '/std' + else: + cflags.append(f'-std={std}') + std_prefix = '-std' # Remove existing -std options to only test ours cmd = (sysconfig.get_config_var('CC') or '') if cmd is not None: cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] cmd = shlex.join(cmd) # CC env var overrides sysconfig CC variable in setuptools os.environ['CC'] = cmd + # Define Py_LIMITED_API macro if limited: version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging for env_name in ('CC', 'CFLAGS'): if env_name in os.environ: print(f"{env_name} env var: {os.environ[env_name]!r}") @@ -60,7 +83,9 @@ def main(): ext = Extension( module_name, sources=[SOURCE], - extra_compile_args=cflags) + extra_compile_args=cflags, + include_dirs=include_dirs, + library_dirs=library_dirs) setup(name=f'internal_{module_name}', version='0.0', ext_modules=[ext]) diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 66ead9a3d78c66..b8414f217c7dde 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,9 +1,10 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import shlex import shutil -import unittest import subprocess +import unittest from test import support @@ -11,9 +12,10 @@ SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') -# With MSVC, the linker fails with: cannot open file 'python311.lib' -# https://github.com/python/cpython/pull/32175#issuecomment-1111175897 -@unittest.skipIf(support.MS_WINDOWS, 'test fails on Windows') +# With MSVC on a debug build, the linker fails with: cannot open file +# 'python311.lib', it should look 'python311_d.lib'. +@unittest.skipIf(support.MS_WINDOWS and support.Py_DEBUG, + 'test fails on Windows debug build') # Building and running an extension in clang sanitizing mode is not # straightforward @support.skip_if_sanitizer('test does not work with analyzing builds', @@ -23,18 +25,25 @@ @support.requires_subprocess() @support.requires_resource('cpu') class TestCPPExt(unittest.TestCase): - def test_build_cpp11(self): - self.check_build(False, '_testcpp11ext') + def test_build(self): + self.check_build('_testcppext') def test_build_cpp03(self): - self.check_build(True, '_testcpp03ext') + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') - def check_build(self, std_cpp03, extension_name): + def check_build(self, extension_name, std=None): venv_dir = 'env' with support.setup_venv_with_pip_setuptools_wheel(venv_dir) as python_exe: - self._check_build(std_cpp03, extension_name, python_exe) + self._check_build(extension_name, python_exe, std=std) - def _check_build(self, std_cpp03, extension_name, python_exe): + def _check_build(self, extension_name, python_exe, std): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -42,10 +51,11 @@ def _check_build(self, std_cpp03, extension_name, python_exe): def run_cmd(operation, cmd): env = os.environ.copy() - env['CPYTHON_TEST_CPP_STD'] = 'c++03' if std_cpp03 else 'c++11' + if std: + env['CPYTHON_TEST_CPP_STD'] = std env['CPYTHON_TEST_EXT_NAME'] = extension_name if support.verbose: - print('Run:', ' '.join(cmd)) + print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) else: proc = subprocess.run(cmd, @@ -54,6 +64,7 @@ def run_cmd(operation, cmd): stderr=subprocess.STDOUT, text=True) if proc.returncode: + print('Run:', ' '.join(map(shlex.quote, cmd))) print(proc.stdout, end='') self.fail( f"{operation} failed with exit code {proc.returncode}") diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 90669b10cb2c6d..a569c2251d1ad7 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -8,10 +8,8 @@ #include "Python.h" -#if __cplusplus >= 201103 -# define NAME _testcpp11ext -#else -# define NAME _testcpp03ext +#ifndef MODULE_NAME +# error "MODULE_NAME macro must be defined" #endif #define _STR(NAME) #NAME @@ -160,7 +158,7 @@ PyType_Slot VirtualPyObject_Slots[] = { }; PyType_Spec VirtualPyObject_Spec = { - /* .name */ STR(NAME) ".VirtualPyObject", + /* .name */ STR(MODULE_NAME) ".VirtualPyObject", /* .basicsize */ sizeof(VirtualPyObject), /* .itemsize */ 0, /* .flags */ Py_TPFLAGS_DEFAULT, @@ -240,7 +238,7 @@ PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); static struct PyModuleDef _testcppext_module = { PyModuleDef_HEAD_INIT, // m_base - STR(NAME), // m_name + STR(MODULE_NAME), // m_name _testcppext_doc, // m_doc 0, // m_size _testcppext_methods, // m_methods @@ -254,7 +252,7 @@ static struct PyModuleDef _testcppext_module = { #define FUNC_NAME(NAME) _FUNC_NAME(NAME) PyMODINIT_FUNC -FUNC_NAME(NAME)(void) +FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcppext_module); } diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index c7ba1efb4dd05a..77e47bcd0cbe16 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,8 +1,8 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import platform import shlex -import sys import sysconfig from test import support @@ -25,28 +25,62 @@ def main(): cppflags = list(CPPFLAGS) - std = os.environ["CPYTHON_TEST_CPP_STD"] - name = os.environ["CPYTHON_TEST_EXT_NAME"] - - cppflags = [*CPPFLAGS, f'-std={std}'] - - # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 - # option emits a C++ compiler warning. Remove "-std11" option from the - # CC command. - cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: - cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith('-std=')] - cmd = shlex.join(cmd) - # CC env var overrides sysconfig CC variable in setuptools - os.environ['CC'] = cmd - - cpp_ext = Extension( - name, + std = os.environ.get("CPYTHON_TEST_CPP_STD", "") + module_name = os.environ["CPYTHON_TEST_EXT_NAME"] + + cppflags = list(CPPFLAGS) + cppflags.append(f'-DMODULE_NAME={module_name}') + + # Add -std=STD or /std:STD (MSVC) compiler flag + if std: + if support.MS_WINDOWS: + cppflags.append(f'/std:{std}') + std_prefix = '/std' + else: + cppflags.append(f'-std={std}') + std_prefix = '-std' + + # Remove existing -std options to only test ours + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + + # On Windows, add PCbuild\amd64\ to include and library directories + include_dirs = [] + library_dirs = [] + if support.MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + machine = platform.uname().machine + pcbuild = os.path.join(srcdir, 'PCbuild', machine) + if os.path.exists(pcbuild): + # pyconfig.h is generated in PCbuild\amd64\ + include_dirs.append(pcbuild) + # python313.lib is generated in PCbuild\amd64\ + library_dirs.append(pcbuild) + print(f"Add PCbuild directory: {pcbuild}") + + # Display information to help debugging + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + if env_name in os.environ: + print(f"{env_name} env var: {os.environ[env_name]!r}") + else: + print(f"{env_name} env var: ") + print(f"extra_compile_args: {cppflags!r}") + + ext = Extension( + module_name, sources=[SOURCE], language='c++', - extra_compile_args=cppflags) - setup(name='internal' + name, version='0.0', ext_modules=[cpp_ext]) + extra_compile_args=cppflags, + include_dirs=include_dirs, + library_dirs=library_dirs) + setup(name=f'internal_{module_name}', + version='0.0', + ext_modules=[ext]) if __name__ == "__main__": From 3cac2af5ecfa9e2a47bfdd15e114b65780836b9d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 15:04:23 +0100 Subject: [PATCH 33/82] gh-116417: Move limited C API long.c tests to _testlimitedcapi (#117001) * Split long.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. * Move testcapi_long.h from Modules/_testcapi/ to Modules/_testlimitedcapi/. * Add MODULE__TESTLIMITEDCAPI_DEPS to Makefile.pre.in. --- Lib/test/test_capi/test_long.py | 41 +- Makefile.pre.in | 3 +- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/clinic/long.c.h | 141 +--- Modules/_testcapi/long.c | 755 +---------------- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/clinic/long.c.h | 143 ++++ Modules/_testlimitedcapi/long.c | 786 ++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + .../testcapi_long.h | 0 PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 2 + 12 files changed, 962 insertions(+), 916 deletions(-) create mode 100644 Modules/_testlimitedcapi/clinic/long.c.h create mode 100644 Modules/_testlimitedcapi/long.c rename Modules/{_testcapi => _testlimitedcapi}/testcapi_long.h (100%) diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 9f5ee507a8eb85..4ac6ea6b725ff1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -4,8 +4,9 @@ from test.support import import_helper -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi and _testlimitedcapi modules isn't available. _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None @@ -56,7 +57,7 @@ def test_compact_known(self): def test_long_check(self): # Test PyLong_Check() - check = _testcapi.pylong_check + check = _testlimitedcapi.pylong_check self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -68,7 +69,7 @@ def test_long_check(self): def test_long_checkexact(self): # Test PyLong_CheckExact() - check = _testcapi.pylong_checkexact + check = _testlimitedcapi.pylong_checkexact self.assertTrue(check(1)) self.assertTrue(check(123456789012345678901234567890)) self.assertTrue(check(-1)) @@ -80,7 +81,7 @@ def test_long_checkexact(self): def test_long_fromdouble(self): # Test PyLong_FromDouble() - fromdouble = _testcapi.pylong_fromdouble + fromdouble = _testlimitedcapi.pylong_fromdouble float_max = sys.float_info.max for value in (5.0, 5.1, 5.9, -5.1, -5.9, 0.0, -0.0, float_max, -float_max): with self.subTest(value=value): @@ -91,7 +92,7 @@ def test_long_fromdouble(self): def test_long_fromvoidptr(self): # Test PyLong_FromVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) @@ -103,7 +104,7 @@ def test_long_fromvoidptr(self): def test_long_fromstring(self): # Test PyLong_FromString() - fromstring = _testcapi.pylong_fromstring + fromstring = _testlimitedcapi.pylong_fromstring self.assertEqual(fromstring(b'123', 10), (123, 3)) self.assertEqual(fromstring(b'cafe', 16), (0xcafe, 4)) self.assertEqual(fromstring(b'xyz', 36), (44027, 3)) @@ -163,7 +164,7 @@ def test_long_fromunicodeobject(self): def test_long_asint(self): # Test PyLong_AsInt() - PyLong_AsInt = _testcapi.PyLong_AsInt + PyLong_AsInt = _testlimitedcapi.PyLong_AsInt from _testcapi import INT_MIN, INT_MAX # round trip (object -> int -> object) @@ -186,7 +187,7 @@ def test_long_asint(self): def test_long_aslong(self): # Test PyLong_AsLong() and PyLong_FromLong() - aslong = _testcapi.pylong_aslong + aslong = _testlimitedcapi.pylong_aslong from _testcapi import LONG_MIN, LONG_MAX # round trip (object -> long -> object) for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): @@ -206,7 +207,7 @@ def test_long_aslong(self): def test_long_aslongandoverflow(self): # Test PyLong_AsLongAndOverflow() - aslongandoverflow = _testcapi.pylong_aslongandoverflow + aslongandoverflow = _testlimitedcapi.pylong_aslongandoverflow from _testcapi import LONG_MIN, LONG_MAX # round trip (object -> long -> object) for value in (LONG_MIN, LONG_MAX, -1, 0, 1, 1234): @@ -224,7 +225,7 @@ def test_long_aslongandoverflow(self): def test_long_asunsignedlong(self): # Test PyLong_AsUnsignedLong() and PyLong_FromUnsignedLong() - asunsignedlong = _testcapi.pylong_asunsignedlong + asunsignedlong = _testlimitedcapi.pylong_asunsignedlong from _testcapi import ULONG_MAX # round trip (object -> unsigned long -> object) for value in (ULONG_MAX, 0, 1, 1234): @@ -244,7 +245,7 @@ def test_long_asunsignedlong(self): def test_long_asunsignedlongmask(self): # Test PyLong_AsUnsignedLongMask() - asunsignedlongmask = _testcapi.pylong_asunsignedlongmask + asunsignedlongmask = _testlimitedcapi.pylong_asunsignedlongmask from _testcapi import ULONG_MAX # round trip (object -> unsigned long -> object) for value in (ULONG_MAX, 0, 1, 1234): @@ -264,7 +265,7 @@ def test_long_asunsignedlongmask(self): def test_long_aslonglong(self): # Test PyLong_AsLongLong() and PyLong_FromLongLong() - aslonglong = _testcapi.pylong_aslonglong + aslonglong = _testlimitedcapi.pylong_aslonglong from _testcapi import LLONG_MIN, LLONG_MAX # round trip (object -> long long -> object) for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): @@ -284,7 +285,7 @@ def test_long_aslonglong(self): def test_long_aslonglongandoverflow(self): # Test PyLong_AsLongLongAndOverflow() - aslonglongandoverflow = _testcapi.pylong_aslonglongandoverflow + aslonglongandoverflow = _testlimitedcapi.pylong_aslonglongandoverflow from _testcapi import LLONG_MIN, LLONG_MAX # round trip (object -> long long -> object) for value in (LLONG_MIN, LLONG_MAX, -1, 0, 1, 1234): @@ -302,7 +303,7 @@ def test_long_aslonglongandoverflow(self): def test_long_asunsignedlonglong(self): # Test PyLong_AsUnsignedLongLong() and PyLong_FromUnsignedLongLong() - asunsignedlonglong = _testcapi.pylong_asunsignedlonglong + asunsignedlonglong = _testlimitedcapi.pylong_asunsignedlonglong from _testcapi import ULLONG_MAX # round trip (object -> unsigned long long -> object) for value in (ULLONG_MAX, 0, 1, 1234): @@ -322,7 +323,7 @@ def test_long_asunsignedlonglong(self): def test_long_asunsignedlonglongmask(self): # Test PyLong_AsUnsignedLongLongMask() - asunsignedlonglongmask = _testcapi.pylong_asunsignedlonglongmask + asunsignedlonglongmask = _testlimitedcapi.pylong_asunsignedlonglongmask from _testcapi import ULLONG_MAX # round trip (object -> unsigned long long -> object) for value in (ULLONG_MAX, 0, 1, 1234): @@ -342,7 +343,7 @@ def test_long_asunsignedlonglongmask(self): def test_long_as_ssize_t(self): # Test PyLong_AsSsize_t() and PyLong_FromSsize_t() - as_ssize_t = _testcapi.pylong_as_ssize_t + as_ssize_t = _testlimitedcapi.pylong_as_ssize_t from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX # round trip (object -> Py_ssize_t -> object) for value in (PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1, 0, 1, 1234): @@ -362,7 +363,7 @@ def test_long_as_ssize_t(self): def test_long_as_size_t(self): # Test PyLong_AsSize_t() and PyLong_FromSize_t() - as_size_t = _testcapi.pylong_as_size_t + as_size_t = _testlimitedcapi.pylong_as_size_t from _testcapi import SIZE_MAX # round trip (object -> size_t -> object) for value in (SIZE_MAX, 0, 1, 1234): @@ -382,7 +383,7 @@ def test_long_as_size_t(self): def test_long_asdouble(self): # Test PyLong_AsDouble() - asdouble = _testcapi.pylong_asdouble + asdouble = _testlimitedcapi.pylong_asdouble MAX = int(sys.float_info.max) for value in (-MAX, MAX, -1, 0, 1, 1234): with self.subTest(value=value): @@ -402,8 +403,8 @@ def test_long_asdouble(self): def test_long_asvoidptr(self): # Test PyLong_AsVoidPtr() - fromvoidptr = _testcapi.pylong_fromvoidptr - asvoidptr = _testcapi.pylong_asvoidptr + fromvoidptr = _testlimitedcapi.pylong_fromvoidptr + asvoidptr = _testlimitedcapi.pylong_asvoidptr obj = object() x = fromvoidptr(obj) y = fromvoidptr(NULL) diff --git a/Makefile.pre.in b/Makefile.pre.in index 404e7ee3e42054..b9f790a14af4cd 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3051,7 +3051,8 @@ MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_ MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA3.h Modules/_hacl/Hacl_Hash_SHA3.c MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h -MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h +MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 99aef6f39cfd1e..d92b21777d521d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/clinic/long.c.h b/Modules/_testcapi/clinic/long.c.h index e2f7042be12c48..767c671abb8eae 100644 --- a/Modules/_testcapi/clinic/long.c.h +++ b/Modules/_testcapi/clinic/long.c.h @@ -2,137 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(_testcapi_test_long_api__doc__, -"test_long_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_API_METHODDEF \ - {"test_long_api", (PyCFunction)_testcapi_test_long_api, METH_NOARGS, _testcapi_test_long_api__doc__}, - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_longlong_api__doc__, -"test_longlong_api($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONGLONG_API_METHODDEF \ - {"test_longlong_api", (PyCFunction)_testcapi_test_longlong_api, METH_NOARGS, _testcapi_test_longlong_api__doc__}, - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module); - -static PyObject * -_testcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_longlong_api_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_and_overflow__doc__, -"test_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongAndOverflow API.\n" -"\n" -"General conversion to PY_LONG is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_and_overflow", (PyCFunction)_testcapi_test_long_and_overflow, METH_NOARGS, _testcapi_test_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_long_and_overflow__doc__, -"test_long_long_and_overflow($module, /)\n" -"--\n" -"\n" -"Test the PyLong_AsLongLongAndOverflow API.\n" -"\n" -"General conversion to long long is tested by test_long_api_inner.\n" -"This test will concentrate on proper handling of overflow."); - -#define _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ - {"test_long_long_and_overflow", (PyCFunction)_testcapi_test_long_long_and_overflow, METH_NOARGS, _testcapi_test_long_long_and_overflow__doc__}, - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_long_and_overflow_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_size_t__doc__, -"test_long_as_size_t($module, /)\n" -"--\n" -"\n" -"Test the PyLong_As{Size,Ssize}_t API.\n" -"\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); - -#define _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ - {"test_long_as_size_t", (PyCFunction)_testcapi_test_long_as_size_t, METH_NOARGS, _testcapi_test_long_as_size_t__doc__}, - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_size_t_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_unsigned_long_long_mask__doc__, -"test_long_as_unsigned_long_long_mask($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ - {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testcapi_test_long_as_unsigned_long_long_mask__doc__}, - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_unsigned_long_long_mask_impl(module); -} - -PyDoc_STRVAR(_testcapi_test_long_as_double__doc__, -"test_long_as_double($module, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ - {"test_long_as_double", (PyCFunction)_testcapi_test_long_as_double, METH_NOARGS, _testcapi_test_long_as_double__doc__}, - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module); - -static PyObject * -_testcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _testcapi_test_long_as_double_impl(module); -} - PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, "call_long_compact_api($module, arg, /)\n" "--\n" @@ -140,12 +9,4 @@ PyDoc_STRVAR(_testcapi_call_long_compact_api__doc__, #define _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF \ {"call_long_compact_api", (PyCFunction)_testcapi_call_long_compact_api, METH_O, _testcapi_call_long_compact_api__doc__}, - -PyDoc_STRVAR(_testcapi_PyLong_AsInt__doc__, -"PyLong_AsInt($module, arg, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_PYLONG_ASINT_METHODDEF \ - {"PyLong_AsInt", (PyCFunction)_testcapi_PyLong_AsInt, METH_O, _testcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=de762870526e241d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ddcbc6ea06e5e21 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index dc21cf9f475228..8e4e1f2246f725 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -12,529 +12,6 @@ module _testcapi /*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ -static PyObject * -raiseTestError(const char* test_name, const char* msg) -{ - PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); - return NULL; -} - -/* Tests of PyLong_{As, From}{Unsigned,}Long(), and - PyLong_{As, From}{Unsigned,}LongLong(). - - Note that the meat of the test is contained in testcapi_long.h. - This is revolting, but delicate code duplication is worse: "almost - exactly the same" code is needed to test long long, but the ubiquitous - dependence on type names makes it impossible to use a parameterized - function. A giant macro would be even worse than this. A C++ template - would be perfect. - - The "report an error" functions are deliberately not part of the #include - file: if the test fails, you can set a breakpoint in the appropriate - error function directly, and crawl back from there in the debugger. -*/ - -#define UNBIND(X) Py_DECREF(X); (X) = NULL - -static PyObject * -raise_test_long_error(const char* msg) -{ - return raiseTestError("test_long_api", msg); -} - -// Test PyLong_FromLong()/PyLong_AsLong() -// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). - -#define TESTNAME test_long_api_inner -#define TYPENAME long -#define F_S_TO_PY PyLong_FromLong -#define F_PY_TO_S PyLong_AsLong -#define F_U_TO_PY PyLong_FromUnsignedLong -#define F_PY_TO_U PyLong_AsUnsignedLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_long_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_api_impl(PyObject *module) -/*[clinic end generated code: output=4405798ca1e9f444 input=e9b8880d7331c688]*/ -{ - return TESTNAME(raise_test_long_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - -// Test PyLong_FromLongLong()/PyLong_AsLongLong() -// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). - -static PyObject * -raise_test_longlong_error(const char* msg) -{ - return raiseTestError("test_longlong_api", msg); -} - -#define TESTNAME test_longlong_api_inner -#define TYPENAME long long -#define F_S_TO_PY PyLong_FromLongLong -#define F_PY_TO_S PyLong_AsLongLong -#define F_U_TO_PY PyLong_FromUnsignedLongLong -#define F_PY_TO_U PyLong_AsUnsignedLongLong - -#include "testcapi_long.h" - -/*[clinic input] -_testcapi.test_longlong_api -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_longlong_api_impl(PyObject *module) -/*[clinic end generated code: output=2b3414ba8c31dfe6 input=ccbb2a48c2b3c4a5]*/ -{ - return TESTNAME(raise_test_longlong_error); -} - -#undef TESTNAME -#undef TYPENAME -#undef F_S_TO_PY -#undef F_PY_TO_S -#undef F_U_TO_PY -#undef F_PY_TO_U - - -/*[clinic input] -_testcapi.test_long_and_overflow - -Test the PyLong_AsLongAndOverflow API. - -General conversion to PY_LONG is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=f8460ca115e31d8e input=762f6b62da0a3cdc]*/ -{ - PyObject *num, *one, *temp; - long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LONG_MAX + 1 */ - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LONG_MIN even on 64-bit platforms */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LONG_MIN - 1 */ - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLong(LONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MAX) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLong(LONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LONG_MIN) - return raiseTestError("test_long_and_overflow", - "expected return value LONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_long_and_overflow - -Test the PyLong_AsLongLongAndOverflow API. - -General conversion to long long is tested by test_long_api_inner. -This test will concentrate on proper handling of overflow. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_long_and_overflow_impl(PyObject *module) -/*[clinic end generated code: output=0b92330786f45483 input=544bb0aefe5e8a9e]*/ -{ - PyObject *num, *one, *temp; - long long value; - int overflow; - - /* Test that overflow is set properly for a large value. */ - /* num is a number larger than LLONG_MAX on a typical machine. */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Same again, with num = LLONG_MAX + 1 */ - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Add(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != 1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to 1"); - - /* Test that overflow is set properly for a large negative value. */ - /* num is a number smaller than LLONG_MIN on a typical platform */ - num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Same again, with num = LLONG_MIN - 1 */ - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - one = PyLong_FromLong(1L); - if (one == NULL) { - Py_DECREF(num); - return NULL; - } - temp = PyNumber_Subtract(num, one); - Py_DECREF(one); - Py_SETREF(num, temp); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -1) - return raiseTestError("test_long_long_and_overflow", - "return value was not set to -1"); - if (overflow != -1) - return raiseTestError("test_long_long_and_overflow", - "overflow was not set to -1"); - - /* Test that overflow is cleared properly for small values. */ - num = PyLong_FromString("FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != 0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromString("-FF", NULL, 16); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != -0xFF) - return raiseTestError("test_long_long_and_overflow", - "expected return value 0xFF"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was set incorrectly"); - - num = PyLong_FromLongLong(LLONG_MAX); - if (num == NULL) - return NULL; - overflow = 1234; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MAX) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MAX"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - num = PyLong_FromLongLong(LLONG_MIN); - if (num == NULL) - return NULL; - overflow = 0; - value = PyLong_AsLongLongAndOverflow(num, &overflow); - Py_DECREF(num); - if (value == -1 && PyErr_Occurred()) - return NULL; - if (value != LLONG_MIN) - return raiseTestError("test_long_long_and_overflow", - "expected return value LLONG_MIN"); - if (overflow != 0) - return raiseTestError("test_long_long_and_overflow", - "overflow was not cleared"); - - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_size_t - -Test the PyLong_As{Size,Ssize}_t API. - -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=f6490ea2b41e6173 input=922990c4a3edfb0d]*/ -{ - size_t out_u; - Py_ssize_t out_s; - - Py_INCREF(Py_None); - - out_u = PyLong_AsSize_t(Py_None); - if (out_u != (size_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - out_s = PyLong_AsSsize_t(Py_None); - if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_size_t", - "PyLong_AsSsize_t(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - -/*[clinic input] -_testcapi.test_long_as_unsigned_long_long_mask -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) -/*[clinic end generated code: output=e3e16cd0189440cc input=eb2438493ae7b9af]*/ -{ - unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); - - if (res != (unsigned long long)-1 || !PyErr_Occurred()) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) didn't " - "complain"); - } - if (!PyErr_ExceptionMatches(PyExc_SystemError)) { - return raiseTestError("test_long_as_unsigned_long_long_mask", - "PyLong_AsUnsignedLongLongMask(NULL) raised " - "something other than SystemError"); - } - PyErr_Clear(); - Py_RETURN_NONE; -} - -/*[clinic input] -_testcapi.test_long_as_double -[clinic start generated code]*/ - -static PyObject * -_testcapi_test_long_as_double_impl(PyObject *module) -/*[clinic end generated code: output=deca0898e15adde5 input=c77bc88ef5a1de76]*/ -{ - double out; - - Py_INCREF(Py_None); - - out = PyLong_AsDouble(Py_None); - if (out != -1.0 || !PyErr_Occurred()) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) didn't complain"); - if (!PyErr_ExceptionMatches(PyExc_TypeError)) - return raiseTestError("test_long_as_double", - "PyLong_AsDouble(None) raised " - "something other than TypeError"); - PyErr_Clear(); - - /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ - return Py_None; -} - /*[clinic input] _testcapi.call_long_compact_api arg: object @@ -555,48 +32,6 @@ _testcapi_call_long_compact_api(PyObject *module, PyObject *arg) return Py_BuildValue("in", is_compact, value); } -static PyObject * -pylong_check(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_Check(obj)); -} - -static PyObject * -pylong_checkexact(PyObject *module, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyLong_CheckExact(obj)); -} - -static PyObject * -pylong_fromdouble(PyObject *module, PyObject *arg) -{ - double value; - if (!PyArg_Parse(arg, "d", &value)) { - return NULL; - } - return PyLong_FromDouble(value); -} - -static PyObject * -pylong_fromstring(PyObject *module, PyObject *args) -{ - const char *str; - Py_ssize_t len; - int base; - char *end = UNINITIALIZED_PTR; - if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { - return NULL; - } - - PyObject *result = PyLong_FromString(str, &end, base); - if (result == NULL) { - // XXX 'end' is not always set. - return NULL; - } - return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); -} static PyObject * pylong_fromunicodeobject(PyObject *module, PyObject *args) @@ -611,170 +46,6 @@ pylong_fromunicodeobject(PyObject *module, PyObject *args) return PyLong_FromUnicodeObject(unicode, base); } -static PyObject * -pylong_fromvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - return PyLong_FromVoidPtr((void *)arg); -} - -/*[clinic input] -_testcapi.PyLong_AsInt - arg: object - / -[clinic start generated code]*/ - -static PyObject * -_testcapi_PyLong_AsInt(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=0df9f19de5fa575b input=9561b97105493a67]*/ -{ - NULLABLE(arg); - assert(!PyErr_Occurred()); - int value = PyLong_AsInt(arg); - if (value == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long value = PyLong_AsLong(arg); - if (value == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLong(value); -} - -static PyObject * -pylong_aslongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long value = PyLong_AsLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); - return NULL; - } - return Py_BuildValue("li", value, overflow); -} - -static PyObject * -pylong_asunsignedlong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLong(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLong(value); -} - -static PyObject * -pylong_asunsignedlongmask(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long value = PyLong_AsUnsignedLongMask(arg); - if (value == (unsigned long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLong(value); -} - -static PyObject * -pylong_aslonglong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - long long value = PyLong_AsLongLong(arg); - if (value == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromLongLong(value); -} - -static PyObject * -pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - int overflow = UNINITIALIZED_INT; - long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); - if (value == -1 && PyErr_Occurred()) { - assert(overflow == -1); - return NULL; - } - return Py_BuildValue("Li", value, overflow); -} - -static PyObject * -pylong_asunsignedlonglong(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLong(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); -} - -static PyObject * -pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); - if (value == (unsigned long long)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromUnsignedLongLong(value); -} - -static PyObject * -pylong_as_ssize_t(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - Py_ssize_t value = PyLong_AsSsize_t(arg); - if (value == -1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromSsize_t(value); -} - -static PyObject * -pylong_as_size_t(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - size_t value = PyLong_AsSize_t(arg); - if (value == (size_t)-1 && PyErr_Occurred()) { - return NULL; - } - return PyLong_FromSize_t(value); -} - -static PyObject * -pylong_asdouble(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - double value = PyLong_AsDouble(arg); - if (value == -1.0 && PyErr_Occurred()) { - return NULL; - } - return PyFloat_FromDouble(value); -} - -static PyObject * -pylong_asvoidptr(PyObject *module, PyObject *arg) -{ - NULLABLE(arg); - void *value = PyLong_AsVoidPtr(arg); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; - } - return Py_NewRef((PyObject *)value); -} static PyObject * pylong_asnativebytes(PyObject *module, PyObject *args) @@ -800,6 +71,7 @@ pylong_asnativebytes(PyObject *module, PyObject *args) return res >= 0 ? PyLong_FromSsize_t(res) : NULL; } + static PyObject * pylong_fromnativebytes(PyObject *module, PyObject *args) { @@ -822,33 +94,8 @@ pylong_fromnativebytes(PyObject *module, PyObject *args) static PyMethodDef test_methods[] = { - _TESTCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONG_API_METHODDEF - _TESTCAPI_TEST_LONG_AS_DOUBLE_METHODDEF - _TESTCAPI_TEST_LONG_AS_SIZE_T_METHODDEF - _TESTCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF - _TESTCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF - _TESTCAPI_TEST_LONGLONG_API_METHODDEF _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF - {"pylong_check", pylong_check, METH_O}, - {"pylong_checkexact", pylong_checkexact, METH_O}, - {"pylong_fromdouble", pylong_fromdouble, METH_O}, - {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, - {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, - _TESTCAPI_PYLONG_ASINT_METHODDEF - {"pylong_aslong", pylong_aslong, METH_O}, - {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, - {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, - {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, - {"pylong_aslonglong", pylong_aslonglong, METH_O}, - {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, - {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, - {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, - {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, - {"pylong_as_size_t", pylong_as_size_t, METH_O}, - {"pylong_asdouble", pylong_asdouble, METH_O}, - {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {NULL}, diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index c038f7a9e2d1a5..35f153aa6d21cd 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -44,6 +44,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_List(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Long(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h new file mode 100644 index 00000000000000..ebaeb53921a82f --- /dev/null +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -0,0 +1,143 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_testlimitedcapi_test_long_api__doc__, +"test_long_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF \ + {"test_long_api", (PyCFunction)_testlimitedcapi_test_long_api, METH_NOARGS, _testlimitedcapi_test_long_api__doc__}, + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_longlong_api__doc__, +"test_longlong_api($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF \ + {"test_longlong_api", (PyCFunction)_testlimitedcapi_test_longlong_api, METH_NOARGS, _testlimitedcapi_test_longlong_api__doc__}, + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_longlong_api(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_longlong_api_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_and_overflow__doc__, +"test_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongAndOverflow API.\n" +"\n" +"General conversion to PY_LONG is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_long_and_overflow__doc__, +"test_long_long_and_overflow($module, /)\n" +"--\n" +"\n" +"Test the PyLong_AsLongLongAndOverflow API.\n" +"\n" +"General conversion to long long is tested by test_long_api_inner.\n" +"This test will concentrate on proper handling of overflow."); + +#define _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF \ + {"test_long_long_and_overflow", (PyCFunction)_testlimitedcapi_test_long_long_and_overflow, METH_NOARGS, _testlimitedcapi_test_long_long_and_overflow__doc__}, + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_long_and_overflow_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, +"test_long_as_size_t($module, /)\n" +"--\n" +"\n" +"Test the PyLong_As{Size,Ssize}_t API.\n" +"\n" +"At present this just tests that non-integer arguments are handled correctly.\n" +"It should be extended to test overflow handling."); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ + {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_size_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_size_t_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__, +"test_long_as_unsigned_long_long_mask($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF \ + {"test_long_as_unsigned_long_long_mask", (PyCFunction)_testlimitedcapi_test_long_as_unsigned_long_long_mask, METH_NOARGS, _testlimitedcapi_test_long_as_unsigned_long_long_mask__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_test_long_as_double__doc__, +"test_long_as_double($module, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF \ + {"test_long_as_double", (PyCFunction)_testlimitedcapi_test_long_as_double, METH_NOARGS, _testlimitedcapi_test_long_as_double__doc__}, + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module); + +static PyObject * +_testlimitedcapi_test_long_as_double(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _testlimitedcapi_test_long_as_double_impl(module); +} + +PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, +"PyLong_AsInt($module, arg, /)\n" +"--\n" +"\n"); + +#define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ + {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, +/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c new file mode 100644 index 00000000000000..16d41b1d4b16dc --- /dev/null +++ b/Modules/_testlimitedcapi/long.c @@ -0,0 +1,786 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.13 to test PyLong_AsInt() +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" +#include "clinic/long.c.h" + +/*[clinic input] +module _testlimitedcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2700057f9c1135ba]*/ + + +static PyObject * +raiseTestError(const char* test_name, const char* msg) +{ + PyErr_Format(PyExc_AssertionError, "%s: %s", test_name, msg); + return NULL; +} + +/* Tests of PyLong_{As, From}{Unsigned,}Long(), and + PyLong_{As, From}{Unsigned,}LongLong(). + + Note that the meat of the test is contained in testcapi_long.h. + This is revolting, but delicate code duplication is worse: "almost + exactly the same" code is needed to test long long, but the ubiquitous + dependence on type names makes it impossible to use a parameterized + function. A giant macro would be even worse than this. A C++ template + would be perfect. + + The "report an error" functions are deliberately not part of the #include + file: if the test fails, you can set a breakpoint in the appropriate + error function directly, and crawl back from there in the debugger. +*/ + +#define UNBIND(X) Py_DECREF(X); (X) = NULL + +static PyObject * +raise_test_long_error(const char* msg) +{ + return raiseTestError("test_long_api", msg); +} + +// Test PyLong_FromLong()/PyLong_AsLong() +// and PyLong_FromUnsignedLong()/PyLong_AsUnsignedLong(). + +#define TESTNAME test_long_api_inner +#define TYPENAME long +#define F_S_TO_PY PyLong_FromLong +#define F_PY_TO_S PyLong_AsLong +#define F_U_TO_PY PyLong_FromUnsignedLong +#define F_PY_TO_U PyLong_AsUnsignedLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_long_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_api_impl(PyObject *module) +/*[clinic end generated code: output=06a2c02366d1853a input=9012b3d6a483df63]*/ +{ + return TESTNAME(raise_test_long_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + +// Test PyLong_FromLongLong()/PyLong_AsLongLong() +// and PyLong_FromUnsignedLongLong()/PyLong_AsUnsignedLongLong(). + +static PyObject * +raise_test_longlong_error(const char* msg) +{ + return raiseTestError("test_longlong_api", msg); +} + +#define TESTNAME test_longlong_api_inner +#define TYPENAME long long +#define F_S_TO_PY PyLong_FromLongLong +#define F_PY_TO_S PyLong_AsLongLong +#define F_U_TO_PY PyLong_FromUnsignedLongLong +#define F_PY_TO_U PyLong_AsUnsignedLongLong + +#include "testcapi_long.h" + +/*[clinic input] +_testlimitedcapi.test_longlong_api +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_longlong_api_impl(PyObject *module) +/*[clinic end generated code: output=8faa10e1c35214bf input=2b582a9d25bd68e7]*/ +{ + return TESTNAME(raise_test_longlong_error); +} + +#undef TESTNAME +#undef TYPENAME +#undef F_S_TO_PY +#undef F_PY_TO_S +#undef F_U_TO_PY +#undef F_PY_TO_U + + +/*[clinic input] +_testlimitedcapi.test_long_and_overflow + +Test the PyLong_AsLongAndOverflow API. + +General conversion to PY_LONG is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=fdfd3c1eeabb6d14 input=e3a18791de6519fe]*/ +{ + PyObject *num, *one, *temp; + long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LONG_MAX even on 64-bit platforms */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LONG_MAX + 1 */ + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); + num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LONG_MIN even on 64-bit platforms */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LONG_MIN - 1 */ + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLong(LONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MAX) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLong(LONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LONG_MIN) + return raiseTestError("test_long_and_overflow", + "expected return value LONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_long_and_overflow + +Test the PyLong_AsLongLongAndOverflow API. + +General conversion to long long is tested by test_long_api_inner. +This test will concentrate on proper handling of overflow. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_long_and_overflow_impl(PyObject *module) +/*[clinic end generated code: output=3d2721a49c09a307 input=741c593b606cc6b3]*/ +{ + PyObject *num, *one, *temp; + long long value; + int overflow; + + /* Test that overflow is set properly for a large value. */ + /* num is a number larger than LLONG_MAX on a typical machine. */ + num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Same again, with num = LLONG_MAX + 1 */ + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Add(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != 1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to 1"); + + /* Test that overflow is set properly for a large negative value. */ + /* num is a number smaller than LLONG_MIN on a typical platform */ + num = PyLong_FromString("-FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Same again, with num = LLONG_MIN - 1 */ + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + one = PyLong_FromLong(1L); + if (one == NULL) { + Py_DECREF(num); + return NULL; + } + temp = PyNumber_Subtract(num, one); + Py_DECREF(one); + Py_DECREF(num); num = temp; + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -1) + return raiseTestError("test_long_long_and_overflow", + "return value was not set to -1"); + if (overflow != -1) + return raiseTestError("test_long_long_and_overflow", + "overflow was not set to -1"); + + /* Test that overflow is cleared properly for small values. */ + num = PyLong_FromString("FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != 0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromString("-FF", NULL, 16); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != -0xFF) + return raiseTestError("test_long_long_and_overflow", + "expected return value 0xFF"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was set incorrectly"); + + num = PyLong_FromLongLong(LLONG_MAX); + if (num == NULL) + return NULL; + overflow = 1234; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MAX) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MAX"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + num = PyLong_FromLongLong(LLONG_MIN); + if (num == NULL) + return NULL; + overflow = 0; + value = PyLong_AsLongLongAndOverflow(num, &overflow); + Py_DECREF(num); + if (value == -1 && PyErr_Occurred()) + return NULL; + if (value != LLONG_MIN) + return raiseTestError("test_long_long_and_overflow", + "expected return value LLONG_MIN"); + if (overflow != 0) + return raiseTestError("test_long_long_and_overflow", + "overflow was not cleared"); + + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_size_t + +Test the PyLong_As{Size,Ssize}_t API. + +At present this just tests that non-integer arguments are handled correctly. +It should be extended to test overflow handling. +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_size_t_impl(PyObject *module) +/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +{ + size_t out_u; + Py_ssize_t out_s; + + Py_INCREF(Py_None); + + out_u = PyLong_AsSize_t(Py_None); + if (out_u != (size_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + out_s = PyLong_AsSsize_t(Py_None); + if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_unsigned_long_long_mask +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_unsigned_long_long_mask_impl(PyObject *module) +/*[clinic end generated code: output=90be09ffeec8ecab input=17c660bd58becad5]*/ +{ + unsigned long long res = PyLong_AsUnsignedLongLongMask(NULL); + + if (res != (unsigned long long)-1 || !PyErr_Occurred()) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) didn't " + "complain"); + } + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + return raiseTestError("test_long_as_unsigned_long_long_mask", + "PyLong_AsUnsignedLongLongMask(NULL) raised " + "something other than SystemError"); + } + PyErr_Clear(); + Py_RETURN_NONE; +} + +/*[clinic input] +_testlimitedcapi.test_long_as_double +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_test_long_as_double_impl(PyObject *module) +/*[clinic end generated code: output=0e688c2acf224f88 input=e7b5712385064a48]*/ +{ + double out; + + Py_INCREF(Py_None); + + out = PyLong_AsDouble(Py_None); + if (out != -1.0 || !PyErr_Occurred()) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +static PyObject * +pylong_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_Check(obj)); +} + +static PyObject * +pylong_checkexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyLong_CheckExact(obj)); +} + +static PyObject * +pylong_fromdouble(PyObject *module, PyObject *arg) +{ + double value; + if (!PyArg_Parse(arg, "d", &value)) { + return NULL; + } + return PyLong_FromDouble(value); +} + +static PyObject * +pylong_fromstring(PyObject *module, PyObject *args) +{ + const char *str; + Py_ssize_t len; + int base; + char *end = UNINITIALIZED_PTR; + if (!PyArg_ParseTuple(args, "z#i", &str, &len, &base)) { + return NULL; + } + + PyObject *result = PyLong_FromString(str, &end, base); + if (result == NULL) { + // XXX 'end' is not always set. + return NULL; + } + return Py_BuildValue("Nn", result, (Py_ssize_t)(end - str)); +} + +static PyObject * +pylong_fromvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + return PyLong_FromVoidPtr((void *)arg); +} + +/*[clinic input] +_testlimitedcapi.PyLong_AsInt + arg: object + / +[clinic start generated code]*/ + +static PyObject * +_testlimitedcapi_PyLong_AsInt(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=d91db4c1287f85fa input=32c66be86f3265a1]*/ +{ + NULLABLE(arg); + assert(!PyErr_Occurred()); + int value = PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long value = PyLong_AsLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(value); +} + +static PyObject * +pylong_aslongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long value = PyLong_AsLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("li", value, overflow); +} + +static PyObject * +pylong_asunsignedlong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLong(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_asunsignedlongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long value = PyLong_AsUnsignedLongMask(arg); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLong(value); +} + +static PyObject * +pylong_aslonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + long long value = PyLong_AsLongLong(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLongLong(value); +} + +static PyObject * +pylong_aslonglongandoverflow(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + int overflow = UNINITIALIZED_INT; + long long value = PyLong_AsLongLongAndOverflow(arg, &overflow); + if (value == -1 && PyErr_Occurred()) { + assert(overflow == -1); + return NULL; + } + return Py_BuildValue("Li", value, overflow); +} + +static PyObject * +pylong_asunsignedlonglong(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLong(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_asunsignedlonglongmask(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + unsigned long long value = PyLong_AsUnsignedLongLongMask(arg); + if (value == (unsigned long long)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromUnsignedLongLong(value); +} + +static PyObject * +pylong_as_ssize_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + Py_ssize_t value = PyLong_AsSsize_t(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSsize_t(value); +} + +static PyObject * +pylong_as_size_t(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + size_t value = PyLong_AsSize_t(arg); + if (value == (size_t)-1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromSize_t(value); +} + +static PyObject * +pylong_asdouble(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + double value = PyLong_AsDouble(arg); + if (value == -1.0 && PyErr_Occurred()) { + return NULL; + } + return PyFloat_FromDouble(value); +} + +static PyObject * +pylong_asvoidptr(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + void *value = PyLong_AsVoidPtr(arg); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef((PyObject *)value); +} + + +static PyMethodDef test_methods[] = { + _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_API_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_DOUBLE_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_AS_UNSIGNED_LONG_LONG_MASK_METHODDEF + _TESTLIMITEDCAPI_TEST_LONG_LONG_AND_OVERFLOW_METHODDEF + _TESTLIMITEDCAPI_TEST_LONGLONG_API_METHODDEF + {"pylong_check", pylong_check, METH_O}, + {"pylong_checkexact", pylong_checkexact, METH_O}, + {"pylong_fromdouble", pylong_fromdouble, METH_O}, + {"pylong_fromstring", pylong_fromstring, METH_VARARGS}, + {"pylong_fromvoidptr", pylong_fromvoidptr, METH_O}, + _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF + {"pylong_aslong", pylong_aslong, METH_O}, + {"pylong_aslongandoverflow", pylong_aslongandoverflow, METH_O}, + {"pylong_asunsignedlong", pylong_asunsignedlong, METH_O}, + {"pylong_asunsignedlongmask", pylong_asunsignedlongmask, METH_O}, + {"pylong_aslonglong", pylong_aslonglong, METH_O}, + {"pylong_aslonglongandoverflow", pylong_aslonglongandoverflow, METH_O}, + {"pylong_asunsignedlonglong", pylong_asunsignedlonglong, METH_O}, + {"pylong_asunsignedlonglongmask", pylong_asunsignedlonglongmask, METH_O}, + {"pylong_as_ssize_t", pylong_as_ssize_t, METH_O}, + {"pylong_as_size_t", pylong_as_size_t, METH_O}, + {"pylong_asdouble", pylong_asdouble, METH_O}, + {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Long(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index f8b84e9388184e..8b3b0669d5aede 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -28,6 +28,7 @@ int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); int _PyTestLimitedCAPI_Init_List(PyObject *module); +int _PyTestLimitedCAPI_Init_Long(PyObject *module); int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); int _PyTestLimitedCAPI_Init_Set(PyObject *module); int _PyTestLimitedCAPI_Init_Sys(PyObject *module); diff --git a/Modules/_testcapi/testcapi_long.h b/Modules/_testlimitedcapi/testcapi_long.h similarity index 100% rename from Modules/_testcapi/testcapi_long.h rename to Modules/_testlimitedcapi/testcapi_long.h diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 3715d82f1e35ab..ca1fbd079ecd08 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -100,6 +100,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 29177dfcc44067..898084d9a334ac 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -15,9 +15,11 @@ + + From 75935746be0cbd32b9d710b93db9bd49c8d634ba Mon Sep 17 00:00:00 2001 From: et-repositories <142979605+et-repositories@users.noreply.github.com> Date: Tue, 19 Mar 2024 22:58:40 +0800 Subject: [PATCH 34/82] gh-116647: Fix recursive child in dataclasses (#116790) --- Lib/dataclasses.py | 4 +++- Lib/test/test_dataclasses/__init__.py | 9 +++++++++ .../2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index e511eff4125038..7db8a4233df883 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1075,7 +1075,9 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, cmp_fields = (field for field in field_list if field.compare) terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] field_comparisons = ' and '.join(terms) or 'True' - body = [f'if other.__class__ is self.__class__:', + body = [f'if self is other:', + f' return True', + f'if other.__class__ is self.__class__:', f' return {field_comparisons}', f'return NotImplemented'] func = _create_fn('__eq__', diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index e27abac5111394..832e5672c77d0d 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -2471,6 +2471,15 @@ def __repr__(self): class TestEq(unittest.TestCase): + def test_recursive_eq(self): + # Test a class with recursive child + @dataclass + class C: + recursive: object = ... + c = C() + c.recursive = c + self.assertEqual(c, c) + def test_no_eq(self): # Test a class with no __eq__ and eq=False. @dataclass(eq=False) diff --git a/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst b/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst new file mode 100644 index 00000000000000..081f36bff91633 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-14-09-38-51.gh-issue-116647.h0d_zj.rst @@ -0,0 +1 @@ +Fix recursive child in dataclasses From f55e1880c1a13ab907070f05fedfbf674b6369e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 16:06:20 +0100 Subject: [PATCH 35/82] gh-116417: Move limited C API dict.c tests to _testlimitedcapi (#117006) Split dict.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. --- Lib/test/test_capi/test_dict.py | 51 ++-- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/dict.c | 279 +--------------------- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/dict.c | 291 +++++++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 8 files changed, 325 insertions(+), 304 deletions(-) create mode 100644 Modules/_testlimitedcapi/dict.c diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index cca6145bc90c04..bcc978d224a583 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -3,6 +3,7 @@ from types import MappingProxyType from test import support import _testcapi +import _testlimitedcapi NULL = None @@ -25,7 +26,7 @@ def gen(): class CAPITest(unittest.TestCase): def test_dict_check(self): - check = _testcapi.dict_check + check = _testlimitedcapi.dict_check self.assertTrue(check({1: 2})) self.assertTrue(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -34,7 +35,7 @@ def test_dict_check(self): # CRASHES check(NULL) def test_dict_checkexact(self): - check = _testcapi.dict_checkexact + check = _testlimitedcapi.dict_checkexact self.assertTrue(check({1: 2})) self.assertFalse(check(OrderedDict({1: 2}))) self.assertFalse(check(UserDict({1: 2}))) @@ -43,7 +44,7 @@ def test_dict_checkexact(self): # CRASHES check(NULL) def test_dict_new(self): - dict_new = _testcapi.dict_new + dict_new = _testlimitedcapi.dict_new dct = dict_new() self.assertEqual(dct, {}) self.assertIs(type(dct), dict) @@ -51,7 +52,7 @@ def test_dict_new(self): self.assertIsNot(dct2, dct) def test_dictproxy_new(self): - dictproxy_new = _testcapi.dictproxy_new + dictproxy_new = _testlimitedcapi.dictproxy_new for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): proxy = dictproxy_new(dct) self.assertIs(type(proxy), MappingProxyType) @@ -67,7 +68,7 @@ def test_dictproxy_new(self): # CRASHES dictproxy_new(NULL) def test_dict_copy(self): - copy = _testcapi.dict_copy + copy = _testlimitedcapi.dict_copy for dct in {1: 2}, OrderedDict({1: 2}): dct_copy = copy(dct) self.assertIs(type(dct_copy), dict) @@ -79,7 +80,7 @@ def test_dict_copy(self): self.assertRaises(SystemError, copy, NULL) def test_dict_clear(self): - clear = _testcapi.dict_clear + clear = _testlimitedcapi.dict_clear dct = {1: 2} clear(dct) self.assertEqual(dct, {}) @@ -98,7 +99,7 @@ def test_dict_clear(self): # CRASHES? clear(NULL) def test_dict_size(self): - size = _testcapi.dict_size + size = _testlimitedcapi.dict_size self.assertEqual(size({1: 2}), 1) self.assertEqual(size(OrderedDict({1: 2})), 1) @@ -109,7 +110,7 @@ def test_dict_size(self): self.assertRaises(SystemError, size, NULL) def test_dict_getitem(self): - getitem = _testcapi.dict_getitem + getitem = _testlimitedcapi.dict_getitem dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -131,7 +132,7 @@ def test_dict_getitem(self): # CRASHES getitem(NULL, 'a') def test_dict_getitemstring(self): - getitemstring = _testcapi.dict_getitemstring + getitemstring = _testlimitedcapi.dict_getitemstring dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitemstring(dct, b'a'), 1) self.assertIs(getitemstring(dct, b'b'), KeyError) @@ -188,7 +189,7 @@ def test_dict_getitemstringref(self): # CRASHES getitemstring(NULL, b'a') def test_dict_getitemwitherror(self): - getitem = _testcapi.dict_getitemwitherror + getitem = _testlimitedcapi.dict_getitemwitherror dct = {'a': 1, '\U0001f40d': 2} self.assertEqual(getitem(dct, 'a'), 1) self.assertIs(getitem(dct, 'b'), KeyError) @@ -206,7 +207,7 @@ def test_dict_getitemwitherror(self): # CRASHES getitem(NULL, 'a') def test_dict_contains(self): - contains = _testcapi.dict_contains + contains = _testlimitedcapi.dict_contains dct = {'a': 1, '\U0001f40d': 2} self.assertTrue(contains(dct, 'a')) self.assertFalse(contains(dct, 'b')) @@ -238,7 +239,7 @@ def test_dict_contains_string(self): # CRASHES contains(NULL, b'a') def test_dict_setitem(self): - setitem = _testcapi.dict_setitem + setitem = _testlimitedcapi.dict_setitem dct = {} setitem(dct, 'a', 5) self.assertEqual(dct, {'a': 5}) @@ -258,7 +259,7 @@ def test_dict_setitem(self): # CRASHES setitem(NULL, 'a', 5) def test_dict_setitemstring(self): - setitemstring = _testcapi.dict_setitemstring + setitemstring = _testlimitedcapi.dict_setitemstring dct = {} setitemstring(dct, b'a', 5) self.assertEqual(dct, {'a': 5}) @@ -277,7 +278,7 @@ def test_dict_setitemstring(self): # CRASHES setitemstring(NULL, b'a', 5) def test_dict_delitem(self): - delitem = _testcapi.dict_delitem + delitem = _testlimitedcapi.dict_delitem dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitem(dct, 'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -298,7 +299,7 @@ def test_dict_delitem(self): # CRASHES delitem(NULL, 'a') def test_dict_delitemstring(self): - delitemstring = _testcapi.dict_delitemstring + delitemstring = _testlimitedcapi.dict_delitemstring dct = {'a': 1, 'c': 2, '\U0001f40d': 3} delitemstring(dct, b'a') self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) @@ -371,21 +372,21 @@ def items(self): return None dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: - self.assertListEqual(_testcapi.dict_keys(mapping), + self.assertListEqual(_testlimitedcapi.dict_keys(mapping), list(dict_obj.keys())) - self.assertListEqual(_testcapi.dict_values(mapping), + self.assertListEqual(_testlimitedcapi.dict_values(mapping), list(dict_obj.values())) - self.assertListEqual(_testcapi.dict_items(mapping), + self.assertListEqual(_testlimitedcapi.dict_items(mapping), list(dict_obj.items())) def test_dict_keys_valuesitems_bad_arg(self): for mapping in UserDict(), [], object(): - self.assertRaises(SystemError, _testcapi.dict_keys, mapping) - self.assertRaises(SystemError, _testcapi.dict_values, mapping) - self.assertRaises(SystemError, _testcapi.dict_items, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_values, mapping) + self.assertRaises(SystemError, _testlimitedcapi.dict_items, mapping) def test_dict_next(self): - dict_next = _testcapi.dict_next + dict_next = _testlimitedcapi.dict_next self.assertIsNone(dict_next({}, 0)) dct = {'a': 1, 'b': 2, 'c': 3} pos = 0 @@ -402,7 +403,7 @@ def test_dict_next(self): # CRASHES dict_next(NULL, 0) def test_dict_update(self): - update = _testcapi.dict_update + update = _testlimitedcapi.dict_update for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -417,7 +418,7 @@ def test_dict_update(self): self.assertRaises(SystemError, update, NULL, {}) def test_dict_merge(self): - merge = _testcapi.dict_merge + merge = _testlimitedcapi.dict_merge for cls1 in dict, DictSubclass: for cls2 in dict, DictSubclass, UserDict: dct = cls1({'a': 1, 'b': 2}) @@ -435,7 +436,7 @@ def test_dict_merge(self): self.assertRaises(SystemError, merge, NULL, {}, 0) def test_dict_mergefromseq2(self): - mergefromseq2 = _testcapi.dict_mergefromseq2 + mergefromseq2 = _testlimitedcapi.dict_mergefromseq2 for cls1 in dict, DictSubclass: for cls2 in list, iter: dct = cls1({'a': 1, 'b': 2}) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d92b21777d521d..2f98b3e500ab18 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index fe03c24f75e196..4319906dc4fee0 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -2,59 +2,6 @@ #include "util.h" -static PyObject * -dict_check(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_Check(obj)); -} - -static PyObject * -dict_checkexact(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyDict_CheckExact(obj)); -} - -static PyObject * -dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyDict_New(); -} - -static PyObject * -dictproxy_new(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDictProxy_New(obj); -} - -static PyObject * -dict_clear(PyObject *self, PyObject *obj) -{ - PyDict_Clear(obj); - Py_RETURN_NONE; -} - -static PyObject * -dict_copy(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Copy(obj); -} - -static PyObject * -dict_contains(PyObject *self, PyObject *args) -{ - PyObject *obj, *key; - if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { - return NULL; - } - NULLABLE(obj); - NULLABLE(key); - RETURN_INT(PyDict_Contains(obj, key)); -} - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -68,72 +15,6 @@ dict_containsstring(PyObject *self, PyObject *args) RETURN_INT(PyDict_ContainsString(obj, key)); } -static PyObject * -dict_size(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - RETURN_SIZE(PyDict_Size(obj)); -} - -static PyObject * -dict_getitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItem(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - PyObject *value = PyDict_GetItemString(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - -static PyObject * -dict_getitemwitherror(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - PyObject *value = PyDict_GetItemWithError(mapping, key); - if (value == NULL) { - if (PyErr_Occurred()) { - return NULL; - } - return Py_NewRef(PyExc_KeyError); - } - return Py_NewRef(value); -} - - static PyObject * dict_getitemref(PyObject *self, PyObject *args) { @@ -185,33 +66,6 @@ dict_getitemstringref(PyObject *self, PyObject *args) } } -static PyObject * -dict_setitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key, *value; - if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - NULLABLE(value); - RETURN_INT(PyDict_SetItem(mapping, key, value)); -} - -static PyObject * -dict_setitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping, *value; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(value); - RETURN_INT(PyDict_SetItemString(mapping, key, value)); -} - static PyObject * dict_setdefault(PyObject *self, PyObject *args) { @@ -250,112 +104,6 @@ dict_setdefaultref(PyObject *self, PyObject *args) } } -static PyObject * -dict_delitem(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key; - if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(key); - RETURN_INT(PyDict_DelItem(mapping, key)); -} - -static PyObject * -dict_delitemstring(PyObject *self, PyObject *args) -{ - PyObject *mapping; - const char *key; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { - return NULL; - } - NULLABLE(mapping); - RETURN_INT(PyDict_DelItemString(mapping, key)); -} - -static PyObject * -dict_keys(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Keys(obj); -} - -static PyObject * -dict_values(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Values(obj); -} - -static PyObject * -dict_items(PyObject *self, PyObject *obj) -{ - NULLABLE(obj); - return PyDict_Items(obj); -} - -static PyObject * -dict_next(PyObject *self, PyObject *args) -{ - PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; - Py_ssize_t pos; - if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { - return NULL; - } - NULLABLE(mapping); - int rc = PyDict_Next(mapping, &pos, &key, &value); - if (rc != 0) { - return Py_BuildValue("inOO", rc, pos, key, value); - } - assert(key == UNINITIALIZED_PTR); - assert(value == UNINITIALIZED_PTR); - if (PyErr_Occurred()) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -dict_merge(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Merge(mapping, mapping2, override)); -} - -static PyObject * -dict_update(PyObject *self, PyObject *args) -{ - PyObject *mapping, *mapping2; - if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(mapping2); - RETURN_INT(PyDict_Update(mapping, mapping2)); -} - -static PyObject * -dict_mergefromseq2(PyObject *self, PyObject *args) -{ - PyObject *mapping, *seq; - int override; - if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { - return NULL; - } - NULLABLE(mapping); - NULLABLE(seq); - RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); -} - - static PyObject * dict_pop(PyObject *self, PyObject *args) { @@ -382,7 +130,6 @@ dict_pop(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_pop_null(PyObject *self, PyObject *args) { @@ -396,7 +143,6 @@ dict_pop_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_Pop(dict, key, NULL)); } - static PyObject * dict_popstring(PyObject *self, PyObject *args) { @@ -423,7 +169,6 @@ dict_popstring(PyObject *self, PyObject *args) return Py_BuildValue("iN", res, result); } - static PyObject * dict_popstring_null(PyObject *self, PyObject *args) { @@ -439,33 +184,11 @@ dict_popstring_null(PyObject *self, PyObject *args) static PyMethodDef test_methods[] = { - {"dict_check", dict_check, METH_O}, - {"dict_checkexact", dict_checkexact, METH_O}, - {"dict_new", dict_new, METH_NOARGS}, - {"dictproxy_new", dictproxy_new, METH_O}, - {"dict_clear", dict_clear, METH_O}, - {"dict_copy", dict_copy, METH_O}, - {"dict_size", dict_size, METH_O}, - {"dict_getitem", dict_getitem, METH_VARARGS}, - {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, - {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_containsstring", dict_containsstring, METH_VARARGS}, {"dict_getitemref", dict_getitemref, METH_VARARGS}, {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, - {"dict_contains", dict_contains, METH_VARARGS}, - {"dict_containsstring", dict_containsstring, METH_VARARGS}, - {"dict_setitem", dict_setitem, METH_VARARGS}, - {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, - {"dict_delitem", dict_delitem, METH_VARARGS}, - {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, {"dict_setdefault", dict_setdefault, METH_VARARGS}, {"dict_setdefaultref", dict_setdefaultref, METH_VARARGS}, - {"dict_keys", dict_keys, METH_O}, - {"dict_values", dict_values, METH_O}, - {"dict_items", dict_items, METH_O}, - {"dict_next", dict_next, METH_VARARGS}, - {"dict_merge", dict_merge, METH_VARARGS}, - {"dict_update", dict_update, METH_VARARGS}, - {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, {"dict_pop", dict_pop, METH_VARARGS}, {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 35f153aa6d21cd..3820d5d1a9fde6 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -35,6 +35,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Float(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/dict.c b/Modules/_testlimitedcapi/dict.c new file mode 100644 index 00000000000000..ec32712eef6434 --- /dev/null +++ b/Modules/_testlimitedcapi/dict.c @@ -0,0 +1,291 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key = UNINITIALIZED_PTR, *value = UNINITIALIZED_PTR; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + assert(key == UNINITIALIZED_PTR); + assert(value == UNINITIALIZED_PTR); + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 8b3b0669d5aede..4de9f8903f186a 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Dict(PyObject *module); int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); int _PyTestLimitedCAPI_Init_List(PyObject *module); diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index ca1fbd079ecd08..e793b93a6c1318 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 898084d9a334ac..7c1226bc2796cb 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ + From 2c82592ab463f1f38237919a12145f34eaadda23 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 19 Mar 2024 08:47:28 -0700 Subject: [PATCH 36/82] GH-116017: Put JIT code and data on the same page (GH-116845) --- Python/jit.c | 52 +++++++++++++++++++----------------------- Tools/jit/_stencils.py | 7 +++--- Tools/jit/_targets.py | 2 +- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index dae25166b1f106..f67d641fe129e1 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -112,26 +112,6 @@ mark_executable(unsigned char *memory, size_t size) return 0; } -static int -mark_readable(unsigned char *memory, size_t size) -{ - if (size == 0) { - return 0; - } - assert(size % get_page_size() == 0); -#ifdef MS_WINDOWS - DWORD old; - int failed = !VirtualProtect(memory, size, PAGE_READONLY, &old); -#else - int failed = mprotect(memory, size, PROT_READ); -#endif - if (failed) { - jit_error("unable to protect readable memory"); - return -1; - } - return 0; -} - // JIT compiler stuff: ///////////////////////////////////////////////////////// // Warning! AArch64 requires you to get your hands dirty. These are your gloves: @@ -409,12 +389,14 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size code_size += group->code.body_size; data_size += group->data.body_size; } - // Round up to the nearest page (code and data need separate pages): + code_size += stencil_groups[_FATAL_ERROR].code.body_size; + data_size += stencil_groups[_FATAL_ERROR].data.body_size; + // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); - code_size += page_size - (code_size & (page_size - 1)); - data_size += page_size - (data_size & (page_size - 1)); - unsigned char *memory = jit_alloc(code_size + data_size); + size_t padding = page_size - ((code_size + data_size) & (page_size - 1)); + size_t total_size = code_size + data_size + padding; + unsigned char *memory = jit_alloc(total_size); if (memory == NULL) { return -1; } @@ -444,14 +426,26 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size code += group->code.body_size; data += group->data.body_size; } - if (mark_executable(memory, code_size) || - mark_readable(memory + code_size, data_size)) - { - jit_free(memory, code_size + data_size); + // Protect against accidental buffer overrun into data: + const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; + uint64_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uint64_t)code; + patches[HoleValue_CONTINUE] = (uint64_t)code; + patches[HoleValue_DATA] = (uint64_t)data; + patches[HoleValue_EXECUTOR] = (uint64_t)executor; + patches[HoleValue_TOP] = (uint64_t)code; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + assert(code == memory + code_size); + assert(data == memory + code_size + data_size); + if (mark_executable(memory, total_size)) { + jit_free(memory, total_size); return -1; } executor->jit_code = memory; - executor->jit_size = code_size + data_size; + executor->jit_size = total_size; return 0; } diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 78c566d9c8a7ef..05c4ce8249f687 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -124,7 +124,7 @@ def emit_aarch64_trampoline(self, hole: Hole) -> None: ): self.holes.append(hole.replace(offset=base + 4 * i, kind=kind)) - def remove_jump(self) -> None: + def remove_jump(self, *, alignment: int = 1) -> None: """Remove a zero-length continuation jump, if it exists.""" hole = max(self.holes, key=lambda hole: hole.offset) match hole: @@ -170,7 +170,7 @@ def remove_jump(self) -> None: offset -= 2 case _: return - if self.body[offset:] == jump: + if self.body[offset:] == jump and offset % alignment == 0: self.body = self.body[:offset] self.holes.remove(hole) @@ -199,9 +199,8 @@ def process_relocations(self, *, alignment: int = 1) -> None: ): self.code.pad(alignment) self.code.emit_aarch64_trampoline(hole) - self.code.pad(alignment) self.code.holes.remove(hole) - self.code.remove_jump() + self.code.remove_jump(alignment=alignment) self.code.pad(alignment) self.data.pad(8) for stencil in [self.code, self.data]: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 417fdb56ccf7a1..66db358679239e 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -89,7 +89,7 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) - group.process_relocations() + group.process_relocations(alignment=self.alignment) return group def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: From 7f64ae30ddc22577ce4101ce0b6601b3548b036f Mon Sep 17 00:00:00 2001 From: Sunghyun Kim Date: Tue, 19 Mar 2024 11:51:12 -0400 Subject: [PATCH 37/82] gh-107607: Update comment about utf-8 BOM being ignored (#107858) --------- Co-authored-by: Terry Jan Reedy --- Doc/reference/lexical_analysis.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index f0b3d0a7458cbe..41ea89fd234122 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -96,10 +96,9 @@ which is recognized also by GNU Emacs, and :: which is recognized by Bram Moolenaar's VIM. -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). +If no encoding declaration is found, the default encoding is UTF-8. If the +implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order +mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The From 61c659e2dcc073dda6d9a88f735af21622b1c844 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 17:23:12 +0100 Subject: [PATCH 38/82] gh-116417: Move limited C API complex.c tests to _testlimitedcapi (#117014) Split complex.c tests of _testcapi into two parts: limited C API tests in _testlimitedcapi and non-limited C API tests in _testcapi. --- Lib/test/test_capi/test_complex.py | 11 ++-- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/complex.c | 61 ------------------ Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/complex.c | 79 ++++++++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 8 files changed, 92 insertions(+), 67 deletions(-) create mode 100644 Modules/_testlimitedcapi/complex.c diff --git a/Lib/test/test_capi/test_complex.py b/Lib/test/test_capi/test_complex.py index a5b59558e7f851..328ea12f97462c 100644 --- a/Lib/test/test_capi/test_complex.py +++ b/Lib/test/test_capi/test_complex.py @@ -10,6 +10,7 @@ _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') NULL = None INF = float("inf") @@ -25,7 +26,7 @@ def __complex__(self): class CAPIComplexTest(unittest.TestCase): def test_check(self): # Test PyComplex_Check() - check = _testcapi.complex_check + check = _testlimitedcapi.complex_check self.assertTrue(check(1+2j)) self.assertTrue(check(ComplexSubclass(1+2j))) @@ -38,7 +39,7 @@ def test_check(self): def test_checkexact(self): # PyComplex_CheckExact() - checkexact = _testcapi.complex_checkexact + checkexact = _testlimitedcapi.complex_checkexact self.assertTrue(checkexact(1+2j)) self.assertFalse(checkexact(ComplexSubclass(1+2j))) @@ -57,13 +58,13 @@ def test_fromccomplex(self): def test_fromdoubles(self): # Test PyComplex_FromDoubles() - fromdoubles = _testcapi.complex_fromdoubles + fromdoubles = _testlimitedcapi.complex_fromdoubles self.assertEqual(fromdoubles(1.0, 2.0), 1.0+2.0j) def test_realasdouble(self): # Test PyComplex_RealAsDouble() - realasdouble = _testcapi.complex_realasdouble + realasdouble = _testlimitedcapi.complex_realasdouble self.assertEqual(realasdouble(1+2j), 1.0) self.assertEqual(realasdouble(-1+0j), -1.0) @@ -98,7 +99,7 @@ def test_realasdouble(self): def test_imagasdouble(self): # Test PyComplex_ImagAsDouble() - imagasdouble = _testcapi.complex_imagasdouble + imagasdouble = _testlimitedcapi.complex_imagasdouble self.assertEqual(imagasdouble(1+2j), 2.0) self.assertEqual(imagasdouble(1-1j), -1.0) diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 2f98b3e500ab18..1b7ad0e5d95a13 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/complex.c b/Modules/_testcapi/complex.c index 4a70217eb90d62..eceb1310bfe874 100644 --- a/Modules/_testcapi/complex.c +++ b/Modules/_testcapi/complex.c @@ -2,20 +2,6 @@ #include "util.h" -static PyObject * -complex_check(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_Check(obj)); -} - -static PyObject * -complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) -{ - NULLABLE(obj); - return PyLong_FromLong(PyComplex_CheckExact(obj)); -} - static PyObject * complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -28,48 +14,6 @@ complex_fromccomplex(PyObject *Py_UNUSED(module), PyObject *obj) return PyComplex_FromCComplex(complex); } -static PyObject * -complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) -{ - double real, imag; - - if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { - return NULL; - } - - return PyComplex_FromDoubles(real, imag); -} - -static PyObject * -complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double real; - - NULLABLE(obj); - real = PyComplex_RealAsDouble(obj); - - if (real == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(real); -} - -static PyObject * -complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) -{ - double imag; - - NULLABLE(obj); - imag = PyComplex_ImagAsDouble(obj); - - if (imag == -1. && PyErr_Occurred()) { - return NULL; - } - - return PyFloat_FromDouble(imag); -} - static PyObject * complex_asccomplex(PyObject *Py_UNUSED(module), PyObject *obj) { @@ -139,12 +83,7 @@ _py_c_abs(PyObject *Py_UNUSED(module), PyObject* obj) static PyMethodDef test_methods[] = { - {"complex_check", complex_check, METH_O}, - {"complex_checkexact", complex_checkexact, METH_O}, {"complex_fromccomplex", complex_fromccomplex, METH_O}, - {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, - {"complex_realasdouble", complex_realasdouble, METH_O}, - {"complex_imagasdouble", complex_imagasdouble, METH_O}, {"complex_asccomplex", complex_asccomplex, METH_O}, {"_py_c_sum", _py_c_sum, METH_VARARGS}, {"_py_c_diff", _py_c_diff, METH_VARARGS}, diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 3820d5d1a9fde6..bfd41070eedd55 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -35,6 +35,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Complex(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Dict(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/complex.c b/Modules/_testlimitedcapi/complex.c new file mode 100644 index 00000000000000..e4c244e5c88d06 --- /dev/null +++ b/Modules/_testlimitedcapi/complex.c @@ -0,0 +1,79 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +complex_check(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_Check(obj)); +} + +static PyObject * +complex_checkexact(PyObject *Py_UNUSED(module), PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyComplex_CheckExact(obj)); +} + +static PyObject * +complex_fromdoubles(PyObject *Py_UNUSED(module), PyObject *args) +{ + double real, imag; + + if (!PyArg_ParseTuple(args, "dd", &real, &imag)) { + return NULL; + } + + return PyComplex_FromDoubles(real, imag); +} + +static PyObject * +complex_realasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double real; + + NULLABLE(obj); + real = PyComplex_RealAsDouble(obj); + + if (real == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(real); +} + +static PyObject * +complex_imagasdouble(PyObject *Py_UNUSED(module), PyObject *obj) +{ + double imag; + + NULLABLE(obj); + imag = PyComplex_ImagAsDouble(obj); + + if (imag == -1. && PyErr_Occurred()) { + return NULL; + } + + return PyFloat_FromDouble(imag); +} + + +static PyMethodDef test_methods[] = { + {"complex_check", complex_check, METH_O}, + {"complex_checkexact", complex_checkexact, METH_O}, + {"complex_fromdoubles", complex_fromdoubles, METH_VARARGS}, + {"complex_realasdouble", complex_realasdouble, METH_O}, + {"complex_imagasdouble", complex_imagasdouble, METH_O}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Complex(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 4de9f8903f186a..4b65912489661b 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index e793b93a6c1318..58a8bcbdbce4e8 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 7c1226bc2796cb..e203edaf123e8d 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ + From b85572c47dc7a8c65fc366a87a3660fc7a3ed244 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Mar 2024 19:26:32 +0200 Subject: [PATCH 39/82] gh-90300: Improve the Python CLI help output (GH-115853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * document equivalent command-line options for all environment variables * document equivalent environment variables for all command-line options * reduce the size of variable and option descriptions to minimum * remove the ending period in single-sentence descriptions Co-authored-by: Éric Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Python/initconfig.c | 118 +++++++++++++++----------------------------- 1 file changed, 40 insertions(+), 78 deletions(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index bbd611f7f7a48c..95234eed5edff1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -161,7 +161,7 @@ Options (and corresponding environment variables):\n\ -P : don't prepend a potentially unsafe path to sys.path; also\n\ PYTHONSAFEPATH\n\ -q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ +-s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x\n\ -S : don't imply 'import site' on initialization\n\ -u : force the stdout and stderr streams to be unbuffered;\n\ this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ @@ -187,67 +187,41 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ --X cpu_count=[n|default]: Override the return value of os.cpu_count(),\n\ - os.process_cpu_count(), and multiprocessing.cpu_count(). This can\n\ - help users who need to limit resources in a container.\n\ --X dev : enable CPython's \"development mode\", introducing additional runtime\n\ - checks which are too expensive to be enabled by default. Effect of\n\ - the developer mode:\n\ - * Add default warning filter, as -W default\n\ - * Install debug hooks on memory allocators: see the\n\ - PyMem_SetupDebugHooks() C function\n\ - * Enable the faulthandler module to dump the Python traceback on\n\ - a crash\n\ - * Enable asyncio debug mode\n\ - * Set the dev_mode attribute of sys.flags to True\n\ - * io.IOBase destructor logs close() exceptions\n\ --X faulthandler: enable faulthandler\n\ --X frozen_modules=[on|off]: whether or not frozen modules should be used.\n\ - The default is \"on\" (or \"off\" if you are running a local build).\n\ --X importtime: show how long each import takes. It shows module name,\n\ - cumulative time (including nested imports) and self time (excluding\n\ - nested imports). Note that its output may be broken in\n\ - multi-threaded application.\n\ - Typical usage is python3 -X importtime -c 'import asyncio'\n\ --X int_max_str_digits=number: limit the size of int<->str conversions.\n\ - This helps avoid denial of service attacks when parsing untrusted\n\ - data. The default is sys.int_info.default_max_str_digits.\n\ - 0 disables.\n\ --X no_debug_ranges: disable the inclusion of the tables mapping extra location\n\ - information (end line, start column offset and end column offset) to\n\ - every instruction in code objects. This is useful when smaller code\n\ - objects and pyc files are desired as well as suppressing the extra\n\ - visual location indicators when the interpreter displays tracebacks.\n\ --X perf: activate support for the Linux \"perf\" profiler by activating the\n\ - \"perf\" trampoline. When this option is activated, the Linux \"perf\"\n\ - profiler will be able to report Python calls. This option is only\n\ - available on some platforms and will do nothing if is not supported\n\ - on the current system. The default value is \"off\".\n\ +-X cpu_count=N: override the return value of os.cpu_count();\n\ + -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ +-X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ +-X faulthandler: dump the Python traceback on fatal errors;\n\ + also PYTHONFAULTHANDLER\n\ +-X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ + for installed Python and \"off\" for a local build;\n\ + also PYTHON_FROZEN_MODULES\n\ +-X importtime: show how long each import takes; also PYTHONPROFILEIMPORTTIME\n\ +-X int_max_str_digits=N: limit the size of int<->str conversions;\n\ + 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ +-X no_debug_ranges: don't include extra location information in code objects;\n\ + also PYTHONNODEBUGRANGES\n\ +-X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ " #ifdef Py_DEBUG -"-X presite=package.module: import this module before site.py is run.\n" +"-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" #endif "\ --X pycache_prefix=PATH: enable writing .pyc files to a parallel tree rooted\n\ - at the given directory instead of to the code tree\n\ +-X pycache_prefix=PATH: write .pyc files to a parallel tree instead of to the\n\ + code tree; also PYTHONPYCACHEPREFIX\n\ " #ifdef Py_STATS -"-X pystats: Enable pystats collection at startup.\n" +"-X pystats: enable pystats collection at startup; also PYTHONSTATS\n" #endif "\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter. This only works on debug builds\n\ --X tracemalloc: start tracing Python memory allocations using the\n\ - tracemalloc module. By default, only the most recent frame is stored\n\ - in a traceback of a trace. Use -X tracemalloc=NFRAME to start\n\ - tracing with a traceback limit of NFRAME frames\n\ --X utf8: enable UTF-8 mode for operating system interfaces, overriding the\n\ - default locale-aware mode. -X utf8=0 explicitly disables UTF-8 mode\n\ - (even when it would otherwise activate automatically)\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None'\ -" -; + the interactive interpreter; only works on debug builds\n\ +-X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ + of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ +-X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ +-X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ + also PYTHONWARNDEFAULTENCODING\ +"; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = @@ -257,9 +231,9 @@ static const char usage_envvars[] = " default module search path. The result is sys.path.\n" "PYTHONHOME : alternate directory (or %lc).\n" " The default module search path uses %s.\n" -"PYTHONPLATLIBDIR: override sys.platlibdir.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" -"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" +"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" "PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" " to seed the hashes of str and bytes objects. It can also be\n" " set to an integer in the range [0,4294967295] to get hash\n" @@ -280,39 +254,27 @@ static const char usage_envvars[] = "PYTHON_HISTORY : the location of a .python_history file.\n" "\n" "These variables have equivalent command-line options (see --help for details):\n" -"PYTHON_CPU_COUNT: Overrides the return value of os.process_cpu_count(),\n" -" os.cpu_count(), and multiprocessing.cpu_count() if set to\n" -" a positive integer. (-X cpu_count)\n" +"PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDEVMODE : enable the development mode (-X dev)\n" +"PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" "PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" "PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" -"PYTHON_FROZEN_MODULES: if this variable is set, it determines whether or not\n" -" frozen modules should be used. The default is \"on\" (or\n" -" \"off\" if you are running a local build).\n" +"PYTHON_FROZEN_MODULES: whether to use frozen modules; the default is \"on\"\n" +" for installed Python and \"off\" for a local build\n" " (-X frozen_modules)\n" #ifdef Py_GIL_DISABLED "PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" #endif "PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS: limits the maximum digit characters in an int value\n" -" when converting from a string and when converting an int\n" -" back to a str. A value of 0 disables the limit.\n" -" Conversions to or from bases 2, 4, 8, 16, and 32 are never\n" -" limited.\n" -" (-X int_max_str_digits=number)\n" -"PYTHONNODEBUGRANGES: if this variable is set, it disables the inclusion of\n" -" the tables mapping extra location information (end line,\n" -" start column offset and end column offset) to every\n" -" instruction in code objects. This is useful when smaller\n" -" code objects and pyc files are desired as well as\n" -" suppressing the extra visual location indicators when the\n" -" interpreter displays tracebacks. (-X no_debug_ranges)\n" +"PYTHONINTMAXSTRDIGITS: limit the size of int<->str conversions;\n" +" 0 disables the limit (-X int_max_str_digits=N)\n" +"PYTHONNODEBUGRANGES: don't include extra location information in code objects\n" +" (-X no_debug_ranges)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" "PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" #ifdef Py_DEBUG -"PYTHON_PRESITE=pkg.mod: import this module before site.py is run (-X presite)\n" +"PYTHON_PRESITE: import this module before site (-X presite)\n" #endif "PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" "PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" @@ -323,11 +285,11 @@ static const char usage_envvars[] = #endif "PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" -"PYTHONUTF8 : if set to 1, enable the UTF-8 mode (-X utf8)\n" +"PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" "PYTHONVERBOSE : trace import statements (-v)\n" "PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" " (-X warn_default_encoding)\n" -"PYTHONWARNINGS=arg: warning control (-W arg)\n" +"PYTHONWARNINGS : warning control (-W)\n" ; #if defined(MS_WINDOWS) From 025ef7a5f7b424fba8713e448244b952bf897df3 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 19 Mar 2024 13:55:21 -0400 Subject: [PATCH 40/82] gh-56374: Clarify documentation of nonlocal (#116942) Define 'nonlocal scopes' in a way that excludes class scopes. Rearrange the rest of the doc. Add "Programmer's note". --- Doc/reference/simple_stmts.rst | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 810232e09dc5fa..a253482156d3b4 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -1006,25 +1006,29 @@ The :keyword:`!nonlocal` statement .. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. - -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). - -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. +When the definition of a function or class is nested (enclosed) within +the definitions of other functions, its nonlocal scopes are the local +scopes of the enclosing functions. The :keyword:`nonlocal` statement +causes the listed identifiers to refer to names previously bound in +nonlocal scopes. It allows encapsulated code to rebind such nonlocal +identifiers. If a name is bound in more than one nonlocal scope, the +nearest binding is used. If a name is not bound in any nonlocal scope, +or if there is no nonlocal scope, a :exc:`SyntaxError` is raised. + +The nonlocal statement applies to the entire scope of a function or +class body. A :exc:`SyntaxError` is raised if a variable is used or +assigned to prior to its nonlocal declaration in the scope. .. seealso:: :pep:`3104` - Access to Names in Outer Scopes The specification for the :keyword:`nonlocal` statement. +**Programmer's note:** :keyword:`nonlocal` is a directive to the parser +and applies only to code parsed along with it. See the note for the +:keyword:`global` statement. + + .. _type: The :keyword:`!type` statement From 60e105c1c11ecca1680d03c38aa06bcc77a28714 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 19 Mar 2024 14:40:20 -0400 Subject: [PATCH 41/82] gh-113964: Don't prevent new threads until all non-daemon threads exit (#116677) Starting in Python 3.12, we prevented calling fork() and starting new threads during interpreter finalization (shutdown). This has led to a number of regressions and flaky tests. We should not prevent starting new threads (or `fork()`) until all non-daemon threads exit and finalization starts in earnest. This changes the checks to use `_PyInterpreterState_GetFinalizing(interp)`, which is set immediately before terminating non-daemon threads. --- Lib/test/test_os.py | 17 +++++---- Lib/test/test_subprocess.py | 13 ++++--- Lib/test/test_threading.py | 38 +++++++++++++++---- ...-03-12-20-31-57.gh-issue-113964.bJppzg.rst | 2 + Modules/_posixsubprocess.c | 4 +- Modules/_threadmodule.c | 2 +- Modules/posixmodule.c | 6 +-- Objects/unicodeobject.c | 2 +- 8 files changed, 57 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4c157842d95523..4bf158247fa2ec 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -5357,20 +5357,21 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - def test_fork_at_exit(self): + def test_fork_at_finalization(self): code = """if 1: import atexit import os - def exit_handler(): - pid = os.fork() - if pid != 0: - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + pid = os.fork() + if pid != 0: + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(b"", out) + self.assertEqual(b"OK\n", out) self.assertIn(b"can't fork at interpreter shutdown", err) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index d20b987961ea6f..70452ca94a6a8a 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3398,14 +3398,15 @@ def test_preexec_at_exit(self): def dummy(): pass - def exit_handler(): - subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) - print("shouldn't be printed") - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + subprocess.Popen({ZERO_RETURN_CMD}, preexec_fn=dummy) + print("shouldn't be printed") + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"preexec_fn not supported at interpreter shutdown", err) @unittest.skipIf(not sysconfig.get_config_var("HAVE_VFORK"), diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f1dc12944cb6cc..4414d2bb9cdb59 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1154,21 +1154,21 @@ def import_threading(): self.assertEqual(out, b'') self.assertEqual(err, b'') - def test_start_new_thread_at_exit(self): + def test_start_new_thread_at_finalization(self): code = """if 1: - import atexit import _thread def f(): print("shouldn't be printed") - def exit_handler(): - _thread.start_new_thread(f, ()) - - atexit.register(exit_handler) + class AtFinalization: + def __del__(self): + print("OK") + _thread.start_new_thread(f, ()) + at_finalization = AtFinalization() """ _, out, err = assert_python_ok("-c", code) - self.assertEqual(out, b'') + self.assertEqual(out.strip(), b"OK") self.assertIn(b"can't create new thread at interpreter shutdown", err) class ThreadJoinOnShutdown(BaseTestCase): @@ -1297,6 +1297,30 @@ def main(): rc, out, err = assert_python_ok('-c', script) self.assertFalse(err) + def test_thread_from_thread(self): + script = """if True: + import threading + import time + + def thread2(): + time.sleep(0.05) + print("OK") + + def thread1(): + time.sleep(0.05) + t2 = threading.Thread(target=thread2) + t2.start() + + t = threading.Thread(target=thread1) + t.start() + # do not join() -- the interpreter waits for non-daemon threads to + # finish. + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"OK") + self.assertEqual(rc, 0) + @skip_unless_reliable_fork def test_reinit_tls_after_fork(self): # Issue #13817: fork() would deadlock in a multithreaded program with diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst new file mode 100644 index 00000000000000..ab370d4aa1baee --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-20-31-57.gh-issue-113964.bJppzg.rst @@ -0,0 +1,2 @@ +Starting new threads and process creation through :func:`os.fork` are now +only prevented once all non-daemon threads exit. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index bcbbe70680b8e7..b160cd78177a17 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1031,7 +1031,9 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, Py_ssize_t fds_to_keep_len = PyTuple_GET_SIZE(py_fds_to_keep); PyInterpreterState *interp = _PyInterpreterState_GET(); - if ((preexec_fn != Py_None) && interp->finalizing) { + if ((preexec_fn != Py_None) && + _PyInterpreterState_GetFinalizing(interp) != NULL) + { PyErr_SetString(PyExc_PythonFinalizationError, "preexec_fn not supported at interpreter shutdown"); return NULL; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 6889e8f6e12126..4912cd776ef5ae 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1729,7 +1729,7 @@ do_start_new_thread(thread_module_state *state, PyObject *func, PyObject *args, "thread is not supported for isolated subinterpreters"); return -1; } - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't create new thread at interpreter shutdown"); return -1; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7b2d3661ee5546..2498b61d6412d5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7842,7 +7842,7 @@ os_fork1_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; @@ -7886,7 +7886,7 @@ os_fork_impl(PyObject *module) { pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; @@ -8719,7 +8719,7 @@ os_forkpty_impl(PyObject *module) pid_t pid; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { PyErr_SetString(PyExc_PythonFinalizationError, "can't fork at interpreter shutdown"); return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index c8f647a7a71135..e412af5f797e7a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -505,7 +505,7 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) /* Disable checks during Python finalization. For example, it allows to call _PyObject_Dump() during finalization for debugging purpose. */ - if (interp->finalizing) { + if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } From 1ab0d0b1167d78bf19661a3b5e533a2b68a57604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 19 Mar 2024 19:06:06 +0000 Subject: [PATCH 42/82] gh-116886: Temporarily disable CIfuzz (memory) (#117018) --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e236534ae3770..205ba7357abbcc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -514,7 +514,8 @@ jobs: strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] + # sanitizer: [address, undefined, memory] -- memory skipped temporarily until GH-116886 is solved. + sanitizer: [address, undefined] steps: - name: Build fuzzers (${{ matrix.sanitizer }}) id: build From 9080e9ed50d1d3c7933cda88658306f6872d2f87 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 19 Mar 2024 21:48:08 +0200 Subject: [PATCH 43/82] gh-101100: Fix Sphinx warnings in `library/pydoc.rst` (#116913) --- Doc/library/pydoc.rst | 14 +++++++------- Doc/tools/.nitignore | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index 03e0915bf6d135..df969b2fc7c04c 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -16,19 +16,19 @@ -------------- -The :mod:`pydoc` module automatically generates documentation from Python +The :mod:`!pydoc` module automatically generates documentation from Python modules. The documentation can be presented as pages of text on the console, served to a web browser, or saved to HTML files. For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, +derived from the docstring (i.e. the :attr:`!__doc__` attribute) of the object, and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just +:mod:`!pydoc` tries to obtain a description from the block of comment lines just above the definition of the class, function or method in the source file, or at the top of the module (see :func:`inspect.getcomments`). The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation +interactive interpreter, which uses :mod:`!pydoc` to generate its documentation as text on the console. The same text documentation can also be viewed from outside the Python interpreter by running :program:`pydoc` as a script at the operating system's command prompt. For example, running :: @@ -46,7 +46,7 @@ produced for that file. .. note:: - In order to find objects and their documentation, :mod:`pydoc` imports the + In order to find objects and their documentation, :mod:`!pydoc` imports the module(s) to be documented. Therefore, any code on module level will be executed on that occasion. Use an ``if __name__ == '__main__':`` guard to only execute code when a file is invoked as a script and not just imported. @@ -90,7 +90,7 @@ Python interpreter and typed ``import spam``. Module docs for core modules are assumed to reside in ``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable +be overridden by setting the :envvar:`!PYTHONDOCS` environment variable to a different URL or to a local directory containing the Library Reference Manual pages. @@ -101,7 +101,7 @@ Reference Manual pages. The ``-g`` command line option was removed. .. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than + :mod:`!pydoc` now uses :func:`inspect.signature` rather than :func:`inspect.getfullargspec` to extract signature information from callables. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 5fbc24c6ee65a5..6f38f36d3311c0 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -44,7 +44,6 @@ Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/profile.rst -Doc/library/pydoc.rst Doc/library/pyexpat.rst Doc/library/readline.rst Doc/library/resource.rst From 332ac46c09cd500a16a5f03b53f038b1d9ce77ef Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Tue, 19 Mar 2024 15:55:20 -0400 Subject: [PATCH 44/82] docs: announce venv creation before installing packages (#117036) --- Doc/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/Makefile b/Doc/Makefile index 38fd60f2ae01d1..dd068c520ad60c 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -163,6 +163,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + echo "Creating venv in $(VENVDIR)"; \ $(PYTHON) -m venv $(VENVDIR); \ $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ From 2d17309cc719c41e02ffd1d6cac10f95a7e2359c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Mar 2024 23:30:25 +0200 Subject: [PATCH 45/82] gh-117041: Add "-X gil" in the Python CLI help (GH-117042) --- Python/initconfig.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/initconfig.c b/Python/initconfig.c index 95234eed5edff1..215d6a1d4e0dba 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -195,6 +195,11 @@ The following implementation-specific options are available:\n\ -X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ for installed Python and \"off\" for a local build;\n\ also PYTHON_FROZEN_MODULES\n\ +" +#ifdef Py_GIL_DISABLED +"-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" +#endif +"\ -X importtime: show how long each import takes; also PYTHONPROFILEIMPORTTIME\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ From 438de10c160c188fced64317df9118100c329ffe Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 22:58:13 +0100 Subject: [PATCH 46/82] gh-116869: Fix test_cext on RHEL7 (#117010) Remove -std option from CC command line. Skip C++14 test for now on non-Windows platforms (like RHEL7). --- Lib/test/test_cext/setup.py | 23 +++++++++++++---------- Lib/test/test_cppext/__init__.py | 3 +++ Lib/test/test_cppext/setup.py | 24 ++++++++++++++---------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index d6c4410fa5f1e3..17a90caa98b566 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -39,19 +39,22 @@ def main(): if std: if support.MS_WINDOWS: cflags.append(f'/std:{std}') - std_prefix = '/std' else: cflags.append(f'-std={std}') - std_prefix = '-std' - # Remove existing -std options to only test ours - cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: - cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] - cmd = shlex.join(cmd) - # CC env var overrides sysconfig CC variable in setuptools - os.environ['CC'] = cmd + # Remove existing -std or /std options from CC command line. + # Python adds -std=c11 option. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd # Define Py_LIMITED_API macro if limited: diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index b8414f217c7dde..00a2840d49c779 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -35,6 +35,9 @@ def test_build_cpp03(self): def test_build_cpp11(self): self.check_build('_testcpp11ext', std='c++11') + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") def test_build_cpp14(self): self.check_build('_testcpp14ext', std='c++14') diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 77e47bcd0cbe16..80b3e0d5212f7b 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -35,19 +35,23 @@ def main(): if std: if support.MS_WINDOWS: cppflags.append(f'/std:{std}') - std_prefix = '/std' else: cppflags.append(f'-std={std}') - std_prefix = '-std' - # Remove existing -std options to only test ours - cmd = (sysconfig.get_config_var('CC') or '') - if cmd is not None: - cmd = shlex.split(cmd) - cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] - cmd = shlex.join(cmd) - # CC env var overrides sysconfig CC variable in setuptools - os.environ['CC'] = cmd + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 + # option emits a C++ compiler warning. Remove "-std11" option from the + # CC command. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + if support.MS_WINDOWS: + std_prefix = '/std' + else: + std_prefix = '-std' + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith(std_prefix)] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] From d5ebf8b71fd18d7a1f2f6b670a2c18749dc2b55e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 19 Mar 2024 23:23:17 +0100 Subject: [PATCH 47/82] gh-116869: Fix test_cext for Free Threading (#117043) Don't check for warnings on Free Threading. --- Lib/test/test_cext/__init__.py | 2 ++ Lib/test/test_cext/setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 7bf2381d844d1c..e4472b3355759c 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -40,9 +40,11 @@ def test_build_c11(self): def test_build_c99(self): self.check_build('_test_c99_cext', std='c99') + @unittest.skipIf(support.Py_GIL_DISABLED, 'incompatible with Free Threading') def test_build_limited(self): self.check_build('_test_limited_cext', limited=True) + @unittest.skipIf(support.Py_GIL_DISABLED, 'broken for now with Free Threading') def test_build_limited_c11(self): self.check_build('_test_limited_c11_cext', limited=True, std='c11') diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 17a90caa98b566..1a4ec26f9985d3 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -11,7 +11,7 @@ SOURCE = 'extension.c' -if not support.MS_WINDOWS: +if not support.MS_WINDOWS and not support.Py_GIL_DISABLED: # C compiler flags for GCC and clang CFLAGS = [ # The purpose of test_cext extension is to check that building a C From 15309329b65a285cb7b3071f0f08ac964b61411b Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 08:54:42 +0000 Subject: [PATCH 48/82] GH-108362: Incremental Cycle GC (GH-116206) --- Doc/whatsnew/3.13.rst | 30 + Include/internal/pycore_gc.h | 41 +- Include/internal/pycore_object.h | 18 +- Include/internal/pycore_runtime_init.h | 8 +- Lib/test/test_gc.py | 72 +- ...-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst | 12 + Modules/gcmodule.c | 25 +- Objects/object.c | 21 + Objects/structseq.c | 5 +- Python/gc.c | 806 +++++++++++------- Python/gc_free_threading.c | 23 +- Python/import.c | 2 +- Python/optimizer.c | 2 +- Tools/gdb/libpython.py | 7 +- 14 files changed, 684 insertions(+), 388 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0e04dcd196d306..40e2e6a8e03be9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -111,6 +111,14 @@ Improved Error Messages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^ TypeError: split() got an unexpected keyword argument 'max_split'. Did you mean 'maxsplit'? +Incremental Garbage Collection +------------------------------ + +* The cycle garbage collector is now incremental. + This means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + + Other Language Changes ====================== @@ -350,6 +358,28 @@ fractions sign handling, minimum width and grouping. (Contributed by Mark Dickinson in :gh:`111320`.) +gc +-- + +* The cyclic garbage collector is now incremental, which changes the meanings + of the results of :meth:`gc.get_threshold` and :meth:`gc.get_threshold` as + well as :meth:`gc.get_count` and :meth:`gc.get_stats`. +* :meth:`gc.get_threshold` returns a three-tuple for backwards compatibility, + the first value is the threshold for young collections, as before, the second + value determines the rate at which the old collection is scanned; the + default is 10 and higher values mean that the old collection is scanned more slowly. + The third value is meangless and is always zero. +* :meth:`gc.set_threshold` ignores any items after the second. +* :meth:`gc.get_count` and :meth:`gc.get_stats`. + These functions return the same format of results as before. + The only difference is that instead of the results refering to + the young, aging and old generations, the results refer to the + young generation and the aging and collecting spaces of the old generation. + +In summary, code that attempted to manipulate the behavior of the cycle GC may +not work exactly as intended, but it is very unlikely to harmful. +All other code will work just fine. + glob ---- diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 4a7191a562cc10..9d66e62ba8b5e3 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -109,11 +109,14 @@ static inline void _PyObject_GC_SET_SHARED_INLINE(PyObject *op) { /* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ -#define _PyGC_PREV_MASK_FINALIZED (1) +#define _PyGC_PREV_MASK_FINALIZED 1 /* Bit 1 is set when the object is in generation which is GCed currently. */ -#define _PyGC_PREV_MASK_COLLECTING (2) -/* The (N-2) most significant bits contain the real address. */ -#define _PyGC_PREV_SHIFT (2) +#define _PyGC_PREV_MASK_COLLECTING 2 + +/* Bit 0 is set if the object belongs to old space 1 */ +#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 + +#define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) /* set for debugging information */ @@ -139,11 +142,13 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next; + uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - gc->_gc_next = (uintptr_t)next; + uintptr_t unext = (uintptr_t)next; + assert((unext & ~_PyGC_PREV_MASK) == 0); + gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -151,6 +156,7 @@ static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) { uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK); return (PyGC_Head*)prev; } + static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) { uintptr_t uprev = (uintptr_t)prev; assert((uprev & ~_PyGC_PREV_MASK) == 0); @@ -236,6 +242,13 @@ struct gc_generation { generations */ }; +struct gc_collection_stats { + /* number of collected objects */ + Py_ssize_t collected; + /* total number of uncollectable objects (put into gc.garbage) */ + Py_ssize_t uncollectable; +}; + /* Running stats per generation */ struct gc_generation_stats { /* total number of collections */ @@ -257,8 +270,8 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ - struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; + struct gc_generation young; + struct gc_generation old[2]; /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; @@ -268,6 +281,12 @@ struct _gc_runtime_state { PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + + Py_ssize_t work_to_do; + /* Which of the old spaces is the visited space */ + int visited_space; + +#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -279,6 +298,7 @@ struct _gc_runtime_state { collections, and are awaiting to undergo a full collection for the first time. */ Py_ssize_t long_lived_pending; +#endif }; #ifdef Py_GIL_DISABLED @@ -291,9 +311,8 @@ struct _gc_thread_state { extern void _PyGC_InitState(struct _gc_runtime_state *); -extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, - _PyGC_Reason reason); -extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); +extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); +extern void _PyGC_CollectNoFail(PyThreadState *tstate); /* Freeze objects tracked by the GC and ignore them in future collections. */ extern void _PyGC_Freeze(PyInterpreterState *interp); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 9809f5f2e0271a..759ec4d17b5eb4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -125,19 +125,8 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) -static inline void _Py_SetImmortal(PyObject *op) -{ - if (op) { -#ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; -#else - op->ob_refcnt = _Py_IMMORTAL_REFCNT; -#endif - } -} -#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) +extern void _Py_SetImmortal(PyObject *op); +extern void _Py_SetImmortalUntracked(PyObject *op); // Makes an immortal object mortal again with the specified refcnt. Should only // be used during runtime finalization. @@ -325,11 +314,12 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = interp->gc.generation0; + PyGC_Head *generation0 = &interp->gc.young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); _PyGCHead_SET_NEXT(gc, generation0); + assert((gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1) == 0); generation0->_gc_prev = (uintptr_t)gc; #endif } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index cc47b9a82e2879..88d888943d28b1 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -168,12 +168,12 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .generations = { \ - /* .head is set in _PyGC_InitState(). */ \ - { .threshold = 700, }, \ - { .threshold = 10, }, \ + .young = { .threshold = 2000, }, \ + .old = { \ { .threshold = 10, }, \ + { .threshold = 0, }, \ }, \ + .work_to_do = -5000, \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index f1a7afac0bcd19..ce01916bcabe4f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -384,19 +384,11 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in gen 1 + # x is now in the old gen a, b, c = gc.get_count() - gc.collect(1) - # x is now in gen 2 - d, e, f = gc.get_count() - gc.collect(2) - # x is now in gen 3 - g, h, i = gc.get_count() - # We don't check a, d, g since their exact values depends on + # We don't check a since its exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) - self.assertEqual((e, f), (0, 1)) - self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -847,16 +839,6 @@ def test_get_objects_generations(self): self.assertFalse( any(l is element for element in gc.get_objects(generation=2)) ) - gc.collect(generation=1) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=0)) - ) - self.assertFalse( - any(l is element for element in gc.get_objects(generation=1)) - ) - self.assertTrue( - any(l is element for element in gc.get_objects(generation=2)) - ) gc.collect(generation=2) self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) @@ -1076,6 +1058,56 @@ class Z: callback.assert_not_called() gc.enable() + @unittest.skipIf(Py_GIL_DISABLED, "Free threading does not support incremental GC") + def test_incremental_gc_handles_fast_cycle_creation(self): + + class LinkedList: + + #Use slots to reduce number of implicit objects + __slots__ = "next", "prev", "surprise" + + def __init__(self, next=None, prev=None): + self.next = next + if next is not None: + next.prev = self + self.prev = prev + if prev is not None: + prev.next = self + + def make_ll(depth): + head = LinkedList() + for i in range(depth): + head = LinkedList(head, head.prev) + return head + + head = make_ll(10000) + count = 10000 + + # We expect the counts to go negative eventually + # as there will some objects we aren't counting, + # e.g. the gc stats dicts. The test merely checks + # that the counts don't grow. + + enabled = gc.isenabled() + gc.enable() + olds = [] + for i in range(1000): + newhead = make_ll(200) + count += 200 + newhead.surprise = head + olds.append(newhead) + if len(olds) == 50: + stats = gc.get_stats() + young = stats[0] + incremental = stats[1] + old = stats[2] + collected = young['collected'] + incremental['collected'] + old['collected'] + live = count - collected + self.assertLess(live, 25000) + del olds[:] + if not enabled: + gc.disable() + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst new file mode 100644 index 00000000000000..893904bcecea8a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-07-04-22-51.gh-issue-108362.oB9Gcf.rst @@ -0,0 +1,12 @@ +Implement an incremental cyclic garbage collector. By collecting the old +generation in increments, there is no need for a full heap scan. This can +hugely reduce maximum pause time for programs with large heaps. + +Reduce the number of generations from three to two. The old generation is +split into two spaces, "visited" and "pending". + +Collection happens in two steps:: +* An increment is formed from the young generation and a small part of the pending space. +* This increment is scanned and the survivors moved to the end of the visited space. + +When the collecting space becomes empty, the two spaces are swapped. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 9807d2e7d48a36..3320e54dd9fe93 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -158,17 +158,12 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, { GCState *gcstate = get_gc_state(); - gcstate->generations[0].threshold = threshold0; + gcstate->young.threshold = threshold0; if (group_right_1) { - gcstate->generations[1].threshold = threshold1; + gcstate->old[0].threshold = threshold1; } if (group_right_2) { - gcstate->generations[2].threshold = threshold2; - - /* generations higher than 2 get the same threshold */ - for (int i = 3; i < NUM_GENERATIONS; i++) { - gcstate->generations[i].threshold = gcstate->generations[2].threshold; - } + gcstate->old[1].threshold = threshold2; } Py_RETURN_NONE; } @@ -185,9 +180,9 @@ gc_get_threshold_impl(PyObject *module) { GCState *gcstate = get_gc_state(); return Py_BuildValue("(iii)", - gcstate->generations[0].threshold, - gcstate->generations[1].threshold, - gcstate->generations[2].threshold); + gcstate->young.threshold, + gcstate->old[0].threshold, + 0); } /*[clinic input] @@ -207,14 +202,14 @@ gc_get_count_impl(PyObject *module) struct _gc_thread_state *gc = &tstate->gc; // Flush the local allocation count to the global count - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; #endif return Py_BuildValue("(iii)", - gcstate->generations[0].count, - gcstate->generations[1].count, - gcstate->generations[2].count); + gcstate->young.count, + gcstate->old[gcstate->visited_space].count, + gcstate->old[gcstate->visited_space^1].count); } /*[clinic input] diff --git a/Objects/object.c b/Objects/object.c index df14fe0c6fbfec..fcb8cf481657e5 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2401,6 +2401,27 @@ _Py_NewReferenceNoTotal(PyObject *op) new_reference(op); } +void +_Py_SetImmortalUntracked(PyObject *op) +{ +#ifdef Py_GIL_DISABLED + op->ob_tid = _Py_UNOWNED_TID; + op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; + op->ob_ref_shared = 0; +#else + op->ob_refcnt = _Py_IMMORTAL_REFCNT; +#endif +} + +void +_Py_SetImmortal(PyObject *op) +{ + if (PyObject_IS_GC(op) && _PyObject_GC_IS_TRACKED(op)) { + _PyObject_GC_UNTRACK(op); + } + _Py_SetImmortalUntracked(op); +} + void _Py_ResurrectReference(PyObject *op) { diff --git a/Objects/structseq.c b/Objects/structseq.c index 581d6ad240885a..661d96a968fb80 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -603,6 +603,9 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, PyStructSequence_Desc *desc, unsigned long tp_flags) { + if (Py_TYPE(type) == NULL) { + Py_SET_TYPE(type, &PyType_Type); + } Py_ssize_t n_unnamed_members; Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; @@ -618,7 +621,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, } initialize_static_fields(type, desc, members, tp_flags); - _Py_SetImmortal(type); + _Py_SetImmortal((PyObject *)type); } #ifndef NDEBUG else { diff --git a/Python/gc.c b/Python/gc.c index 6b3316b642ea9e..d0f4ce38bbe567 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -46,7 +46,7 @@ typedef struct _gc_runtime_state GCState; // move_legacy_finalizers() removes this flag instead. // Between them, unreachable list is not normal list and we can not use // most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE (1) +#define NEXT_MASK_UNREACHABLE 2 #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) @@ -96,9 +96,48 @@ gc_decref(PyGC_Head *g) g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; } +static inline int +gc_old_space(PyGC_Head *g) +{ + return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; +} -#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) +static inline int +flip_old_space(int space) +{ + assert(space == 0 || space == 1); + return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; +} +static inline void +gc_flip_old_space(PyGC_Head *g) +{ + g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1; +} + +static inline void +gc_set_old_space(PyGC_Head *g, int space) +{ + assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); + g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; + g->_gc_next |= space; +} + +static PyGC_Head * +GEN_HEAD(GCState *gcstate, int n) +{ + assert((gcstate->visited_space & (~1)) == 0); + switch(n) { + case 0: + return &gcstate->young.head; + case 1: + return &gcstate->old[gcstate->visited_space].head; + case 2: + return &gcstate->old[gcstate->visited_space^1].head; + default: + Py_UNREACHABLE(); + } +} static GCState * get_gc_state(void) @@ -117,11 +156,12 @@ _PyGC_InitState(GCState *gcstate) GEN.head._gc_prev = (uintptr_t)&GEN.head; \ } while (0) - for (int i = 0; i < NUM_GENERATIONS; i++) { - assert(gcstate->generations[i].count == 0); - INIT_HEAD(gcstate->generations[i]); - }; - gcstate->generation0 = GEN_HEAD(gcstate, 0); + assert(gcstate->young.count == 0); + assert(gcstate->old[0].count == 0); + assert(gcstate->old[1].count == 0); + INIT_HEAD(gcstate->young); + INIT_HEAD(gcstate->old[0]); + INIT_HEAD(gcstate->old[1]); INIT_HEAD(gcstate->permanent_generation); #undef INIT_HEAD @@ -219,6 +259,7 @@ gc_list_is_empty(PyGC_Head *list) static inline void gc_list_append(PyGC_Head *node, PyGC_Head *list) { + assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0); PyGC_Head *last = (PyGC_Head *)list->_gc_prev; // last <-> node @@ -276,6 +317,8 @@ gc_list_merge(PyGC_Head *from, PyGC_Head *to) PyGC_Head *from_tail = GC_PREV(from); assert(from_head != from); assert(from_tail != from); + assert(gc_list_is_empty(to) || + gc_old_space(to_tail) == gc_old_space(from_tail)); _PyGCHead_SET_NEXT(to_tail, from_head); _PyGCHead_SET_PREV(from_head, to_tail); @@ -344,8 +387,8 @@ enum flagstates {collecting_clear_unreachable_clear, static void validate_list(PyGC_Head *head, enum flagstates flags) { - assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); - assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); + assert((head->_gc_prev & ~_PyGC_PREV_MASK) == 0); + assert((head->_gc_next & ~_PyGC_PREV_MASK) == 0); uintptr_t prev_value = 0, next_value = 0; switch (flags) { case collecting_clear_unreachable_clear: @@ -367,7 +410,7 @@ validate_list(PyGC_Head *head, enum flagstates flags) PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + PyGC_Head *truenext = GC_NEXT(gc); assert(truenext != NULL); assert(trueprev == prev); assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); @@ -377,8 +420,44 @@ validate_list(PyGC_Head *head, enum flagstates flags) } assert(prev == GC_PREV(head)); } + +static void +validate_old(GCState *gcstate) +{ + for (int space = 0; space < 2; space++) { + PyGC_Head *head = &gcstate->old[space].head; + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + PyGC_Head *next = GC_NEXT(gc); + assert(gc_old_space(gc) == space); + gc = next; + } + } +} + +static void +validate_consistent_old_space(PyGC_Head *head) +{ + PyGC_Head *prev = head; + PyGC_Head *gc = GC_NEXT(head); + if (gc == head) { + return; + } + int old_space = gc_old_space(gc); + while (gc != head) { + PyGC_Head *truenext = GC_NEXT(gc); + assert(truenext != NULL); + assert(gc_old_space(gc) == old_space); + prev = gc; + gc = truenext; + } + assert(prev == GC_PREV(head)); +} + #else #define validate_list(x, y) do{}while(0) +#define validate_old(g) do{}while(0) +#define validate_consistent_old_space(l) do{}while(0) #endif /*** end of list stuff ***/ @@ -396,10 +475,6 @@ update_refs(PyGC_Head *containers) while (gc != containers) { next = GC_NEXT(gc); PyObject *op = FROM_GC(gc); - /* Move any object that might have become immortal to the - * permanent generation as the reference count is not accurately - * reflecting the actual number of live references to this object - */ if (_Py_IsImmortal(op)) { gc_list_move(gc, &get_gc_state()->permanent_generation.head); gc = next; @@ -502,12 +577,13 @@ visit_reachable(PyObject *op, void *arg) // Manually unlink gc from unreachable list because the list functions // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + PyGC_Head *next = GC_NEXT(gc); _PyObject_ASSERT(FROM_GC(prev), prev->_gc_next & NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(next), next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE + prev->_gc_next = gc->_gc_next; // copy flag bits + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; _PyGCHead_SET_PREV(next, prev); gc_list_append(gc, reachable); @@ -559,6 +635,9 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) * or to the right have been scanned yet. */ + validate_consistent_old_space(young); + /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ + uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); while (gc != young) { if (gc_get_refs(gc)) { /* gc is definitely reachable from outside the @@ -604,17 +683,18 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) // But this may pollute the unreachable list head's 'next' pointer // too. That's semantically senseless but expedient here - the // damage is repaired when this function ends. - last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); + last->_gc_next = flags | (uintptr_t)gc; _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); + gc->_gc_next = flags | (uintptr_t)unreachable; unreachable->_gc_prev = (uintptr_t)gc; } - gc = (PyGC_Head*)prev->_gc_next; + gc = _PyGCHead_NEXT(prev); } // young->_gc_prev must be last element remained in the list. young->_gc_prev = (uintptr_t)prev; + young->_gc_next &= _PyGC_PREV_MASK; // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; + unreachable->_gc_next &= _PyGC_PREV_MASK; } static void @@ -673,8 +753,8 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; if (has_legacy_finalizer(op)) { gc_clear_collecting(gc); @@ -697,8 +777,8 @@ clear_unreachable_mask(PyGC_Head *unreachable) PyGC_Head *gc, *next; for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); + next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; - next = (PyGC_Head*)gc->_gc_next; } validate_list(unreachable, collecting_set_unreachable_clear); } @@ -1030,25 +1110,6 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - /* Deduce which objects among "base" are unreachable from outside the list and move them to 'unreachable'. The process consist in the following steps: @@ -1122,7 +1183,6 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * the reachable objects instead. But this is a one-time cost, probably not * worth complicating the code to speed just a little. */ - gc_list_init(unreachable); move_unreachable(base, unreachable); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); @@ -1161,219 +1221,292 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, } -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping +#define UNTRACK_TUPLES 1 +#define UNTRACK_DICTS 2 + +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + int untrack, + struct gc_collection_stats *stats); + +static inline Py_ssize_t +gc_list_set_space(PyGC_Head *list, uintptr_t space) +{ + Py_ssize_t size = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { + gc_set_old_space(gc, space); + size++; + } + return size; +} + +/* Making progress in the incremental collector + * In order to eventually collect all cycles + * the incremental collector must progress through the old + * space faster than objects are added to the old space. + * + * Each young or incremental collection adds a numebr of + * objects, S (for survivors) to the old space, and + * incremental collectors scan I objects from the old space. + * I > S must be true. We also want I > S * N to be where + * N > 1. Higher values of N mean that the old space is + * scanned more rapidly. + * The default incremental threshold of 10 translates to + * N == 1.4 (1 + 4/threshold) */ + +/* Multiply by 4 so that the default incremental threshold of 10 + * scans objects at 20% the rate of object creation */ +#define SCAN_RATE_MULTIPLIER 2 + static void -invoke_gc_callback(PyThreadState *tstate, const char *phase, - int generation, Py_ssize_t collected, - Py_ssize_t uncollectable) +add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) { - assert(!_PyErr_Occurred(tstate)); + gcstate->generation_stats[gen].collected += stats->collected; + gcstate->generation_stats[gen].uncollectable += stats->uncollectable; + gcstate->generation_stats[gen].collections += 1; +} - /* we may get called very early */ +static void +gc_collect_young(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ GCState *gcstate = &tstate->interp->gc; - if (gcstate->callbacks == NULL) { - return; - } - - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", collected, - "uncollectable", uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; +#ifdef Py_STATS + { + Py_ssize_t count = 0; + PyGC_Head *gc; + for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { + count++; } + GC_STAT_ADD(0, objects_queued, count); } +#endif - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); - return; + PyGC_Head survivors; + gc_list_init(&survivors); + gc_collect_region(tstate, young, &survivors, UNTRACK_TUPLES, stats); + Py_ssize_t survivor_count = 0; + if (gcstate->visited_space) { + /* objects in visited space have bit set, so we set it here */ + survivor_count = gc_list_set_space(&survivors, 1); } - - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; icallbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_WriteUnraisable(cb); - } - else { - Py_DECREF(r); + else { + PyGC_Head *gc; + for (gc = GC_NEXT(&survivors); gc != &survivors; gc = GC_NEXT(gc)) { +#ifdef GC_DEBUG + assert(gc_old_space(gc) == 0); +#endif + survivor_count++; } - Py_DECREF(cb); } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!_PyErr_Occurred(tstate)); + gc_list_merge(&survivors, visited); + validate_old(gcstate); + gcstate->young.count = 0; + gcstate->old[gcstate->visited_space].count++; + Py_ssize_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 1) { + scale_factor = 1; + } + gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor; + add_stats(gcstate, 0, stats); } +static inline int +IS_IN_VISITED(PyGC_Head *gc, int visited_space) +{ + assert(visited_space == 0 || flip_old_space(visited_space) == 0); + return gc_old_space(gc) == visited_space; +} + +struct container_and_flag { + PyGC_Head *container; + int visited_space; + uintptr_t size; +}; -/* Find the oldest generation (highest numbered) where the count - * exceeds the threshold. Objects in the that generation and - * generations younger than it will be collected. */ +/* A traversal callback for adding to container) */ static int -gc_select_generation(GCState *gcstate) -{ - for (int i = NUM_GENERATIONS-1; i >= 0; i--) { - if (gcstate->generations[i].count > gcstate->generations[i].threshold) { - /* Avoid quadratic performance degradation in number - of tracked objects (see also issue #4074): - - To limit the cost of garbage collection, there are two strategies; - - make each collection faster, e.g. by scanning fewer objects - - do less collections - This heuristic is about the latter strategy. - - In addition to the various configurable thresholds, we only trigger a - full collection if the ratio - - long_lived_pending / long_lived_total - - is above a given value (hardwired to 25%). - - The reason is that, while "non-full" collections (i.e., collections of - the young and middle generations) will always examine roughly the same - number of objects -- determined by the aforementioned thresholds --, - the cost of a full collection is proportional to the total number of - long-lived objects, which is virtually unbounded. - - Indeed, it has been remarked that doing a full collection every - of object creations entails a dramatic performance - degradation in workloads which consist in creating and storing lots of - long-lived objects (e.g. building a large list of GC-tracked objects would - show quadratic performance, instead of linear as expected: see issue #4074). - - Using the above ratio, instead, yields amortized linear performance in - the total number of objects (the effect of which can be summarized - thusly: "each full garbage collection is more and more costly as the - number of objects grows, but we do fewer and fewer of them"). - - This heuristic was suggested by Martin von Löwis on python-dev in - June 2008. His original analysis and proposal can be found at: - http://mail.python.org/pipermail/python-dev/2008-June/080579.html - */ - if (i == NUM_GENERATIONS - 1 - && gcstate->long_lived_pending < gcstate->long_lived_total / 4) - { - continue; - } - return i; +visit_add_to_container(PyObject *op, void *arg) +{ + OBJECT_STAT_INC(object_visits); + struct container_and_flag *cf = (struct container_and_flag *)arg; + int visited = cf->visited_space; + assert(visited == get_gc_state()->visited_space); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited) { + gc_flip_old_space(gc); + gc_list_move(gc, cf->container); + cf->size++; } } - return -1; + return 0; } - -/* This is the main function. Read this to understand how the - * collection process works. */ -static Py_ssize_t -gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) +static uintptr_t +expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) { - int i; - Py_ssize_t m = 0; /* # objects collected */ - Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ - PyGC_Head *young; /* the generation we are examining */ - PyGC_Head *old; /* next older generation */ - PyGC_Head unreachable; /* non-problematic unreachable trash */ - PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; - PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ - GCState *gcstate = &tstate->interp->gc; - - // gc_collect_main() must not be called before _PyGC_Init - // or after _PyGC_Fini() - assert(gcstate->garbage != NULL); - assert(!_PyErr_Occurred(tstate)); + validate_list(container, collecting_clear_unreachable_clear); + struct container_and_flag arg = { + .container = container, + .visited_space = gcstate->visited_space, + .size = 0 + }; + assert(GC_NEXT(gc) == container); + while (gc != container) { + /* Survivors will be moved to visited space, so they should + * have been marked as visited */ + assert(IS_IN_VISITED(gc, gcstate->visited_space)); + PyObject *op = FROM_GC(gc); + if (_Py_IsImmortal(op)) { + PyGC_Head *next = GC_NEXT(gc); + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + gc = GC_NEXT(gc); + } + return arg.size; +} - int expected = 0; - if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { - // Don't start a garbage collection if one is already in progress. - return 0; +/* Do bookkeeping for a completed GC cycle */ +static void +completed_cycle(GCState *gcstate) +{ + PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; + assert(gc_list_is_empty(not_visited)); + gcstate->visited_space = flip_old_space(gcstate->visited_space); + if (gcstate->work_to_do > 0) { + gcstate->work_to_do = 0; } +} - if (generation == GENERATION_AUTO) { - // Select the oldest generation that needs collecting. We will collect - // objects from that generation and all generations younger than it. - generation = gc_select_generation(gcstate); - if (generation < 0) { - // No generation needs to be collected. - _Py_atomic_store_int(&gcstate->collecting, 0); - return 0; +static void +gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) +{ + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + PyGC_Head increment; + gc_list_init(&increment); + Py_ssize_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 1) { + scale_factor = 1; + } + Py_ssize_t increment_size = 0; + gc_list_merge(&gcstate->young.head, &increment); + gcstate->young.count = 0; + if (gcstate->visited_space) { + /* objects in visited space have bit set, so we set it here */ + gc_list_set_space(&increment, 1); + } + while (increment_size < gcstate->work_to_do) { + if (gc_list_is_empty(not_visited)) { + break; } + PyGC_Head *gc = _PyGCHead_NEXT(not_visited); + gc_list_move(gc, &increment); + increment_size++; + gc_set_old_space(gc, gcstate->visited_space); + increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); } - - assert(generation >= 0 && generation < NUM_GENERATIONS); - -#ifdef Py_STATS - if (_Py_stats) { - _Py_stats->object_stats.object_visits = 0; + GC_STAT_ADD(1, objects_queued, region_size); + PyGC_Head survivors; + gc_list_init(&survivors); + gc_collect_region(tstate, &increment, &survivors, UNTRACK_TUPLES, stats); + Py_ssize_t survivor_count = gc_list_size(&survivors); + gc_list_merge(&survivors, visited); + assert(gc_list_is_empty(&increment)); + gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor; + gcstate->work_to_do -= increment_size; + if (gcstate->work_to_do < 0) { + gcstate->work_to_do = 0; } -#endif - GC_STAT_ADD(generation, collections, 1); - - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "start", generation, 0, 0); + validate_old(gcstate); + add_stats(gcstate, 1, stats); + if (gc_list_is_empty(not_visited)) { + completed_cycle(gcstate); } +} - if (gcstate->debug & _PyGC_DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - t1 = _PyTime_PerfCounterUnchecked(); - } - if (PyDTrace_GC_START_ENABLED()) { - PyDTrace_GC_START(generation); +static void +gc_collect_full(PyThreadState *tstate, + struct gc_collection_stats *stats) +{ + GCState *gcstate = &tstate->interp->gc; + validate_old(gcstate); + PyGC_Head *young = &gcstate->young.head; + PyGC_Head *old0 = &gcstate->old[0].head; + PyGC_Head *old1 = &gcstate->old[1].head; + /* merge all generations into old0 */ + gc_list_merge(young, old0); + gcstate->young.count = 0; + PyGC_Head *gc = GC_NEXT(old1); + while (gc != old1) { + PyGC_Head *next = GC_NEXT(gc); + gc_set_old_space(gc, 0); + gc = next; } + gc_list_merge(old1, old0); - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) { - gcstate->generations[generation+1].count += 1; - } - for (i = 0; i <= generation; i++) { - gcstate->generations[i].count = 0; - } + gc_collect_region(tstate, old0, old0, + UNTRACK_TUPLES | UNTRACK_DICTS, + stats); + gcstate->visited_space = 1; + gcstate->young.count = 0; + gcstate->old[0].count = 0; + gcstate->old[1].count = 0; - /* merge younger generations with one we are currently collecting */ - for (i = 0; i < generation; i++) { - gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); - } + gcstate->work_to_do = - gcstate->young.threshold * 2; + _PyGC_ClearAllFreeLists(tstate->interp); + validate_old(gcstate); + add_stats(gcstate, 2, stats); +} - /* handy references */ - young = GEN_HEAD(gcstate, generation); - if (generation < NUM_GENERATIONS-1) { - old = GEN_HEAD(gcstate, generation+1); - } - else { - old = young; - } - validate_list(old, collecting_clear_unreachable_clear); +/* This is the main function. Read this to understand how the + * collection process works. */ +static void +gc_collect_region(PyThreadState *tstate, + PyGC_Head *from, + PyGC_Head *to, + int untrack, + struct gc_collection_stats *stats) +{ + PyGC_Head unreachable; /* non-problematic unreachable trash */ + PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ + PyGC_Head *gc; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; - deduce_unreachable(young, &unreachable); + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); - untrack_tuples(young); - /* Move reachable objects to next generation. */ - if (young != old) { - if (generation == NUM_GENERATIONS - 2) { - gcstate->long_lived_pending += gc_list_size(young); - } - gc_list_merge(young, old); + gc_list_init(&unreachable); + deduce_unreachable(from, &unreachable); + validate_consistent_old_space(from); + if (untrack & UNTRACK_TUPLES) { + untrack_tuples(from); } - else { - /* We only un-track dicts in full collections, to avoid quadratic - dict build-up. See issue #14775. */ - untrack_dicts(young); - gcstate->long_lived_pending = 0; - gcstate->long_lived_total = gc_list_size(young); + if (untrack & UNTRACK_DICTS) { + untrack_dicts(from); + } + validate_consistent_old_space(to); + if (from != to) { + gc_list_merge(from, to); } + validate_consistent_old_space(to); + /* Move reachable objects to next generation. */ /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1387,10 +1520,8 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) * and we move those into the finalizers list too. */ move_legacy_finalizer_reachable(&finalizers); - validate_list(&finalizers, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); - /* Print debugging information. */ if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { @@ -1399,89 +1530,99 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } /* Clear weakrefs and invoke callbacks as necessary. */ - m += handle_weakrefs(&unreachable, old); - - validate_list(old, collecting_clear_unreachable_clear); + stats->collected += handle_weakrefs(&unreachable, to); + validate_list(to, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); /* Call tp_finalize on objects which have one. */ finalize_garbage(tstate, &unreachable); - /* Handle any objects that may have resurrected after the call * to 'finalize_garbage' and continue the collection with the * objects that are still unreachable */ PyGC_Head final_unreachable; - handle_resurrected_objects(&unreachable, &final_unreachable, old); + gc_list_init(&final_unreachable); + handle_resurrected_objects(&unreachable, &final_unreachable, to); /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. */ - m += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, old); + stats->collected += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, to); /* Collect statistics on uncollectable objects found and print * debugging information. */ + Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } - if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); - PySys_WriteStderr( - "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", - n+m, n, d); - } - + stats->uncollectable = n; /* Append instances in the uncollectable set to a Python * reachable list of garbage. The programmer has to deal with * this if they insist on creating this type of structure. */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, old); - validate_list(old, collecting_clear_unreachable_clear); + handle_legacy_finalizers(tstate, gcstate, &finalizers, to); + validate_list(to, collecting_clear_unreachable_clear); +} - /* Clear free list only during the collection of the highest - * generation */ - if (generation == NUM_GENERATIONS-1) { - _PyGC_ClearAllFreeLists(tstate->interp); - } +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping + */ +static void +do_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + assert(!PyErr_Occurred()); - if (_PyErr_Occurred(tstate)) { - if (reason == _Py_GC_REASON_SHUTDOWN) { - _PyErr_Clear(tstate); - } - else { - PyErr_FormatUnraisable("Exception ignored in garbage collection"); + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", stats->collected, + "uncollectable", stats->uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; } } - /* Update stats */ - struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; - stats->collections++; - stats->collected += m; - stats->uncollectable += n; - - GC_STAT_ADD(generation, objects_collected, m); -#ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; } -#endif - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(n + m); + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; icallbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_WriteUnraisable(cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!PyErr_Occurred()); +} - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(tstate, "stop", generation, m, n); +static void +invoke_gc_callback(GCState *gcstate, const char *phase, + int generation, struct gc_collection_stats *stats) +{ + if (gcstate->callbacks == NULL) { + return; } - - assert(!_PyErr_Occurred(tstate)); - _Py_atomic_store_int(&gcstate->collecting, 0); - return n + m; + do_gc_callback(gcstate, phase, generation, stats); } static int @@ -1571,10 +1712,16 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - for (int i = 0; i < NUM_GENERATIONS; ++i) { - gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); - gcstate->generations[i].count = 0; - } + gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); + gcstate->young.count = 0; + PyGC_Head*old0 = &gcstate->old[0].head; + PyGC_Head*old1 = &gcstate->old[1].head; + gc_list_merge(old0, &gcstate->permanent_generation.head); + gcstate->old[0].count = 0; + gc_list_set_space(old1, 0); + gc_list_merge(old1, &gcstate->permanent_generation.head); + gcstate->old[1].count = 0; + validate_old(gcstate); } void @@ -1582,7 +1729,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - GEN_HEAD(gcstate, NUM_GENERATIONS-1)); + &gcstate->old[0].head); + validate_old(gcstate); } Py_ssize_t @@ -1618,32 +1766,66 @@ PyGC_IsEnabled(void) return gcstate->enabled; } -/* Public API to invoke gc.collect() from C */ Py_ssize_t -PyGC_Collect(void) +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - PyThreadState *tstate = _PyThreadState_GET(); GCState *gcstate = &tstate->interp->gc; - if (!gcstate->enabled) { + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. return 0; } - Py_ssize_t n; + struct gc_collection_stats stats = { 0 }; + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "start", generation, &stats); + } + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); + } PyObject *exc = _PyErr_GetRaisedException(tstate); - n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); + switch(generation) { + case 0: + gc_collect_young(tstate, &stats); + break; + case 1: + gc_collect_increment(tstate, &stats); + break; + case 2: + gc_collect_full(tstate, &stats); + break; + default: + Py_UNREACHABLE(); + } + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(stats.uncollectable + stats.collected); + } + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(gcstate, "stop", generation, &stats); + } _PyErr_SetRaisedException(tstate, exc); - - return n; + GC_STAT_ADD(generation, objects_collected, stats.collected); +#ifdef Py_STATS + if (_py_stats) { + GC_STAT_ADD(generation, object_visits, + _py_stats->object_stats.object_visits); + _py_stats->object_stats.object_visits = 0; + } +#endif + validate_old(gcstate); + _Py_atomic_store_int(&gcstate->collecting, 0); + return stats.uncollectable + stats.collected; } +/* Public API to invoke gc.collect() from C */ Py_ssize_t -_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +PyGC_Collect(void) { - return gc_collect_main(tstate, generation, reason); + return _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_MANUAL); } -Py_ssize_t +void _PyGC_CollectNoFail(PyThreadState *tstate) { /* Ideally, this function is only called on interpreter shutdown, @@ -1652,7 +1834,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); + _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_SHUTDOWN); } void @@ -1791,10 +1973,10 @@ _PyObject_GC_Link(PyObject *op) GCState *gcstate = &tstate->interp->gc; gc->_gc_next = 0; gc->_gc_prev = 0; - gcstate->generations[0].count++; /* number of allocated GC objects */ - if (gcstate->generations[0].count > gcstate->generations[0].threshold && + gcstate->young.count++; /* number of allocated GC objects */ + if (gcstate->young.count > gcstate->young.threshold && gcstate->enabled && - gcstate->generations[0].threshold && + gcstate->young.threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -1805,11 +1987,9 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { - GCState *gcstate = get_gc_state(); - if (!gcstate->enabled) { - return; + if (tstate->interp->gc.enabled) { + _PyGC_Collect(tstate, 1, _Py_GC_REASON_HEAP); } - gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } static PyObject * @@ -1912,8 +2092,8 @@ PyObject_GC_Del(void *op) #endif } GCState *gcstate = get_gc_state(); - if (gcstate->generations[0].count > 0) { - gcstate->generations[0].count--; + if (gcstate->young.count > 0) { + gcstate->young.count--; } PyObject_Free(((char *)op)-presize); } @@ -1936,26 +2116,36 @@ PyObject_GC_IsFinalized(PyObject *obj) return 0; } +static int +visit_generation(gcvisitobjects_t callback, void *arg, struct gc_generation *gen) +{ + PyGC_Head *gc_list, *gc; + gc_list = &gen->head; + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { + PyObject *op = FROM_GC(gc); + Py_INCREF(op); + int res = callback(op, arg); + Py_DECREF(op); + if (!res) { + return -1; + } + } + return 0; +} + void PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) { - size_t i; GCState *gcstate = get_gc_state(); int origenstate = gcstate->enabled; gcstate->enabled = 0; - for (i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gc_list, *gc; - gc_list = GEN_HEAD(gcstate, i); - for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { - PyObject *op = FROM_GC(gc); - Py_INCREF(op); - int res = callback(op, arg); - Py_DECREF(op); - if (!res) { - goto done; - } - } + if (visit_generation(callback, arg, &gcstate->young)) { + goto done; + } + if (visit_generation(callback, arg, &gcstate->old[0])) { + goto done; } + visit_generation(callback, arg, &gcstate->old[1]); done: gcstate->enabled = origenstate; } diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 2b13d1f005dd97..52c79c02099b53 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -675,7 +675,7 @@ void _PyGC_InitState(GCState *gcstate) { // TODO: move to pycore_runtime_init.h once the incremental GC lands. - gcstate->generations[0].threshold = 2000; + gcstate->young.threshold = 2000; } @@ -970,8 +970,8 @@ cleanup_worklist(struct worklist *worklist) static bool gc_should_collect(GCState *gcstate) { - int count = _Py_atomic_load_int_relaxed(&gcstate->generations[0].count); - int threshold = gcstate->generations[0].threshold; + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int threshold = gcstate->young.threshold; if (count <= threshold || threshold == 0 || !gcstate->enabled) { return false; } @@ -979,7 +979,7 @@ gc_should_collect(GCState *gcstate) // objects. A few tests rely on immediate scheduling of the GC so we ignore // the scaled threshold if generations[1].threshold is set to zero. return (count > gcstate->long_lived_total / 4 || - gcstate->generations[1].threshold == 0); + gcstate->old[0].threshold == 0); } static void @@ -993,7 +993,7 @@ record_allocation(PyThreadState *tstate) if (gc->alloc_count >= LOCAL_ALLOC_COUNT_THRESHOLD) { // TODO: Use Py_ssize_t for the generation count. GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; if (gc_should_collect(gcstate) && @@ -1012,7 +1012,7 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->generations[0].count, (int)gc->alloc_count); + _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); gc->alloc_count = 0; } } @@ -1137,10 +1137,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) /* update collection and allocation counters */ if (generation+1 < NUM_GENERATIONS) { - gcstate->generations[generation+1].count += 1; + gcstate->old[generation].count += 1; } - for (i = 0; i <= generation; i++) { - gcstate->generations[i].count = 0; + gcstate->young.count = 0; + for (i = 1; i <= generation; i++) { + gcstate->old[i-1].count = 0; } PyInterpreterState *interp = tstate->interp; @@ -1463,7 +1464,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) return gc_collect_main(tstate, generation, reason); } -Py_ssize_t +void _PyGC_CollectNoFail(PyThreadState *tstate) { /* Ideally, this function is only called on interpreter shutdown, @@ -1472,7 +1473,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - return gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); + gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void diff --git a/Python/import.c b/Python/import.c index dc92708c8b6ea0..6544a84d895d4a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1031,7 +1031,7 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) if (!already_set) { /* We assume that all module defs are statically allocated and will never be freed. Otherwise, we would incref here. */ - _Py_SetImmortal(def); + _Py_SetImmortal((PyObject *)def); } res = 0; diff --git a/Python/optimizer.c b/Python/optimizer.c index bb00e0d2575784..4a3cd46ce80a26 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -983,7 +983,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, const _PyBloomFilter *depende static int init_cold_exit_executor(_PyExecutorObject *executor, int oparg) { - _Py_SetImmortal(executor); + _Py_SetImmortalUntracked((PyObject *)executor); Py_SET_TYPE(executor, &_PyUOpExecutor_Type); executor->trace = (_PyUOpInstruction *)executor->exits; executor->code_size = 1; diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 483f28b46dfec7..656667ac93970c 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1753,8 +1753,11 @@ def is_waiting_for_gil(self): return (name == 'take_gil') def is_gc_collect(self): - '''Is this frame gc_collect_main() within the garbage-collector?''' - return self._gdbframe.name() in ('collect', 'gc_collect_main') + '''Is this frame a collector within the garbage-collector?''' + return self._gdbframe.name() in ( + 'collect', 'gc_collect_full', 'gc_collect_main', + 'gc_collect_young', 'gc_collect_increment', + ) def get_pyop(self): try: From 8182319de33a9519a2f243ac8c35a20ef82a4d2d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 20 Mar 2024 13:43:20 +0300 Subject: [PATCH 49/82] gh-94808: add tests covering `PyFunction_{Get,Set}Closure` (GH-99429) --- Lib/test/test_capi/test_misc.py | 120 +++++++++++++++++++++++++++++++- Modules/_testcapimodule.c | 29 ++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 7365ead1c4749c..5b4f67e7f5f58d 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1170,7 +1170,6 @@ class MyType: self.assertEqual(get_type_fullyqualname(MyType), 'my_qualname') - @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): """Test API for extending opaque types (PEP 697)""" @@ -1326,6 +1325,125 @@ def test_pyobject_getitemdata_error(self): _testcapi.pyobject_getitemdata(0) + def test_function_get_closure(self): + from types import CellType + + def regular_function(): ... + def unused_one_level(arg1): + def inner(arg2, arg3): ... + return inner + def unused_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): ... + return inner + return decorator + def with_one_level(arg1): + def inner(arg2, arg3): + return arg1 + arg2 + arg3 + return inner + def with_two_levels(arg1, arg2): + def decorator(arg3, arg4): + def inner(arg5, arg6): + return arg1 + arg2 + arg3 + arg4 + arg5 + arg6 + return inner + return decorator + + # Functions without closures: + self.assertIsNone(_testcapi.function_get_closure(regular_function)) + self.assertIsNone(regular_function.__closure__) + + func = unused_one_level(1) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + func = unused_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertIsNone(closure) + self.assertIsNone(func.__closure__) + + # Functions with closures: + func = with_one_level(5) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 1) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertTrue(closure[0].cell_contents, 5) + + func = with_two_levels(1, 2)(3, 4) + closure = _testcapi.function_get_closure(func) + self.assertEqual(closure, func.__closure__) + self.assertIsInstance(closure, tuple) + self.assertEqual(len(closure), 4) + self.assertEqual(len(closure), len(func.__code__.co_freevars)) + self.assertTrue(all(isinstance(cell, CellType) for cell in closure)) + self.assertEqual([cell.cell_contents for cell in closure], + [1, 2, 3, 4]) + + def test_function_get_closure_error(self): + with self.assertRaises(SystemError): + _testcapi.function_get_closure(1) + with self.assertRaises(SystemError): + _testcapi.function_get_closure(None) + + def test_function_set_closure(self): + from types import CellType + + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + func = function_without_closure + _testcapi.function_set_closure(func, (CellType(1), CellType(1))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 1]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 1]) + + func = function_with_closure(1) + _testcapi.function_set_closure(func, + (CellType(1), CellType(2), CellType(3))) + closure = _testcapi.function_get_closure(func) + self.assertEqual([c.cell_contents for c in closure], [1, 2, 3]) + self.assertEqual([c.cell_contents for c in func.__closure__], [1, 2, 3]) + + def test_function_set_closure_none(self): + def function_without_closure(): ... + def function_with_closure(arg): + def inner(): + return arg + return inner + + _testcapi.function_set_closure(function_without_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_without_closure)) + self.assertIsNone(function_without_closure.__closure__) + + _testcapi.function_set_closure(function_with_closure, None) + self.assertIsNone( + _testcapi.function_get_closure(function_with_closure)) + self.assertIsNone(function_with_closure.__closure__) + + def test_function_set_closure_errors(self): + def function_without_closure(): ... + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(None, ()) # not a function + + with self.assertRaises(SystemError): + _testcapi.function_set_closure(function_without_closure, 1) + self.assertIsNone(function_without_closure.__closure__) # no change + + # NOTE: this works, but goes against the docs: + _testcapi.function_set_closure(function_without_closure, (1, 2)) + self.assertEqual( + _testcapi.function_get_closure(function_without_closure), (1, 2)) + self.assertEqual(function_without_closure.__closure__, (1, 2)) + + class TestPendingCalls(unittest.TestCase): # See the comment in ceval.c (at the "handle_eval_breaker" label) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 7928cd7d6fe1ae..9621c654a7713f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3094,6 +3094,33 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +function_get_closure(PyObject *self, PyObject *func) +{ + PyObject *closure = PyFunction_GetClosure(func); + if (closure != NULL) { + return Py_NewRef(closure); + } else if (PyErr_Occurred()) { + return NULL; + } else { + Py_RETURN_NONE; // This can happen when `closure` is set to `None` + } +} + +static PyObject * +function_set_closure(PyObject *self, PyObject *args) +{ + PyObject *func = NULL, *closure = NULL; + if (!PyArg_ParseTuple(args, "OO", &func, &closure)) { + return NULL; + } + int result = PyFunction_SetClosure(func, closure); + if (result == -1) { + return NULL; + } + Py_RETURN_NONE; +} + static PyObject * check_pyimport_addmodule(PyObject *self, PyObject *args) { @@ -3379,6 +3406,8 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, + {"function_get_closure", function_get_closure, METH_O, NULL}, + {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, {NULL, NULL} /* sentinel */ From 519b2ae22b54760475bbf62b9558d453c703f9c6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 20 Mar 2024 15:39:53 +0200 Subject: [PATCH 50/82] gh-117021: Fix integer overflow in PyLong_AsPid() on non-Windows 64-bit platforms (GH-117064) --- Include/Python.h | 2 +- Include/longobject.h | 19 ++++++++++++- Lib/test/test_capi/test_long.py | 28 +++++++++++++++++++ ...-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst | 2 ++ Modules/_testcapi/long.c | 12 ++++++++ Modules/_testcapimodule.c | 1 + Modules/_testlimitedcapi/long.c | 12 ++++++++ 7 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst diff --git a/Include/Python.h b/Include/Python.h index 01fc45137a17bb..ca38a98d8c4eca 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -63,6 +63,7 @@ #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" +#include "pyerrors.h" #include "longobject.h" #include "cpython/longintrepr.h" #include "boolobject.h" @@ -99,7 +100,6 @@ #include "cpython/picklebufobject.h" #include "cpython/pytime.h" #include "codecs.h" -#include "pyerrors.h" #include "pythread.h" #include "cpython/context.h" #include "modsupport.h" diff --git a/Include/longobject.h b/Include/longobject.h index 51005efff636fa..19104cd9d1bef9 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -40,7 +40,24 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); #if !defined(SIZEOF_PID_T) || SIZEOF_PID_T == SIZEOF_INT #define _Py_PARSE_PID "i" #define PyLong_FromPid PyLong_FromLong -#define PyLong_AsPid PyLong_AsLong +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define PyLong_AsPid PyLong_AsInt +# elif SIZEOF_INT == SIZEOF_LONG +# define PyLong_AsPid PyLong_AsLong +# else +static inline int +PyLong_AsPid(PyObject *obj) +{ + int overflow; + long result = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow || result > INT_MAX || result < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return -1; + } + return (int)result; +} +# endif #elif SIZEOF_PID_T == SIZEOF_LONG #define _Py_PARSE_PID "l" #define PyLong_FromPid PyLong_FromLong diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 4ac6ea6b725ff1..d2140154d811b4 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -425,6 +425,34 @@ def test_long_asvoidptr(self): self.assertRaises(OverflowError, asvoidptr, -2**1000) # CRASHES asvoidptr(NULL) + def _test_long_aspid(self, aspid): + # Test PyLong_AsPid() + from _testcapi import SIZEOF_PID_T + bits = 8 * SIZEOF_PID_T + PID_T_MIN = -2**(bits-1) + PID_T_MAX = 2**(bits-1) - 1 + # round trip (object -> long -> object) + for value in (PID_T_MIN, PID_T_MAX, -1, 0, 1, 1234): + with self.subTest(value=value): + self.assertEqual(aspid(value), value) + + self.assertEqual(aspid(IntSubclass(42)), 42) + self.assertEqual(aspid(Index(42)), 42) + self.assertEqual(aspid(MyIndexAndInt()), 10) + + self.assertRaises(OverflowError, aspid, PID_T_MIN - 1) + self.assertRaises(OverflowError, aspid, PID_T_MAX + 1) + self.assertRaises(TypeError, aspid, 1.0) + self.assertRaises(TypeError, aspid, b'2') + self.assertRaises(TypeError, aspid, '3') + self.assertRaises(SystemError, aspid, NULL) + + def test_long_aspid(self): + self._test_long_aspid(_testcapi.pylong_aspid) + + def test_long_aspid_limited(self): + self._test_long_aspid(_testlimitedcapi.pylong_aspid) + def test_long_asnativebytes(self): import math from _testcapi import ( diff --git a/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst b/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst new file mode 100644 index 00000000000000..2f93e1e6da00aa --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-20-13-13-22.gh-issue-117021.0Q5jBx.rst @@ -0,0 +1,2 @@ +Fix integer overflow in :c:func:`PyLong_AsPid` on non-Windows 64-bit +platforms. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 8e4e1f2246f725..28dca01bee09a0 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -92,12 +92,24 @@ pylong_fromnativebytes(PyObject *module, PyObject *args) return res; } +static PyObject * +pylong_aspid(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + pid_t value = PyLong_AsPid(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromPid(value); +} + static PyMethodDef test_methods[] = { _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, {"pylong_asnativebytes", pylong_asnativebytes, METH_VARARGS}, {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, + {"pylong_aspid", pylong_aspid, METH_O}, {NULL}, }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 9621c654a7713f..b73085bb8f67ce 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3975,6 +3975,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_VOID_P", PyLong_FromSsize_t(sizeof(void*))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); + PyModule_AddObject(m, "SIZEOF_PID_T", PyLong_FromSsize_t(sizeof(pid_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index 16d41b1d4b16dc..5953009b6ef9b7 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -746,6 +746,17 @@ pylong_asvoidptr(PyObject *module, PyObject *arg) return Py_NewRef((PyObject *)value); } +static PyObject * +pylong_aspid(PyObject *module, PyObject *arg) +{ + NULLABLE(arg); + pid_t value = PyLong_AsPid(arg); + if (value == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromPid(value); +} + static PyMethodDef test_methods[] = { _TESTLIMITEDCAPI_TEST_LONG_AND_OVERFLOW_METHODDEF @@ -773,6 +784,7 @@ static PyMethodDef test_methods[] = { {"pylong_as_size_t", pylong_as_size_t, METH_O}, {"pylong_asdouble", pylong_asdouble, METH_O}, {"pylong_asvoidptr", pylong_asvoidptr, METH_O}, + {"pylong_aspid", pylong_aspid, METH_O}, {NULL}, }; From fc4599800778f9b130d5e336deadbdeb5bd3e5ee Mon Sep 17 00:00:00 2001 From: jkriegshauser Date: Wed, 20 Mar 2024 07:33:28 -0700 Subject: [PATCH 51/82] gh-116773: Ensure overlapped objects on Windows are not deallocated too early by asyncio (GH-116774) --- Lib/asyncio/windows_events.py | 14 +++--- Lib/test/test_asyncio/test_windows_events.py | 50 +++++++++++++++++-- ...-03-14-01-58-22.gh-issue-116773.H2UldY.rst | 1 + Modules/overlapped.c | 18 +++++++ 4 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index b62ea75fee3858..aca063e54c73f6 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -324,13 +324,13 @@ def _run_forever_cleanup(self): if self._self_reading_future is not None: ov = self._self_reading_future._ov self._self_reading_future.cancel() - # self_reading_future was just cancelled so if it hasn't been - # finished yet, it never will be (it's possible that it has - # already finished and its callback is waiting in the queue, - # where it could still happen if the event loop is restarted). - # Unregister it otherwise IocpProactor.close will wait for it - # forever - if ov is not None: + # self_reading_future always uses IOCP, so even though it's + # been cancelled, we need to make sure that the IOCP message + # is received so that the kernel is not holding on to the + # memory, possibly causing memory corruption later. Only + # unregister it if IO is complete in all respects. Otherwise + # we need another _poll() later to complete the IO. + if ov is not None and not ov.pending: self._proactor._unregister(ov) self._self_reading_future = None diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 6e6c90a247b291..0c128c599ba011 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -36,7 +36,23 @@ def data_received(self, data): self.trans.close() -class ProactorLoopCtrlC(test_utils.TestCase): +class WindowsEventsTestCase(test_utils.TestCase): + def _unraisablehook(self, unraisable): + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self._unraisable = unraisable + print(unraisable) + + def setUp(self): + self._prev_unraisablehook = sys.unraisablehook + self._unraisable = None + sys.unraisablehook = self._unraisablehook + + def tearDown(self): + sys.unraisablehook = self._prev_unraisablehook + self.assertIsNone(self._unraisable) + +class ProactorLoopCtrlC(WindowsEventsTestCase): def test_ctrl_c(self): @@ -58,7 +74,7 @@ def SIGINT_after_delay(): thread.join() -class ProactorMultithreading(test_utils.TestCase): +class ProactorMultithreading(WindowsEventsTestCase): def test_run_from_nonmain_thread(self): finished = False @@ -79,7 +95,7 @@ def func(): self.assertTrue(finished) -class ProactorTests(test_utils.TestCase): +class ProactorTests(WindowsEventsTestCase): def setUp(self): super().setUp() @@ -283,8 +299,32 @@ async def probe(): return "done" - -class WinPolicyTests(test_utils.TestCase): + def test_loop_restart(self): + # We're fishing for the "RuntimeError: <_overlapped.Overlapped object at XXX> + # still has pending operation at deallocation, the process may crash" error + stop = threading.Event() + def threadMain(): + while not stop.is_set(): + self.loop.call_soon_threadsafe(lambda: None) + time.sleep(0.01) + thr = threading.Thread(target=threadMain) + + # In 10 60-second runs of this test prior to the fix: + # time in seconds until failure: (none), 15.0, 6.4, (none), 7.6, 8.3, 1.7, 22.2, 23.5, 8.3 + # 10 seconds had a 50% failure rate but longer would be more costly + end_time = time.time() + 10 # Run for 10 seconds + self.loop.call_soon(thr.start) + while not self._unraisable: # Stop if we got an unraisable exc + self.loop.stop() + self.loop.run_forever() + if time.time() >= end_time: + break + + stop.set() + thr.join() + + +class WinPolicyTests(WindowsEventsTestCase): def test_selector_win_policy(self): async def main(): diff --git a/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst b/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst new file mode 100644 index 00000000000000..8fc3fe80041d26 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-03-14-01-58-22.gh-issue-116773.H2UldY.rst @@ -0,0 +1 @@ +Fix instances of ``<_overlapped.Overlapped object at 0xXXX> still has pending operation at deallocation, the process may crash``. diff --git a/Modules/overlapped.c b/Modules/overlapped.c index fd40e91d0f50c4..4c6c7531cdeab0 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -723,6 +723,24 @@ Overlapped_dealloc(OverlappedObject *self) if (!HasOverlappedIoCompleted(&self->overlapped) && self->type != TYPE_NOT_STARTED) { + // NOTE: We should not get here, if we do then something is wrong in + // the IocpProactor or ProactorEventLoop. Since everything uses IOCP if + // the overlapped IO hasn't completed yet then we should not be + // deallocating! + // + // The problem is likely that this OverlappedObject was removed from + // the IocpProactor._cache before it was complete. The _cache holds a + // reference while IO is pending so that it does not get deallocated + // while the kernel has retained the OVERLAPPED structure. + // + // CancelIoEx (likely called from self.cancel()) may have successfully + // completed, but the OVERLAPPED is still in use until either + // HasOverlappedIoCompleted() is true or GetQueuedCompletionStatus has + // returned this OVERLAPPED object. + // + // NOTE: Waiting when IOCP is in use can hang indefinitely, but this + // CancelIoEx is superfluous in that self.cancel() was already called, + // so I've only ever seen this return FALSE with GLE=ERROR_NOT_FOUND Py_BEGIN_ALLOW_THREADS if (CancelIoEx(self->handle, &self->overlapped)) wait = TRUE; From 9221ef2d8cb7f4cf37592eb650d4c8f972033000 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Wed, 20 Mar 2024 08:18:26 -0700 Subject: [PATCH 52/82] gh-116908: Only write to `_pending_calls.calls_to_do` with atomic operations (#117044) These writes to `pending->calls_to_do` need to be atomic, because other threads can read (atomically) from `calls_to_do` without holding `pending->mutex`. --- Python/ceval_gil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index d2cd35dfa86833..78c13d619e6ee0 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -671,7 +671,7 @@ _push_pending_call(struct _pending_calls *pending, pending->calls[i].flags = flags; pending->last = j; assert(pending->calls_to_do < NPENDINGCALLS); - pending->calls_to_do++; + _Py_atomic_add_int32(&pending->calls_to_do, 1); return 0; } @@ -701,7 +701,7 @@ _pop_pending_call(struct _pending_calls *pending, pending->calls[i] = (struct _pending_call){0}; pending->first = (i + 1) % NPENDINGCALLS; assert(pending->calls_to_do > 0); - pending->calls_to_do--; + _Py_atomic_add_int32(&pending->calls_to_do, -1); } } From 44fbab43d8f3f2df07091d237824cf4fa1f6c57c Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 20 Mar 2024 23:32:56 +0800 Subject: [PATCH 53/82] gh-117058: Update GUI and packaging recommendations for macOS. (#117059) --- Doc/using/mac.rst | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index e99993238895f9..8f3372b8e017f5 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -152,26 +152,41 @@ Tk toolkit (https://www.tcl.tk). An Aqua-native version of Tk is bundled with macOS by Apple, and the latest version can be downloaded and installed from https://www.activestate.com; it can also be built from source. -*wxPython* is another popular cross-platform GUI toolkit that runs natively on -macOS. Packages and documentation are available from https://www.wxpython.org. +A number of alternative macOS GUI toolkits are available: -*PyQt* is another popular cross-platform GUI toolkit that runs natively on -macOS. More information can be found at -https://riverbankcomputing.com/software/pyqt/intro. +* `PySide `__: Official Python bindings to the + `Qt GUI toolkit `__. -*PySide* is another cross-platform Qt-based toolkit. More information at -https://www.qt.io/qt-for-python. +* `PyQt `__: Alternative + Python bindings to Qt. +* `Kivy `__: A cross-platform GUI toolkit that supports + desktop and mobile platforms. + +* `Toga `__: Part of the `BeeWare Project + `__; supports desktop, mobile, web and console apps. + +* `wxPython `__: A cross-platform toolkit that + supports desktop operating systems. .. _distributing-python-applications-on-the-mac: Distributing Python Applications ================================ -The standard tool for deploying standalone Python applications on the Mac is -:program:`py2app`. More information on installing and using :program:`py2app` -can be found at https://pypi.org/project/py2app/. +A range of tools exist for converting your Python code into a standalone +distributable application: + +* `py2app `__: Supports creating macOS ``.app`` + bundles from a Python project. +* `Briefcase `__: Part of the `BeeWare Project + `__; a cross-platform packaging tool that supports + creation of ``.app`` bundles on macOS, as well as managing signing and + notarization. + +* `PyInstaller `__: A cross-platform packaging tool that creates + a single file or folder as a distributable artifact. Other Resources =============== @@ -184,4 +199,3 @@ https://www.python.org/community/sigs/current/pythonmac-sig/ Another useful resource is the MacPython wiki: https://wiki.python.org/moin/MacPython - From dcaf33a41d5d220523d71c9b35bc08f5b8405dac Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 20 Mar 2024 17:33:08 +0100 Subject: [PATCH 54/82] gh-114314: ctypes: remove stgdict and switch to heap types (GH-116458) Before this change, ctypes classes used a custom dict subclass, `StgDict`, as their `tp_dict`. This acts like a regular dict but also includes extra information about the type. This replaces stgdict by `StgInfo`, a C struct on the type, accessed by `PyObject_GetTypeData()` (PEP-697). All usage of `StgDict` (mainly variables named `stgdict`, `dict`, `edict` etc.) is converted to `StgInfo` (named `stginfo`, `info`, `einfo`, etc.). Where the dict is actually used for class attributes (as a regular PyDict), it's now called `attrdict`. This change -- not overriding `tp_dict` -- is made to make me comfortable with the next part of this PR: moving the initialization logic from `tp_new` to `tp_init`. The `StgInfo` is set up in `__init__` of each class, with a guard that prevents calling `__init__` more than once. Note that abstract classes (like `Array` or `Structure`) are created using `PyType_FromMetaclass` and do not have `__init__` called. Previously, this was done in `__new__`, which also wasn't called for abstract classes. Since `__init__` can be called from Python code or skipped, there is a tested guard to ensure `StgInfo` is initialized exactly once before it's used. Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> Co-authored-by: Erlend E. Aasland --- Lib/test/test_ctypes/test_arrays.py | 16 + Lib/test/test_ctypes/test_callbacks.py | 2 +- Lib/test/test_ctypes/test_funcptr.py | 6 + Lib/test/test_ctypes/test_pointers.py | 5 + Lib/test/test_ctypes/test_simplesubclasses.py | 23 + Lib/test/test_ctypes/test_struct_fields.py | 2 +- Lib/test/test_ctypes/test_structures.py | 25 +- Lib/test/test_ctypes/test_unions.py | 19 +- ...-03-07-11-10-27.gh-issue-114314.iEhAMH.rst | 3 + Modules/_ctypes/_ctypes.c | 1986 +++++++++-------- Modules/_ctypes/callbacks.c | 42 +- Modules/_ctypes/callproc.c | 124 +- Modules/_ctypes/cfield.c | 55 +- Modules/_ctypes/ctypes.h | 172 +- Modules/_ctypes/stgdict.c | 427 ++-- 15 files changed, 1496 insertions(+), 1411 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 774316e227ff73..3568cf97f40b50 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -37,6 +37,22 @@ def test_type_flags(self): self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index 64f92ffdca6a3f..8038169afe4304 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -106,7 +106,7 @@ def test_pyobject(self): def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 0eed39484fb39e..03cfddea6ea61a 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -29,6 +29,12 @@ def test_type_flags(self): self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 8cf2114c282cab..3a5f3660dbbe23 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -33,6 +33,11 @@ def test_type_flags(self): self.assertTrue(_Pointer.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_Pointer.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCPointerType.__init__(POINTER(c_byte), 'ptr', (), {}) + def test_pointer_crash(self): class A(POINTER(c_ulong)): diff --git a/Lib/test/test_ctypes/test_simplesubclasses.py b/Lib/test/test_ctypes/test_simplesubclasses.py index c96798e67f23f7..4e4bef3690f66a 100644 --- a/Lib/test/test_ctypes/test_simplesubclasses.py +++ b/Lib/test/test_ctypes/test_simplesubclasses.py @@ -26,6 +26,29 @@ def test_type_flags(self): self.assertTrue(_SimpleCData.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(_SimpleCData.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewT = PyCSimpleType.__new__(PyCSimpleType, 'NewT', (_SimpleCData,), {}) + for cls in _SimpleCData, NewT: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(_SimpleCData): + _type_ = "i" + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCSimpleType.__init__(T, 'ptr', (), {}) + + def test_swapped_type_creation(self): + cls = PyCSimpleType.__new__(PyCSimpleType, '', (), {'_type_': 'i'}) + with self.assertRaises(TypeError): + PyCSimpleType.__init__(cls) + PyCSimpleType.__init__(cls, '', (), {'_type_': 'i'}) + self.assertEqual(cls.__ctype_le__.__dict__.get('_type_'), 'i') + self.assertEqual(cls.__ctype_be__.__dict__.get('_type_'), 'i') + def test_compare(self): self.assertEqual(MyInt(3), MyInt(3)) self.assertNotEqual(MyInt(42), MyInt(43)) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index f474a02fa8db06..7adab794809def 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -69,7 +69,7 @@ def test_cfield_inheritance_hierarchy(self): def test_gh99275(self): class BrokenStructure(Structure): def __init_subclass__(cls, **kwargs): - cls._fields_ = [] # This line will fail, `stgdict` is not ready + cls._fields_ = [] # This line will fail, `stginfo` is not ready with self.assertRaisesRegex(TypeError, 'ctypes state is not initialized'): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 98bc4bdcac9306..8d83ce4f281b16 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -85,6 +85,23 @@ def test_type_flags(self): self.assertTrue(Structure.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(Structure.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewStructure = PyCStructType.__new__(PyCStructType, 'NewStructure', + (Structure,), {}) + for cls in Structure, NewStructure: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Structure): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCStructType.__init__(T, 'ptr', (), {}) + def test_simple_structs(self): for code, tp in self.formats.items(): class X(Structure): @@ -507,8 +524,8 @@ def _test_issue18060(self, Vector): @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_a(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each - # _fields_ assignment, and PyCStgDict_clone() + # PyCStructUnionType_update_stginfo() for each + # _fields_ assignment, and PyCStgInfo_clone() # for the Mid and Vector class definitions. class Base(Structure): _fields_ = [('y', c_double), @@ -523,7 +540,7 @@ class Vector(Mid): pass @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_b(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each + # PyCStructUnionType_update_stginfo() for each # _fields_ assignment. class Base(Structure): _fields_ = [('y', c_double), @@ -538,7 +555,7 @@ class Vector(Mid): @unittest.skipUnless(sys.byteorder == 'little', "can't test on this platform") def test_issue18060_c(self): # This test case calls - # PyCStructUnionType_update_stgdict() for each + # PyCStructUnionType_update_stginfo() for each # _fields_ assignment. class Base(Structure): _fields_ = [('y', c_double)] diff --git a/Lib/test/test_ctypes/test_unions.py b/Lib/test/test_ctypes/test_unions.py index cf5344bdf19165..e2dff0f22a9213 100644 --- a/Lib/test/test_ctypes/test_unions.py +++ b/Lib/test/test_ctypes/test_unions.py @@ -1,5 +1,5 @@ import unittest -from ctypes import Union +from ctypes import Union, c_char from ._support import (_CData, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -16,3 +16,20 @@ def test_type_flags(self): with self.subTest(cls=Union): self.assertTrue(Union.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) self.assertFalse(Union.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewUnion = UnionType.__new__(UnionType, 'NewUnion', + (Union,), {}) + for cls in Union, NewUnion: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Union): + _fields_ = [("x", c_char), + ("y", c_char)] + with self.assertRaisesRegex(SystemError, "already initialized"): + UnionType.__init__(T, 'ptr', (), {}) diff --git a/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst b/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst new file mode 100644 index 00000000000000..c241d966f9087d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-07-11-10-27.gh-issue-114314.iEhAMH.rst @@ -0,0 +1,3 @@ +In :mod:`ctypes`, ctype data is now stored in type objects directly rather +than in a dict subclass. This is an internal change that should not affect +usage. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 94245ae41afffc..af094a0fb59e27 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2,7 +2,7 @@ ToDo: Get rid of the checker (and also the converters) field in PyCFuncPtrObject and - StgDictObject, and replace them by slot functions in StgDictObject. + StgInfo, and replace them by slot functions in StgInfo. think about a buffer-like object (memory? bytes?) @@ -36,7 +36,6 @@ PyCData_Type Simple_Type __new__(), __init__(), _as_parameter_ PyCField_Type -PyCStgDict_Type ============================================================================== @@ -82,7 +81,6 @@ bytes(cdata) */ /* - * PyCStgDict_Type * PyCStructType_Type * UnionType_Type * PyCPointerType_Type @@ -128,20 +126,7 @@ bytes(cdata) #include "pycore_long.h" // _PyLong_GetZero() -static PyTypeObject Union_Type; -static PyTypeObject Struct_Type; -static PyTypeObject Simple_Type; - -ctypes_state global_state = { - .PyCStgDict_Type = &PyCStgDict_Type, - .PyCData_Type = &PyCData_Type, - .Struct_Type = &Struct_Type, - .Union_Type = &Union_Type, - .PyCArray_Type = &PyCArray_Type, - .Simple_Type = &Simple_Type, - .PyCPointer_Type = &PyCPointer_Type, - .PyCFuncPtr_Type = &PyCFuncPtr_Type, -}; +ctypes_state global_state = {0}; PyObject *PyExc_ArgError = NULL; @@ -459,12 +444,121 @@ static PyType_Spec structparam_spec = { .slots = structparam_slots, }; +/* + CType_Type - a base metaclass. Its instances (classes) have a StgInfo. + */ + +static int +CType_Type_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +_ctype_clear_stginfo(StgInfo *info) +{ + assert(info); + Py_CLEAR(info->proto); + Py_CLEAR(info->argtypes); + Py_CLEAR(info->converters); + Py_CLEAR(info->restype); + Py_CLEAR(info->checker); +} + +static int +CType_Type_clear(PyObject *self) +{ + ctypes_state *st = GLOBAL_STATE(); + if (st && st->PyCType_Type) { + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return -1; + } + if (info) { + _ctype_clear_stginfo(info); + } + } + return 0; +} + +static void +CType_Type_dealloc(PyObject *self) +{ + ctypes_state *st = GLOBAL_STATE(); + + if (st && st->PyCType_Type) { + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + PyErr_WriteUnraisable(self); + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + _ctype_clear_stginfo(info); + } + } + + PyTypeObject *tp = Py_TYPE(self); + PyType_Type.tp_dealloc(self); + Py_DECREF(tp); +} + +static PyObject * +CType_Type_sizeof(PyObject *self) +{ + Py_ssize_t size = Py_TYPE(self)->tp_basicsize; + size += Py_TYPE(self)->tp_itemsize * Py_SIZE(self); + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, self, &info) < 0) { + return NULL; + } + if (info) { + if (info->format) { + size += strlen(info->format) + 1; + } + if (info->ffi_type_pointer.elements) { + size += (info->length + 1) * sizeof(ffi_type *); + } + size += info->ndim * sizeof(Py_ssize_t); + } + + return PyLong_FromSsize_t(size); +} + +static PyMethodDef ctype_methods[] = { + {"__sizeof__", _PyCFunction_CAST(CType_Type_sizeof), + METH_NOARGS, PyDoc_STR("Return memory consumption of the type object.")}, + {0}, +}; + +static PyType_Slot ctype_type_slots[] = { + {Py_tp_traverse, CType_Type_traverse}, + {Py_tp_clear, CType_Type_clear}, + {Py_tp_dealloc, CType_Type_dealloc}, + {Py_tp_methods, ctype_methods}, + {0, NULL}, +}; + +static PyType_Spec pyctype_type_spec = { + .name = "_ctypes.CType_Type", + .basicsize = -(Py_ssize_t)sizeof(StgInfo), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_BASETYPE ), + .slots = ctype_type_slots, +}; /* PyCStructType_Type - a meta type/class. Creating a new class using this one as - __metaclass__ will call the constructor StructUnionType_new. It replaces the - tp_dict member with a new instance of StgDict, and initializes the C - accessible fields somehow. + __metaclass__ will call the constructor StructUnionType_new. + It initializes the C accessible fields somehow. */ static PyCArgObject * @@ -472,7 +566,6 @@ StructUnionType_paramfunc(CDataObject *self) { PyCArgObject *parg; PyObject *obj; - StgDictObject *stgdict; void *ptr; if ((size_t)self->b_size > sizeof(void*)) { @@ -507,104 +600,104 @@ StructUnionType_paramfunc(CDataObject *self) return NULL; } + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + Py_DECREF(obj); + return NULL; + } + assert(stginfo); /* Cannot be NULL for structure/union instances */ + parg->tag = 'V'; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for structure/union instances */ - parg->pffi_type = &stgdict->ffi_type_pointer; + parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; parg->obj = obj; return parg; } -static PyObject * -StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isStruct) +static int +StructUnionType_init(PyObject *self, PyObject *args, PyObject *kwds, int isStruct) { - PyTypeObject *result; PyObject *fields; - StgDictObject *dict; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (!result) - return NULL; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } /* keep this for bw compatibility */ - int r = PyDict_Contains(result->tp_dict, &_Py_ID(_abstract_)); + int r = PyDict_Contains(attrdict, &_Py_ID(_abstract_)); if (r > 0) { - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } if (r < 0) { - Py_DECREF(result); - return NULL; + Py_DECREF(attrdict); + return -1; } ctypes_state *st = GLOBAL_STATE(); - dict = (StgDictObject *)_PyObject_CallNoArgs((PyObject *)st->PyCStgDict_Type); - if (!dict) { - Py_DECREF(result); - return NULL; + StgInfo *info = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!info) { + Py_DECREF(attrdict); + return -1; } if (!isStruct) { - dict->flags |= TYPEFLAG_HASUNION; - } - /* replace the class dict by our updated stgdict, which holds info - about storage requirements of the instances */ - if (-1 == PyDict_Update((PyObject *)dict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)dict); - return NULL; + info->flags |= TYPEFLAG_HASUNION; } - Py_SETREF(result->tp_dict, (PyObject *)dict); - dict->format = _ctypes_alloc_format_string(NULL, "B"); - if (dict->format == NULL) { - Py_DECREF(result); - return NULL; + + info->format = _ctypes_alloc_format_string(NULL, "B"); + if (info->format == NULL) { + Py_DECREF(attrdict); + return -1; } - dict->paramfunc = StructUnionType_paramfunc; + info->paramfunc = StructUnionType_paramfunc; - if (PyDict_GetItemRef((PyObject *)dict, &_Py_ID(_fields_), &fields) < 0) { - Py_DECREF(result); - return NULL; + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_fields_), &fields) < 0) { + Py_DECREF(attrdict); + return -1; } + Py_CLEAR(attrdict); if (fields) { - if (PyObject_SetAttr((PyObject *)result, &_Py_ID(_fields_), fields) < 0) { - Py_DECREF(result); + if (PyObject_SetAttr(self, &_Py_ID(_fields_), fields) < 0) { Py_DECREF(fields); - return NULL; + return -1; } Py_DECREF(fields); - return (PyObject *)result; + return 0; } else { - StgDictObject *basedict = PyType_stgdict((PyObject *)result->tp_base); - - if (basedict == NULL) { - return (PyObject *)result; + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)self)->tp_base, + &baseinfo) < 0) { + return -1; } - /* copy base dict */ - if (-1 == PyCStgDict_clone(dict, basedict)) { - Py_DECREF(result); - return NULL; + if (baseinfo == NULL) { + return 0; } - dict->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass dict */ - basedict->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass dict */ - return (PyObject *)result; + + /* copy base info */ + if (PyCStgInfo_clone(info, baseinfo) < 0) { + return -1; + } + info->flags &= ~DICTFLAG_FINAL; /* clear the 'final' flag in the subclass info */ + baseinfo->flags |= DICTFLAG_FINAL; /* set the 'final' flag in the baseclass info */ } + return 0; } -static PyObject * -PyCStructType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCStructType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 1); + return StructUnionType_init(self, args, kwds, 1); } -static PyObject * -UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +UnionType_init(PyObject *self, PyObject *args, PyObject *kwds) { - return StructUnionType_new(type, args, kwds, 0); + return StructUnionType_init(self, args, kwds, 0); } PyDoc_STRVAR(from_address_doc, @@ -640,8 +733,12 @@ CDataType_from_buffer(PyObject *type, PyObject *args) Py_buffer *buffer; Py_ssize_t offset = 0; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } @@ -676,11 +773,11 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return NULL; } - if (dict->size > buffer->len - offset) { + if (info->size > buffer->len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small " "(%zd instead of at least %zd bytes)", - buffer->len, dict->size + offset); + buffer->len, info->size + offset); Py_DECREF(mv); return NULL; } @@ -717,8 +814,13 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) Py_buffer buffer; Py_ssize_t offset = 0; PyObject *result; - StgDictObject *dict = PyType_stgdict(type); - if (!dict) { + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } @@ -733,10 +835,10 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) return NULL; } - if (dict->size > buffer.len - offset) { + if (info->size > buffer.len - offset) { PyErr_Format(PyExc_ValueError, "Buffer size too small (%zd instead of at least %zd bytes)", - buffer.len, dict->size + offset); + buffer.len, info->size + offset); PyBuffer_Release(&buffer); return NULL; } @@ -750,7 +852,7 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) result = GenericPyCData_new((PyTypeObject *)type, NULL, NULL); if (result != NULL) { memcpy(((CDataObject *)result)->b_ptr, - (char *)buffer.buf + offset, dict->size); + (char *)buffer.buf + offset, info->size); } PyBuffer_Release(&buffer); return result; @@ -836,13 +938,14 @@ CDataType_from_param(PyObject *type, PyObject *value) PyCArgObject *p = (PyCArgObject *)value; PyObject *ob = p->obj; const char *ob_name; - StgDictObject *dict; - dict = PyType_stgdict(type); - + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } /* If we got a PyCArgObject, we must check if the object packed in it - is an instance of the type's dict->proto */ - if(dict && ob) { - res = PyObject_IsInstance(ob, dict->proto); + is an instance of the type's info->proto */ + if(info && ob) { + res = PyObject_IsInstance(ob, info->proto); if (res == -1) return NULL; if (res) { @@ -893,18 +996,27 @@ CDataType_repeat(PyObject *self, Py_ssize_t length) static int CDataType_clear(PyTypeObject *self) { - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) - Py_CLEAR(dict->proto); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + return -1; + } + if (info) { + Py_CLEAR(info->proto); + } return PyType_Type.tp_clear((PyObject *)self); } static int CDataType_traverse(PyTypeObject *self, visitproc visit, void *arg) { - StgDictObject *dict = PyType_stgdict((PyObject *)self); - if (dict) { - Py_VISIT(dict->proto); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + return -1; + } + if (info) { + Py_VISIT(info->proto); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse((PyObject *)self, visit, arg); @@ -919,7 +1031,7 @@ PyCStructType_setattro(PyObject *self, PyObject *key, PyObject *value) if (value && PyUnicode_Check(key) && _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 1); + return PyCStructUnionType_update_stginfo(self, value, 1); return 0; } @@ -933,7 +1045,7 @@ UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) if (PyUnicode_Check(key) && _PyUnicode_EqualToASCIIString(key, "_fields_")) - return PyCStructUnionType_update_stgdict(self, value, 0); + return PyCStructUnionType_update_stginfo(self, value, 0); return 0; } @@ -943,14 +1055,14 @@ static PyType_Slot pycstruct_type_slots[] = { {Py_tp_traverse, CDataType_traverse}, {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCStructType_new}, + {Py_tp_init, PyCStructType_init}, // Sequence protocol. {Py_sq_repeat, CDataType_repeat}, {0, NULL}, }; -PyType_Spec pycstruct_type_spec = { +static PyType_Spec pycstruct_type_spec = { .name = "_ctypes.PyCStructType", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), @@ -963,7 +1075,7 @@ static PyType_Slot union_type_slots[] = { {Py_tp_traverse, CDataType_traverse}, {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, UnionType_new}, + {Py_tp_init, UnionType_init}, // Sequence protocol. {Py_sq_repeat, CDataType_repeat}, @@ -994,20 +1106,25 @@ size property/method, and the sequence protocol. */ static int -PyCPointerType_SetProto(StgDictObject *stgdict, PyObject *proto) +PyCPointerType_SetProto(StgInfo *stginfo, PyObject *proto) { + ctypes_state *st = GLOBAL_STATE(); if (!proto || !PyType_Check(proto)) { PyErr_SetString(PyExc_TypeError, "_type_ must be a type"); return -1; } - if (!PyType_stgdict(proto)) { + StgInfo *info; + if (PyStgInfo_FromType(st, proto, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); return -1; } Py_INCREF(proto); - Py_XSETREF(stgdict->proto, proto); + Py_XSETREF(stginfo->proto, proto); return 0; } @@ -1027,110 +1144,103 @@ PyCPointerType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCPointerType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCPointerType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; PyObject *typedict; - typedict = PyTuple_GetItem(args, 2); if (!typedict) { - return NULL; + return -1; } + /* - stgdict items size, align, length contain info about pointers itself, - stgdict->proto has info about the pointed to type! + stginfo items size, align, length contain info about pointers itself, + stginfo->proto has info about the pointed to type! */ ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + return -1; } - stgdict->size = sizeof(void *); - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->ffi_type_pointer = ffi_type_pointer; - stgdict->paramfunc = PyCPointerType_paramfunc; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->size = sizeof(void *); + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->ffi_type_pointer = ffi_type_pointer; + stginfo->paramfunc = PyCPointerType_paramfunc; + stginfo->flags |= TYPEFLAG_ISPOINTER; if (PyDict_GetItemRef(typedict, &_Py_ID(_type_), &proto) < 0) { - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } if (proto) { - StgDictObject *itemdict; const char *current_format; - if (-1 == PyCPointerType_SetProto(stgdict, proto)) { + if (-1 == PyCPointerType_SetProto(stginfo, proto)) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - itemdict = PyType_stgdict(proto); - /* PyCPointerType_SetProto has verified proto has a stgdict. */ - assert(itemdict); - /* If itemdict->format is NULL, then this is a pointer to an + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + Py_DECREF(proto); + return -1; + } + /* PyCPointerType_SetProto has verified proto has a stginfo. */ + assert(iteminfo); + /* If iteminfo->format is NULL, then this is a pointer to an incomplete type. We create a generic format string 'pointer to bytes' in this case. XXX Better would be to fix the format string later... */ - current_format = itemdict->format ? itemdict->format : "B"; - if (itemdict->shape != NULL) { + current_format = iteminfo->format ? iteminfo->format : "B"; + if (iteminfo->shape != NULL) { /* pointer to an array: the shape needs to be prefixed */ - stgdict->format = _ctypes_alloc_format_string_with_shape( - itemdict->ndim, itemdict->shape, "&", current_format); + stginfo->format = _ctypes_alloc_format_string_with_shape( + iteminfo->ndim, iteminfo->shape, "&", current_format); } else { - stgdict->format = _ctypes_alloc_format_string("&", current_format); + stginfo->format = _ctypes_alloc_format_string("&", current_format); } Py_DECREF(proto); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; + if (stginfo->format == NULL) { + return -1; } } - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); - - return (PyObject *)result; + return 0; } static PyObject * PyCPointerType_set_type(PyTypeObject *self, PyObject *type) { - StgDictObject *dict; - - - dict = PyType_stgdict((PyObject *)self); - if (!dict) { + PyObject *attrdict = PyType_GetDict(self); + if (!attrdict) { + return NULL; + } + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)self, &info) < 0) { + Py_DECREF(attrdict); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); + Py_DECREF(attrdict); return NULL; } - if (-1 == PyCPointerType_SetProto(dict, type)) + if (-1 == PyCPointerType_SetProto(info, type)) { + Py_DECREF(attrdict); return NULL; + } - if (-1 == PyDict_SetItem((PyObject *)dict, &_Py_ID(_type_), type)) + if (-1 == PyDict_SetItem(attrdict, &_Py_ID(_type_), type)) { + Py_DECREF(attrdict); return NULL; + } + Py_DECREF(attrdict); Py_RETURN_NONE; } @@ -1139,15 +1249,17 @@ static PyObject *_byref(PyObject *); static PyObject * PyCPointerType_from_param(PyObject *type, PyObject *value) { - StgDictObject *typedict; - if (value == Py_None) { /* ConvParam will convert to a NULL pointer later */ return Py_NewRef(value); } - typedict = PyType_stgdict(type); - if (!typedict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *typeinfo; + if (PyStgInfo_FromType(st, type, &typeinfo) < 0) { + return NULL; + } + if (!typeinfo) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; @@ -1156,7 +1268,8 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) /* If we expect POINTER(), but receive a instance, accept it by calling byref(). */ - switch (PyObject_IsInstance(value, typedict->proto)) { + assert(typeinfo->proto); + switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ return _byref(value); @@ -1166,14 +1279,16 @@ PyCPointerType_from_param(PyObject *type, PyObject *value) break; } - ctypes_state *st = GLOBAL_STATE(); if (PointerObject_Check(st, value) || ArrayObject_Check(st, value)) { /* Array instances are also pointers when the item types are the same. */ - StgDictObject *v = PyObject_stgdict(value); + StgInfo *v; + if (PyStgInfo_FromObject(st, value, &v) < 0) { + return NULL; + } assert(v); /* Cannot be NULL for pointer or array objects */ - int ret = PyObject_IsSubclass(v->proto, typedict->proto); + int ret = PyObject_IsSubclass(v->proto, typeinfo->proto); if (ret < 0) { return NULL; } @@ -1199,7 +1314,7 @@ static PyType_Slot pycpointer_type_slots[] = { {Py_tp_traverse, CDataType_traverse}, {Py_tp_clear, CDataType_clear}, {Py_tp_methods, PyCPointerType_methods}, - {Py_tp_new, PyCPointerType_new}, + {Py_tp_init, PyCPointerType_init}, // Sequence protocol. {Py_sq_repeat, CDataType_repeat}, @@ -1219,7 +1334,7 @@ static PyType_Spec pycpointer_type_spec = { PyCArrayType_Type */ /* - PyCArrayType_new ensures that the new Array subclass created has a _length_ + PyCArrayType_init ensures that the new Array subclass created has a _length_ attribute, and a _type_ attribute. */ @@ -1403,28 +1518,18 @@ PyCArrayType_paramfunc(CDataObject *self) return p; } -static PyObject * -PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCArrayType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; - StgDictObject *itemdict; PyObject *length_attr, *type_attr; Py_ssize_t length; Py_ssize_t itemsize, itemalign; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - /* Initialize these variables to NULL so that we can simplify error handling by using Py_XDECREF. */ - stgdict = NULL; type_attr = NULL; - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_length_), &length_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_length_), &length_attr) < 0) { goto error; } if (!length_attr) { @@ -1457,7 +1562,7 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &type_attr) < 0) { + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &type_attr) < 0) { goto error; } if (!type_attr) { @@ -1467,88 +1572,83 @@ PyCArrayType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject*)self); + if (!stginfo) { + goto error; + } + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, type_attr, &iteminfo) < 0) { goto error; } - itemdict = PyType_stgdict(type_attr); - if (!itemdict) { + if (!iteminfo) { PyErr_SetString(PyExc_TypeError, "_type_ must have storage info"); goto error; } - assert(itemdict->format); - stgdict->format = _ctypes_alloc_format_string(NULL, itemdict->format); - if (stgdict->format == NULL) + assert(iteminfo->format); + stginfo->format = _ctypes_alloc_format_string(NULL, iteminfo->format); + if (stginfo->format == NULL) goto error; - stgdict->ndim = itemdict->ndim + 1; - stgdict->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stgdict->ndim); - if (stgdict->shape == NULL) { + stginfo->ndim = iteminfo->ndim + 1; + stginfo->shape = PyMem_Malloc(sizeof(Py_ssize_t) * stginfo->ndim); + if (stginfo->shape == NULL) { PyErr_NoMemory(); goto error; } - stgdict->shape[0] = length; - if (stgdict->ndim > 1) { - memmove(&stgdict->shape[1], itemdict->shape, - sizeof(Py_ssize_t) * (stgdict->ndim - 1)); + stginfo->shape[0] = length; + if (stginfo->ndim > 1) { + memmove(&stginfo->shape[1], iteminfo->shape, + sizeof(Py_ssize_t) * (stginfo->ndim - 1)); } - itemsize = itemdict->size; + itemsize = iteminfo->size; if (itemsize != 0 && length > PY_SSIZE_T_MAX / itemsize) { PyErr_SetString(PyExc_OverflowError, "array too large"); goto error; } - itemalign = itemdict->align; + itemalign = iteminfo->align; - if (itemdict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; + if (iteminfo->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; - stgdict->size = itemsize * length; - stgdict->align = itemalign; - stgdict->length = length; - stgdict->proto = type_attr; + stginfo->size = itemsize * length; + stginfo->align = itemalign; + stginfo->length = length; + stginfo->proto = type_attr; type_attr = NULL; - stgdict->paramfunc = &PyCArrayType_paramfunc; + stginfo->paramfunc = &PyCArrayType_paramfunc; /* Arrays are passed as pointers to function calls. */ - stgdict->ffi_type_pointer = ffi_type_pointer; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) - goto error; - Py_SETREF(result->tp_dict, (PyObject *)stgdict); /* steal the reference */ - stgdict = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; /* Special case for character arrays. A permanent annoyance: char arrays are also strings! */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - if (-1 == add_getset(result, CharArray_getsets)) + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, CharArray_getsets)) goto error; } - else if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - if (-1 == add_getset(result, WCharArray_getsets)) + else if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (-1 == add_getset((PyTypeObject*)self, WCharArray_getsets)) goto error; } - return (PyObject *)result; + return 0; error: - Py_XDECREF((PyObject*)stgdict); Py_XDECREF(type_attr); - Py_DECREF(result); - return NULL; + return -1; } static PyType_Slot pycarray_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the Array Objects")}, {Py_tp_traverse, CDataType_traverse}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCArrayType_new}, + {Py_tp_init, PyCArrayType_init}, {Py_tp_clear, CDataType_clear}, // Sequence protocol. @@ -1569,7 +1669,7 @@ static PyType_Spec pycarray_type_spec = { */ /* -PyCSimpleType_new ensures that the new Simple_Type subclass created has a valid +PyCSimpleType_init ensures that the new Simple_Type subclass created has a valid _type_ attribute. */ @@ -1609,19 +1709,29 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_wchar array instance or pointer(c_wchar(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("u")->setfunc)) { return Py_NewRef(value); } } @@ -1673,19 +1783,29 @@ c_char_p_from_param(PyObject *type, PyObject *value) ctypes_state *st = GLOBAL_STATE(); if (ArrayObject_Check(st, value) || PointerObject_Check(st, value)) { /* c_char array instance or pointer(c_char(...)) */ - StgDictObject *dt = PyObject_stgdict(value); - StgDictObject *dict; - assert(dt); /* Cannot be NULL for pointer or array objects */ - dict = dt && dt->proto ? PyType_stgdict(dt->proto) : NULL; - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *it; + if (PyStgInfo_FromObject(st, value, &it) < 0) { + return NULL; + } + assert(it); /* Cannot be NULL for pointer or array objects */ + StgInfo *info = NULL; + if (it && it->proto) { + if (PyStgInfo_FromType(st, it->proto, &info) < 0) { + return NULL; + } + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } if (PyCArg_CheckExact(st, value)) { /* byref(c_char(...)) */ PyCArgObject *a = (PyCArgObject *)value; - StgDictObject *dict = PyObject_stgdict(a->obj); - if (dict && (dict->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { + StgInfo *info; + if (PyStgInfo_FromObject(st, a->obj, &info) < 0) { + return NULL; + } + if (info && (info->setfunc == _ctypes_get_fielddesc("c")->setfunc)) { return Py_NewRef(value); } } @@ -1707,7 +1827,6 @@ c_char_p_from_param(PyObject *type, PyObject *value) static PyObject * c_void_p_from_param(PyObject *type, PyObject *value) { - StgDictObject *stgd; PyObject *as_parameter; int res; @@ -1806,15 +1925,18 @@ c_void_p_from_param(PyObject *type, PyObject *value) return (PyObject *)parg; } /* c_char_p, c_wchar_p */ - stgd = PyObject_stgdict(value); - if (stgd + StgInfo *stgi; + if (PyStgInfo_FromObject(st, value, &stgi) < 0) { + return NULL; + } + if (stgi && CDataObject_Check(st, value) - && stgd->proto - && PyUnicode_Check(stgd->proto)) + && stgi->proto + && PyUnicode_Check(stgi->proto)) { PyCArgObject *parg; - switch (PyUnicode_AsUTF8(stgd->proto)[0]) { + switch (PyUnicode_AsUTF8(stgi->proto)[0]) { case 'z': /* c_char_p */ case 'Z': /* c_wchar_p */ parg = PyCArgObject_new(); @@ -1851,7 +1973,6 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject PyObject *proto, struct fielddesc *fmt) { PyTypeObject *result; - StgDictObject *stgdict; PyObject *name = PyTuple_GET_ITEM(args, 0); PyObject *newname; PyObject *swapped_args; @@ -1894,29 +2015,21 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject return NULL; ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + + StgInfo *stginfo = PyStgInfo_Init(st, result); + if (!stginfo) { Py_DECREF(result); return NULL; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc_swapped; - stgdict->getfunc = fmt->getfunc_swapped; + stginfo->ffi_type_pointer = *fmt->pffi_type; + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc_swapped; + stginfo->getfunc = fmt->getfunc_swapped; - stgdict->proto = Py_NewRef(proto); - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = Py_NewRef(proto); return (PyObject *)result; } @@ -1924,14 +2037,17 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject static PyCArgObject * PyCSimpleType_paramfunc(CDataObject *self) { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - fmt = PyUnicode_AsUTF8(dict->proto); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); @@ -1948,33 +2064,27 @@ PyCSimpleType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; PyObject *proto; const char *proto_str; Py_ssize_t proto_len; PyMethodDef *ml; struct fielddesc *fmt; - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) - return NULL; - - if (PyObject_GetOptionalAttr((PyObject *)result, &_Py_ID(_type_), &proto) < 0) { - return NULL; + if (PyType_Type.tp_init(self, args, kwds) < 0) { + return -1; + } + if (PyObject_GetOptionalAttr(self, &_Py_ID(_type_), &proto) < 0) { + return -1; } if (!proto) { PyErr_SetString(PyExc_AttributeError, "class must define a '_type_' attribute"); error: Py_XDECREF(proto); - Py_DECREF(result); - return NULL; + return -1; } if (PyUnicode_Check(proto)) { proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); @@ -2006,70 +2116,60 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { goto error; } - stgdict->ffi_type_pointer = *fmt->pffi_type; - stgdict->align = fmt->pffi_type->alignment; - stgdict->length = 0; - stgdict->size = fmt->pffi_type->size; - stgdict->setfunc = fmt->setfunc; - stgdict->getfunc = fmt->getfunc; + + stginfo->ffi_type_pointer = *fmt->pffi_type; + stginfo->align = fmt->pffi_type->alignment; + stginfo->length = 0; + stginfo->size = fmt->pffi_type->size; + stginfo->setfunc = fmt->setfunc; + stginfo->getfunc = fmt->getfunc; #ifdef WORDS_BIGENDIAN - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1); #else - stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); + stginfo->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0); #endif - if (stgdict->format == NULL) { - Py_DECREF(result); + if (stginfo->format == NULL) { Py_DECREF(proto); - Py_DECREF((PyObject *)stgdict); - return NULL; + return -1; } - stgdict->paramfunc = PyCSimpleType_paramfunc; + stginfo->paramfunc = PyCSimpleType_paramfunc; /* - if (result->tp_base != st->Simple_Type) { - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; + if (self->tp_base != st->Simple_Type) { + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; } */ /* This consumes the refcount on proto which we have */ - stgdict->proto = proto; - - /* replace the class dict by our updated spam dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; - } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->proto = proto; /* Install from_param class methods in ctypes base classes. Overrides the PyCSimpleType_from_param generic method. */ - if (result->tp_base == st->Simple_Type) { + if (((PyTypeObject *)self)->tp_base == st->Simple_Type) { switch (*proto_str) { case 'z': /* c_char_p */ ml = &c_char_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'Z': /* c_wchar_p */ ml = &c_wchar_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 'P': /* c_void_p */ ml = &c_void_p_method; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; case 's': case 'X': case 'O': ml = NULL; - stgdict->flags |= TYPEFLAG_ISPOINTER; + stginfo->flags |= TYPEFLAG_ISPOINTER; break; default: ml = NULL; @@ -2079,57 +2179,57 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (ml) { PyObject *meth; int x; - meth = PyDescr_NewClassMethod(result, ml); + meth = PyDescr_NewClassMethod((PyTypeObject*)self, ml); if (!meth) { - Py_DECREF(result); - return NULL; + return -1; } - x = PyDict_SetItemString(result->tp_dict, + x = PyDict_SetItemString(((PyTypeObject*)self)->tp_dict, ml->ml_name, meth); Py_DECREF(meth); if (x == -1) { - Py_DECREF(result); - return NULL; + return -1; } } } + PyTypeObject *type = Py_TYPE(self); if (type == st->PyCSimpleType_Type && fmt->setfunc_swapped && fmt->getfunc_swapped) { PyObject *swapped = CreateSwappedType(type, args, kwds, proto, fmt); - StgDictObject *sw_dict; if (swapped == NULL) { - Py_DECREF(result); - return NULL; + return -1; } - sw_dict = PyType_stgdict(swapped); + StgInfo *sw_info; + if (PyStgInfo_FromType(st, swapped, &sw_info) < 0) { + return -1; + } + assert(sw_info); #ifdef WORDS_BIGENDIAN - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_be__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_le__", swapped); + PyObject_SetAttrString(self, "__ctype_be__", self); + PyObject_SetAttrString(swapped, "__ctype_be__", self); PyObject_SetAttrString(swapped, "__ctype_le__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string("<", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string("<", stginfo->format+1); #else - PyObject_SetAttrString((PyObject *)result, "__ctype_be__", swapped); - PyObject_SetAttrString((PyObject *)result, "__ctype_le__", (PyObject *)result); - PyObject_SetAttrString(swapped, "__ctype_le__", (PyObject *)result); + PyObject_SetAttrString(self, "__ctype_be__", swapped); + PyObject_SetAttrString(self, "__ctype_le__", self); + PyObject_SetAttrString(swapped, "__ctype_le__", self); PyObject_SetAttrString(swapped, "__ctype_be__", swapped); /* We are creating the type for the OTHER endian */ - sw_dict->format = _ctypes_alloc_format_string(">", stgdict->format+1); + sw_info->format = _ctypes_alloc_format_string(">", stginfo->format+1); #endif Py_DECREF(swapped); if (PyErr_Occurred()) { - Py_DECREF(result); - return NULL; + return -1; } }; - return (PyObject *)result; + return 0; } /* @@ -2139,7 +2239,6 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * PyCSimpleType_from_param(PyObject *type, PyObject *value) { - StgDictObject *dict; const char *fmt; PyCArgObject *parg; struct fielddesc *fd; @@ -2155,15 +2254,19 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) return Py_NewRef(value); } - dict = PyType_stgdict(type); - if (!dict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } /* I think we can rely on this being a one-character string */ - fmt = PyUnicode_AsUTF8(dict->proto); + fmt = PyUnicode_AsUTF8(info->proto); assert(fmt); fd = _ctypes_get_fielddesc(fmt); @@ -2218,7 +2321,7 @@ static PyMethodDef PyCSimpleType_methods[] = { static PyType_Slot pycsimple_type_slots[] = { {Py_tp_doc, PyDoc_STR("metatype for the PyCSimpleType Objects")}, {Py_tp_methods, PyCSimpleType_methods}, - {Py_tp_new, PyCSimpleType_new}, + {Py_tp_init, PyCSimpleType_init}, {Py_tp_traverse, CDataType_traverse}, {Py_tp_clear, CDataType_clear}, @@ -2227,7 +2330,7 @@ static PyType_Slot pycsimple_type_slots[] = { {0, NULL}, }; -PyType_Spec pycsimple_type_spec = { +static PyType_Spec pycsimple_type_spec = { .name = "_ctypes.PyCSimpleType", .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), @@ -2292,10 +2395,13 @@ converters_from_argtypes(PyObject *ob) * not bitfields, the bitfields check is also being disabled as a * precaution. - StgDictObject *stgdict = PyType_stgdict(tp); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, tp, &stginfo) < 0) { + return -1; + } - if (stgdict != NULL) { - if (stgdict->flags & TYPEFLAG_HASUNION) { + if (stginfo != NULL) { + if (stginfo->flags & TYPEFLAG_HASUNION) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2306,7 +2412,7 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - if (stgdict->flags & TYPEFLAG_HASBITFIELD) { + if (stginfo->flags & TYPEFLAG_HASBITFIELD) { Py_DECREF(converters); Py_DECREF(ob); if (!PyErr_Occurred()) { @@ -2338,19 +2444,19 @@ converters_from_argtypes(PyObject *ob) } static int -make_funcptrtype_dict(StgDictObject *stgdict) +make_funcptrtype_dict(PyObject *attrdict, StgInfo *stginfo) { PyObject *ob; PyObject *converters = NULL; - stgdict->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; - stgdict->length = 1; - stgdict->size = sizeof(void *); - stgdict->setfunc = NULL; - stgdict->getfunc = NULL; - stgdict->ffi_type_pointer = ffi_type_pointer; + stginfo->align = _ctypes_get_fielddesc("P")->pffi_type->alignment; + stginfo->length = 1; + stginfo->size = sizeof(void *); + stginfo->setfunc = NULL; + stginfo->getfunc = NULL; + stginfo->ffi_type_pointer = ffi_type_pointer; - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_flags_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_flags_), &ob) < 0) { return -1; } if (!ob || !PyLong_Check(ob)) { @@ -2359,11 +2465,11 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_XDECREF(ob); return -1; } - stgdict->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; + stginfo->flags = PyLong_AsUnsignedLongMask(ob) | TYPEFLAG_ISPOINTER; Py_DECREF(ob); /* _argtypes_ is optional... */ - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_argtypes_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_argtypes_), &ob) < 0) { return -1; } if (ob) { @@ -2372,29 +2478,34 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_DECREF(ob); return -1; } - stgdict->argtypes = ob; - stgdict->converters = converters; + stginfo->argtypes = ob; + stginfo->converters = converters; } - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py_ID(_restype_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py_ID(_restype_), &ob) < 0) { return -1; } if (ob) { - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + StgInfo *info; + ctypes_state *st = GLOBAL_STATE(); + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "_restype_ must be a type, a callable, or None"); Py_DECREF(ob); return -1; } - stgdict->restype = ob; + stginfo->restype = ob; if (PyObject_GetOptionalAttr(ob, &_Py_ID(_check_retval_), - &stgdict->checker) < 0) + &stginfo->checker) < 0) { return -1; } } /* XXX later, maybe. - if (PyDict_GetItemRef((PyObject *)stgdict, &_Py _ID(_errcheck_), &ob) < 0) { + if (PyDict_GetItemRef((PyObject *)attrdict, &_Py _ID(_errcheck_), &ob) < 0) { return -1; } if (ob) { @@ -2404,7 +2515,7 @@ make_funcptrtype_dict(StgDictObject *stgdict) Py_DECREF(ob); return -1; } - stgdict->errcheck = ob; + stginfo->errcheck = ob; } */ return 0; @@ -2426,54 +2537,43 @@ PyCFuncPtrType_paramfunc(CDataObject *self) return parg; } -static PyObject * -PyCFuncPtrType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static int +PyCFuncPtrType_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyTypeObject *result; - StgDictObject *stgdict; + PyObject *attrdict = PyType_GetDict((PyTypeObject *)self); + if (!attrdict) { + return -1; + } ctypes_state *st = GLOBAL_STATE(); - stgdict = (StgDictObject *)_PyObject_CallNoArgs( - (PyObject *)st->PyCStgDict_Type); - if (!stgdict) { - return NULL; + StgInfo *stginfo = PyStgInfo_Init(st, (PyTypeObject *)self); + if (!stginfo) { + Py_DECREF(attrdict); + return -1; } - stgdict->paramfunc = PyCFuncPtrType_paramfunc; + + stginfo->paramfunc = PyCFuncPtrType_paramfunc; + /* We do NOT expose the function signature in the format string. It is impossible, generally, because the only requirement for the argtypes items is that they have a .from_param method - we do not know the types of the arguments (although, in practice, most argtypes would be a ctypes type). */ - stgdict->format = _ctypes_alloc_format_string(NULL, "X{}"); - if (stgdict->format == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - stgdict->flags |= TYPEFLAG_ISPOINTER; - - /* create the new instance (which is a class, - since we are a metatype!) */ - result = (PyTypeObject *)PyType_Type.tp_new(type, args, kwds); - if (result == NULL) { - Py_DECREF((PyObject *)stgdict); - return NULL; - } - - /* replace the class dict by our updated storage dict */ - if (-1 == PyDict_Update((PyObject *)stgdict, result->tp_dict)) { - Py_DECREF(result); - Py_DECREF((PyObject *)stgdict); - return NULL; + stginfo->format = _ctypes_alloc_format_string(NULL, "X{}"); + if (stginfo->format == NULL) { + Py_DECREF(attrdict); + return -1; } - Py_SETREF(result->tp_dict, (PyObject *)stgdict); + stginfo->flags |= TYPEFLAG_ISPOINTER; - if (-1 == make_funcptrtype_dict(stgdict)) { - Py_DECREF(result); - return NULL; + if (-1 == make_funcptrtype_dict(attrdict, stginfo)) { + Py_DECREF(attrdict); + return -1; } - return (PyObject *)result; + Py_DECREF(attrdict); + return 0; } static PyType_Slot pycfuncptr_type_slots[] = { @@ -2481,7 +2581,7 @@ static PyType_Slot pycfuncptr_type_slots[] = { {Py_tp_traverse, CDataType_traverse}, {Py_tp_clear, CDataType_clear}, {Py_tp_methods, CDataType_methods}, - {Py_tp_new, PyCFuncPtrType_new}, + {Py_tp_init, PyCFuncPtrType_init}, // Sequence protocol. {Py_sq_repeat, CDataType_repeat}, @@ -2609,6 +2709,8 @@ PyCData_traverse(CDataObject *self, visitproc visit, void *arg) { Py_VISIT(self->b_objects); Py_VISIT((PyObject *)self->b_base); + PyTypeObject *type = Py_TYPE(self); + Py_VISIT(type); return 0; } @@ -2627,8 +2729,11 @@ PyCData_clear(CDataObject *self) static void PyCData_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); PyCData_clear((CDataObject *)self); - Py_TYPE(self)->tp_free(self); + type->tp_free(self); + Py_DECREF(type); } static PyMemberDef PyCData_members[] = { @@ -2650,13 +2755,15 @@ PyCData_item_type(PyObject *type) { ctypes_state *st = GLOBAL_STATE(); if (PyCArrayTypeObject_Check(st, type)) { - StgDictObject *stg_dict; PyObject *elem_type; /* asserts used here as these are all guaranteed by construction */ - stg_dict = PyType_stgdict(type); - assert(stg_dict); - elem_type = stg_dict->proto; + StgInfo *stg_info; + if (PyStgInfo_FromType(st, type, &stg_info) < 0) { + return NULL; + } + assert(stg_info); + elem_type = stg_info->proto; assert(elem_type); return PyCData_item_type(elem_type); } @@ -2669,32 +2776,42 @@ static int PyCData_NewGetBuffer(PyObject *myself, Py_buffer *view, int flags) { CDataObject *self = (CDataObject *)myself; - StgDictObject *dict = PyObject_stgdict(myself); + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return -1; + } + assert(info); + PyObject *item_type = PyCData_item_type((PyObject*)Py_TYPE(myself)); - StgDictObject *item_dict = PyType_stgdict(item_type); + if (item_type == NULL) { + return 0; + } if (view == NULL) return 0; + StgInfo *item_info; + if (PyStgInfo_FromType(st, item_type, &item_info) < 0) { + return -1; + } + assert(item_info); + view->buf = self->b_ptr; view->obj = Py_NewRef(myself); view->len = self->b_size; view->readonly = 0; /* use default format character if not set */ - view->format = dict->format ? dict->format : "B"; - view->ndim = dict->ndim; - view->shape = dict->shape; - view->itemsize = item_dict->size; + view->format = info->format ? info->format : "B"; + view->ndim = info->ndim; + view->shape = info->shape; + view->itemsize = item_info->size; view->strides = NULL; view->suboffsets = NULL; view->internal = NULL; return 0; } -static PyBufferProcs PyCData_as_buffer = { - PyCData_NewGetBuffer, - NULL, -}; - /* * CData objects are mutable, so they cannot be hashable! */ @@ -2710,7 +2827,14 @@ PyCData_reduce(PyObject *myself, PyObject *args) { CDataObject *self = (CDataObject *)myself; - if (PyObject_stgdict(myself)->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, myself, &info) < 0) { + return NULL; + } + assert(info); + + if (info->flags & (TYPEFLAG_ISPOINTER|TYPEFLAG_HASPOINTER)) { PyErr_SetString(PyExc_ValueError, "ctypes objects containing pointers cannot be pickled"); return NULL; @@ -2773,51 +2897,30 @@ static PyMethodDef PyCData_methods[] = { { NULL, NULL }, }; -PyTypeObject PyCData_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._CData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - PyCData_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyCData_nohash, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCData_methods, /* tp_methods */ - PyCData_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycdata_slots[] = { + {Py_tp_dealloc, PyCData_dealloc}, + {Py_tp_hash, PyCData_nohash}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_traverse, PyCData_traverse}, + {Py_tp_clear, PyCData_clear}, + {Py_tp_methods, PyCData_methods}, + {Py_tp_members, PyCData_members}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, }; -static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) +static PyType_Spec pycdata_spec = { + .name = "_ctypes._CData", + .basicsize = sizeof(CDataObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_DISALLOW_INSTANTIATION), + .slots = pycdata_slots, +}; + +static int +PyCData_MallocBuffer(CDataObject *obj, StgInfo *info) { - if ((size_t)dict->size <= sizeof(obj->b_value)) { + if ((size_t)info->size <= sizeof(obj->b_value)) { /* No need to call malloc, can use the default buffer */ obj->b_ptr = (char *)&obj->b_value; /* The b_needsfree flag does not mean that we actually did @@ -2831,15 +2934,15 @@ static int PyCData_MallocBuffer(CDataObject *obj, StgDictObject *dict) /* In python 2.4, and ctypes 0.9.6, the malloc call took about 33% of the creation time for c_int(). */ - obj->b_ptr = (char *)PyMem_Malloc(dict->size); + obj->b_ptr = (char *)PyMem_Malloc(info->size); if (obj->b_ptr == NULL) { PyErr_NoMemory(); return -1; } obj->b_needsfree = 1; - memset(obj->b_ptr, 0, dict->size); + memset(obj->b_ptr, 0, info->size); } - obj->b_size = dict->size; + obj->b_size = info->size; return 0; } @@ -2847,23 +2950,28 @@ PyObject * PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) { CDataObject *cmem; - StgDictObject *dict; assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; cmem = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (cmem == NULL) { return NULL; } assert(CDataObject_Check(GLOBAL_STATE(), cmem)); - cmem->b_length = dict->length; - cmem->b_size = dict->size; + cmem->b_length = info->length; + cmem->b_size = info->size; if (base) { /* use base's buffer */ assert(CDataObject_Check(GLOBAL_STATE(), base)); cmem->b_ptr = adr; @@ -2871,11 +2979,11 @@ PyCData_FromBaseObj(PyObject *type, PyObject *base, Py_ssize_t index, char *adr) cmem->b_base = (CDataObject *)Py_NewRef(base); cmem->b_index = index; } else { /* copy contents of adr */ - if (-1 == PyCData_MallocBuffer(cmem, dict)) { + if (-1 == PyCData_MallocBuffer(cmem, info)) { Py_DECREF(cmem); return NULL; } - memcpy(cmem->b_ptr, adr, dict->size); + memcpy(cmem->b_ptr, adr, info->size); cmem->b_index = index; } return (PyObject *)cmem; @@ -2888,20 +2996,25 @@ PyObject * PyCData_AtAddress(PyObject *type, void *buf) { CDataObject *pd; - StgDictObject *dict; if (PySys_Audit("ctypes.cdata", "n", (Py_ssize_t)buf) < 0) { return NULL; } assert(PyType_Check(type)); - dict = PyType_stgdict(type); - if (!dict) { + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; pd = (CDataObject *)((PyTypeObject *)type)->tp_alloc((PyTypeObject *)type, 0); if (!pd) { @@ -2909,8 +3022,8 @@ PyCData_AtAddress(PyObject *type, void *buf) } assert(CDataObject_Check(GLOBAL_STATE(), pd)); pd->b_ptr = (char *)buf; - pd->b_length = dict->length; - pd->b_size = dict->size; + pd->b_length = info->length; + pd->b_size = info->size; return (PyObject *)pd; } @@ -2934,13 +3047,16 @@ PyObject * PyCData_get(PyObject *type, GETFUNC getfunc, PyObject *src, Py_ssize_t index, Py_ssize_t size, char *adr) { - StgDictObject *dict; if (getfunc) return getfunc(adr, size); assert(type); - dict = PyType_stgdict(type); - if (dict && dict->getfunc && !_ctypes_simple_instance(type)) - return dict->getfunc(adr, size); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->getfunc && !_ctypes_simple_instance(type)) + return info->getfunc(adr, size); return PyCData_FromBaseObj(type, src, index, adr); } @@ -2959,9 +3075,12 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, } ctypes_state *st = GLOBAL_STATE(); if (!CDataObject_Check(st, value)) { - StgDictObject *dict = PyType_stgdict(type); - if (dict && dict->setfunc) - return dict->setfunc(ptr, value, size); + StgInfo *info; + if (PyStgInfo_FromType(st, type, &info) < 0) { + return NULL; + } + if (info && info->setfunc) + return info->setfunc(ptr, value, size); /* If value is a tuple, we try to call the type with the tuple and use the result! @@ -3014,11 +3133,16 @@ _PyCData_set(CDataObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, if (PyCPointerTypeObject_Check(st, type) && ArrayObject_Check(st, value)) { - StgDictObject *p1, *p2; PyObject *keep; - p1 = PyObject_stgdict(value); + + StgInfo *p1, *p2; + if (PyStgInfo_FromObject(st, value, &p1) < 0) { + return NULL; + } assert(p1); /* Cannot be NULL for array instances */ - p2 = PyType_stgdict(type); + if (PyStgInfo_FromType(st, type, &p2) < 0) { + return NULL; + } assert(p2); /* Cannot be NULL for pointer types */ if (p1->proto != p2->proto) { @@ -3086,15 +3210,19 @@ static PyObject * GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { CDataObject *obj; - StgDictObject *dict; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return NULL; } - dict->flags |= DICTFLAG_FINAL; + + info->flags |= DICTFLAG_FINAL; obj = (CDataObject *)type->tp_alloc(type, 0); if (!obj) @@ -3103,9 +3231,9 @@ GenericPyCData_new(PyTypeObject *type, PyObject *args, PyObject *kwds) obj->b_base = NULL; obj->b_index = 0; obj->b_objects = NULL; - obj->b_length = dict->length; + obj->b_length = info->length; - if (-1 == PyCData_MallocBuffer(obj, dict)) { + if (-1 == PyCData_MallocBuffer(obj, info)) { Py_DECREF(obj); return NULL; } @@ -3149,7 +3277,12 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign Py_XDECREF(oldchecker); return 0; } - if (ob != Py_None && !PyType_stgdict(ob) && !PyCallable_Check(ob)) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + return -1; + } + if (ob != Py_None && !info && !PyCallable_Check(ob)) { PyErr_SetString(PyExc_TypeError, "restype must be a type, a callable, or None"); return -1; @@ -3168,14 +3301,17 @@ PyCFuncPtr_set_restype(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ign static PyObject * PyCFuncPtr_get_restype(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->restype) { return Py_NewRef(self->restype); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->restype) { - return Py_NewRef(dict->restype); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->restype) { + return Py_NewRef(info->restype); } else { Py_RETURN_NONE; } @@ -3203,14 +3339,17 @@ PyCFuncPtr_set_argtypes(PyCFuncPtrObject *self, PyObject *ob, void *Py_UNUSED(ig static PyObject * PyCFuncPtr_get_argtypes(PyCFuncPtrObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; if (self->argtypes) { return Py_NewRef(self->argtypes); } - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - if (dict->argtypes) { - return Py_NewRef(dict->argtypes); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + if (info->argtypes) { + return Py_NewRef(info->argtypes); } else { Py_RETURN_NONE; } @@ -3242,7 +3381,6 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) #else char *mangled_name; int i; - StgDictObject *dict; Py_BEGIN_ALLOW_THREADS address = (PPROC)GetProcAddress(handle, name); @@ -3253,9 +3391,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) return NULL; } - dict = PyType_stgdict((PyObject *)type); - /* It should not happen that dict is NULL, but better be safe */ - if (dict==NULL || dict->flags & FUNCFLAG_CDECL) + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + /* It should not happen that info is NULL, but better be safe */ + if (info==NULL || info->flags & FUNCFLAG_CDECL) return address; /* for stdcall, try mangled names: @@ -3282,7 +3424,6 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) static int _check_outarg_type(PyObject *arg, Py_ssize_t index) { - StgDictObject *dict; ctypes_state *st = GLOBAL_STATE(); if (PyCPointerTypeObject_Check(st, arg)) { @@ -3291,12 +3432,15 @@ _check_outarg_type(PyObject *arg, Py_ssize_t index) if (PyCArrayTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return -1; + } + if (info /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ - && PyUnicode_Check(dict->proto) + && PyUnicode_Check(info->proto) /* We only allow c_void_p, c_char_p and c_wchar_p as a simple output parameter type */ - && (strchr("PzZ", PyUnicode_AsUTF8(dict->proto)[0]))) { + && (strchr("PzZ", PyUnicode_AsUTF8(info->proto)[0]))) { return 1; } @@ -3314,18 +3458,21 @@ static int _validate_paramflags(PyTypeObject *type, PyObject *paramflags) { Py_ssize_t i, len; - StgDictObject *dict; PyObject *argtypes; - dict = PyType_stgdict((PyObject *)type); - if (!dict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "abstract class"); return 0; } - argtypes = dict->argtypes; + argtypes = info->argtypes; - if (paramflags == NULL || dict->argtypes == NULL) + if (paramflags == NULL || info->argtypes == NULL) return 1; if (!PyTuple_Check(paramflags)) { @@ -3335,7 +3482,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(dict->argtypes)) { + if (len != PyTuple_GET_SIZE(info->argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3565,7 +3712,6 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyCFuncPtrObject *self; PyObject *callable; - StgDictObject *dict; CThunkObject *thunk; if (PyTuple_GET_SIZE(args) == 0) @@ -3616,9 +3762,13 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } */ - dict = PyType_stgdict((PyObject *)type); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } /* XXXX Fails if we do: 'PyCFuncPtr(lambda x: x)' */ - if (!dict || !dict->argtypes) { + if (!info || !info->argtypes) { PyErr_SetString(PyExc_TypeError, "cannot construct instance of this class:" " no argtypes"); @@ -3626,9 +3776,9 @@ PyCFuncPtr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } thunk = _ctypes_alloc_callback(callable, - dict->argtypes, - dict->restype, - dict->flags); + info->argtypes, + info->restype, + info->flags); if (!thunk) return NULL; @@ -3737,7 +3887,6 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, { PyObject *paramflags = self->paramflags; PyObject *callargs; - StgDictObject *dict; Py_ssize_t i, len; int inargs_index = 0; /* It's a little bit difficult to determine how many arguments the @@ -3826,15 +3975,18 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, break; } ob = PyTuple_GET_ITEM(argtypes, i); - dict = PyType_stgdict(ob); - if (dict == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, ob, &info) < 0) { + goto error; + } + if (info == NULL) { /* Cannot happen: _validate_paramflags() would not accept such an object */ PyErr_Format(PyExc_RuntimeError, - "NULL stgdict unexpected"); + "NULL stginfo unexpected"); goto error; } - if (PyUnicode_Check(dict->proto)) { + if (PyUnicode_Check(info->proto)) { PyErr_Format( PyExc_TypeError, "%s 'out' parameter must be passed as default value", @@ -3846,7 +3998,7 @@ _build_callargs(PyCFuncPtrObject *self, PyObject *argtypes, } else { /* Create an instance of the pointed-to type */ - ob = _PyObject_CallNoArgs(dict->proto); + ob = _PyObject_CallNoArgs(info->proto); } /* XXX Is the following correct any longer? @@ -3966,7 +4118,6 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) PyObject *converters; PyObject *checker; PyObject *argtypes; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); PyObject *result; PyObject *callargs; PyObject *errcheck; @@ -3979,13 +4130,19 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int outmask; unsigned int numretvals; - assert(dict); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : dict->restype; - converters = self->converters ? self->converters : dict->converters; - checker = self->checker ? self->checker : dict->checker; - argtypes = self->argtypes ? self->argtypes : dict->argtypes; -/* later, we probably want to have an errcheck field in stgdict */ - errcheck = self->errcheck /* ? self->errcheck : dict->errcheck */; + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ + + restype = self->restype ? self->restype : info->restype; + converters = self->converters ? self->converters : info->converters; + checker = self->checker ? self->checker : info->checker; + argtypes = self->argtypes ? self->argtypes : info->argtypes; +/* later, we probably want to have an errcheck field in stginfo */ + errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; pProc = *(void **)self->b_ptr; @@ -4033,7 +4190,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) int actual = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(callargs), Py_ssize_t, int); - if ((dict->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { + if ((info->flags & FUNCFLAG_CDECL) == FUNCFLAG_CDECL) { /* For cdecl functions, we allow more actual arguments than the length of the argtypes tuple. */ @@ -4063,7 +4220,7 @@ PyCFuncPtr_call(PyCFuncPtrObject *self, PyObject *inargs, PyObject *kwds) piunk, self->iid, #endif - dict->flags, + info->flags, converters, restype, checker); @@ -4123,8 +4280,11 @@ PyCFuncPtr_clear(PyCFuncPtrObject *self) static void PyCFuncPtr_dealloc(PyCFuncPtrObject *self) { + PyObject_GC_UnTrack(self); PyCFuncPtr_clear(self); - Py_TYPE(self)->tp_free((PyObject *)self); + PyTypeObject *type = Py_TYPE(self); + type->tp_free((PyObject *)self); + Py_DECREF(type); } static PyObject * @@ -4152,59 +4312,26 @@ PyCFuncPtr_bool(PyCFuncPtrObject *self) ); } -static PyNumberMethods PyCFuncPtr_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)PyCFuncPtr_bool, /* nb_bool */ +static PyType_Slot pycfuncptr_slots[] = { + {Py_tp_dealloc, PyCFuncPtr_dealloc}, + {Py_tp_repr, PyCFuncPtr_repr}, + {Py_tp_call, PyCFuncPtr_call}, + {Py_tp_doc, PyDoc_STR("Function Pointer")}, + {Py_tp_traverse, PyCFuncPtr_traverse}, + {Py_tp_clear, PyCFuncPtr_clear}, + {Py_tp_getset, PyCFuncPtr_getsets}, + {Py_tp_new, PyCFuncPtr_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, PyCFuncPtr_bool}, + {0, NULL}, }; -PyTypeObject PyCFuncPtr_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.CFuncPtr", - sizeof(PyCFuncPtrObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PyCFuncPtr_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)PyCFuncPtr_repr, /* tp_repr */ - &PyCFuncPtr_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - (ternaryfunc)PyCFuncPtr_call, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Function Pointer"), /* tp_doc */ - (traverseproc)PyCFuncPtr_traverse, /* tp_traverse */ - (inquiry)PyCFuncPtr_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - PyCFuncPtr_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyCFuncPtr_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycfuncptr_spec = { + .name = "_ctypes.CFuncPtr", + .basicsize = sizeof(PyCFuncPtrObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycfuncptr_slots, }; /*****************************************************************/ @@ -4224,11 +4351,15 @@ _init_pos_args(PyObject *self, PyTypeObject *type, PyObject *args, PyObject *kwds, Py_ssize_t index) { - StgDictObject *dict; PyObject *fields; Py_ssize_t i; - if (PyType_stgdict((PyObject *)type->tp_base)) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)type->tp_base, &baseinfo) < 0) { + return -1; + } + if (baseinfo) { index = _init_pos_args(self, type->tp_base, args, kwds, index); @@ -4236,8 +4367,17 @@ _init_pos_args(PyObject *self, PyTypeObject *type, return -1; } - dict = PyType_stgdict((PyObject *)type); - fields = PyDict_GetItemWithError((PyObject *)dict, &_Py_ID(_fields_)); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return -1; + } + assert(info); + + PyObject *attrdict = PyType_GetDict(type); + assert(attrdict); + + fields = PyDict_GetItemWithError((PyObject *)attrdict, &_Py_ID(_fields_)); + Py_CLEAR(attrdict); if (fields == NULL) { if (PyErr_Occurred()) { return -1; @@ -4246,7 +4386,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, } for (i = index; - i < dict->length && i < PyTuple_GET_SIZE(args); + i < info->length && i < PyTuple_GET_SIZE(args); ++i) { PyObject *pair = PySequence_GetItem(fields, i - index); PyObject *name, *val; @@ -4279,7 +4419,7 @@ _init_pos_args(PyObject *self, PyTypeObject *type, if (res == -1) return -1; } - return dict->length; + return info->length; } static int @@ -4316,88 +4456,34 @@ Struct_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject Struct_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Structure", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Structure base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycstruct_slots[] = { + {Py_tp_doc, PyDoc_STR("Structure base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, +}; + +static PyType_Spec pycstruct_spec = { + .name = "_ctypes.Structure", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycstruct_slots, +}; + +static PyType_Slot pycunion_slots[] = { + {Py_tp_doc, PyDoc_STR("Union base class")}, + {Py_tp_init, Struct_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {0, NULL}, }; -static PyTypeObject Union_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Union", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Union base class"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - Struct_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycunion_spec = { + .name = "_ctypes.Union", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycunion_slots, }; @@ -4431,8 +4517,6 @@ Array_item(PyObject *myself, Py_ssize_t index) { CDataObject *self = (CDataObject *)myself; Py_ssize_t offset, size; - StgDictObject *stgdict; - if (index < 0 || index >= self->b_length) { PyErr_SetString(PyExc_IndexError, @@ -4440,15 +4524,19 @@ Array_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array instances */ + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + /* Would it be clearer if we got the item size from - stgdict->proto's stgdict? + stginfo->proto's stginfo? */ - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; - return PyCData_get(stgdict->proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(stginfo->proto, stginfo->getfunc, (PyObject *)self, index, size, self->b_ptr + offset); } @@ -4467,7 +4555,6 @@ Array_subscript(PyObject *myself, PyObject *item) return Array_item(myself, i); } else if (PySlice_Check(item)) { - StgDictObject *stgdict, *itemdict; PyObject *proto; PyObject *np; Py_ssize_t start, stop, step, slicelen, i; @@ -4478,14 +4565,21 @@ Array_subscript(PyObject *myself, PyObject *item) } slicelen = PySlice_AdjustIndices(self->b_length, &start, &stop, step); - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - proto = stgdict->proto; - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the array, a + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + proto = stginfo->proto; + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the array, a ctypes type, so this cannot be NULL */ - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = (char *)self->b_ptr; char *dest; @@ -4509,7 +4603,7 @@ Array_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = (wchar_t *)self->b_ptr; wchar_t *dest; @@ -4564,7 +4658,6 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) { CDataObject *self = (CDataObject *)myself; Py_ssize_t size, offset; - StgDictObject *stgdict; char *ptr; if (value == NULL) { @@ -4573,18 +4666,23 @@ Array_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for array object instances */ - if (index < 0 || index >= stgdict->length) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for array object instances */ + + if (index < 0 || index >= stginfo->length) { PyErr_SetString(PyExc_IndexError, "invalid index"); return -1; } - size = stgdict->size / stgdict->length; + size = stginfo->size / stginfo->length; offset = index * size; ptr = self->b_ptr + offset; - return PyCData_set((PyObject *)self, stgdict->proto, stgdict->setfunc, value, + return PyCData_set((PyObject *)self, stginfo->proto, stginfo->setfunc, value, index, size, ptr); } @@ -4658,26 +4756,6 @@ static PyMethodDef Array_methods[] = { { NULL, NULL } }; -static PySequenceMethods Array_as_sequence = { - Array_length, /* sq_length; */ - 0, /* sq_concat; */ - 0, /* sq_repeat; */ - Array_item, /* sq_item; */ - 0, /* sq_slice; */ - Array_ass_item, /* sq_ass_item; */ - 0, /* sq_ass_slice; */ - 0, /* sq_contains; */ - - 0, /* sq_inplace_concat; */ - 0, /* sq_inplace_repeat; */ -}; - -static PyMappingMethods Array_as_mapping = { - Array_length, - Array_subscript, - Array_ass_subscript, -}; - PyDoc_STRVAR(array_doc, "Abstract base class for arrays.\n" "\n" @@ -4688,46 +4766,26 @@ PyDoc_STRVAR(array_doc, "reads, the resulting object is not itself an Array." ); -PyTypeObject PyCArray_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.Array", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - &Array_as_sequence, /* tp_as_sequence */ - &Array_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - array_doc, /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Array_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Array_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycarray_slots[] = { + {Py_tp_doc, (char*)array_doc}, + {Py_tp_methods, Array_methods}, + {Py_tp_init, Array_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_sq_length, Array_length}, + {Py_sq_item, Array_item}, + {Py_sq_ass_item, Array_ass_item}, + {Py_mp_length, Array_length}, + {Py_mp_subscript, Array_subscript}, + {Py_mp_ass_subscript, Array_ass_subscript}, + {0, NULL}, +}; + +static PyType_Spec pycarray_spec = { + .name = "_ctypes.Array", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycarray_slots, }; PyObject * @@ -4804,16 +4862,22 @@ static int Simple_set_value(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) { PyObject *result; - StgDictObject *dict = PyObject_stgdict((PyObject *)self); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->setfunc); - result = dict->setfunc(self->b_ptr, value, dict->size); + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return -1; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->setfunc); + + result = info->setfunc(self->b_ptr, value, info->size); if (!result) return -1; @@ -4835,11 +4899,14 @@ Simple_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Simple_get_value(CDataObject *self, void *Py_UNUSED(ignored)) { - StgDictObject *dict; - dict = PyObject_stgdict((PyObject *)self); - assert(dict); /* Cannot be NULL for CDataObject instances */ - assert(dict->getfunc); - return dict->getfunc(self->b_ptr, self->b_size); + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromObject(st, (PyObject *)self, &info) < 0) { + return NULL; + } + assert(info); /* Cannot be NULL for CDataObject instances */ + assert(info->getfunc); + return info->getfunc(self->b_ptr, self->b_size); } static PyGetSetDef Simple_getsets[] = { @@ -4854,7 +4921,7 @@ Simple_from_outparm(PyObject *self, PyObject *args) if (_ctypes_simple_instance((PyObject *)Py_TYPE(self))) { return Py_NewRef(self); } - /* call stgdict->getfunc */ + /* call stginfo->getfunc */ return Simple_get_value((CDataObject *)self, NULL); } @@ -4868,19 +4935,6 @@ static int Simple_bool(CDataObject *self) return memcmp(self->b_ptr, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", self->b_size); } -static PyNumberMethods Simple_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Simple_bool, /* nb_bool */ -}; - /* "%s(%s)" % (self.__class__.__name__, self.value) */ static PyObject * Simple_repr(CDataObject *self) @@ -4903,48 +4957,26 @@ Simple_repr(CDataObject *self) return result; } -static PyTypeObject Simple_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._SimpleCData", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)&Simple_repr, /* tp_repr */ - &Simple_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Simple_methods, /* tp_methods */ - 0, /* tp_members */ - Simple_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Simple_init, /* tp_init */ - 0, /* tp_alloc */ - GenericPyCData_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Slot pycsimple_slots[] = { + {Py_tp_repr, &Simple_repr}, + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_methods, Simple_methods}, + {Py_tp_getset, Simple_getsets}, + {Py_tp_init, Simple_init}, + {Py_tp_new, GenericPyCData_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Simple_bool}, + {0, NULL}, +}; + +static PyType_Spec pycsimple_spec = { + .name = "_ctypes._SimpleCData", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycsimple_slots, }; + /******************************************************************/ /* PyCPointer_Type @@ -4955,7 +4987,6 @@ Pointer_item(PyObject *myself, Py_ssize_t index) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (*(void **)self->b_ptr == NULL) { @@ -4964,19 +4995,27 @@ Pointer_item(PyObject *myself, Py_ssize_t index) return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer object instances */ + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer object instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* proto is the item type of the pointer, a ctypes + + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); /* proto is the item type of the pointer, a ctypes type, so this cannot be NULL */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_get(proto, stgdict->getfunc, (PyObject *)self, + return PyCData_get(proto, stginfo->getfunc, (PyObject *)self, index, size, (*(char **)self->b_ptr) + offset); } @@ -4986,7 +5025,6 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) CDataObject *self = (CDataObject *)myself; Py_ssize_t size; Py_ssize_t offset; - StgDictObject *stgdict, *itemdict; PyObject *proto; if (value == NULL) { @@ -5001,37 +5039,47 @@ Pointer_ass_item(PyObject *myself, Py_ssize_t index, PyObject *value) return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); /* Cannot be NULL because the itemtype of a pointer + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return -1; + } + assert(iteminfo); /* Cannot be NULL because the itemtype of a pointer is always a ctypes type */ - size = itemdict->size; - offset = index * itemdict->size; + size = iteminfo->size; + offset = index * iteminfo->size; - return PyCData_set((PyObject *)self, proto, stgdict->setfunc, value, + return PyCData_set((PyObject *)self, proto, stginfo->setfunc, value, index, size, (*(char **)self->b_ptr) + offset); } static PyObject * Pointer_get_contents(CDataObject *self, void *closure) { - StgDictObject *stgdict; - if (*(void **)self->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, "NULL pointer access"); return NULL; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - return PyCData_FromBaseObj(stgdict->proto, + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + + return PyCData_FromBaseObj(stginfo->proto, (PyObject *)self, 0, *(void **)self->b_ptr); } @@ -5039,7 +5087,6 @@ Pointer_get_contents(CDataObject *self, void *closure) static int Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) { - StgDictObject *stgdict; CDataObject *dst; PyObject *keep; @@ -5048,18 +5095,21 @@ Pointer_set_contents(CDataObject *self, PyObject *value, void *closure) "Pointer does not support item deletion"); return -1; } - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - assert(stgdict->proto); ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return -1; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + assert(stginfo->proto); if (!CDataObject_Check(st, value)) { - int res = PyObject_IsInstance(value, stgdict->proto); + int res = PyObject_IsInstance(value, stginfo->proto); if (res == -1) return -1; if (!res) { PyErr_Format(PyExc_TypeError, "expected %s instead of %s", - ((PyTypeObject *)(stgdict->proto))->tp_name, + ((PyTypeObject *)(stginfo->proto))->tp_name, Py_TYPE(value)->tp_name); return -1; } @@ -5107,8 +5157,12 @@ Pointer_init(CDataObject *self, PyObject *args, PyObject *kw) static PyObject * Pointer_new(PyTypeObject *type, PyObject *args, PyObject *kw) { - StgDictObject *dict = PyType_stgdict((PyObject *)type); - if (!dict || !dict->proto) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { + return NULL; + } + if (!info || !info->proto) { PyErr_SetString(PyExc_TypeError, "Cannot create instance: has no _type_"); return NULL; @@ -5130,7 +5184,6 @@ Pointer_subscript(PyObject *myself, PyObject *item) PySliceObject *slice = (PySliceObject *)item; Py_ssize_t start, stop, step; PyObject *np; - StgDictObject *stgdict, *itemdict; PyObject *proto; Py_ssize_t i, len; size_t cur; @@ -5184,13 +5237,20 @@ Pointer_subscript(PyObject *myself, PyObject *item) else len = (stop - start + 1) / step + 1; - stgdict = PyObject_stgdict((PyObject *)self); - assert(stgdict); /* Cannot be NULL for pointer instances */ - proto = stgdict->proto; + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromObject(st, (PyObject *)self, &stginfo) < 0) { + return NULL; + } + assert(stginfo); /* Cannot be NULL for pointer instances */ + proto = stginfo->proto; assert(proto); - itemdict = PyType_stgdict(proto); - assert(itemdict); - if (itemdict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + StgInfo *iteminfo; + if (PyStgInfo_FromType(st, proto, &iteminfo) < 0) { + return NULL; + } + assert(iteminfo); + if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = *(char **)self->b_ptr; char *dest; @@ -5210,7 +5270,7 @@ Pointer_subscript(PyObject *myself, PyObject *item) PyMem_Free(dest); return np; } - if (itemdict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = *(wchar_t **)self->b_ptr; wchar_t *dest; @@ -5248,87 +5308,32 @@ Pointer_subscript(PyObject *myself, PyObject *item) } } -static PySequenceMethods Pointer_as_sequence = { - 0, /* inquiry sq_length; */ - 0, /* binaryfunc sq_concat; */ - 0, /* intargfunc sq_repeat; */ - Pointer_item, /* intargfunc sq_item; */ - 0, /* intintargfunc sq_slice; */ - Pointer_ass_item, /* intobjargproc sq_ass_item; */ - 0, /* intintobjargproc sq_ass_slice; */ - 0, /* objobjproc sq_contains; */ - /* Added in release 2.0 */ - 0, /* binaryfunc sq_inplace_concat; */ - 0, /* intargfunc sq_inplace_repeat; */ -}; - -static PyMappingMethods Pointer_as_mapping = { - 0, - Pointer_subscript, -}; - static int Pointer_bool(CDataObject *self) { return (*(void **)self->b_ptr != NULL); } -static PyNumberMethods Pointer_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - (inquiry)Pointer_bool, /* nb_bool */ +static PyType_Slot pycpointer_slots[] = { + {Py_tp_doc, PyDoc_STR("XXX to be provided")}, + {Py_tp_getset, Pointer_getsets}, + {Py_tp_init, Pointer_init}, + {Py_tp_new, Pointer_new}, + {Py_bf_getbuffer, PyCData_NewGetBuffer}, + {Py_nb_bool, Pointer_bool}, + {Py_mp_subscript, Pointer_subscript}, + {Py_sq_item, Pointer_item}, + {Py_sq_ass_item, Pointer_ass_item}, + {0, NULL}, }; -PyTypeObject PyCPointer_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes._Pointer", - sizeof(CDataObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - &Pointer_as_number, /* tp_as_number */ - &Pointer_as_sequence, /* tp_as_sequence */ - &Pointer_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ - (traverseproc)PyCData_traverse, /* tp_traverse */ - (inquiry)PyCData_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - Pointer_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Pointer_init, /* tp_init */ - 0, /* tp_alloc */ - Pointer_new, /* tp_new */ - 0, /* tp_free */ +static PyType_Spec pycpointer_spec = { + .name = "_ctypes._Pointer", + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pycpointer_slots, }; - /******************************************************************/ /* * Module initialization. @@ -5433,7 +5438,6 @@ string_at(const char *ptr, int size) static int cast_check_pointertype(PyObject *arg) { - StgDictObject *dict; ctypes_state *st = GLOBAL_STATE(); if (PyCPointerTypeObject_Check(st, arg)) { @@ -5442,10 +5446,13 @@ cast_check_pointertype(PyObject *arg) if (PyCFuncPtrTypeObject_Check(st, arg)) { return 1; } - dict = PyType_stgdict(arg); - if (dict != NULL && dict->proto != NULL) { - if (PyUnicode_Check(dict->proto) - && (strchr("sPzUZXO", PyUnicode_AsUTF8(dict->proto)[0]))) { + StgInfo *info; + if (PyStgInfo_FromType(st, arg, &info) < 0) { + return 0; + } + if (info != NULL && info->proto != NULL) { + if (PyUnicode_Check(info->proto) + && (strchr("sPzUZXO", PyUnicode_AsUTF8(info->proto)[0]))) { /* simple pointer types, c_void_p, c_wchar_p, BSTR, ... */ return 1; } @@ -5544,26 +5551,8 @@ _ctypes_add_types(PyObject *mod) if (PyType_Ready(TYPE) < 0) { \ return -1; \ } - -#define TYPE_READY_BASE(TYPE_EXPR, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - type->tp_base = (TP_BASE); \ - TYPE_READY(type); \ - } while (0) - -#define MOD_ADD_TYPE(TYPE_EXPR, TP_TYPE, TP_BASE) \ - do { \ - PyTypeObject *type = (TYPE_EXPR); \ - Py_SET_TYPE(type, TP_TYPE); \ - type->tp_base = TP_BASE; \ - if (PyModule_AddType(mod, type) < 0) { \ - return -1; \ - } \ - } while (0) - -#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ +#define CREATE_TYPE(TP, SPEC, META, BASE) do { \ + PyObject *type = PyType_FromMetaclass(META, mod, SPEC, \ (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ @@ -5571,67 +5560,82 @@ _ctypes_add_types(PyObject *mod) TP = (PyTypeObject *)type; \ } while (0) +#define MOD_ADD_TYPE(TP, SPEC, META, BASE) do { \ + CREATE_TYPE(TP, SPEC, META, BASE); \ + if (PyModule_AddType(mod, (PyTypeObject *)(TP)) < 0) { \ + return -1; \ + } \ +} while (0) + ctypes_state *st = GLOBAL_STATE(); /* Note: ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); - TYPE_READY(st->PyCData_Type); - /* StgDict is derived from PyDict_Type */ - TYPE_READY_BASE(st->PyCStgDict_Type, &PyDict_Type); + CREATE_TYPE(st->PyCArg_Type, &carg_spec, NULL, NULL); + CREATE_TYPE(st->PyCThunk_Type, &cthunk_spec, NULL, NULL); + CREATE_TYPE(st->PyCData_Type, &pycdata_spec, NULL, NULL); + + // Common Metaclass + CREATE_TYPE(st->PyCType_Type, &pyctype_type_spec, + NULL, &PyType_Type); /************************************************* * * Metaclasses */ - CREATE_TYPE(mod, st->PyCStructType_Type, &pycstruct_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->UnionType_Type, &union_type_spec, &PyType_Type); - CREATE_TYPE(mod, st->PyCPointerType_Type, &pycpointer_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCArrayType_Type, &pycarray_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCSimpleType_Type, &pycsimple_type_spec, - &PyType_Type); - CREATE_TYPE(mod, st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, - &PyType_Type); + CREATE_TYPE(st->PyCStructType_Type, &pycstruct_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->UnionType_Type, &union_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCPointerType_Type, &pycpointer_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCArrayType_Type, &pycarray_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCSimpleType_Type, &pycsimple_type_spec, + NULL, st->PyCType_Type); + CREATE_TYPE(st->PyCFuncPtrType_Type, &pycfuncptr_type_spec, + NULL, st->PyCType_Type); /************************************************* * * Classes using a custom metaclass */ - MOD_ADD_TYPE(st->Struct_Type, st->PyCStructType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Union_Type, st->UnionType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCPointer_Type, st->PyCPointerType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCArray_Type, st->PyCArrayType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->Simple_Type, st->PyCSimpleType_Type, st->PyCData_Type); - MOD_ADD_TYPE(st->PyCFuncPtr_Type, st->PyCFuncPtrType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Struct_Type, &pycstruct_spec, + st->PyCStructType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Union_Type, &pycunion_spec, + st->UnionType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCPointer_Type, &pycpointer_spec, + st->PyCPointerType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCArray_Type, &pycarray_spec, + st->PyCArrayType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->Simple_Type, &pycsimple_spec, + st->PyCSimpleType_Type, st->PyCData_Type); + MOD_ADD_TYPE(st->PyCFuncPtr_Type, &pycfuncptr_spec, + st->PyCFuncPtrType_Type, st->PyCData_Type); /************************************************* * * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); + CREATE_TYPE(st->PyCField_Type, &cfield_spec, NULL, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); + CREATE_TYPE(st->DictRemover_Type, &dictremover_spec, NULL, NULL); + CREATE_TYPE(st->StructParam_Type, &structparam_spec, NULL, NULL); #ifdef MS_WIN32 - CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); + CREATE_TYPE(st->PyComError_Type, &comerror_spec, NULL, PyExc_Exception); #endif #undef TYPE_READY -#undef TYPE_READY_BASE #undef MOD_ADD_TYPE #undef CREATE_TYPE return 0; diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index f70479435915ff..08d068e47ee2bf 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -109,10 +109,14 @@ PrintError(const char *msg, ...) * slower. */ static void -TryAddRef(StgDictObject *dict, CDataObject *obj) +TryAddRef(PyObject *cnv, CDataObject *obj) { IUnknown *punk; - int r = PyDict_Contains((PyObject *)dict, &_Py_ID(_needs_com_addref_)); + PyObject *attrdict = _PyType_GetDict((PyTypeObject *)cnv); + if (!attrdict) { + return; + } + int r = PyDict_Contains(attrdict, &_Py_ID(_needs_com_addref_)); if (r <= 0) { if (r < 0) { PrintError("getting _needs_com_addref_"); @@ -154,22 +158,26 @@ static void _CallPythonObject(void *mem, ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < nargs; i++) { PyObject *cnv = cnvs[i]; // borrowed ref - StgDictObject *dict; - dict = PyType_stgdict(cnv); - if (dict && dict->getfunc && !_ctypes_simple_instance(cnv)) { - PyObject *v = dict->getfunc(*pArgs, dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, cnv, &info) < 0) { + goto Done; + } + + if (info && info->getfunc && !_ctypes_simple_instance(cnv)) { + PyObject *v = info->getfunc(*pArgs, info->size); if (!v) { PrintError("create argument %zd:\n", i); goto Done; } args[i] = v; /* XXX XXX XX - We have the problem that c_byte or c_short have dict->size of + We have the problem that c_byte or c_short have info->size of 1 resp. 4, but these parameters are pushed as sizeof(int) bytes. BTW, the same problem occurs when they are pushed as parameters */ - } else if (dict) { + } + else if (info) { /* Hm, shouldn't we use PyCData_AtAddress() or something like that instead? */ CDataObject *obj = (CDataObject *)_PyObject_CallNoArgs(cnv); if (!obj) { @@ -181,10 +189,10 @@ static void _CallPythonObject(void *mem, PrintError("unexpected result of create argument %zd:\n", i); goto Done; } - memcpy(obj->b_ptr, *pArgs, dict->size); + memcpy(obj->b_ptr, *pArgs, info->size); args[i] = (PyObject *)obj; #ifdef MS_WIN32 - TryAddRef(dict, obj); + TryAddRef(cnv, obj); #endif } else { PyErr_SetString(PyExc_TypeError, @@ -348,10 +356,8 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, if (p == NULL) return NULL; -#ifdef Py_DEBUG ctypes_state *st = GLOBAL_STATE(); assert(CThunk_CheckExact(st, (PyObject *)p)); -#endif p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { @@ -372,14 +378,18 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, p->setfunc = NULL; p->ffi_restype = &ffi_type_void; } else { - StgDictObject *dict = PyType_stgdict(restype); - if (dict == NULL || dict->setfunc == NULL) { + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + goto error; + } + + if (info == NULL || info->setfunc == NULL) { PyErr_SetString(PyExc_TypeError, "invalid result type for callback function"); goto error; } - p->setfunc = dict->setfunc; - p->ffi_restype = &dict->ffi_type_pointer; + p->setfunc = info->setfunc; + p->ffi_restype = &info->ffi_type_pointer; } cc = FFI_DEFAULT_ABI; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 97d1dbaae03d4f..6ebbb64d61b07a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -664,15 +664,20 @@ struct argument { */ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) { - StgDictObject *dict; pa->keep = NULL; /* so we cannot forget it later */ + ctypes_state *st = GLOBAL_STATE(); - dict = PyObject_stgdict(obj); - if (dict) { + StgInfo *info; + int result = PyStgInfo_FromObject(st, obj, &info); + if (result < 0) { + return -1; + } + if (info) { + assert(info); PyCArgObject *carg; - assert(dict->paramfunc); - /* If it has an stgdict, it is a CDataObject */ - carg = dict->paramfunc((CDataObject *)obj); + assert(info->paramfunc); + /* If it has an stginfo, it is a CDataObject */ + carg = info->paramfunc((CDataObject *)obj); if (carg == NULL) return -1; pa->ffi_type = carg->pffi_type; @@ -681,7 +686,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) return 0; } - ctypes_state *st = GLOBAL_STATE(); if (PyCArg_CheckExact(st, obj)) { PyCArgObject *carg = (PyCArgObject *)obj; pa->ffi_type = carg->pffi_type; @@ -778,26 +782,34 @@ int can_return_struct_as_sint64(size_t s) #endif +// returns NULL with exception set on error ffi_type *_ctypes_get_ffi_type(PyObject *obj) { - StgDictObject *dict; - if (obj == NULL) + if (obj == NULL) { return &ffi_type_sint; - dict = PyType_stgdict(obj); - if (dict == NULL) + } + + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; + } + + if (info == NULL) { return &ffi_type_sint; + } #if defined(MS_WIN32) && !defined(_WIN32_WCE) /* This little trick works correctly with MSVC. It returns small structures in registers */ - if (dict->ffi_type_pointer.type == FFI_TYPE_STRUCT) { - if (can_return_struct_as_int(dict->ffi_type_pointer.size)) + if (info->ffi_type_pointer.type == FFI_TYPE_STRUCT) { + if (can_return_struct_as_int(info->ffi_type_pointer.size)) return &ffi_type_sint32; - else if (can_return_struct_as_sint64 (dict->ffi_type_pointer.size)) + else if (can_return_struct_as_sint64 (info->ffi_type_pointer.size)) return &ffi_type_sint64; } #endif - return &dict->ffi_type_pointer; + return &info->ffi_type_pointer; } @@ -983,7 +995,6 @@ static int _call_function_pointer(int flags, */ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) { - StgDictObject *dict; PyObject *retval, *v; if (restype == NULL) @@ -993,17 +1004,22 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) Py_RETURN_NONE; } - dict = PyType_stgdict(restype); - if (dict == NULL) + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromType(st, restype, &info) < 0) { + return NULL; + } + if (info == NULL) { return PyObject_CallFunction(restype, "i", *(int *)result); + } - if (dict->getfunc && !_ctypes_simple_instance(restype)) { - retval = dict->getfunc(result, dict->size); + if (info->getfunc && !_ctypes_simple_instance(restype)) { + retval = info->getfunc(result, info->size); /* If restype is py_object (detected by comparing getfunc with O_get), we have to call Py_DECREF because O_get has already called Py_INCREF. */ - if (dict->getfunc == _ctypes_get_fielddesc("O")->getfunc) { + if (info->getfunc == _ctypes_get_fielddesc("O")->getfunc) { Py_DECREF(retval); } } else @@ -1240,6 +1256,9 @@ PyObject *_ctypes_callproc(PPROC pProc, } else { rtype = _ctypes_get_ffi_type(restype); } + if (!rtype) { + goto cleanup; + } resbuf = alloca(max(rtype->size, sizeof(ffi_arg))); @@ -1683,13 +1702,16 @@ PyDoc_STRVAR(sizeof_doc, static PyObject * sizeof_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; + ctypes_state *st = GLOBAL_STATE(); - dict = PyType_stgdict(obj); - if (dict) { - return PyLong_FromSsize_t(dict->size); + StgInfo *info; + if (PyStgInfo_FromType(st, obj, &info) < 0) { + return NULL; } - ctypes_state *st = GLOBAL_STATE(); + if (info) { + return PyLong_FromSsize_t(info->size); + } + if (CDataObject_Check(st, obj)) { return PyLong_FromSsize_t(((CDataObject *)obj)->b_size); } @@ -1706,16 +1728,14 @@ PyDoc_STRVAR(alignment_doc, static PyObject * align_func(PyObject *self, PyObject *obj) { - StgDictObject *dict; - - dict = PyType_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - - dict = PyObject_stgdict(obj); - if (dict) - return PyLong_FromSsize_t(dict->align); - + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromAny(st, obj, &info) < 0) { + return NULL; + } + if (info) { + return PyLong_FromSsize_t(info->align); + } PyErr_SetString(PyExc_TypeError, "no alignment info"); return NULL; @@ -1824,7 +1844,6 @@ static PyObject * resize(PyObject *self, PyObject *args) { CDataObject *obj; - StgDictObject *dict; Py_ssize_t size; if (!PyArg_ParseTuple(args, @@ -1832,16 +1851,21 @@ resize(PyObject *self, PyObject *args) &obj, &size)) return NULL; - dict = PyObject_stgdict((PyObject *)obj); - if (dict == NULL) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + int result = PyStgInfo_FromObject(st, (PyObject *)obj, &info); + if (result < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "expected ctypes instance"); return NULL; } - if (size < dict->size) { + if (size < info->size) { PyErr_Format(PyExc_ValueError, "minimum size is %zd", - dict->size); + info->size); return NULL; } if (obj->b_needsfree == 0) { @@ -2004,28 +2028,30 @@ create_pointer_inst(PyObject *module, PyObject *arg) static PyObject * buffer_info(PyObject *self, PyObject *arg) { - StgDictObject *dict = PyType_stgdict(arg); PyObject *shape; Py_ssize_t i; - if (dict == NULL) - dict = PyObject_stgdict(arg); - if (dict == NULL) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *info; + if (PyStgInfo_FromAny(st, arg, &info) < 0) { + return NULL; + } + if (info == NULL) { PyErr_SetString(PyExc_TypeError, "not a ctypes type or object"); return NULL; } - shape = PyTuple_New(dict->ndim); + shape = PyTuple_New(info->ndim); if (shape == NULL) return NULL; - for (i = 0; i < (int)dict->ndim; ++i) - PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(dict->shape[i])); + for (i = 0; i < (int)info->ndim; ++i) + PyTuple_SET_ITEM(shape, i, PyLong_FromSsize_t(info->shape[i])); if (PyErr_Occurred()) { Py_DECREF(shape); return NULL; } - return Py_BuildValue("siN", dict->format, dict->ndim, shape); + return Py_BuildValue("siN", info->format, info->ndim, shape); } diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 1d5b0b14bc39e5..16b66382bfe33f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -54,7 +54,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t size, align; SETFUNC setfunc = NULL; GETFUNC getfunc = NULL; - StgDictObject *dict; int fieldtype; #define NO_BITFIELD 0 #define NEW_BITFIELD 1 @@ -66,21 +65,27 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, self = (CFieldObject *)tp->tp_alloc(tp, 0); if (self == NULL) return NULL; - dict = PyType_stgdict(desc); - if (!dict) { + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(self); + return NULL; + } + if (!info) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); Py_DECREF(self); return NULL; } + if (bitsize /* this is a bitfield request */ && *pfield_size /* we have a bitfield open */ #ifdef MS_WIN32 /* MSVC, GCC with -mms-bitfields */ - && dict->size * 8 == *pfield_size + && info->size * 8 == *pfield_size #else /* GCC */ - && dict->size * 8 <= *pfield_size + && info->size * 8 <= *pfield_size #endif && (*pbitofs + bitsize) <= *pfield_size) { /* continue bit field */ @@ -88,8 +93,8 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, #ifndef MS_WIN32 } else if (bitsize /* this is a bitfield request */ && *pfield_size /* we have a bitfield open */ - && dict->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= dict->size * 8) { + && info->size * 8 >= *pfield_size + && (*pbitofs + bitsize) <= info->size * 8) { /* expand bit field */ fieldtype = EXPAND_BITFIELD; #endif @@ -97,7 +102,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, /* start new bitfield */ fieldtype = NEW_BITFIELD; *pbitofs = 0; - *pfield_size = dict->size * 8; + *pfield_size = info->size * 8; } else { /* not a bit field */ fieldtype = NO_BITFIELD; @@ -105,29 +110,37 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, *pfield_size = 0; } - size = dict->size; + size = info->size; proto = desc; /* Field descriptors for 'c_char * n' are be scpecial cased to return a Python string instead of an Array object instance... */ if (PyCArrayTypeObject_Check(st, proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { + StgInfo *ainfo; + if (PyStgInfo_FromType(st, proto, &ainfo) < 0) { + Py_DECREF(self); + return NULL; + } + + if (ainfo && ainfo->proto) { + StgInfo *iinfo; + if (PyStgInfo_FromType(st, ainfo->proto, &iinfo) < 0) { + Py_DECREF(self); + return NULL; + } + if (!iinfo) { PyErr_SetString(PyExc_TypeError, "has no _stginfo_"); Py_DECREF(self); return NULL; } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("s"); getfunc = fd->getfunc; setfunc = fd->setfunc; } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + if (iinfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { struct fielddesc *fd = _ctypes_get_fielddesc("U"); getfunc = fd->getfunc; setfunc = fd->setfunc; @@ -151,9 +164,9 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, /* fall through */ case NO_BITFIELD: if (pack) - align = min(pack, dict->align); + align = min(pack, info->align); else - align = dict->align; + align = info->align; if (align && *poffset % align) { Py_ssize_t delta = align - (*poffset % align); *psize += delta; @@ -171,10 +184,10 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, break; case EXPAND_BITFIELD: - *poffset += dict->size - *pfield_size/8; - *psize += dict->size - *pfield_size/8; + *poffset += info->size - *pfield_size/8; + *psize += info->size - *pfield_size/8; - *pfield_size = dict->size * 8; + *pfield_size = info->size * 8; if (big_endian) self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 02f48a9ed55843..d7d725a4fdf669 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -41,7 +41,6 @@ typedef struct { PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; - PyTypeObject *PyCStgDict_Type; PyTypeObject *StructParam_Type; PyTypeObject *PyCStructType_Type; PyTypeObject *UnionType_Type; @@ -59,6 +58,7 @@ typedef struct { #ifdef MS_WIN32 PyTypeObject *PyComError_Type; #endif + PyTypeObject *PyCType_Type; } ctypes_state; extern ctypes_state global_state; @@ -144,7 +144,7 @@ typedef struct { CThunkObject *thunk; PyObject *callable; - /* These two fields will override the ones in the type's stgdict if + /* These two fields will override the ones in the type's stginfo if they are set */ PyObject *converters; PyObject *argtypes; @@ -158,17 +158,12 @@ typedef struct { PyObject *paramflags; } PyCFuncPtrObject; -extern PyTypeObject PyCStgDict_Type; -#define PyCStgDict_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCStgDict_Type) -#define PyCStgDict_Check(st, v) PyObject_TypeCheck((v), (st)->PyCStgDict_Type) - -extern int PyCStructUnionType_update_stgdict(PyObject *fields, PyObject *type, int isStruct); +extern int PyCStructUnionType_update_stginfo(PyObject *fields, PyObject *type, int isStruct); extern int PyType_stginfo(PyTypeObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palign, Py_ssize_t *plength); -extern PyTypeObject PyCData_Type; #define CDataObject_CheckExact(st, v) Py_IS_TYPE((v), (st)->PyCData_Type) #define CDataObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCData_Type) #define _CDataObject_HasExternalBuffer(v) ((v)->b_ptr != (char *)&(v)->b_value) @@ -188,10 +183,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); -extern PyTypeObject PyCArray_Type; -extern PyTypeObject PyCPointer_Type; -extern PyTypeObject PyCFuncPtr_Type; - #define PyCArrayTypeObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArrayType_Type) #define ArrayObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCArray_Type) #define PointerObject_Check(st, v) PyObject_TypeCheck((v), (st)->PyCPointer_Type) @@ -231,45 +222,22 @@ typedef struct { int anonymous; } CFieldObject; -/* A subclass of PyDictObject, used as the instance dictionary of ctypes - metatypes */ -typedef struct { - PyDictObject dict; /* first part identical to PyDictObject */ -/* The size and align fields are unneeded, they are in ffi_type as well. As - an experiment shows, it's trivial to get rid of them, the only thing to - remember is that in PyCArrayType_new the ffi_type fields must be filled in - - so far it was unneeded because libffi doesn't support arrays at all - (because they are passed as pointers to function calls anyway). But it's - too much risk to change that now, and there are other fields which doesn't - belong into this structure anyway. Maybe in ctypes 2.0... (ctypes 2000?) -*/ - Py_ssize_t size; /* number of bytes */ - Py_ssize_t align; /* alignment requirements */ - Py_ssize_t length; /* number of fields */ - ffi_type ffi_type_pointer; - PyObject *proto; /* Only for Pointer/ArrayObject */ - SETFUNC setfunc; /* Only for simple objects */ - GETFUNC getfunc; /* Only for simple objects */ - PARAMFUNC paramfunc; +/**************************************************************** + StgInfo - /* Following fields only used by PyCFuncPtrType_Type instances */ - PyObject *argtypes; /* tuple of CDataObjects */ - PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ - PyObject *restype; /* CDataObject or NULL */ - PyObject *checker; - int flags; /* calling convention and such */ + Since Python 3.13, ctypes-specific type information is stored in the + corresponding type object, in a `StgInfo` struct accessed by the helpers + below. + Before that, each type's `tp_dict` was set to a dict *subclass* that included + the fields that are now in StgInfo. The mechanism was called "StgDict"; a few + references to that name might remain. - /* pep3118 fields, pointers need PyMem_Free */ - char *format; - int ndim; - Py_ssize_t *shape; -/* Py_ssize_t *strides; */ /* unused in ctypes */ -/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ + Functions for accessing StgInfo are `static inline` for performance; + see later in this file. -} StgDictObject; + **************************************************************** -/**************************************************************** - StgDictObject fields + StgInfo fields setfunc and getfunc is only set for simple data types, it is copied from the corresponding fielddesc entry. These are functions to set and get the value @@ -280,11 +248,11 @@ typedef struct { object. Probably all the magic ctypes methods (like from_param) should have C - callable wrappers in the StgDictObject. For simple data type, for example, + callable wrappers in the StgInfo. For simple data type, for example, the fielddesc table could have entries for C codec from_param functions or other methods as well, if a subtype overrides this method in Python at construction time, or assigns to it later, tp_setattro should update the - StgDictObject function to a generic one. + StgInfo function to a generic one. Currently, PyCFuncPtr types have 'converters' and 'checker' entries in their type dict. They are only used to cache attributes from other entries, which @@ -308,13 +276,33 @@ typedef struct { *****************************************************************/ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyType_stgdict(PyObject *obj); +typedef struct { + int initialized; + Py_ssize_t size; /* number of bytes */ + Py_ssize_t align; /* alignment requirements */ + Py_ssize_t length; /* number of fields */ + ffi_type ffi_type_pointer; + PyObject *proto; /* Only for Pointer/ArrayObject */ + SETFUNC setfunc; /* Only for simple objects */ + GETFUNC getfunc; /* Only for simple objects */ + PARAMFUNC paramfunc; + + /* Following fields only used by PyCFuncPtrType_Type instances */ + PyObject *argtypes; /* tuple of CDataObjects */ + PyObject *converters; /* tuple([t.from_param for t in argtypes]) */ + PyObject *restype; /* CDataObject or NULL */ + PyObject *checker; + int flags; /* calling convention and such */ -/* May return NULL, but does not set an exception! */ -extern StgDictObject *PyObject_stgdict(PyObject *self); + /* pep3118 fields, pointers need PyMem_Free */ + char *format; + int ndim; + Py_ssize_t *shape; +/* Py_ssize_t *strides; */ /* unused in ctypes */ +/* Py_ssize_t *suboffsets; */ /* unused in ctypes */ +} StgInfo; -extern int PyCStgDict_clone(StgDictObject *src, StgDictObject *dst); +extern int PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info); typedef int(* PPROC)(void); @@ -416,8 +404,74 @@ void *Py_ffi_closure_alloc(size_t size, void** codeloc); #define Py_ffi_closure_alloc ffi_closure_alloc #endif -/* - Local Variables: - compile-command: "python setup.py -q build install --home ~" - End: -*/ + +/**************************************************************** + * Accessing StgInfo -- these are inlined for performance reasons. + */ + +// `PyStgInfo_From**` functions get a PyCTypeDataObject. +// These return -1 on error, 0 if "not found", 1 on OK. +// (Currently, these do not return -1 in practice. This might change +// in the future.) + +// +// Common helper: +static inline int +_stginfo_from_type(ctypes_state *state, PyTypeObject *type, StgInfo **result) +{ + *result = NULL; + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + // not a ctypes class. + return 0; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + assert(info != NULL); + if (!info->initialized) { + // StgInfo is not initialized. This happens in abstract classes. + return 0; + } + *result = info; + return 1; +} +// from a type: +static inline int +PyStgInfo_FromType(ctypes_state *state, PyObject *type, StgInfo **result) +{ + return _stginfo_from_type(state, (PyTypeObject *)type, result); +} +// from an instance: +static inline int +PyStgInfo_FromObject(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + return _stginfo_from_type(state, Py_TYPE(obj), result); +} +// from either a type or an instance: +static inline int +PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) +{ + if (PyType_Check(obj)) { + return _stginfo_from_type(state, (PyTypeObject *)obj, result); + } + return _stginfo_from_type(state, Py_TYPE(obj), result); +} + +// Initialize StgInfo on a newly created type +static inline StgInfo * +PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) +{ + if (!PyObject_IsInstance((PyObject *)type, (PyObject *)state->PyCType_Type)) { + PyErr_Format(PyExc_SystemError, + "'%s' is not a ctypes class.", + type->tp_name); + return NULL; + } + StgInfo *info = PyObject_GetTypeData((PyObject *)type, state->PyCType_Type); + if (info->initialized) { + PyErr_Format(PyExc_SystemError, + "StgInfo of '%s' is already initialized.", + type->tp_name); + return NULL; + } + info->initialized = 1; + return info; +} diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 32ee414a7a0cdd..8666ded5c2b3f2 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -16,201 +16,62 @@ #endif #include "ctypes.h" -/******************************************************************/ -/* - StdDict - a dictionary subclass, containing additional C accessible fields - - XXX blabla more -*/ - -/* Seems we need this, otherwise we get problems when calling - * PyDict_SetItem() (ma_lookup is NULL) +/* This file relates to StgInfo -- type-specific information for ctypes. + * See ctypes.h for details. */ -static int -PyCStgDict_init(StgDictObject *self, PyObject *args, PyObject *kwds) -{ - if (PyDict_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->format = NULL; - self->ndim = 0; - self->shape = NULL; - return 0; -} -static int -PyCStgDict_clear(StgDictObject *self) +int +PyCStgInfo_clone(StgInfo *dst_info, StgInfo *src_info) { - Py_CLEAR(self->proto); - Py_CLEAR(self->argtypes); - Py_CLEAR(self->converters); - Py_CLEAR(self->restype); - Py_CLEAR(self->checker); - return 0; -} + Py_ssize_t size; -static void -PyCStgDict_dealloc(StgDictObject *self) -{ - PyCStgDict_clear(self); - PyMem_Free(self->format); - PyMem_Free(self->shape); - PyMem_Free(self->ffi_type_pointer.elements); - PyDict_Type.tp_dealloc((PyObject *)self); -} + PyMem_Free(dst_info->ffi_type_pointer.elements); + PyMem_Free(dst_info->format); + dst_info->format = NULL; + PyMem_Free(dst_info->shape); + dst_info->shape = NULL; + dst_info->ffi_type_pointer.elements = NULL; -static PyObject * -PyCStgDict_sizeof(StgDictObject *self, void *unused) -{ - Py_ssize_t res; - - res = _PyDict_SizeOf((PyDictObject *)self); - res += sizeof(StgDictObject) - sizeof(PyDictObject); - if (self->format) - res += strlen(self->format) + 1; - res += self->ndim * sizeof(Py_ssize_t); - if (self->ffi_type_pointer.elements) - res += (self->length + 1) * sizeof(ffi_type *); - return PyLong_FromSsize_t(res); -} + memcpy(dst_info, src_info, sizeof(StgInfo)); -int -PyCStgDict_clone(StgDictObject *dst, StgDictObject *src) -{ - char *d, *s; - Py_ssize_t size; + Py_XINCREF(dst_info->proto); + Py_XINCREF(dst_info->argtypes); + Py_XINCREF(dst_info->converters); + Py_XINCREF(dst_info->restype); + Py_XINCREF(dst_info->checker); - PyCStgDict_clear(dst); - PyMem_Free(dst->ffi_type_pointer.elements); - PyMem_Free(dst->format); - dst->format = NULL; - PyMem_Free(dst->shape); - dst->shape = NULL; - dst->ffi_type_pointer.elements = NULL; - - d = (char *)dst; - s = (char *)src; - memcpy(d + sizeof(PyDictObject), - s + sizeof(PyDictObject), - sizeof(StgDictObject) - sizeof(PyDictObject)); - - Py_XINCREF(dst->proto); - Py_XINCREF(dst->argtypes); - Py_XINCREF(dst->converters); - Py_XINCREF(dst->restype); - Py_XINCREF(dst->checker); - - if (src->format) { - dst->format = PyMem_Malloc(strlen(src->format) + 1); - if (dst->format == NULL) { + if (src_info->format) { + dst_info->format = PyMem_Malloc(strlen(src_info->format) + 1); + if (dst_info->format == NULL) { PyErr_NoMemory(); return -1; } - strcpy(dst->format, src->format); + strcpy(dst_info->format, src_info->format); } - if (src->shape) { - dst->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src->ndim); - if (dst->shape == NULL) { + if (src_info->shape) { + dst_info->shape = PyMem_Malloc(sizeof(Py_ssize_t) * src_info->ndim); + if (dst_info->shape == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->shape, src->shape, - sizeof(Py_ssize_t) * src->ndim); + memcpy(dst_info->shape, src_info->shape, + sizeof(Py_ssize_t) * src_info->ndim); } - if (src->ffi_type_pointer.elements == NULL) + if (src_info->ffi_type_pointer.elements == NULL) return 0; - size = sizeof(ffi_type *) * (src->length + 1); - dst->ffi_type_pointer.elements = PyMem_Malloc(size); - if (dst->ffi_type_pointer.elements == NULL) { + size = sizeof(ffi_type *) * (src_info->length + 1); + dst_info->ffi_type_pointer.elements = PyMem_Malloc(size); + if (dst_info->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memcpy(dst->ffi_type_pointer.elements, - src->ffi_type_pointer.elements, + memcpy(dst_info->ffi_type_pointer.elements, + src_info->ffi_type_pointer.elements, size); return 0; } -static struct PyMethodDef PyCStgDict_methods[] = { - {"__sizeof__", (PyCFunction)PyCStgDict_sizeof, METH_NOARGS}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyCStgDict_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "StgDict", - sizeof(StgDictObject), - 0, - (destructor)PyCStgDict_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyCStgDict_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyCStgDict_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ -}; - -/* May return NULL, but does not set an exception! */ -StgDictObject * -PyType_stgdict(PyObject *obj) -{ - PyTypeObject *type; - - if (!PyType_Check(obj)) { - return NULL; - } - ctypes_state *st = GLOBAL_STATE(); - type = (PyTypeObject *)obj; - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - -/* May return NULL, but does not set an exception! */ -/* - This function should be as fast as possible, so we don't call PyType_stgdict - above but inline the code, and avoid the PyType_Check(). -*/ -StgDictObject * -PyObject_stgdict(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - ctypes_state *st = GLOBAL_STATE(); - if (!type->tp_dict || !PyCStgDict_CheckExact(st, type->tp_dict)) { - return NULL; - } - return (StgDictObject *)type->tp_dict; -} - /* descr is the descriptor for a field marked as anonymous. Get all the _fields_ descriptors from descr->proto, create new descriptors with offset and index adjusted, and stuff them into type. @@ -372,12 +233,11 @@ _ctypes_alloc_format_padding(const char *prefix, Py_ssize_t padding) /* Retrieve the (optional) _pack_ attribute from a type, the _fields_ attribute, - and create an StgDictObject. Used for Structure and Union subclasses. + and initialize StgInfo. Used for Structure and Union subclasses. */ int -PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct) +PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct) { - StgDictObject *stgdict, *basedict; Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align, aligned_size; Py_ssize_t field_size = 0; @@ -456,90 +316,97 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } - stgdict = PyType_stgdict(type); - if (!stgdict) { + ctypes_state *st = GLOBAL_STATE(); + StgInfo *stginfo; + if (PyStgInfo_FromType(st, type, &stginfo) < 0) { + return -1; + } + if (!stginfo) { PyErr_SetString(PyExc_TypeError, "ctypes state is not initialized"); return -1; } + /* If this structure/union is already marked final we cannot assign _fields_ anymore. */ - if (stgdict->flags & DICTFLAG_FINAL) {/* is final ? */ + if (stginfo->flags & DICTFLAG_FINAL) {/* is final ? */ PyErr_SetString(PyExc_AttributeError, "_fields_ is final"); return -1; } - if (stgdict->format) { - PyMem_Free(stgdict->format); - stgdict->format = NULL; + if (stginfo->format) { + PyMem_Free(stginfo->format); + stginfo->format = NULL; } - if (stgdict->ffi_type_pointer.elements) - PyMem_Free(stgdict->ffi_type_pointer.elements); + if (stginfo->ffi_type_pointer.elements) + PyMem_Free(stginfo->ffi_type_pointer.elements); - basedict = PyType_stgdict((PyObject *)((PyTypeObject *)type)->tp_base); - if (basedict) { - stgdict->flags |= (basedict->flags & + StgInfo *baseinfo; + if (PyStgInfo_FromType(st, (PyObject *)((PyTypeObject *)type)->tp_base, + &baseinfo) < 0) { + return -1; + } + if (baseinfo) { + stginfo->flags |= (baseinfo->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); } if (!isStruct) { - stgdict->flags |= TYPEFLAG_HASUNION; + stginfo->flags |= TYPEFLAG_HASUNION; } - if (basedict) { - size = offset = basedict->size; - align = basedict->align; + if (baseinfo) { + size = offset = baseinfo->size; + align = baseinfo->align; union_size = 0; total_align = align ? align : 1; total_align = max(total_align, forced_alignment); - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, basedict->length + len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, baseinfo->length + len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memset(stgdict->ffi_type_pointer.elements, 0, - sizeof(ffi_type *) * (basedict->length + len + 1)); - if (basedict->length > 0) { - memcpy(stgdict->ffi_type_pointer.elements, - basedict->ffi_type_pointer.elements, - sizeof(ffi_type *) * (basedict->length)); + memset(stginfo->ffi_type_pointer.elements, 0, + sizeof(ffi_type *) * (baseinfo->length + len + 1)); + if (baseinfo->length > 0) { + memcpy(stginfo->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, + sizeof(ffi_type *) * (baseinfo->length)); } - ffi_ofs = basedict->length; + ffi_ofs = baseinfo->length; } else { offset = 0; size = 0; align = 0; union_size = 0; total_align = forced_alignment; - stgdict->ffi_type_pointer.type = FFI_TYPE_STRUCT; - stgdict->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); - if (stgdict->ffi_type_pointer.elements == NULL) { + stginfo->ffi_type_pointer.type = FFI_TYPE_STRUCT; + stginfo->ffi_type_pointer.elements = PyMem_New(ffi_type *, len + 1); + if (stginfo->ffi_type_pointer.elements == NULL) { PyErr_NoMemory(); return -1; } - memset(stgdict->ffi_type_pointer.elements, 0, + memset(stginfo->ffi_type_pointer.elements, 0, sizeof(ffi_type *) * (len + 1)); ffi_ofs = 0; } - assert(stgdict->format == NULL); + assert(stginfo->format == NULL); if (isStruct) { - stgdict->format = _ctypes_alloc_format_string(NULL, "T{"); + stginfo->format = _ctypes_alloc_format_string(NULL, "T{"); } else { /* PEP3118 doesn't support union. Use 'B' for bytes. */ - stgdict->format = _ctypes_alloc_format_string(NULL, "B"); + stginfo->format = _ctypes_alloc_format_string(NULL, "B"); } - if (stgdict->format == NULL) + if (stginfo->format == NULL) return -1; - ctypes_state *st = GLOBAL_STATE(); for (i = 0; i < len; ++i) { PyObject *name = NULL, *desc = NULL; PyObject *pair = PySequence_GetItem(fields, i); PyObject *prop; - StgDictObject *dict; int bitsize = 0; if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { @@ -551,22 +418,28 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (PyCArrayTypeObject_Check(st, desc)) { arrays_seen = 1; } - dict = PyType_stgdict(desc); - if (dict == NULL) { + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + return -1; + } + if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); return -1; } - stgdict->ffi_type_pointer.elements[ffi_ofs + i] = &dict->ffi_type_pointer; - if (dict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) - stgdict->flags |= TYPEFLAG_HASPOINTER; - stgdict->flags |= dict->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); - dict->flags |= DICTFLAG_FINAL; /* mark field type final */ + + stginfo->ffi_type_pointer.elements[ffi_ofs + i] = &info->ffi_type_pointer; + if (info->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) + stginfo->flags |= TYPEFLAG_HASPOINTER; + stginfo->flags |= info->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); + info->flags |= DICTFLAG_FINAL; /* mark field type final */ if (PyTuple_Size(pair) == 3) { /* bits specified */ - stgdict->flags |= TYPEFLAG_HASBITFIELD; - switch(dict->ffi_type_pointer.type) { + stginfo->flags |= TYPEFLAG_HASBITFIELD; + switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: case FFI_TYPE_UINT32: @@ -577,8 +450,8 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: case FFI_TYPE_SINT32: - if (dict->getfunc != _ctypes_get_fielddesc("c")->getfunc - && dict->getfunc != _ctypes_get_fielddesc("u")->getfunc + if (info->getfunc != _ctypes_get_fielddesc("c")->getfunc + && info->getfunc != _ctypes_get_fielddesc("u")->getfunc ) break; /* else fall through */ @@ -589,7 +462,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_DECREF(pair); return -1; } - if (bitsize <= 0 || bitsize > dict->size * 8) { + if (bitsize <= 0 || bitsize > info->size * 8) { PyErr_SetString(PyExc_ValueError, "number of bits invalid for bit field"); Py_DECREF(pair); @@ -599,7 +472,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct bitsize = 0; if (isStruct) { - const char *fieldfmt = dict->format ? dict->format : "B"; + const char *fieldfmt = info->format ? info->format : "B"; const char *fieldname = PyUnicode_AsUTF8(name); char *ptr; Py_ssize_t len; @@ -629,10 +502,10 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct padding = ((CFieldObject *)prop)->offset - last_size; if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); return -1; @@ -650,17 +523,17 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct } sprintf(buf, "%s:%s:", fieldfmt, fieldname); - ptr = stgdict->format; - if (dict->shape != NULL) { - stgdict->format = _ctypes_alloc_format_string_with_shape( - dict->ndim, dict->shape, stgdict->format, buf); + ptr = stginfo->format; + if (info->shape != NULL) { + stginfo->format = _ctypes_alloc_format_string_with_shape( + info->ndim, info->shape, stginfo->format, buf); } else { - stgdict->format = _ctypes_alloc_format_string(stgdict->format, buf); + stginfo->format = _ctypes_alloc_format_string(stginfo->format, buf); } PyMem_Free(ptr); PyMem_Free(buf); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { Py_DECREF(pair); Py_DECREF(prop); return -1; @@ -704,29 +577,29 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct /* Pad up to the full size of the struct */ padding = aligned_size - size; if (padding > 0) { - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_padding(ptr, padding); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_padding(ptr, padding); PyMem_Free(ptr); - if (stgdict->format == NULL) { + if (stginfo->format == NULL) { return -1; } } - ptr = stgdict->format; - stgdict->format = _ctypes_alloc_format_string(stgdict->format, "}"); + ptr = stginfo->format; + stginfo->format = _ctypes_alloc_format_string(stginfo->format, "}"); PyMem_Free(ptr); - if (stgdict->format == NULL) + if (stginfo->format == NULL) return -1; } - stgdict->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, + stginfo->ffi_type_pointer.alignment = Py_SAFE_DOWNCAST(total_align, Py_ssize_t, unsigned short); - stgdict->ffi_type_pointer.size = aligned_size; + stginfo->ffi_type_pointer.size = aligned_size; - stgdict->size = aligned_size; - stgdict->align = total_align; - stgdict->length = ffi_ofs + len; + stginfo->size = aligned_size; + stginfo->align = total_align; + stginfo->length = ffi_ofs + len; /* * The value of MAX_STRUCT_SIZE depends on the platform Python is running on. @@ -817,7 +690,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct for (i = 0; i < len; ++i) { PyObject *name, *desc; PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; int bitsize = 0; if (pair == NULL) { @@ -829,25 +701,34 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_DECREF(pair); return -1; } - dict = PyType_stgdict(desc); - if (dict == NULL) { + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + return -1; + } + if (info == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", i); return -1; } + if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just need an ffi_type pointer. */ num_ffi_type_pointers++; } else { /* It's an array. */ - Py_ssize_t length = dict->length; - StgDictObject *edict; + Py_ssize_t length = info->length; - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + Py_DECREF(pair); + return -1; + } + if (einfo == NULL) { Py_DECREF(pair); PyErr_Format(PyExc_TypeError, "second item in _fields_ tuple (index %zd) must be a C type", @@ -895,9 +776,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (num_ffi_types > 0) { memset(structs, 0, num_ffi_types * sizeof(ffi_type)); } - if (ffi_ofs && (basedict != NULL)) { + if (ffi_ofs && (baseinfo != NULL)) { memcpy(element_types, - basedict->ffi_type_pointer.elements, + baseinfo->ffi_type_pointer.elements, ffi_ofs * sizeof(ffi_type *)); } element_index = ffi_ofs; @@ -906,7 +787,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct for (i = 0; i < len; ++i) { PyObject *name, *desc; PyObject *pair = PySequence_GetItem(fields, i); - StgDictObject *dict; int bitsize = 0; if (pair == NULL) { @@ -926,9 +806,16 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct PyMem_Free(type_block); return -1; } - dict = PyType_stgdict(desc); + + StgInfo *info; + if (PyStgInfo_FromType(st, desc, &info) < 0) { + Py_DECREF(pair); + PyMem_Free(type_block); + return -1; + } + /* Possibly this check could be avoided, but see above comment. */ - if (dict == NULL) { + if (info == NULL) { Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, @@ -936,17 +823,21 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct i); return -1; } + assert(element_index < (ffi_ofs + len)); /* will be used below */ if (!PyCArrayTypeObject_Check(st, desc)) { /* Not an array. Just copy over the element ffi_type. */ - element_types[element_index++] = &dict->ffi_type_pointer; + element_types[element_index++] = &info->ffi_type_pointer; } else { - Py_ssize_t length = dict->length; - StgDictObject *edict; - - edict = PyType_stgdict(dict->proto); - if (edict == NULL) { + Py_ssize_t length = info->length; + StgInfo *einfo; + if (PyStgInfo_FromType(st, info->proto, &einfo) < 0) { + Py_DECREF(pair); + PyMem_Free(type_block); + return -1; + } + if (einfo == NULL) { Py_DECREF(pair); PyMem_Free(type_block); PyErr_Format(PyExc_TypeError, @@ -955,15 +846,15 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } element_types[element_index++] = &structs[struct_index]; - structs[struct_index].size = length * edict->ffi_type_pointer.size; - structs[struct_index].alignment = edict->ffi_type_pointer.alignment; + structs[struct_index].size = length * einfo->ffi_type_pointer.size; + structs[struct_index].alignment = einfo->ffi_type_pointer.alignment; structs[struct_index].type = FFI_TYPE_STRUCT; structs[struct_index].elements = &dummy_types[dummy_index]; ++struct_index; /* Copy over the element's type, length times. */ while (length > 0) { assert(dummy_index < (num_ffi_type_pointers)); - dummy_types[dummy_index++] = &edict->ffi_type_pointer; + dummy_types[dummy_index++] = &einfo->ffi_type_pointer; length--; } assert(dummy_index < (num_ffi_type_pointers)); @@ -977,19 +868,19 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct * Replace the old elements with the new, taking into account * base class elements where necessary. */ - assert(stgdict->ffi_type_pointer.elements); - PyMem_Free(stgdict->ffi_type_pointer.elements); - stgdict->ffi_type_pointer.elements = element_types; + assert(stginfo->ffi_type_pointer.elements); + PyMem_Free(stginfo->ffi_type_pointer.elements); + stginfo->ffi_type_pointer.elements = element_types; } /* We did check that this flag was NOT set above, it must not have been set until now. */ - if (stgdict->flags & DICTFLAG_FINAL) { + if (stginfo->flags & DICTFLAG_FINAL) { PyErr_SetString(PyExc_AttributeError, "Structure or union cannot contain itself"); return -1; } - stgdict->flags |= DICTFLAG_FINAL; + stginfo->flags |= DICTFLAG_FINAL; return MakeAnonFields(type); } From 63289b9dfbc7d87e81f1517422ee91b6b6d19531 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 20 Mar 2024 18:24:02 +0000 Subject: [PATCH 55/82] GH-117066: Tier 2 optimizer: Don't throw away good traces if we can't optimize them perfectly. (GH-117067) --- Python/optimizer_analysis.c | 16 ++++++++++------ Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 4 +++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 0c95616848a85b..603ac6815665ca 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -406,24 +406,28 @@ optimize_uops( out_of_space: DPRINTF(3, "\n"); DPRINTF(1, "Out of space in abstract interpreter\n"); - _Py_uop_abstractcontext_fini(ctx); - return 0; - + goto done; error: DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); - return 0; + return -1; hit_bottom: // Attempted to push a "bottom" (contradition) symbol onto the stack. // This means that the abstract interpreter has hit unreachable code. - // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's - // simpler to just admit failure and not create the executor. + // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting + // bottom indicates type instability, so we are probably better off + // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); return 0; +done: + /* Cannot optimize further, but there would be no benefit + * in retrying later */ + _Py_uop_abstractcontext_fini(ctx); + return 1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index ef08c0d8897c9f..a1ef644e4621e2 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -546,7 +546,9 @@ dummy_func(void) { PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; DPRINTF(3, "func: %p ", func); if (func == NULL) { - goto error; + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + goto done; } PyCodeObject *co = (PyCodeObject *)func->func_code; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 610d1b1aede9cc..a0ecf58905f287 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1599,7 +1599,9 @@ PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; DPRINTF(3, "func: %p ", func); if (func == NULL) { - goto error; + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + goto done; } PyCodeObject *co = (PyCodeObject *)func->func_code; assert(self_or_null != NULL); From 7d446548ef53f6c3de1097c6d44cada6642ddc85 Mon Sep 17 00:00:00 2001 From: Carol Willing Date: Wed, 20 Mar 2024 14:00:59 -0700 Subject: [PATCH 56/82] Fix sort order for "locale encoding" glossary item (#115794) Co-authored-by: C.A.M. Gerlach --- Doc/glossary.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 72fb09ef6207c9..ee8b26665d6921 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -727,18 +727,6 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. - - See also the :term:`filesystem encoding and error handler`. - list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to @@ -758,6 +746,18 @@ Glossary :term:`finder`. See :pep:`302` for details and :class:`importlib.abc.Loader` for an :term:`abstract base class`. + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + :func:`locale.setlocale(locale.LC_CTYPE, new_locale) `. + + On Windows, it is the ANSI code page (ex: ``"cp1252"``). + + On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. + + :func:`locale.getencoding` can be used to get the locale encoding. + + See also the :term:`filesystem encoding and error handler`. + magic method .. index:: pair: magic; method From 8ad88984200b2ccddc0a08229dd2f4c14d1a71fc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 17:11:00 -0400 Subject: [PATCH 57/82] gh-117089: Move importlib.metadata tests to their own package (#117092) * Ensure importlib.metadata tests do not leak references in sys.modules. * Move importlib.metadata tests to their own package for easier syncing with importlib_metadata. * Update owners and makefile for new directories. * Add blurb --- .github/CODEOWNERS | 2 +- .../test_importlib/{data => metadata}/__init__.py | 0 Lib/test/test_importlib/{ => metadata}/_context.py | 0 Lib/test/test_importlib/{ => metadata}/_path.py | 0 Lib/test/test_importlib/metadata/data/__init__.py | 0 .../data/example-21.12-py3-none-any.whl | Bin .../{ => metadata}/data/example-21.12-py3.6.egg | Bin .../data/example2-1.0.0-py3-none-any.whl | Bin Lib/test/test_importlib/{ => metadata}/fixtures.py | 4 +++- Lib/test/test_importlib/{ => metadata}/stubs.py | 0 .../{test_metadata_api.py => metadata/test_api.py} | 0 Lib/test/test_importlib/{ => metadata}/test_main.py | 0 Lib/test/test_importlib/{ => metadata}/test_zip.py | 0 Makefile.pre.in | 3 ++- .../2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst | 1 + 15 files changed, 7 insertions(+), 3 deletions(-) rename Lib/test/test_importlib/{data => metadata}/__init__.py (100%) rename Lib/test/test_importlib/{ => metadata}/_context.py (100%) rename Lib/test/test_importlib/{ => metadata}/_path.py (100%) create mode 100644 Lib/test/test_importlib/metadata/data/__init__.py rename Lib/test/test_importlib/{ => metadata}/data/example-21.12-py3-none-any.whl (100%) rename Lib/test/test_importlib/{ => metadata}/data/example-21.12-py3.6.egg (100%) rename Lib/test/test_importlib/{ => metadata}/data/example2-1.0.0-py3-none-any.whl (100%) rename Lib/test/test_importlib/{ => metadata}/fixtures.py (98%) rename Lib/test/test_importlib/{ => metadata}/stubs.py (100%) rename Lib/test/test_importlib/{test_metadata_api.py => metadata/test_api.py} (100%) rename Lib/test/test_importlib/{ => metadata}/test_main.py (100%) rename Lib/test/test_importlib/{ => metadata}/test_zip.py (100%) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e8eed400d961fc..235bc78599400e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -119,7 +119,7 @@ Python/dynload_*.c @ericsnowcurrently Lib/test/test_module/ @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently **/*importlib/resources/* @jaraco @warsaw @FFY00 -**/importlib/metadata/* @jaraco @warsaw +**/*importlib/metadata/* @jaraco @warsaw # Dates and times **/*datetime* @pganssle @abalkin diff --git a/Lib/test/test_importlib/data/__init__.py b/Lib/test/test_importlib/metadata/__init__.py similarity index 100% rename from Lib/test/test_importlib/data/__init__.py rename to Lib/test/test_importlib/metadata/__init__.py diff --git a/Lib/test/test_importlib/_context.py b/Lib/test/test_importlib/metadata/_context.py similarity index 100% rename from Lib/test/test_importlib/_context.py rename to Lib/test/test_importlib/metadata/_context.py diff --git a/Lib/test/test_importlib/_path.py b/Lib/test/test_importlib/metadata/_path.py similarity index 100% rename from Lib/test/test_importlib/_path.py rename to Lib/test/test_importlib/metadata/_path.py diff --git a/Lib/test/test_importlib/metadata/data/__init__.py b/Lib/test/test_importlib/metadata/data/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/test/test_importlib/data/example-21.12-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example-21.12-py3-none-any.whl diff --git a/Lib/test/test_importlib/data/example-21.12-py3.6.egg b/Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg similarity index 100% rename from Lib/test/test_importlib/data/example-21.12-py3.6.egg rename to Lib/test/test_importlib/metadata/data/example-21.12-py3.6.egg diff --git a/Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl b/Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl similarity index 100% rename from Lib/test/test_importlib/data/example2-1.0.0-py3-none-any.whl rename to Lib/test/test_importlib/metadata/data/example2-1.0.0-py3-none-any.whl diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py similarity index 98% rename from Lib/test/test_importlib/fixtures.py rename to Lib/test/test_importlib/metadata/fixtures.py index 8c973356b5660d..f23ac5d9019d17 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -9,6 +9,7 @@ import functools import contextlib +from test.support import import_helper from test.support.os_helper import FS_NONASCII from test.support import requires_zlib @@ -85,6 +86,7 @@ def add_sys_path(dir): def setUp(self): super().setUp() self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + self.fixtures.enter_context(import_helper.isolated_modules()) class SiteBuilder(SiteDir): @@ -348,7 +350,7 @@ def DALS(str): @requires_zlib() class ZipFixtures: - root = 'test.test_importlib.data' + root = 'test.test_importlib.metadata.data' def _fixture_on_path(self, filename): pkg_file = resources.files(self.root).joinpath(filename) diff --git a/Lib/test/test_importlib/stubs.py b/Lib/test/test_importlib/metadata/stubs.py similarity index 100% rename from Lib/test/test_importlib/stubs.py rename to Lib/test/test_importlib/metadata/stubs.py diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/metadata/test_api.py similarity index 100% rename from Lib/test/test_importlib/test_metadata_api.py rename to Lib/test/test_importlib/metadata/test_api.py diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/metadata/test_main.py similarity index 100% rename from Lib/test/test_importlib/test_main.py rename to Lib/test/test_importlib/metadata/test_main.py diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/metadata/test_zip.py similarity index 100% rename from Lib/test/test_importlib/test_zip.py rename to Lib/test/test_importlib/metadata/test_zip.py diff --git a/Makefile.pre.in b/Makefile.pre.in index b9f790a14af4cd..1451cf3a31ff02 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2351,10 +2351,11 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ - test/test_importlib/data \ test/test_importlib/extension \ test/test_importlib/frozen \ test/test_importlib/import_ \ + test/test_importlib/metadata \ + test/test_importlib/metadata/data \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ diff --git a/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst b/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst new file mode 100644 index 00000000000000..ab0baec8c96035 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-20-14-19-32.gh-issue-117089.WwR1Z1.rst @@ -0,0 +1 @@ +Consolidated tests for importlib.metadata in their own ``metadata`` package. From 104602a6078564765b7b8f42888f8eaa37b129b1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Mar 2024 23:52:23 +0100 Subject: [PATCH 58/82] gh-105927: Limit PyWeakref_GetRef() to limited C API 3.13 (#117091) --- Include/weakrefobject.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 727ba6934bbacb..a6e71eb178b124 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -28,7 +28,10 @@ PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, PyObject *callback); Py_DEPRECATED(3.13) PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030D0000 PyAPI_FUNC(int) PyWeakref_GetRef(PyObject *ref, PyObject **pobj); +#endif #ifndef Py_LIMITED_API From f4cc77d494ee0e10ed84ce369f0910c70a2f6d44 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Mar 2024 00:06:24 +0100 Subject: [PATCH 59/82] gh-116869: Enable -Werror in test_cext for Free Threading (#117106) Check for warnings, but don't enable the compiler flag -Werror=declaration-after-statement. --- Lib/test/test_cext/setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 1a4ec26f9985d3..ccad3fa62ad086 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -11,17 +11,19 @@ SOURCE = 'extension.c' -if not support.MS_WINDOWS and not support.Py_GIL_DISABLED: +if not support.MS_WINDOWS: # C compiler flags for GCC and clang CFLAGS = [ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', - - # gh-116869: The Python C API must be compatible with building - # with the -Werror=declaration-after-statement compiler flag. - '-Werror=declaration-after-statement', ] + if not support.Py_GIL_DISABLED: + CFLAGS.append( + # gh-116869: The Python C API must be compatible with building + # with the -Werror=declaration-after-statement compiler flag. + '-Werror=declaration-after-statement', + ) else: # Don't pass any compiler flag to MSVC CFLAGS = [] From 667294d5b2ee812ebe0c9c1efd58e2006b61f827 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 20 Mar 2024 23:01:24 -0400 Subject: [PATCH 60/82] gh-117089: Apply changes from importlib_metadata 7.1.0 (#117094) * Apply changes from importlib_metadata 7.1.0 * Include the data sources in the makefile (even though they're not needed) --- Lib/importlib/metadata/__init__.py | 155 ++++++++++++++---- Lib/importlib/metadata/_meta.py | 52 +++--- Lib/test/test_importlib/metadata/_path.py | 15 +- .../data/sources/example/example/__init__.py | 2 + .../metadata/data/sources/example/setup.py | 11 ++ .../sources/example2/example2/__init__.py | 2 + .../data/sources/example2/pyproject.toml | 10 ++ Lib/test/test_importlib/metadata/fixtures.py | 22 +-- Lib/test/test_importlib/metadata/test_main.py | 33 ++-- Makefile.pre.in | 5 + 10 files changed, 212 insertions(+), 95 deletions(-) create mode 100644 Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example/setup.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py create mode 100644 Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 41c2a4a6088b5d..c8e59cac44194d 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import re import abc @@ -26,7 +28,7 @@ from importlib import import_module from importlib.abc import MetaPathFinder from itertools import starmap -from typing import Iterable, List, Mapping, Optional, Set, Union, cast +from typing import Any, Iterable, List, Mapping, Match, Optional, Set, cast __all__ = [ 'Distribution', @@ -163,17 +165,17 @@ class EntryPoint: value: str group: str - dist: Optional['Distribution'] = None + dist: Optional[Distribution] = None def __init__(self, name: str, value: str, group: str) -> None: vars(self).update(name=name, value=value, group=group) - def load(self): + def load(self) -> Any: """Load the entry point from its definition. If only a module is indicated by the value, return that module. Otherwise, return the named object. """ - match = self.pattern.match(self.value) + match = cast(Match, self.pattern.match(self.value)) module = import_module(match.group('module')) attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) @@ -268,7 +270,7 @@ def __repr__(self): """ return '%s(%r)' % (self.__class__.__name__, tuple(self)) - def select(self, **params): + def select(self, **params) -> EntryPoints: """ Select entry points from self that match the given parameters (typically group and/or name). @@ -304,19 +306,17 @@ def _from_text(text): class PackagePath(pathlib.PurePosixPath): """A reference to a path in a package""" - hash: Optional["FileHash"] + hash: Optional[FileHash] size: int - dist: "Distribution" + dist: Distribution def read_text(self, encoding: str = 'utf-8') -> str: # type: ignore[override] - with self.locate().open(encoding=encoding) as stream: - return stream.read() + return self.locate().read_text(encoding=encoding) def read_binary(self) -> bytes: - with self.locate().open('rb') as stream: - return stream.read() + return self.locate().read_bytes() - def locate(self) -> pathlib.Path: + def locate(self) -> SimplePath: """Return a path-like object for this path""" return self.dist.locate_file(self) @@ -330,6 +330,7 @@ def __repr__(self) -> str: class DeprecatedNonAbstract: + # Required until Python 3.14 def __new__(cls, *args, **kwargs): all_names = { name for subclass in inspect.getmro(cls) for name in vars(subclass) @@ -349,25 +350,48 @@ def __new__(cls, *args, **kwargs): class Distribution(DeprecatedNonAbstract): - """A Python distribution package.""" + """ + An abstract Python distribution package. + + Custom providers may derive from this class and define + the abstract methods to provide a concrete implementation + for their environment. Some providers may opt to override + the default implementation of some properties to bypass + the file-reading mechanism. + """ @abc.abstractmethod def read_text(self, filename) -> Optional[str]: """Attempt to load metadata file given by the name. + Python distribution metadata is organized by blobs of text + typically represented as "files" in the metadata directory + (e.g. package-1.0.dist-info). These files include things + like: + + - METADATA: The distribution metadata including fields + like Name and Version and Description. + - entry_points.txt: A series of entry points as defined in + `the entry points spec `_. + - RECORD: A record of files according to + `this recording spec `_. + + A package may provide any set of files, including those + not listed here or none at all. + :param filename: The name of the file in the distribution info. :return: The text if found, otherwise None. """ @abc.abstractmethod - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: """ - Given a path to a file in this distribution, return a path + Given a path to a file in this distribution, return a SimplePath to it. """ @classmethod - def from_name(cls, name: str) -> "Distribution": + def from_name(cls, name: str) -> Distribution: """Return the Distribution for the given package name. :param name: The name of the distribution package to search for. @@ -385,16 +409,18 @@ def from_name(cls, name: str) -> "Distribution": raise PackageNotFoundError(name) @classmethod - def discover(cls, **kwargs) -> Iterable["Distribution"]: + def discover( + cls, *, context: Optional[DistributionFinder.Context] = None, **kwargs + ) -> Iterable[Distribution]: """Return an iterable of Distribution objects for all packages. Pass a ``context`` or pass keyword arguments for constructing a context. :context: A ``DistributionFinder.Context`` object. - :return: Iterable of Distribution objects for all packages. + :return: Iterable of Distribution objects for packages matching + the context. """ - context = kwargs.pop('context', None) if context and kwargs: raise ValueError("cannot accept context and kwargs") context = context or DistributionFinder.Context(**kwargs) @@ -403,8 +429,8 @@ def discover(cls, **kwargs) -> Iterable["Distribution"]: ) @staticmethod - def at(path: Union[str, os.PathLike[str]]) -> "Distribution": - """Return a Distribution for the indicated metadata path + def at(path: str | os.PathLike[str]) -> Distribution: + """Return a Distribution for the indicated metadata path. :param path: a string or path-like object :return: a concrete Distribution instance for the path @@ -413,7 +439,7 @@ def at(path: Union[str, os.PathLike[str]]) -> "Distribution": @staticmethod def _discover_resolvers(): - """Search the meta_path for resolvers.""" + """Search the meta_path for resolvers (MetadataPathFinders).""" declared = ( getattr(finder, 'find_distributions', None) for finder in sys.meta_path ) @@ -424,7 +450,11 @@ def metadata(self) -> _meta.PackageMetadata: """Return the parsed metadata for this Distribution. The returned object will have keys that name the various bits of - metadata. See PEP 566 for details. + metadata per the + `Core metadata specifications `_. + + Custom providers may provide the METADATA file or override this + property. """ opt_text = ( self.read_text('METADATA') @@ -454,6 +484,12 @@ def version(self) -> str: @property def entry_points(self) -> EntryPoints: + """ + Return EntryPoints for this distribution. + + Custom providers may provide the ``entry_points.txt`` file + or override this property. + """ return EntryPoints._from_text_for(self.read_text('entry_points.txt'), self) @property @@ -466,6 +502,10 @@ def files(self) -> Optional[List[PackagePath]]: (i.e. RECORD for dist-info, or installed-files.txt or SOURCES.txt for egg-info) is missing. Result may be empty if the metadata exists but is empty. + + Custom providers are recommended to provide a "RECORD" file (in + ``read_text``) or override this property to allow for callers to be + able to resolve filenames provided by the package. """ def make_file(name, hash=None, size_str=None): @@ -497,7 +537,7 @@ def skip_missing_files(package_paths): def _read_files_distinfo(self): """ - Read the lines of RECORD + Read the lines of RECORD. """ text = self.read_text('RECORD') return text and text.splitlines() @@ -611,6 +651,9 @@ def _load_json(self, filename): class DistributionFinder(MetaPathFinder): """ A MetaPathFinder capable of discovering installed distributions. + + Custom providers should implement this interface in order to + supply metadata. """ class Context: @@ -623,6 +666,17 @@ class Context: Each DistributionFinder may expect any parameters and should attempt to honor the canonical parameters defined below when appropriate. + + This mechanism gives a custom provider a means to + solicit additional details from the caller beyond + "name" and "path" when searching distributions. + For example, imagine a provider that exposes suites + of packages in either a "public" or "private" ``realm``. + A caller may wish to query only for distributions in + a particular realm and could call + ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. """ name = None @@ -658,11 +712,18 @@ def find_distributions(self, context=Context()) -> Iterable[Distribution]: class FastPath: """ - Micro-optimized class for searching a path for - children. + Micro-optimized class for searching a root for children. + + Root is a path on the file system that may contain metadata + directories either as natural directories or within a zip file. >>> FastPath('').children() ['...'] + + FastPath objects are cached and recycled for any given root. + + >>> FastPath('foobar') is FastPath('foobar') + True """ @functools.lru_cache() # type: ignore @@ -704,7 +765,19 @@ def lookup(self, mtime): class Lookup: + """ + A micro-optimized class for searching a (fast) path for metadata. + """ + def __init__(self, path: FastPath): + """ + Calculate all of the children representing metadata. + + From the children in the path, calculate early all of the + children that appear to represent metadata (infos) or legacy + metadata (eggs). + """ + base = os.path.basename(path.root).lower() base_is_egg = base.endswith(".egg") self.infos = FreezableDefaultDict(list) @@ -725,7 +798,10 @@ def __init__(self, path: FastPath): self.infos.freeze() self.eggs.freeze() - def search(self, prepared): + def search(self, prepared: Prepared): + """ + Yield all infos and eggs matching the Prepared query. + """ infos = ( self.infos[prepared.normalized] if prepared @@ -741,13 +817,28 @@ def search(self, prepared): class Prepared: """ - A prepared search for metadata on a possibly-named package. + A prepared search query for metadata on a possibly-named package. + + Pre-calculates the normalization to prevent repeated operations. + + >>> none = Prepared(None) + >>> none.normalized + >>> none.legacy_normalized + >>> bool(none) + False + >>> sample = Prepared('Sample__Pkg-name.foo') + >>> sample.normalized + 'sample_pkg_name_foo' + >>> sample.legacy_normalized + 'sample__pkg_name.foo' + >>> bool(sample) + True """ normalized = None legacy_normalized = None - def __init__(self, name): + def __init__(self, name: Optional[str]): self.name = name if name is None: return @@ -777,7 +868,7 @@ class MetadataPathFinder(DistributionFinder): @classmethod def find_distributions( cls, context=DistributionFinder.Context() - ) -> Iterable["PathDistribution"]: + ) -> Iterable[PathDistribution]: """ Find distributions. @@ -810,7 +901,7 @@ def __init__(self, path: SimplePath) -> None: """ self._path = path - def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: + def read_text(self, filename: str | os.PathLike[str]) -> Optional[str]: with suppress( FileNotFoundError, IsADirectoryError, @@ -824,7 +915,7 @@ def read_text(self, filename: Union[str, os.PathLike[str]]) -> Optional[str]: read_text.__doc__ = Distribution.read_text.__doc__ - def locate_file(self, path: Union[str, os.PathLike[str]]) -> pathlib.Path: + def locate_file(self, path: str | os.PathLike[str]) -> SimplePath: return self._path.parent / path @property diff --git a/Lib/importlib/metadata/_meta.py b/Lib/importlib/metadata/_meta.py index f670016de7fef2..1927d0f624d82f 100644 --- a/Lib/importlib/metadata/_meta.py +++ b/Lib/importlib/metadata/_meta.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +import os from typing import Protocol from typing import Any, Dict, Iterator, List, Optional, TypeVar, Union, overload @@ -6,30 +9,27 @@ class PackageMetadata(Protocol): - def __len__(self) -> int: - ... # pragma: no cover + def __len__(self) -> int: ... # pragma: no cover - def __contains__(self, item: str) -> bool: - ... # pragma: no cover + def __contains__(self, item: str) -> bool: ... # pragma: no cover - def __getitem__(self, key: str) -> str: - ... # pragma: no cover + def __getitem__(self, key: str) -> str: ... # pragma: no cover - def __iter__(self) -> Iterator[str]: - ... # pragma: no cover + def __iter__(self) -> Iterator[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: None = None) -> Optional[str]: - ... # pragma: no cover + def get( + self, name: str, failobj: None = None + ) -> Optional[str]: ... # pragma: no cover @overload - def get(self, name: str, failobj: _T) -> Union[str, _T]: - ... # pragma: no cover + def get(self, name: str, failobj: _T) -> Union[str, _T]: ... # pragma: no cover # overload per python/importlib_metadata#435 @overload - def get_all(self, name: str, failobj: None = None) -> Optional[List[Any]]: - ... # pragma: no cover + def get_all( + self, name: str, failobj: None = None + ) -> Optional[List[Any]]: ... # pragma: no cover @overload def get_all(self, name: str, failobj: _T) -> Union[List[Any], _T]: @@ -44,20 +44,24 @@ def json(self) -> Dict[str, Union[str, List[str]]]: """ -class SimplePath(Protocol[_T]): +class SimplePath(Protocol): """ - A minimal subset of pathlib.Path required by PathDistribution. + A minimal subset of pathlib.Path required by Distribution. """ - def joinpath(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def joinpath( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover - def __truediv__(self, other: Union[str, _T]) -> _T: - ... # pragma: no cover + def __truediv__( + self, other: Union[str, os.PathLike[str]] + ) -> SimplePath: ... # pragma: no cover @property - def parent(self) -> _T: - ... # pragma: no cover + def parent(self) -> SimplePath: ... # pragma: no cover + + def read_text(self, encoding=None) -> str: ... # pragma: no cover + + def read_bytes(self) -> bytes: ... # pragma: no cover - def read_text(self) -> str: - ... # pragma: no cover + def exists(self) -> bool: ... # pragma: no cover diff --git a/Lib/test/test_importlib/metadata/_path.py b/Lib/test/test_importlib/metadata/_path.py index 25c799fa44cd55..b3cfb9cd549d6c 100644 --- a/Lib/test/test_importlib/metadata/_path.py +++ b/Lib/test/test_importlib/metadata/_path.py @@ -17,20 +17,15 @@ class Symlink(str): @runtime_checkable class TreeMaker(Protocol): - def __truediv__(self, *args, **kwargs): - ... # pragma: no cover + def __truediv__(self, *args, **kwargs): ... # pragma: no cover - def mkdir(self, **kwargs): - ... # pragma: no cover + def mkdir(self, **kwargs): ... # pragma: no cover - def write_text(self, content, **kwargs): - ... # pragma: no cover + def write_text(self, content, **kwargs): ... # pragma: no cover - def write_bytes(self, content): - ... # pragma: no cover + def write_bytes(self, content): ... # pragma: no cover - def symlink_to(self, target): - ... # pragma: no cover + def symlink_to(self, target): ... # pragma: no cover def _ensure_tree_maker(obj: Union[str, TreeMaker]) -> TreeMaker: diff --git a/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py new file mode 100644 index 00000000000000..ba73b743394169 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/example/__init__.py @@ -0,0 +1,2 @@ +def main(): + return 'example' diff --git a/Lib/test/test_importlib/metadata/data/sources/example/setup.py b/Lib/test/test_importlib/metadata/data/sources/example/setup.py new file mode 100644 index 00000000000000..479488a0348186 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup( + name='example', + version='21.12', + license='Apache Software License', + packages=['example'], + entry_points={ + 'console_scripts': ['example = example:main', 'Example=example:main'], + }, +) diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py new file mode 100644 index 00000000000000..de645c2e8bc75b --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/example2/__init__.py @@ -0,0 +1,2 @@ +def main(): + return "example" diff --git a/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml new file mode 100644 index 00000000000000..011f4751fb9e32 --- /dev/null +++ b/Lib/test/test_importlib/metadata/data/sources/example2/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +build-backend = 'trampolim' +requires = ['trampolim'] + +[project] +name = 'example2' +version = '1.0.0' + +[project.scripts] +example = 'example2:main' diff --git a/Lib/test/test_importlib/metadata/fixtures.py b/Lib/test/test_importlib/metadata/fixtures.py index f23ac5d9019d17..7ff94c9afe88e1 100644 --- a/Lib/test/test_importlib/metadata/fixtures.py +++ b/Lib/test/test_importlib/metadata/fixtures.py @@ -10,7 +10,7 @@ import contextlib from test.support import import_helper -from test.support.os_helper import FS_NONASCII +from test.support import os_helper from test.support import requires_zlib from . import _path @@ -143,15 +143,13 @@ class DistInfoPkgEditable(DistInfoPkg): some_hash = '524127ce937f7cb65665130c695abd18ca386f60bb29687efb976faa1596fdcc' files: FilesSpec = { 'distinfo_pkg-1.0.0.dist-info': { - 'direct_url.json': json.dumps( - { - "archive_info": { - "hash": f"sha256={some_hash}", - "hashes": {"sha256": f"{some_hash}"}, - }, - "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", - } - ) + 'direct_url.json': json.dumps({ + "archive_info": { + "hash": f"sha256={some_hash}", + "hashes": {"sha256": f"{some_hash}"}, + }, + "url": "file:///path/to/distinfo_pkg-1.0.0.editable-py3-none-any.whl", + }) }, } @@ -340,7 +338,9 @@ def record_names(file_defs): class FileBuilder: def unicode_filename(self): - return FS_NONASCII or self.skip("File system does not support non-ascii.") + return os_helper.FS_NONASCII or self.skip( + "File system does not support non-ascii." + ) def DALS(str): diff --git a/Lib/test/test_importlib/metadata/test_main.py b/Lib/test/test_importlib/metadata/test_main.py index 0a769b89841234..c4accaeb9ba9ed 100644 --- a/Lib/test/test_importlib/metadata/test_main.py +++ b/Lib/test/test_importlib/metadata/test_main.py @@ -2,6 +2,7 @@ import pickle import unittest import warnings +import importlib import importlib.metadata import contextlib from test.support import os_helper @@ -308,12 +309,10 @@ def test_sortable(self): """ EntryPoint objects are sortable, but result is undefined. """ - sorted( - [ - EntryPoint(name='b', value='val', group='group'), - EntryPoint(name='a', value='val', group='group'), - ] - ) + sorted([ + EntryPoint(name='b', value='val', group='group'), + EntryPoint(name='a', value='val', group='group'), + ]) class FileSystem( @@ -380,18 +379,16 @@ def test_packages_distributions_all_module_types(self): 'all_distributions-1.0.0.dist-info': metadata, } for i, suffix in enumerate(suffixes): - files.update( - { - f'importable-name {i}{suffix}': '', - f'in_namespace_{i}': { - f'mod{suffix}': '', - }, - f'in_package_{i}': { - '__init__.py': '', - f'mod{suffix}': '', - }, - } - ) + files.update({ + f'importable-name {i}{suffix}': '', + f'in_namespace_{i}': { + f'mod{suffix}': '', + }, + f'in_package_{i}': { + '__init__.py': '', + f'mod{suffix}': '', + }, + }) metadata.update(RECORD=fixtures.build_record(files)) fixtures.build_files(files, prefix=self.site_dir) diff --git a/Makefile.pre.in b/Makefile.pre.in index 1451cf3a31ff02..cacf14a52cb68e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2356,6 +2356,11 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/import_ \ test/test_importlib/metadata \ test/test_importlib/metadata/data \ + test/test_importlib/metadata/data/sources \ + test/test_importlib/metadata/data/sources/example \ + test/test_importlib/metadata/data/sources/example/example \ + test/test_importlib/metadata/data/sources/example2 \ + test/test_importlib/metadata/data/sources/example2/example2 \ test/test_importlib/namespace_pkgs \ test/test_importlib/namespace_pkgs/both_portions \ test/test_importlib/namespace_pkgs/both_portions/foo \ From 6547330f4e896c6748da23704b617e060e6cc68e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 21 Mar 2024 03:49:10 +0000 Subject: [PATCH 61/82] GH-109653: Defer import of ``importlib.metadata._adapters`` (#109829) * adapters * Add comments for deferred imports with links to rationale. * Add blurb --------- Co-authored-by: Jason R. Coombs --- Lib/importlib/metadata/__init__.py | 5 ++++- Lib/importlib/resources/_common.py | 5 +++-- .../Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index c8e59cac44194d..245f905737cb15 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -18,7 +18,7 @@ import posixpath import collections -from . import _adapters, _meta +from . import _meta from ._collections import FreezableDefaultDict, Pair from ._functools import method_cache, pass_none from ._itertools import always_iterable, unique_everseen @@ -456,6 +456,9 @@ def metadata(self) -> _meta.PackageMetadata: Custom providers may provide the METADATA file or override this property. """ + # deferred for performance (python/cpython#109829) + from . import _adapters + opt_text = ( self.read_text('METADATA') or self.read_text('PKG-INFO') diff --git a/Lib/importlib/resources/_common.py b/Lib/importlib/resources/_common.py index a3902535342612..e18082fb3d26a0 100644 --- a/Lib/importlib/resources/_common.py +++ b/Lib/importlib/resources/_common.py @@ -12,8 +12,6 @@ from typing import Union, Optional, cast from .abc import ResourceReader, Traversable -from ._adapters import wrap_spec - Package = Union[types.ModuleType, str] Anchor = Package @@ -109,6 +107,9 @@ def from_package(package: types.ModuleType): Return a Traversable object for the given package. """ + # deferred for performance (python/cpython#109829) + from ._adapters import wrap_spec + spec = wrap_spec(package) reader = spec.loader.get_resource_reader(spec.name) return reader.files() diff --git a/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst b/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst new file mode 100644 index 00000000000000..38d7634b54c2fe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-20-23-07-58.gh-issue-109653.uu3lrX.rst @@ -0,0 +1,2 @@ +Deferred select imports in importlib.metadata and importlib.resources for a +14% speedup. From d16c9d1278164f04778861814ebc87ed087511fc Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 21 Mar 2024 03:30:10 -0700 Subject: [PATCH 62/82] gh-116987: Support class code objects in inspect.findsource() (GH-117025) --- Lib/inspect.py | 11 ++--------- Lib/test/test_inspect/inspect_fodder2.py | 5 +++++ Lib/test/test_inspect/test_inspect.py | 3 +++ .../2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst | 1 + 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 7336cea0dc3fdc..422c09a92ad141 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1157,15 +1157,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - try: - line = lines[lnum] - except IndexError: - raise OSError('lineno is out of bounds') - if pat.match(line): - break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 8639cf2e72cd7a..bb9d3e88cfbee1 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -310,3 +310,8 @@ def f(): class cls310: def g(): pass + +# line 314 +class ClassWithCodeObject: + import sys + code = sys._getframe(0).f_code diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 21d9f96c8c460e..dc46c0bc8ed353 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -983,6 +983,9 @@ def test_findsource_with_out_of_bounds_lineno(self): def test_getsource_on_method(self): self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + def test_nested_func(self): self.assertSourceEqual(mod2.cls135.func136, 136, 139) diff --git a/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst b/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst new file mode 100644 index 00000000000000..f2da956f66c86b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-19-19-42-25.gh-issue-116987.ZVKUH1.rst @@ -0,0 +1 @@ +Fixed :func:`inspect.findsource` for class code objects. From 1f8b24ef69896680d6ba6005e75e1cc79a744f9e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 21 Mar 2024 13:20:57 +0000 Subject: [PATCH 63/82] gh-71052: Implement `ctypes.util.find_library` on Android (GH-116379) --- Doc/library/ctypes.rst | 5 ++-- Lib/ctypes/util.py | 9 ++++++++ Lib/test/test_ctypes/test_find.py | 23 +++++++++++++++++++ ...4-03-05-19-56-29.gh-issue-71052.PMDK--.rst | 1 + 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 36976470b5a468..9f7d6456e623a2 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1334,8 +1334,9 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: +On macOS and Android, :func:`~ctypes.util.find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: >>> from ctypes.util import find_library >>> find_library("c") diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 12d7428fe9a776..117bf06cb01013 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -89,6 +89,15 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 7732ff37308848..85b28617d2d754 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -129,5 +129,28 @@ def test_gh114257(self): self.assertIsNone(find_library("libc")) +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst b/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst new file mode 100644 index 00000000000000..ddca54c7c9ed7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-05-19-56-29.gh-issue-71052.PMDK--.rst @@ -0,0 +1 @@ +Implement :func:`ctypes.util.find_library` on Android. From e728303532168efab7694c55c82ea19b18bf8385 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 21 Mar 2024 10:01:16 -0400 Subject: [PATCH 64/82] gh-116522: Stop the world before fork() and during shutdown (#116607) This changes the free-threaded build to perform a stop-the-world pause before deleting other thread states when forking and during shutdown. This fixes some crashes when using multiprocessing and during shutdown when running with `PYTHON_GIL=0`. This also changes `PyOS_BeforeFork` to acquire the runtime lock (i.e., `HEAD_LOCK(&_PyRuntime)`) before forking to ensure that data protected by the runtime lock (and not just the GIL or stop-the-world) is in a consistent state before forking. --- Modules/posixmodule.c | 28 +++++++++++++++++++++------- Python/pylifecycle.c | 3 +++ Python/pystate.c | 6 ++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2498b61d6412d5..644d4ba1f65b38 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -613,11 +613,16 @@ PyOS_BeforeFork(void) run_at_forkers(interp->before_forkers, 1); _PyImport_AcquireLock(interp); + _PyEval_StopTheWorldAll(&_PyRuntime); + HEAD_LOCK(&_PyRuntime); } void PyOS_AfterFork_Parent(void) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorldAll(&_PyRuntime); + PyInterpreterState *interp = _PyInterpreterState_GET(); if (_PyImport_ReleaseLock(interp) <= 0) { Py_FatalError("failed releasing import lock after fork"); @@ -632,6 +637,7 @@ PyOS_AfterFork_Child(void) PyStatus status; _PyRuntimeState *runtime = &_PyRuntime; + // re-creates runtime->interpreters.mutex (HEAD_UNLOCK) status = _PyRuntimeState_ReInitThreads(runtime); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; @@ -7731,10 +7737,15 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This code might do an import, thus acquiring the import lock, which -// PyOS_BeforeFork() also does. As this should only be called from -// the parent process, it is in the same thread so that works. -static void warn_about_fork_with_threads(const char* name) { +// This should only be called from the parent process after +// PyOS_AfterFork_Parent(). +static void +warn_about_fork_with_threads(const char* name) +{ + // It's not safe to issue the warning while the world is stopped, because + // other threads might be holding locks that we need, which would deadlock. + assert(!_PyRuntime.stoptheworld.world_stopped); + // TODO: Consider making an `os` module API to return the current number // of threads in the process. That'd presumably use this platform code but // raise an error rather than using the inaccurate fallback. @@ -7858,9 +7869,10 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork1"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork1"); } if (pid == -1) { errno = saved_errno; @@ -7906,9 +7918,10 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("fork"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("fork"); } if (pid == -1) { errno = saved_errno; @@ -8737,9 +8750,10 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { - warn_about_fork_with_threads("forkpty"); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); + // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. + warn_about_fork_with_threads("forkpty"); } if (pid == -1) { return posix_error(); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 3a2c0a450ac9d9..bc76822e72c54a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1911,6 +1911,9 @@ Py_FinalizeEx(void) int malloc_stats = tstate->interp->config.malloc_stats; #endif + /* Ensure that remaining threads are detached */ + _PyEval_StopTheWorldAll(runtime); + /* Remaining daemon threads will automatically exit when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(tstate->interp, tstate); diff --git a/Python/pystate.c b/Python/pystate.c index eedcb920cd1cf2..5a334e8721e63b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1692,6 +1692,10 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyRuntimeState *runtime = interp->runtime; +#ifdef Py_GIL_DISABLED + assert(runtime->stoptheworld.world_stopped); +#endif + HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of thread states. This will allow calling PyThreadState_Clear() @@ -1710,6 +1714,8 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) interp->threads.head = tstate; HEAD_UNLOCK(runtime); + _PyEval_StartTheWorldAll(runtime); + /* Clear and deallocate all stale thread states. Even if this executes Python code, we should be safe since it executes in the current thread, not one of the stale threads. */ From bbee57fa8c318cb26d6c8651254927a1972c9738 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 09:56:12 -0600 Subject: [PATCH 65/82] gh-76785: Clean Up Interpreter ID Conversions (gh-117048) Mostly we unify the two different implementations of the conversion code (from PyObject * to int64_t. We also drop the PyArg_ParseTuple()-style converter function, as well as rename and move PyInterpreterID_LookUp(). --- Include/cpython/interpreteridobject.h | 5 +- Include/internal/pycore_interp.h | 3 + Lib/test/test_capi/test_misc.py | 4 +- Modules/_testcapimodule.c | 26 ------- Modules/_testinternalcapi.c | 32 +++++++- Modules/_xxsubinterpretersmodule.c | 82 +------------------- Objects/interpreteridobject.c | 66 ++++++----------- Python/pystate.c | 103 ++++++++++++++++++++------ 8 files changed, 143 insertions(+), 178 deletions(-) diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h index 4ab9ad5d315f80..d425c909806e44 100644 --- a/Include/cpython/interpreteridobject.h +++ b/Include/cpython/interpreteridobject.h @@ -8,4 +8,7 @@ PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) PyInterpreterID_LookUp(PyObject *); + +#ifdef Py_BUILD_CORE +extern int64_t _PyInterpreterID_GetID(PyObject *); +#endif diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 942f47340b3966..b28e8a3ff45f3f 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -295,8 +295,11 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } +extern int64_t _PyInterpreterState_ObjectToID(PyObject *); + // Export for the _xxinterpchannels module. PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 5b4f67e7f5f58d..fe5e19d46d8b6c 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2303,7 +2303,7 @@ def test_equality(self): def test_linked_lifecycle(self): id1 = _interpreters.create() - _testcapi.unlink_interpreter_refcount(id1) + _testinternalcapi.unlink_interpreter_refcount(id1) self.assertEqual( _testinternalcapi.get_interpreter_refcount(id1), 0) @@ -2319,7 +2319,7 @@ def test_linked_lifecycle(self): _testinternalcapi.get_interpreter_refcount(id1), 0) - _testcapi.link_interpreter_refcount(id1) + _testinternalcapi.link_interpreter_refcount(id1) self.assertEqual( _testinternalcapi.get_interpreter_refcount(id1), 0) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b73085bb8f67ce..e68d083955d64a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1455,30 +1455,6 @@ get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) return Py_NewRef(&PyInterpreterID_Type); } -static PyObject * -link_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 1); - Py_RETURN_NONE; -} - -static PyObject * -unlink_interpreter_refcount(PyObject *self, PyObject *idobj) -{ - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); - if (interp == NULL) { - assert(PyErr_Occurred()); - return NULL; - } - _PyInterpreterState_RequireIDRef(interp, 0); - Py_RETURN_NONE; -} - static PyMethodDef ml; static PyObject * @@ -3324,8 +3300,6 @@ static PyMethodDef TestMethods[] = { {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, - {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, - {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 1c10dd02138f3a..f73a29e5afe801 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -29,8 +29,6 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "interpreteridobject.h" // PyInterpreterID_LookUp() - #include "clinic/_testinternalcapi.c.h" // Include test definitions from _testinternalcapi/ @@ -1112,7 +1110,7 @@ pending_identify(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "O:pending_identify", &interpid)) { return NULL; } - PyInterpreterState *interp = PyInterpreterID_LookUp(interpid); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(interpid); if (interp == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, "interpreter not found"); @@ -1480,13 +1478,37 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) static PyObject * get_interpreter_refcount(PyObject *self, PyObject *idobj) { - PyInterpreterState *interp = PyInterpreterID_LookUp(idobj); + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); if (interp == NULL) { return NULL; } return PyLong_FromLongLong(interp->id_refcount); } +static PyObject * +link_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 1); + Py_RETURN_NONE; +} + +static PyObject * +unlink_interpreter_refcount(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + _PyInterpreterState_RequireIDRef(interp, 0); + Py_RETURN_NONE; +} + static void _xid_capsule_destructor(PyObject *capsule) @@ -1728,6 +1750,8 @@ static PyMethodDef module_functions[] = { _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, + {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, + {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 28c2f9c08bc0da..606b2a36481ce2 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -35,83 +35,8 @@ _get_current_interp(void) return PyInterpreterState_Get(); } -static int64_t -pylong_to_interpid(PyObject *idobj) -{ - assert(PyLong_CheckExact(idobj)); - - if (_PyLong_IsNegative((PyLongObject *)idobj)) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", - idobj); - return -1; - } - - int overflow; - long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow); - if (id == -1) { - if (!overflow) { - assert(PyErr_Occurred()); - return -1; - } - assert(!PyErr_Occurred()); - // For now, we don't worry about if LLONG_MAX < INT64_MAX. - goto bad_id; - } -#if LLONG_MAX > INT64_MAX - if (id > INT64_MAX) { - goto bad_id; - } -#endif - return (int64_t)id; - -bad_id: - PyErr_Format(PyExc_RuntimeError, - "unrecognized interpreter ID %O", idobj); - return -1; -} - -static int64_t -convert_interpid_obj(PyObject *arg) -{ - int64_t id = -1; - if (_PyIndex_Check(arg)) { - PyObject *idobj = PyNumber_Long(arg); - if (idobj == NULL) { - return -1; - } - id = pylong_to_interpid(idobj); - Py_DECREF(idobj); - if (id < 0) { - return -1; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return -1; - } - return id; -} - -static PyInterpreterState * -look_up_interp(PyObject *arg) -{ - int64_t id = convert_interpid_obj(arg); - if (id < 0) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} - +#define look_up_interp _PyInterpreterState_LookUpIDObject -static PyObject * -interpid_to_pylong(int64_t id) -{ - assert(id < LLONG_MAX); - return PyLong_FromLongLong(id); -} static PyObject * get_interpid_obj(PyInterpreterState *interp) @@ -123,7 +48,8 @@ get_interpid_obj(PyInterpreterState *interp) if (id < 0) { return NULL; } - return interpid_to_pylong(id); + assert(id < LLONG_MAX); + return PyLong_FromLongLong(id); } static PyObject * @@ -699,7 +625,7 @@ interp_set___main___attrs(PyObject *self, PyObject *args) } // Look up the interpreter. - PyInterpreterState *interp = PyInterpreterID_LookUp(id); + PyInterpreterState *interp = look_up_interp(id); if (interp == NULL) { return NULL; } diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 16e27b64c0c9c2..4844d6a9bf781c 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -1,8 +1,7 @@ /* InterpreterID object */ #include "Python.h" -#include "pycore_abstract.h" // _PyIndex_Check() -#include "pycore_interp.h" // _PyInterpreterState_LookUpID() +#include "pycore_interp.h" // _PyInterpreterState_LookUpID() #include "interpreteridobject.h" @@ -11,6 +10,21 @@ typedef struct interpid { int64_t id; } interpid; +int64_t +_PyInterpreterID_GetID(PyObject *self) +{ + if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { + PyErr_Format(PyExc_TypeError, + "expected an InterpreterID, got %R", + self); + return -1; + + } + int64_t id = ((interpid *)self)->id; + assert(id >= 0); + return id; +} + static interpid * newinterpid(PyTypeObject *cls, int64_t id, int force) { @@ -42,43 +56,19 @@ newinterpid(PyTypeObject *cls, int64_t id, int force) return self; } -static int -interp_id_converter(PyObject *arg, void *ptr) -{ - int64_t id; - if (PyObject_TypeCheck(arg, &PyInterpreterID_Type)) { - id = ((interpid *)arg)->id; - } - else if (_PyIndex_Check(arg)) { - id = PyLong_AsLongLong(arg); - if (id == -1 && PyErr_Occurred()) { - return 0; - } - if (id < 0) { - PyErr_Format(PyExc_ValueError, - "interpreter ID must be a non-negative int, got %R", arg); - return 0; - } - } - else { - PyErr_Format(PyExc_TypeError, - "interpreter ID must be an int, got %.100s", - Py_TYPE(arg)->tp_name); - return 0; - } - *(int64_t *)ptr = id; - return 1; -} - static PyObject * interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "force", NULL}; - int64_t id; + PyObject *idobj; int force = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&|$p:InterpreterID.__init__", kwlist, - interp_id_converter, &id, &force)) { + "O|$p:InterpreterID.__init__", kwlist, + &idobj, &force)) { + return NULL; + } + int64_t id = _PyInterpreterState_ObjectToID(idobj); + if (id < 0) { return NULL; } @@ -282,13 +272,3 @@ PyInterpreterState_GetIDObject(PyInterpreterState *interp) } return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); } - -PyInterpreterState * -PyInterpreterID_LookUp(PyObject *requested_id) -{ - int64_t id; - if (!interp_id_converter(requested_id, &id)) { - return NULL; - } - return _PyInterpreterState_LookUpID(id); -} diff --git a/Python/pystate.c b/Python/pystate.c index 5a334e8721e63b..5332b8a827d7e8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,6 +2,8 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" +#include "interpreteridobject.h" // PyInterpreterID_Type +#include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" #include "pycore_code.h" // stats #include "pycore_critical_section.h" // _PyCriticalSection_Resume() @@ -1064,6 +1066,73 @@ _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) // accessors //---------- +PyObject * +PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) +{ + PyObject *modules = _PyImport_GetModules(interp); + if (modules == NULL) { + PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); + return NULL; + } + return PyMapping_GetItemString(modules, "__main__"); +} + +PyObject * +PyInterpreterState_GetDict(PyInterpreterState *interp) +{ + if (interp->dict == NULL) { + interp->dict = PyDict_New(); + if (interp->dict == NULL) { + PyErr_Clear(); + } + } + /* Returning NULL means no per-interpreter dict is available. */ + return interp->dict; +} + + +//---------- +// interp ID +//---------- + +int64_t +_PyInterpreterState_ObjectToID(PyObject *idobj) +{ + if (PyObject_TypeCheck(idobj, &PyInterpreterID_Type)) { + return _PyInterpreterID_GetID(idobj); + } + + if (!_PyIndex_Check(idobj)) { + PyErr_Format(PyExc_TypeError, + "interpreter ID must be an int, got %.100s", + Py_TYPE(idobj)->tp_name); + return -1; + } + + // This may raise OverflowError. + // For now, we don't worry about if LLONG_MAX < INT64_MAX. + long long id = PyLong_AsLongLong(idobj); + if (id == -1 && PyErr_Occurred()) { + return -1; + } + + if (id < 0) { + PyErr_Format(PyExc_ValueError, + "interpreter ID must be a non-negative int, got %R", + idobj); + return -1; + } +#if LLONG_MAX > INT64_MAX + else if (id > INT64_MAX) { + PyErr_SetString(PyExc_OverflowError, "int too big to convert"); + return -1; + } +#endif + else { + return (int64_t)id; + } +} + int64_t PyInterpreterState_GetID(PyInterpreterState *interp) { @@ -1142,30 +1211,6 @@ _PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required) interp->requires_idref = required ? 1 : 0; } -PyObject * -PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) -{ - PyObject *modules = _PyImport_GetModules(interp); - if (modules == NULL) { - PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized"); - return NULL; - } - return PyMapping_GetItemString(modules, "__main__"); -} - -PyObject * -PyInterpreterState_GetDict(PyInterpreterState *interp) -{ - if (interp->dict == NULL) { - interp->dict = PyDict_New(); - if (interp->dict == NULL) { - PyErr_Clear(); - } - } - /* Returning NULL means no per-interpreter dict is available. */ - return interp->dict; -} - //----------------------------- // look up an interpreter state @@ -1227,6 +1272,16 @@ _PyInterpreterState_LookUpID(int64_t requested_id) return interp; } +PyInterpreterState * +_PyInterpreterState_LookUpIDObject(PyObject *requested_id) +{ + int64_t id = _PyInterpreterState_ObjectToID(requested_id); + if (id < 0) { + return NULL; + } + return _PyInterpreterState_LookUpID(id); +} + /********************************/ /* the per-thread runtime state */ From 5a76d1be8ef371b75ca65166726923c249b5f615 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 10:06:35 -0600 Subject: [PATCH 66/82] gh-105716: Update interp->threads.main After Fork (gh-117049) I missed this in gh-109921. We also update Py_Exit() to call _PyInterpreterState_SetNotRunningMain(), if necessary. --- Include/internal/pycore_pystate.h | 3 +++ Modules/posixmodule.c | 4 ++++ Python/pylifecycle.c | 4 ++++ Python/pystate.c | 35 +++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 6f9e6a332a7830..9aa439229cc8ea 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -83,6 +83,9 @@ PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); +extern int _PyThreadState_IsRunningMain(PyThreadState *); +extern void _PyInterpreterState_ReinitRunningMain(PyThreadState *); + static inline const PyConfig * _Py_GetMainConfig(void) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 644d4ba1f65b38..88679164fc3aab 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -646,6 +646,7 @@ PyOS_AfterFork_Child(void) PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); + assert(tstate->thread_id == PyThread_get_thread_ident()); #ifdef PY_HAVE_THREAD_NATIVE_ID tstate->native_thread_id = PyThread_get_thread_native_id(); #endif @@ -655,6 +656,9 @@ PyOS_AfterFork_Child(void) _Py_qsbr_after_fork((_PyThreadStateImpl *)tstate); #endif + // Ideally we could guarantee tstate is running main. + _PyInterpreterState_ReinitRunningMain(tstate); + status = _PyEval_ReInitThreads(tstate); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bc76822e72c54a..4bd01ad23e573f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -3138,6 +3138,10 @@ call_ll_exitfuncs(_PyRuntimeState *runtime) void _Py_NO_RETURN Py_Exit(int sts) { + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL && _PyThreadState_IsRunningMain(tstate)) { + _PyInterpreterState_SetNotRunningMain(tstate->interp); + } if (Py_FinalizeEx() < 0) { sts = 120; } diff --git a/Python/pystate.c b/Python/pystate.c index 5332b8a827d7e8..6d63eac22a470c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1050,6 +1050,30 @@ _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) return 0; } +#ifndef NDEBUG +static int +is_running_main(PyThreadState *tstate) +{ + if (tstate->interp->threads.main != NULL) { + return tstate == tstate->interp->threads.main; + } + return 0; +} +#endif + +int +_PyThreadState_IsRunningMain(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + if (interp->threads.main != NULL) { + return tstate == interp->threads.main; + } + if (_Py_IsMainInterpreter(interp)) { + return tstate->thread_id == interp->runtime->main_thread; + } + return 0; +} + int _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) { @@ -1061,6 +1085,15 @@ _PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) return 0; } +void +_PyInterpreterState_ReinitRunningMain(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + if (interp->threads.main != tstate) { + interp->threads.main = NULL; + } +} + //---------- // accessors @@ -1543,6 +1576,7 @@ PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); + assert(!is_running_main(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1641,6 +1675,7 @@ tstate_delete_common(PyThreadState *tstate) assert(tstate->_status.cleared && !tstate->_status.finalized); assert(tstate->state != _Py_THREAD_ATTACHED); tstate_verify_not_active(tstate); + assert(!is_running_main(tstate)); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { From 8bea6c411d65cd987616b4ecdb86373e4f21f1c6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Mar 2024 17:07:00 +0100 Subject: [PATCH 67/82] gh-115754: Add Py_GetConstant() function (#116883) Add Py_GetConstant() and Py_GetConstantBorrowed() functions. In the limited C API version 3.13, getting Py_None, Py_False, Py_True, Py_Ellipsis and Py_NotImplemented singletons is now implemented as function calls at the stable ABI level to hide implementation details. Getting these constants still return borrowed references. Add _testlimitedcapi/object.c and test_capi/test_object.py to test Py_GetConstant() and Py_GetConstantBorrowed() functions. --- Doc/c-api/object.rst | 49 ++++++++++++ Doc/data/stable_abi.dat | 2 + Doc/whatsnew/3.13.rst | 5 ++ Include/boolobject.h | 9 ++- Include/internal/pycore_object.h | 2 + Include/object.h | 31 ++++++- Include/sliceobject.h | 6 +- Lib/test/test_capi/test_object.py | 54 +++++++++++++ Lib/test/test_stable_abi_ctypes.py | 2 + ...-03-15-23-55-24.gh-issue-115754.xnzc__.rst | 3 + ...-03-15-23-57-33.gh-issue-115754.zLdv82.rst | 5 ++ Misc/stable_abi.toml | 4 + Modules/Setup.stdlib.in | 2 +- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/object.c | 80 +++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + Objects/object.c | 51 ++++++++++++ PC/python3dll.c | 2 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + Python/pylifecycle.c | 4 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 22 files changed, 312 insertions(+), 6 deletions(-) create mode 100644 Lib/test/test_capi/test_object.py create mode 100644 Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst create mode 100644 Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst create mode 100644 Modules/_testlimitedcapi/object.c diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 12476412799a4f..ba454db9117504 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -6,6 +6,55 @@ Object Protocol =============== +.. c:function:: PyObject* Py_GetConstant(unsigned int constant_id) + + Get a :term:`strong reference` to a constant. + + Set an exception and return ``NULL`` if *constant_id* is invalid. + + *constant_id* must be one of these constant identifiers: + + .. c:namespace:: NULL + + ======================================== ===== ========================= + Constant Identifier Value Returned object + ======================================== ===== ========================= + .. c:macro:: Py_CONSTANT_NONE ``0`` :py:data:`None` + .. c:macro:: Py_CONSTANT_FALSE ``1`` :py:data:`False` + .. c:macro:: Py_CONSTANT_TRUE ``2`` :py:data:`True` + .. c:macro:: Py_CONSTANT_ELLIPSIS ``3`` :py:data:`Ellipsis` + .. c:macro:: Py_CONSTANT_NOT_IMPLEMENTED ``4`` :py:data:`NotImplemented` + .. c:macro:: Py_CONSTANT_ZERO ``5`` ``0`` + .. c:macro:: Py_CONSTANT_ONE ``6`` ``1`` + .. c:macro:: Py_CONSTANT_EMPTY_STR ``7`` ``''`` + .. c:macro:: Py_CONSTANT_EMPTY_BYTES ``8`` ``b''`` + .. c:macro:: Py_CONSTANT_EMPTY_TUPLE ``9`` ``()`` + ======================================== ===== ========================= + + Numeric values are only given for projects which cannot use the constant + identifiers. + + + .. versionadded:: 3.13 + + .. impl-detail:: + + In CPython, all of these constants are :term:`immortal`. + + +.. c:function:: PyObject* Py_GetConstantBorrowed(unsigned int constant_id) + + Similar to :c:func:`Py_GetConstant`, but return a :term:`borrowed + reference`. + + This function is primarily intended for backwards compatibility: + using :c:func:`Py_GetConstant` is recommended for new code. + + The reference is borrowed from the interpreter, and is valid until the + interpreter finalization. + .. versionadded:: 3.13 + + .. c:var:: PyObject* Py_NotImplemented The ``NotImplemented`` singleton, used to signal that an operation is diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 9d0ad3d036dac3..565f134290802a 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -838,6 +838,8 @@ function,Py_GenericAlias,3.9,, var,Py_GenericAliasType,3.9,, function,Py_GetBuildInfo,3.2,, function,Py_GetCompiler,3.2,, +function,Py_GetConstant,3.13,, +function,Py_GetConstantBorrowed,3.13,, function,Py_GetCopyright,3.2,, function,Py_GetExecPrefix,3.2,, function,Py_GetPath,3.2,, diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 40e2e6a8e03be9..caadf4a3d68fb2 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1731,6 +1731,11 @@ New Features more information. (Contributed by Victor Stinner in :gh:`111696`.) +* Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions + to get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a + :term:`strong reference` to the constant zero. + (Contributed by Victor Stinner in :gh:`115754`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/boolobject.h b/Include/boolobject.h index 19aef5b1b87c6a..3037e61bbf6d0c 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -18,8 +18,13 @@ PyAPI_DATA(PyLongObject) _Py_FalseStruct; PyAPI_DATA(PyLongObject) _Py_TrueStruct; /* Use these macros */ -#define Py_False _PyObject_CAST(&_Py_FalseStruct) -#define Py_True _PyObject_CAST(&_Py_TrueStruct) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_False Py_GetConstantBorrowed(Py_CONSTANT_FALSE) +# define Py_True Py_GetConstantBorrowed(Py_CONSTANT_TRUE) +#else +# define Py_False _PyObject_CAST(&_Py_FalseStruct) +# define Py_True _PyObject_CAST(&_Py_TrueStruct) +#endif // Test if an object is the True singleton, the same as "x is True" in Python. PyAPI_FUNC(int) Py_IsTrue(PyObject *x); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 759ec4d17b5eb4..13fe543133f11e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -716,6 +716,8 @@ PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; // Export for the stable ABI. PyAPI_DATA(int) _Py_SwappedOp[]; +extern void _Py_GetConstant_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index b0c0dba06ca139..67a5e514c421c3 100644 --- a/Include/object.h +++ b/Include/object.h @@ -1068,12 +1068,34 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) #endif +#define Py_CONSTANT_NONE 0 +#define Py_CONSTANT_FALSE 1 +#define Py_CONSTANT_TRUE 2 +#define Py_CONSTANT_ELLIPSIS 3 +#define Py_CONSTANT_NOT_IMPLEMENTED 4 +#define Py_CONSTANT_ZERO 5 +#define Py_CONSTANT_ONE 6 +#define Py_CONSTANT_EMPTY_STR 7 +#define Py_CONSTANT_EMPTY_BYTES 8 +#define Py_CONSTANT_EMPTY_TUPLE 9 + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +PyAPI_FUNC(PyObject*) Py_GetConstant(unsigned int constant_id); +PyAPI_FUNC(PyObject*) Py_GetConstantBorrowed(unsigned int constant_id); +#endif + + /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ -#define Py_None (&_Py_NoneStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_None Py_GetConstantBorrowed(Py_CONSTANT_NONE) +#else +# define Py_None (&_Py_NoneStruct) +#endif // Test if an object is the None singleton, the same as "x is None" in Python. PyAPI_FUNC(int) Py_IsNone(PyObject *x); @@ -1087,7 +1109,12 @@ Py_NotImplemented is a singleton used to signal that an operation is not implemented for a given type combination. */ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ -#define Py_NotImplemented (&_Py_NotImplementedStruct) + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_NotImplemented Py_GetConstantBorrowed(Py_CONSTANT_NOT_IMPLEMENTED) +#else +# define Py_NotImplemented (&_Py_NotImplementedStruct) +#endif /* Macro for returning Py_NotImplemented from a function */ #define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented diff --git a/Include/sliceobject.h b/Include/sliceobject.h index c13863f27c2e63..35e2ea254ca80a 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -8,7 +8,11 @@ extern "C" { PyAPI_DATA(PyObject) _Py_EllipsisObject; /* Don't use this directly */ -#define Py_Ellipsis (&_Py_EllipsisObject) +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000 +# define Py_Ellipsis Py_GetConstantBorrowed(Py_CONSTANT_ELLIPSIS) +#else +# define Py_Ellipsis (&_Py_EllipsisObject) +#endif /* Slice object interface */ diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py new file mode 100644 index 00000000000000..c80e9b653789ad --- /dev/null +++ b/Lib/test/test_capi/test_object.py @@ -0,0 +1,54 @@ +import enum +import unittest +from test.support import import_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class Constant(enum.IntEnum): + Py_CONSTANT_NONE = 0 + Py_CONSTANT_FALSE = 1 + Py_CONSTANT_TRUE = 2 + Py_CONSTANT_ELLIPSIS = 3 + Py_CONSTANT_NOT_IMPLEMENTED = 4 + Py_CONSTANT_ZERO = 5 + Py_CONSTANT_ONE = 6 + Py_CONSTANT_EMPTY_STR = 7 + Py_CONSTANT_EMPTY_BYTES = 8 + Py_CONSTANT_EMPTY_TUPLE = 9 + + INVALID_CONSTANT = Py_CONSTANT_EMPTY_TUPLE + 1 + + +class CAPITest(unittest.TestCase): + def check_get_constant(self, get_constant): + self.assertIs(get_constant(Constant.Py_CONSTANT_NONE), None) + self.assertIs(get_constant(Constant.Py_CONSTANT_FALSE), False) + self.assertIs(get_constant(Constant.Py_CONSTANT_TRUE), True) + self.assertIs(get_constant(Constant.Py_CONSTANT_ELLIPSIS), Ellipsis) + self.assertIs(get_constant(Constant.Py_CONSTANT_NOT_IMPLEMENTED), NotImplemented) + + for constant_id, constant_type, value in ( + (Constant.Py_CONSTANT_ZERO, int, 0), + (Constant.Py_CONSTANT_ONE, int, 1), + (Constant.Py_CONSTANT_EMPTY_STR, str, ""), + (Constant.Py_CONSTANT_EMPTY_BYTES, bytes, b""), + (Constant.Py_CONSTANT_EMPTY_TUPLE, tuple, ()), + ): + with self.subTest(constant_id=constant_id): + obj = get_constant(constant_id) + self.assertEqual(type(obj), constant_type, obj) + self.assertEqual(obj, value) + + with self.assertRaises(SystemError): + get_constant(Constant.INVALID_CONSTANT) + + def test_get_constant(self): + self.check_get_constant(_testlimitedcapi.get_constant) + + def test_get_constant_borrowed(self): + self.check_get_constant(_testlimitedcapi.get_constant_borrowed) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 117c27d27b38dc..0e304853d399aa 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -856,6 +856,8 @@ def test_windows_feature_macros(self): "Py_GetArgcArgv", "Py_GetBuildInfo", "Py_GetCompiler", + "Py_GetConstant", + "Py_GetConstantBorrowed", "Py_GetCopyright", "Py_GetExecPrefix", "Py_GetPath", diff --git a/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst b/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst new file mode 100644 index 00000000000000..d76c98ee54056d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-15-23-55-24.gh-issue-115754.xnzc__.rst @@ -0,0 +1,3 @@ +Add :c:func:`Py_GetConstant` and :c:func:`Py_GetConstantBorrowed` functions to +get constants. For example, ``Py_GetConstant(Py_CONSTANT_ZERO)`` returns a +:term:`strong reference` to the constant zero. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst b/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst new file mode 100644 index 00000000000000..feff0c0897eae1 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-15-23-57-33.gh-issue-115754.zLdv82.rst @@ -0,0 +1,5 @@ +In the limited C API version 3.13, getting ``Py_None``, ``Py_False``, +``Py_True``, ``Py_Ellipsis`` and ``Py_NotImplemented`` singletons is now +implemented as function calls at the stable ABI level to hide implementation +details. Getting these constants still return borrowed references. Patch by +Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index c68adf8db079f9..6d6e484b0b6c67 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2500,3 +2500,7 @@ added = '3.13' [function.PyType_GetModuleName] added = '3.13' +[function.Py_GetConstant] + added = '3.13' +[function.Py_GetConstantBorrowed] + added = '3.13' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 1b7ad0e5d95a13..09d6f3b2bb7e8d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index bfd41070eedd55..598071fe0ddbad 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -53,6 +53,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Long(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Object(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_PyOS(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/object.c b/Modules/_testlimitedcapi/object.c new file mode 100644 index 00000000000000..6e438c811d6e98 --- /dev/null +++ b/Modules/_testlimitedcapi/object.c @@ -0,0 +1,80 @@ +// Need limited C API version 3.13 for Py_GetConstant() +#include "pyconfig.h" // Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API ) +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "parts.h" +#include "util.h" + + +/* Test Py_GetConstant() */ +static PyObject * +get_constant(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstant(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return obj; +} + + +/* Test Py_GetConstantBorrowed() */ +static PyObject * +get_constant_borrowed(PyObject *Py_UNUSED(module), PyObject *args) +{ + int constant_id; + if (!PyArg_ParseTuple(args, "i", &constant_id)) { + return NULL; + } + + PyObject *obj = Py_GetConstantBorrowed(constant_id); + if (obj == NULL) { + assert(PyErr_Occurred()); + return NULL; + } + return Py_NewRef(obj); +} + + +/* Test constants */ +static PyObject * +test_constants(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // Test that implementation of constants in the limited C API: + // check that the C code compiles. + // + // Test also that constants and Py_GetConstant() return the same + // objects. + assert(Py_None == Py_GetConstant(Py_CONSTANT_NONE)); + assert(Py_False == Py_GetConstant(Py_CONSTANT_FALSE)); + assert(Py_True == Py_GetConstant(Py_CONSTANT_TRUE)); + assert(Py_Ellipsis == Py_GetConstant(Py_CONSTANT_ELLIPSIS)); + assert(Py_NotImplemented == Py_GetConstant(Py_CONSTANT_NOT_IMPLEMENTED)); + // Other constants are tested in test_capi.test_object + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"get_constant", get_constant, METH_VARARGS}, + {"get_constant_borrowed", get_constant_borrowed, METH_VARARGS}, + {"test_constants", test_constants, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Object(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 4b65912489661b..d91f174cd31eed 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -29,6 +29,7 @@ int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); int _PyTestLimitedCAPI_Init_Float(PyObject *module); int _PyTestLimitedCAPI_Init_HeaptypeRelative(PyObject *module); +int _PyTestLimitedCAPI_Init_Object(PyObject *module); int _PyTestLimitedCAPI_Init_List(PyObject *module); int _PyTestLimitedCAPI_Init_Long(PyObject *module); int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); diff --git a/Objects/object.c b/Objects/object.c index fcb8cf481657e5..0d03292c9115cd 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -14,6 +14,7 @@ #include "pycore_memoryobject.h" // _PyManagedBuffer_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition +#include "pycore_long.h" // _PyLong_GetZero() #include "pycore_optimizer.h" // _PyUOpExecutor_Type, _PyUOpOptimizer_Type, ... #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() @@ -2991,3 +2992,53 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) { Py_SET_REFCNT(ob, refcnt); } + + +static PyObject* constants[] = { + &_Py_NoneStruct, // Py_CONSTANT_NONE + (PyObject*)(&_Py_FalseStruct), // Py_CONSTANT_FALSE + (PyObject*)(&_Py_TrueStruct), // Py_CONSTANT_TRUE + &_Py_EllipsisObject, // Py_CONSTANT_ELLIPSIS + &_Py_NotImplementedStruct, // Py_CONSTANT_NOT_IMPLEMENTED + NULL, // Py_CONSTANT_ZERO + NULL, // Py_CONSTANT_ONE + NULL, // Py_CONSTANT_EMPTY_STR + NULL, // Py_CONSTANT_EMPTY_BYTES + NULL, // Py_CONSTANT_EMPTY_TUPLE +}; + +void +_Py_GetConstant_Init(void) +{ + constants[Py_CONSTANT_ZERO] = _PyLong_GetZero(); + constants[Py_CONSTANT_ONE] = _PyLong_GetOne(); + constants[Py_CONSTANT_EMPTY_STR] = PyUnicode_New(0, 0); + constants[Py_CONSTANT_EMPTY_BYTES] = PyBytes_FromStringAndSize(NULL, 0); + constants[Py_CONSTANT_EMPTY_TUPLE] = PyTuple_New(0); +#ifndef NDEBUG + for (size_t i=0; i < Py_ARRAY_LENGTH(constants); i++) { + assert(constants[i] != NULL); + assert(_Py_IsImmortal(constants[i])); + } +#endif +} + +PyObject* +Py_GetConstant(unsigned int constant_id) +{ + if (constant_id < Py_ARRAY_LENGTH(constants)) { + return constants[constant_id]; + } + else { + PyErr_BadInternalCall(); + return NULL; + } +} + + +PyObject* +Py_GetConstantBorrowed(unsigned int constant_id) +{ + // All constants are immortal + return Py_GetConstant(constant_id); +} diff --git a/PC/python3dll.c b/PC/python3dll.c index dbfa3f23bb586d..147bfad44c3741 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -55,6 +55,8 @@ EXPORT_FUNC(Py_GenericAlias) EXPORT_FUNC(Py_GetArgcArgv) EXPORT_FUNC(Py_GetBuildInfo) EXPORT_FUNC(Py_GetCompiler) +EXPORT_FUNC(Py_GetConstant) +EXPORT_FUNC(Py_GetConstantBorrowed) EXPORT_FUNC(Py_GetCopyright) EXPORT_FUNC(Py_GetExecPrefix) EXPORT_FUNC(Py_GetPath) diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 58a8bcbdbce4e8..252039d93103bd 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -103,6 +103,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index e203edaf123e8d..7efbb0acf8f960 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -18,6 +18,7 @@ + diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4bd01ad23e573f..683534d342f437 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -687,6 +687,10 @@ pycore_init_global_objects(PyInterpreterState *interp) _PyUnicode_InitState(interp); + if (_Py_IsMainInterpreter(interp)) { + _Py_GetConstant_Init(); + } + return _PyStatus_OK(); } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 0024e2683052c8..965346b9b04a32 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -742,3 +742,4 @@ Modules/_sqlite/module.c - _sqlite3module - Modules/clinic/md5module.c.h _md5_md5 _keywords - Modules/clinic/grpmodule.c.h grp_getgrgid _keywords - Modules/clinic/grpmodule.c.h grp_getgrnam _keywords - +Objects/object.c - constants static PyObject*[] From abdd1f938f08e536864532b2071f144515ecc88b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Mar 2024 17:45:43 +0100 Subject: [PATCH 68/82] gh-85283: Build _testconsole extension with limited C API (#117125) --- Doc/whatsnew/3.13.rst | 2 +- ...4-03-14-10-33-58.gh-issue-85283.LOgmdU.rst | 5 +- PC/_testconsole.c | 35 ++++--- PC/clinic/_testconsole.c.h | 99 ++++--------------- 4 files changed, 43 insertions(+), 98 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index caadf4a3d68fb2..7e6c79dbf50aac 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1527,7 +1527,7 @@ Build Changes * The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, ``termios``, ``winsound``, ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_statistics``, ``_testimportmultiple`` and ``_uuid`` + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) diff --git a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst index ef8a934b435a88..c8e6b1b1e6ed62 100644 --- a/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst +++ b/Misc/NEWS.d/next/C API/2024-03-14-10-33-58.gh-issue-85283.LOgmdU.rst @@ -1,2 +1,3 @@ -The ``fcntl``, ``grp``, ``pwd``, ``termios`` and ``_statistics`` C extensions are now -built with the :ref:`limited C API `. Patch by Victor Stinner. +The ``fcntl``, ``grp``, ``pwd``, ``termios``, ``_statistics`` and +``_testconsole`` C extensions are now built with the :ref:`limited C API +`. Patch by Victor Stinner. diff --git a/PC/_testconsole.c b/PC/_testconsole.c index 1dc0d230c4d7c3..f1ace003df483b 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,17 +1,16 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 +// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030c0000 #endif #include "Python.h" #ifdef MS_WINDOWS -#include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "pycore_runtime.h" // _Py_ID() - #define WIN32_LEAN_AND_MEAN #include #include @@ -57,20 +56,24 @@ module _testconsole _testconsole.write_input file: object - s: PyBytesObject + s: Py_buffer Writes UTF-16-LE encoded bytes to the console as if typed by a user. [clinic start generated code]*/ static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s) -/*[clinic end generated code: output=48f9563db34aedb3 input=4c774f2d05770bc6]*/ +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s) +/*[clinic end generated code: output=58631a8985426ad3 input=68062f1bb2e52206]*/ { INPUT_RECORD *rec = NULL; - PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( - &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); + PyObject *mod = PyImport_ImportModule("_io"); + if (mod == NULL) { + return NULL; + } + + PyTypeObject *winconsoleio_type = (PyTypeObject *)PyObject_GetAttrString(mod, "_WindowsConsoleIO"); + Py_DECREF(mod); if (winconsoleio_type == NULL) { return NULL; } @@ -81,8 +84,8 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, return NULL; } - const wchar_t *p = (const wchar_t *)PyBytes_AS_STRING(s); - DWORD size = (DWORD)PyBytes_GET_SIZE(s) / sizeof(wchar_t); + const wchar_t *p = (const wchar_t *)s->buf; + DWORD size = (DWORD)s->len / sizeof(wchar_t); rec = (INPUT_RECORD*)PyMem_Calloc(size, sizeof(INPUT_RECORD)); if (!rec) @@ -96,9 +99,11 @@ _testconsole_write_input_impl(PyObject *module, PyObject *file, prec->Event.KeyEvent.uChar.UnicodeChar = *p; } - HANDLE hInput = _Py_get_osfhandle(((winconsoleio*)file)->fd); - if (hInput == INVALID_HANDLE_VALUE) + HANDLE hInput = (HANDLE)_get_osfhandle(((winconsoleio*)file)->fd); + if (hInput == INVALID_HANDLE_VALUE) { + PyErr_SetFromErrno(PyExc_OSError); goto error; + } DWORD total = 0; while (total < size) { diff --git a/PC/clinic/_testconsole.c.h b/PC/clinic/_testconsole.c.h index 2c71c11c438b5b..4c11e545499ac5 100644 --- a/PC/clinic/_testconsole.c.h +++ b/PC/clinic/_testconsole.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - #if defined(MS_WINDOWS) PyDoc_STRVAR(_testconsole_write_input__doc__, @@ -17,58 +11,30 @@ PyDoc_STRVAR(_testconsole_write_input__doc__, "Writes UTF-16-LE encoded bytes to the console as if typed by a user."); #define _TESTCONSOLE_WRITE_INPUT_METHODDEF \ - {"write_input", _PyCFunction_CAST(_testconsole_write_input), METH_FASTCALL|METH_KEYWORDS, _testconsole_write_input__doc__}, + {"write_input", (PyCFunction)(void(*)(void))_testconsole_write_input, METH_VARARGS|METH_KEYWORDS, _testconsole_write_input__doc__}, static PyObject * -_testconsole_write_input_impl(PyObject *module, PyObject *file, - PyBytesObject *s); +_testconsole_write_input_impl(PyObject *module, PyObject *file, Py_buffer *s); static PyObject * -_testconsole_write_input(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_write_input(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), &_Py_ID(s), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", "s", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "write_input", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; + static char *_keywords[] = {"file", "s", NULL}; PyObject *file; - PyBytesObject *s; + Py_buffer s = {NULL, NULL}; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - file = args[0]; - if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("write_input", "argument 's'", "bytes", args[1]); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oy*:write_input", _keywords, + &file, &s)) goto exit; - } - s = (PyBytesObject *)args[1]; - return_value = _testconsole_write_input_impl(module, file, s); + return_value = _testconsole_write_input_impl(module, file, &s); exit: + /* Cleanup for s */ + if (s.obj) { + PyBuffer_Release(&s); + } + return return_value; } @@ -83,48 +49,21 @@ PyDoc_STRVAR(_testconsole_read_output__doc__, "Reads a str from the console as written to stdout."); #define _TESTCONSOLE_READ_OUTPUT_METHODDEF \ - {"read_output", _PyCFunction_CAST(_testconsole_read_output), METH_FASTCALL|METH_KEYWORDS, _testconsole_read_output__doc__}, + {"read_output", (PyCFunction)(void(*)(void))_testconsole_read_output, METH_VARARGS|METH_KEYWORDS, _testconsole_read_output__doc__}, static PyObject * _testconsole_read_output_impl(PyObject *module, PyObject *file); static PyObject * -_testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_testconsole_read_output(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(file), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"file", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "read_output", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"file", NULL}; PyObject *file; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:read_output", _keywords, + &file)) goto exit; - } - file = args[0]; return_value = _testconsole_read_output_impl(module, file); exit: @@ -140,4 +79,4 @@ _testconsole_read_output(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef _TESTCONSOLE_READ_OUTPUT_METHODDEF #define _TESTCONSOLE_READ_OUTPUT_METHODDEF #endif /* !defined(_TESTCONSOLE_READ_OUTPUT_METHODDEF) */ -/*[clinic end generated code: output=08a1c844b3657272 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d60ce07157e3741a input=a9049054013a1b77]*/ From 617158e07811edfd6fd552a3d84b0beedd8f1d18 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 11:15:02 -0600 Subject: [PATCH 69/82] gh-76785: Drop PyInterpreterID_Type (gh-117101) I added it quite a while ago as a strategy for managing interpreter lifetimes relative to the PEP 554 (now 734) implementation. Relatively recently I refactored that implementation to no longer rely on InterpreterID objects. Thus now I'm removing it. --- Include/cpython/interpreteridobject.h | 14 - Include/internal/pycore_interp.h | 5 +- Include/interpreteridobject.h | 17 -- Lib/test/test_capi/test_misc.py | 308 ++++++++++++++------ Makefile.pre.in | 3 - Modules/_interpreters_common.h | 17 ++ Modules/_testcapimodule.c | 8 - Modules/_testinternalcapi.c | 119 ++++++++ Modules/_xxinterpchannelsmodule.c | 5 +- Modules/_xxsubinterpretersmodule.c | 17 +- Objects/interpreteridobject.c | 274 ----------------- Objects/object.c | 3 - PCbuild/_freeze_module.vcxproj | 1 - PCbuild/_freeze_module.vcxproj.filters | 3 - PCbuild/pythoncore.vcxproj | 3 - PCbuild/pythoncore.vcxproj.filters | 9 - Python/pystate.c | 5 - Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 18 files changed, 363 insertions(+), 449 deletions(-) delete mode 100644 Include/cpython/interpreteridobject.h delete mode 100644 Include/interpreteridobject.h delete mode 100644 Objects/interpreteridobject.c diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h deleted file mode 100644 index d425c909806e44..00000000000000 --- a/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) PyInterpreterState_GetIDObject(PyInterpreterState *); - -#ifdef Py_BUILD_CORE -extern int64_t _PyInterpreterID_GetID(PyObject *); -#endif diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index b28e8a3ff45f3f..b8d0fdcce11ba8 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -295,12 +295,11 @@ _PyInterpreterState_SetFinalizing(PyInterpreterState *interp, PyThreadState *tst } -extern int64_t _PyInterpreterState_ObjectToID(PyObject *); -// Export for the _xxinterpchannels module. +// Exports for the _testinternalcapi module. +PyAPI_FUNC(int64_t) _PyInterpreterState_ObjectToID(PyObject *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(int64_t); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpIDObject(PyObject *); - PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index 8432632f339e92..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index fe5e19d46d8b6c..55a1ab6d6d9359 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2207,132 +2207,264 @@ def test_module_state_shared_in_global(self): @requires_subinterpreters class InterpreterIDTests(unittest.TestCase): - InterpreterID = _testcapi.get_interpreterid_type() - - def new_interpreter(self): - def ensure_destroyed(interpid): + def add_interp_cleanup(self, interpid): + def ensure_destroyed(): try: _interpreters.destroy(interpid) except _interpreters.InterpreterNotFoundError: pass + self.addCleanup(ensure_destroyed) + + def new_interpreter(self): id = _interpreters.create() - self.addCleanup(lambda: ensure_destroyed(id)) + self.add_interp_cleanup(id) return id - def test_with_int(self): - id = self.InterpreterID(10, force=True) - - self.assertEqual(int(id), 10) + def test_conversion_int(self): + convert = _testinternalcapi.normalize_interp_id + interpid = convert(10) + self.assertEqual(interpid, 10) - def test_coerce_id(self): - class Int(str): + def test_conversion_coerced(self): + convert = _testinternalcapi.normalize_interp_id + class MyInt(str): def __index__(self): return 10 + interpid = convert(MyInt()) + self.assertEqual(interpid, 10) - id = self.InterpreterID(Int(), force=True) - self.assertEqual(int(id), 10) + def test_conversion_from_interpreter(self): + convert = _testinternalcapi.normalize_interp_id + interpid = self.new_interpreter() + converted = convert(interpid) + self.assertEqual(converted, interpid) + + def test_conversion_bad(self): + convert = _testinternalcapi.normalize_interp_id - def test_bad_id(self): for badid in [ object(), 10.0, '10', b'10', ]: - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(TypeError): - self.InterpreterID(badid) + convert(badid) badid = -1 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(ValueError): - self.InterpreterID(badid) + convert(badid) badid = 2**64 - with self.subTest(badid): + with self.subTest(f'bad: {badid!r}'): with self.assertRaises(OverflowError): - self.InterpreterID(badid) + convert(badid) - def test_exists(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_exists(self): + interpid = self.new_interpreter() + self.assertTrue( + _testinternalcapi.interpreter_exists(interpid)) - def test_does_not_exist(self): - id = self.new_interpreter() - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(int(id) + 1) # unforced + def test_lookup_does_not_exist(self): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) - def test_destroyed(self): - id = _interpreters.create() - _interpreters.destroy(id) - with self.assertRaises(_interpreters.InterpreterNotFoundError): - self.InterpreterID(id) # unforced - - def test_str(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(str(id), '10') - - def test_repr(self): - id = self.InterpreterID(10, force=True) - self.assertEqual(repr(id), 'InterpreterID(10)') - - def test_equality(self): - id1 = self.new_interpreter() - id2 = self.InterpreterID(id1) - id3 = self.InterpreterID( - self.new_interpreter()) - - self.assertTrue(id2 == id2) # identity - self.assertTrue(id2 == id1) # int-equivalent - self.assertTrue(id1 == id2) # reversed - self.assertTrue(id2 == int(id2)) - self.assertTrue(id2 == float(int(id2))) - self.assertTrue(float(int(id2)) == id2) - self.assertFalse(id2 == float(int(id2)) + 0.1) - self.assertFalse(id2 == str(int(id2))) - self.assertFalse(id2 == 2**1000) - self.assertFalse(id2 == float('inf')) - self.assertFalse(id2 == 'spam') - self.assertFalse(id2 == id3) - - self.assertFalse(id2 != id2) - self.assertFalse(id2 != id1) - self.assertFalse(id1 != id2) - self.assertTrue(id2 != id3) - - def test_linked_lifecycle(self): - id1 = _interpreters.create() - _testinternalcapi.unlink_interpreter_refcount(id1) + def test_lookup_destroyed(self): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + _testinternalcapi.interpreter_exists(interpid)) + + def test_linked_lifecycle_does_not_exist(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount = _testinternalcapi.get_interpreter_refcount + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref + + with self.subTest('never existed'): + interpid = _testinternalcapi.unused_interpreter_id() + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + with self.subTest('destroyed'): + interpid = _interpreters.create() + _interpreters.destroy(interpid) + self.assertFalse( + exists(interpid)) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + is_linked(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + link(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + unlink(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + get_refcount(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + incref(interpid) + with self.assertRaises(_interpreters.InterpreterNotFoundError): + decref(interpid) + + def test_linked_lifecycle_initial(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount = _testinternalcapi.get_interpreter_refcount + + # A new interpreter will start out not linked, with a refcount of 0. + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) + linked = is_linked(interpid) + refcount = get_refcount(interpid) + + self.assertFalse(linked) + self.assertEqual(refcount, 0) + + def test_linked_lifecycle_never_linked(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + get_refcount = _testinternalcapi.get_interpreter_refcount + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref + + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) + + # Incref will not automatically link it. + incref(interpid) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 1, get_refcount(interpid)) + + # It isn't linked so it isn't destroyed. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) + + def test_linked_lifecycle_link_unlink(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) + + # Linking at refcount 0 does not destroy the interpreter. + link(interpid) + self.assertTrue( + exists(interpid)) + self.assertTrue( + is_linked(interpid)) + + # Unlinking at refcount 0 does not destroy the interpreter. + unlink(interpid) + self.assertTrue( + exists(interpid)) + self.assertFalse( + is_linked(interpid)) + + def test_linked_lifecycle_link_incref_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount = _testinternalcapi.get_interpreter_refcount + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref + + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) + + # Linking it will not change the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 0, get_refcount(interpid)) - id2 = self.InterpreterID(id1) + # Decref with a refcount of 0 is not allowed. + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) - # The interpreter isn't linked to ID objects, so it isn't destroyed. - del id2 + # When linked, decref back to 0 destroys the interpreter. + decref(interpid) + self.assertFalse( + exists(interpid)) + + def test_linked_lifecycle_incref_link(self): + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + get_refcount = _testinternalcapi.get_interpreter_refcount + incref = _testinternalcapi.interpreter_incref + + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) + + incref(interpid) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) - _testinternalcapi.link_interpreter_refcount(id1) + # Linking it will not reset the refcount. + link(interpid) + self.assertTrue( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 0) + 1, get_refcount(interpid)) + + def test_linked_lifecycle_link_incref_unlink_decref(self): + exists = _testinternalcapi.interpreter_exists + is_linked = _testinternalcapi.interpreter_refcount_linked + link = _testinternalcapi.link_interpreter_refcount + unlink = _testinternalcapi.unlink_interpreter_refcount + get_refcount = _testinternalcapi.get_interpreter_refcount + incref = _testinternalcapi.interpreter_incref + decref = _testinternalcapi.interpreter_decref + + interpid = _testinternalcapi.new_interpreter() + self.add_interp_cleanup(interpid) - id3 = self.InterpreterID(id1) + link(interpid) + self.assertTrue( + is_linked(interpid)) + + incref(interpid) + self.assertEqual( + 1, get_refcount(interpid)) + + # Unlinking it will not change the refcount. + unlink(interpid) + self.assertFalse( + is_linked(interpid)) self.assertEqual( - _testinternalcapi.get_interpreter_refcount(id1), - 1) + 1, get_refcount(interpid)) - # The interpreter is linked now so is destroyed. - del id3 - with self.assertRaises(_interpreters.InterpreterNotFoundError): - _testinternalcapi.get_interpreter_refcount(id1) + # Unlinked: decref back to 0 does not destroys the interpreter. + decref(interpid) + self.assertTrue( + exists(interpid)) + self.assertEqual( + 0, get_refcount(interpid)) class BuiltinStaticTypesTests(unittest.TestCase): diff --git a/Makefile.pre.in b/Makefile.pre.in index cacf14a52cb68e..c454f31aae1e57 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -507,7 +507,6 @@ OBJECT_OBJS= \ Objects/floatobject.o \ Objects/frameobject.o \ Objects/funcobject.o \ - Objects/interpreteridobject.o \ Objects/iterobject.o \ Objects/listobject.o \ Objects/longobject.o \ @@ -1003,7 +1002,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/genericaliasobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1077,7 +1075,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index 07120f6ccc7207..de9a60ce657e0c 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -19,3 +19,20 @@ clear_xid_class(PyTypeObject *cls) return _PyCrossInterpreterData_UnregisterClass(cls); } #endif + + +#ifdef RETURNS_INTERPID_OBJECT +static PyObject * +get_interpid_obj(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IDInitref(interp) != 0) { + return NULL; + }; + int64_t id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + assert(id < LLONG_MAX); + return PyLong_FromLongLong(id); +} +#endif diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e68d083955d64a..16b5e1d257eed2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -13,7 +13,6 @@ #include "_testcapi/parts.h" #include "frameobject.h" // PyFrame_New() -#include "interpreteridobject.h" // PyInterpreterID_Type #include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX @@ -1449,12 +1448,6 @@ run_in_subinterp(PyObject *self, PyObject *args) return PyLong_FromLong(r); } -static PyObject * -get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - return Py_NewRef(&PyInterpreterID_Type); -} - static PyMethodDef ml; static PyObject * @@ -3299,7 +3292,6 @@ static PyMethodDef TestMethods[] = { {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, {"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS}, {"run_in_subinterp", run_in_subinterp, METH_VARARGS}, - {"get_interpreterid_type", get_interpreterid_type, METH_NOARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f73a29e5afe801..e1717f7a66b1de 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1475,6 +1475,83 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) } +static PyObject * +normalize_interp_id(PyObject *self, PyObject *idobj) +{ + int64_t interpid = _PyInterpreterState_ObjectToID(idobj); + if (interpid < 0) { + return NULL; + } + return PyLong_FromLongLong(interpid); +} + +static PyObject * +unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int64_t interpid = INT64_MAX; + assert(interpid > _PyRuntime.interpreters.next_id); + return PyLong_FromLongLong(interpid); +} + +static PyObject * +new_interpreter(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + // Unlike _interpreters.create(), we do not automatically link + // the interpreter to its refcount. + PyThreadState *save_tstate = PyThreadState_Get(); + const PyInterpreterConfig config = \ + (PyInterpreterConfig)_PyInterpreterConfig_INIT; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + PyThreadState_Swap(save_tstate); + if (PyStatus_Exception(status)) { + _PyErr_SetFromPyStatus(status); + return NULL; + } + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); + + if (_PyInterpreterState_IDInitref(interp) < 0) { + goto error; + } + + int64_t interpid = PyInterpreterState_GetID(interp); + if (interpid < 0) { + goto error; + } + PyObject *idobj = PyLong_FromLongLong(interpid); + if (idobj == NULL) { + goto error; + } + + PyThreadState_Swap(tstate); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + + return idobj; + +error: + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + return NULL; +} + +static PyObject * +interpreter_exists(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + if (PyErr_ExceptionMatches(PyExc_InterpreterNotFoundError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + assert(PyErr_Occurred()); + return NULL; + } + Py_RETURN_TRUE; +} + static PyObject * get_interpreter_refcount(PyObject *self, PyObject *idobj) { @@ -1509,6 +1586,41 @@ unlink_interpreter_refcount(PyObject *self, PyObject *idobj) Py_RETURN_NONE; } +static PyObject * +interpreter_refcount_linked(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + if (_PyInterpreterState_RequiresIDRef(interp)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static PyObject * +interpreter_incref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDIncref(interp); + Py_RETURN_NONE; +} + +static PyObject * +interpreter_decref(PyObject *self, PyObject *idobj) +{ + PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj); + if (interp == NULL) { + return NULL; + } + _PyInterpreterState_IDDecref(interp); + Py_RETURN_NONE; +} + static void _xid_capsule_destructor(PyObject *capsule) @@ -1749,9 +1861,16 @@ static PyMethodDef module_functions[] = { {"run_in_subinterp_with_config", _PyCFunction_CAST(run_in_subinterp_with_config), METH_VARARGS | METH_KEYWORDS}, + {"normalize_interp_id", normalize_interp_id, METH_O}, + {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, + {"new_interpreter", new_interpreter, METH_NOARGS}, + {"interpreter_exists", interpreter_exists, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"link_interpreter_refcount", link_interpreter_refcount, METH_O}, {"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O}, + {"interpreter_refcount_linked", interpreter_refcount_linked, METH_O}, + {"interpreter_incref", interpreter_incref, METH_O}, + {"interpreter_decref", interpreter_decref, METH_O}, {"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS}, {"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 28ec00a159d6cd..b63a3aab8263bc 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -6,7 +6,6 @@ #endif #include "Python.h" -#include "interpreteridobject.h" #include "pycore_crossinterp.h" // struct _xid #include "pycore_interp.h" // _PyInterpreterState_LookUpID() @@ -18,7 +17,9 @@ #endif #define REGISTERS_HEAP_TYPES +#define RETURNS_INTERPID_OBJECT #include "_interpreters_common.h" +#undef RETURNS_INTERPID_OBJECT #undef REGISTERS_HEAP_TYPES @@ -2908,7 +2909,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) goto except; } if (res) { - interpid_obj = PyInterpreterState_GetIDObject(interp); + interpid_obj = get_interpid_obj(interp); if (interpid_obj == NULL) { goto except; } diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 606b2a36481ce2..befa225c9183c5 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -16,10 +16,11 @@ #include "pycore_pyerrors.h" // _Py_excinfo #include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() -#include "interpreteridobject.h" #include "marshal.h" // PyMarshal_ReadObjectFromString() +#define RETURNS_INTERPID_OBJECT #include "_interpreters_common.h" +#undef RETURNS_INTERPID_OBJECT #define MODULE_NAME _xxsubinterpreters @@ -38,20 +39,6 @@ _get_current_interp(void) #define look_up_interp _PyInterpreterState_LookUpIDObject -static PyObject * -get_interpid_obj(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - assert(id < LLONG_MAX); - return PyLong_FromLongLong(id); -} - static PyObject * _get_current_module(void) { diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c deleted file mode 100644 index 4844d6a9bf781c..00000000000000 --- a/Objects/interpreteridobject.c +++ /dev/null @@ -1,274 +0,0 @@ -/* InterpreterID object */ - -#include "Python.h" -#include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" - - -typedef struct interpid { - PyObject_HEAD - int64_t id; -} interpid; - -int64_t -_PyInterpreterID_GetID(PyObject *self) -{ - if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { - PyErr_Format(PyExc_TypeError, - "expected an InterpreterID, got %R", - self); - return -1; - - } - int64_t id = ((interpid *)self)->id; - assert(id >= 0); - return id; -} - -static interpid * -newinterpid(PyTypeObject *cls, int64_t id, int force) -{ - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp == NULL) { - if (force) { - PyErr_Clear(); - } - else { - return NULL; - } - } - - if (interp != NULL) { - if (_PyInterpreterState_IDIncref(interp) < 0) { - return NULL; - } - } - - interpid *self = PyObject_New(interpid, cls); - if (self == NULL) { - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - return NULL; - } - self->id = id; - - return self; -} - -static PyObject * -interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"id", "force", NULL}; - PyObject *idobj; - int force = 0; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|$p:InterpreterID.__init__", kwlist, - &idobj, &force)) { - return NULL; - } - int64_t id = _PyInterpreterState_ObjectToID(idobj); - if (id < 0) { - return NULL; - } - - return (PyObject *)newinterpid(cls, id, force); -} - -static void -interpid_dealloc(PyObject *v) -{ - int64_t id = ((interpid *)v)->id; - PyInterpreterState *interp = _PyInterpreterState_LookUpID(id); - if (interp != NULL) { - _PyInterpreterState_IDDecref(interp); - } - else { - // already deleted - PyErr_Clear(); - } - Py_TYPE(v)->tp_free(v); -} - -static PyObject * -interpid_repr(PyObject *self) -{ - PyTypeObject *type = Py_TYPE(self); - const char *name = _PyType_Name(type); - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id); -} - -static PyObject * -interpid_str(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyUnicode_FromFormat("%" PRId64 "", id->id); -} - -static PyObject * -interpid_int(PyObject *self) -{ - interpid *id = (interpid *)self; - return PyLong_FromLongLong(id->id); -} - -static PyNumberMethods interpid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)interpid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)interpid_int, /* nb_index */ -}; - -static Py_hash_t -interpid_hash(PyObject *self) -{ - interpid *id = (interpid *)self; - PyObject *obj = PyLong_FromLongLong(id->id); - if (obj == NULL) { - return -1; - } - Py_hash_t hash = PyObject_Hash(obj); - Py_DECREF(obj); - return hash; -} - -static PyObject * -interpid_richcompare(PyObject *self, PyObject *other, int op) -{ - if (op != Py_EQ && op != Py_NE) { - Py_RETURN_NOTIMPLEMENTED; - } - - if (!PyObject_TypeCheck(self, &PyInterpreterID_Type)) { - Py_RETURN_NOTIMPLEMENTED; - } - - interpid *id = (interpid *)self; - int equal; - if (PyObject_TypeCheck(other, &PyInterpreterID_Type)) { - interpid *otherid = (interpid *)other; - equal = (id->id == otherid->id); - } - else if (PyLong_CheckExact(other)) { - /* Fast path */ - int overflow; - long long otherid = PyLong_AsLongLongAndOverflow(other, &overflow); - if (otherid == -1 && PyErr_Occurred()) { - return NULL; - } - equal = !overflow && (otherid >= 0) && (id->id == otherid); - } - else if (PyNumber_Check(other)) { - PyObject *pyid = PyLong_FromLongLong(id->id); - if (pyid == NULL) { - return NULL; - } - PyObject *res = PyObject_RichCompare(pyid, other, op); - Py_DECREF(pyid); - return res; - } - else { - Py_RETURN_NOTIMPLEMENTED; - } - - if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; - } - Py_RETURN_FALSE; -} - -PyDoc_STRVAR(interpid_doc, -"A interpreter ID identifies a interpreter and may be used as an int."); - -PyTypeObject PyInterpreterID_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "InterpreterID", /* tp_name */ - sizeof(interpid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)interpid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)interpid_repr, /* tp_repr */ - &interpid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - interpid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)interpid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - interpid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - interpid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - interpid_new, /* tp_new */ -}; - -PyObject *PyInterpreterID_New(int64_t id) -{ - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} - -PyObject * -PyInterpreterState_GetIDObject(PyInterpreterState *interp) -{ - if (_PyInterpreterState_IDInitref(interp) != 0) { - return NULL; - }; - int64_t id = PyInterpreterState_GetID(interp); - if (id < 0) { - return NULL; - } - return (PyObject *)newinterpid(&PyInterpreterID_Type, id, 0); -} diff --git a/Objects/object.c b/Objects/object.c index 0d03292c9115cd..b4f0fd4d7db941 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -24,8 +24,6 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type - #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() # error "Py_LIMITED_API macro must not be defined" @@ -2240,7 +2238,6 @@ static PyTypeObject* static_types[] = { &PyGen_Type, &PyGetSetDescr_Type, &PyInstanceMethod_Type, - &PyInterpreterID_Type, &PyListIter_Type, &PyListRevIter_Type, &PyList_Type, diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index bce92c91f1ca0d..82471e0f140ec3 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -142,7 +142,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5b34440af9322b..97c52fdadf7c05 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -241,9 +241,6 @@ Source Files - - Source Files - Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9131ce87db6c84..c944bbafdba7e5 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -154,7 +154,6 @@ - @@ -303,7 +302,6 @@ - @@ -504,7 +502,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 27bd1121663398..0afad125ce1e97 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -330,9 +330,6 @@ Include - - Include - Modules @@ -492,9 +489,6 @@ Include - - Include\cpython - Include\cpython @@ -1475,9 +1469,6 @@ Objects - - Objects - Modules diff --git a/Python/pystate.c b/Python/pystate.c index 6d63eac22a470c..3ef405105a8d46 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2,7 +2,6 @@ /* Thread and interpreter state structures and their interfaces */ #include "Python.h" -#include "interpreteridobject.h" // PyInterpreterID_Type #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" #include "pycore_code.h" // stats @@ -1131,10 +1130,6 @@ PyInterpreterState_GetDict(PyInterpreterState *interp) int64_t _PyInterpreterState_ObjectToID(PyObject *idobj) { - if (PyObject_TypeCheck(idobj, &PyInterpreterID_Type)) { - return _PyInterpreterID_GetID(idobj); - } - if (!_PyIndex_Check(idobj)) { PyErr_Format(PyExc_TypeError, "interpreter ID must be an int, got %.100s", diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 686a3d3160cc90..92fab9b3998636 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -54,7 +54,6 @@ Objects/genobject.c - _PyAsyncGenASend_Type - Objects/genobject.c - _PyAsyncGenAThrow_Type - Objects/genobject.c - _PyAsyncGenWrappedValue_Type - Objects/genobject.c - _PyCoroWrapper_Type - -Objects/interpreteridobject.c - PyInterpreterID_Type - Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyAnextAwaitable_Type - From 50369e6c34d05222e5a0ec9443a9f7b230e83112 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 21 Mar 2024 13:27:46 -0400 Subject: [PATCH 70/82] gh-116996: Add pystats about _Py_uop_analyse_and_optimize (GH-116997) --- Include/cpython/pystats.h | 7 ++++++- Include/internal/pycore_code.h | 2 ++ Python/optimizer_analysis.c | 11 +++++++++-- Python/specialize.c | 13 +++++++++++++ Tools/scripts/summarize_stats.py | 26 ++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 887fbbedf88502..5bf7bacd514699 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -19,6 +19,8 @@ // Define _PY_INTERPRETER macro to increment interpreter_increfs and // interpreter_decrefs. Otherwise, increment increfs and decrefs. +#include "pycore_uop_ids.h" + #ifndef Py_CPYTHON_PYSTATS_H # error "this header file must not be included directly" #endif @@ -116,7 +118,7 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t executors_invalidated; - UOpStats opcode[512]; + UOpStats opcode[MAX_UOP_ID]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t trace_run_length_hist[_Py_UOP_HIST_SIZE]; @@ -124,6 +126,9 @@ typedef struct _optimization_stats { uint64_t optimizer_attempts; uint64_t optimizer_successes; uint64_t optimizer_failure_reason_no_memory; + uint64_t remove_globals_builtins_changed; + uint64_t remove_globals_incorrect_keys; + uint64_t error_in_opcode[MAX_UOP_ID]; } OptimizationStats; typedef struct _rare_event_stats { diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8eabd49a18afa9..e004783ee48198 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -308,6 +308,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0) #define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0) #define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0) +#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0) #define OPT_HIST(length, name) \ do { \ if (_Py_stats) { \ @@ -334,6 +335,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OPT_STAT_INC(name) ((void)0) #define UOP_STAT_INC(opname, name) ((void)0) #define OPT_UNSUPPORTED_OPCODE(opname) ((void)0) +#define OPT_ERROR_IN_OPCODE(opname) ((void)0) #define OPT_HIST(length, name) ((void)0) #define RARE_EVENT_STAT_INC(name) ((void)0) #endif // !Py_STATS diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 603ac6815665ca..6c460c5359d71e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -139,6 +139,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *builtins = frame->f_builtins; if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); return 1; } PyObject *globals = frame->f_globals; @@ -170,6 +171,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, switch(opcode) { case _GUARD_BUILTINS_VERSION: if (incorrect_keys(inst, builtins)) { + OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } if (interp->rare_events.builtin_dict >= _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { @@ -190,6 +192,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, break; case _GUARD_GLOBALS_VERSION: if (incorrect_keys(inst, globals)) { + OPT_STAT_INC(remove_globals_incorrect_keys); return 0; } uint64_t watched_mutations = get_mutations(globals); @@ -238,6 +241,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, globals = func->func_globals; builtins = func->func_builtins; if (builtins != interp->builtins) { + OPT_STAT_INC(remove_globals_builtins_changed); return 1; } break; @@ -358,6 +362,7 @@ optimize_uops( _Py_UOpsContext context; _Py_UOpsContext *ctx = &context; + uint32_t opcode = UINT16_MAX; if (_Py_uop_abstractcontext_init(ctx) < 0) { goto out_of_space; @@ -374,8 +379,7 @@ optimize_uops( this_instr++) { int oparg = this_instr->oparg; - uint32_t opcode = this_instr->opcode; - + opcode = this_instr->opcode; _Py_UopsSymbol **stack_pointer = ctx->frame->stack_pointer; #ifdef Py_DEBUG @@ -410,6 +414,9 @@ optimize_uops( error: DPRINTF(3, "\n"); DPRINTF(1, "Encountered error in abstract interpreter\n"); + if (opcode <= MAX_UOP_ID) { + OPT_ERROR_IN_OPCODE(opcode); + } _Py_uop_abstractcontext_fini(ctx); return -1; diff --git a/Python/specialize.c b/Python/specialize.c index b1f9eb756c3665..801ab1f2e64e5d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -247,6 +247,8 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "Optimization optimizer successes: %" PRIu64 "\n", stats->optimizer_successes); fprintf(out, "Optimization optimizer failure no memory: %" PRIu64 "\n", stats->optimizer_failure_reason_no_memory); + fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); + fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); const char* const* names; for (int i = 0; i <= MAX_UOP_ID; i++) { @@ -268,6 +270,17 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) ); } } + + for (int i = 0; i < MAX_UOP_ID; i++) { + if (stats->error_in_opcode[i]) { + fprintf( + out, + "error_in_opcode[%s].count : %" PRIu64 "\n", + _PyUOpName(i), + stats->error_in_opcode[i] + ); + } + } } static void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 6af14e1b769b80..d40106b8682388 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -513,6 +513,8 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: attempts = self._data["Optimization optimizer attempts"] successes = self._data["Optimization optimizer successes"] no_memory = self._data["Optimization optimizer failure no memory"] + builtins_changed = self._data["Optimizer remove globals builtins changed"] + incorrect_keys = self._data["Optimizer remove globals incorrect keys"] return { Doc( @@ -527,6 +529,14 @@ def get_optimizer_stats(self) -> dict[str, tuple[int, int | None]]: "Optimizer no memory", "The number of optimizations that failed due to no memory.", ): (no_memory, attempts), + Doc( + "Remove globals builtins changed", + "The builtins changed during optimization", + ): (builtins_changed, attempts), + Doc( + "Remove globals incorrect keys", + "The keys in the globals dictionary aren't what was expected", + ): (incorrect_keys, attempts), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: @@ -1177,6 +1187,17 @@ def calc_unsupported_opcodes_table(stats: Stats) -> Rows: reverse=True, ) + def calc_error_in_opcodes_table(stats: Stats) -> Rows: + error_in_opcodes = stats.get_opcode_stats("error_in_opcode") + return sorted( + [ + (opcode, Count(count)) + for opcode, count in error_in_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): if not base_stats.get_optimization_stats() or ( head_stats is not None and not head_stats.get_optimization_stats() @@ -1223,6 +1244,11 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) ) ], ) + yield Section( + "Optimizer errored out with opcode", + "Optimization stopped after encountering this opcode", + [Table(("Opcode", "Count:"), calc_error_in_opcodes_table, JoinMode.CHANGE)], + ) return Section( "Optimization (Tier 2) stats", From 1f72fb5447ef3f8892b4a7a6213522579c618e8e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 21 Mar 2024 14:21:02 -0400 Subject: [PATCH 71/82] gh-116522: Refactor `_PyThreadState_DeleteExcept` (#117131) Split `_PyThreadState_DeleteExcept` into two functions: - `_PyThreadState_RemoveExcept` removes all thread states other than one passed as an argument. It returns the removed thread states as a linked list. - `_PyThreadState_DeleteList` deletes those dead thread states. It may call destructors, so we want to "start the world" before calling `_PyThreadState_DeleteList` to avoid potential deadlocks. --- Include/internal/pycore_pystate.h | 3 ++- Modules/posixmodule.c | 8 +++++++ Python/ceval_gil.c | 7 ++---- Python/pylifecycle.c | 7 ++++-- Python/pystate.c | 39 +++++++++++++++++++------------ 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 9aa439229cc8ea..35e266acd3ab60 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -218,7 +218,8 @@ extern PyThreadState * _PyThreadState_New( PyInterpreterState *interp, int whence); extern void _PyThreadState_Bind(PyThreadState *tstate); -extern void _PyThreadState_DeleteExcept(PyThreadState *tstate); +extern PyThreadState * _PyThreadState_RemoveExcept(PyThreadState *tstate); +extern void _PyThreadState_DeleteList(PyThreadState *list); extern void _PyThreadState_ClearMimallocHeaps(PyThreadState *tstate); // Export for '_testinternalcapi' shared extension diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 88679164fc3aab..a4b635ef5bf629 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -664,6 +664,14 @@ PyOS_AfterFork_Child(void) goto fatal_error; } + // Remove the dead thread states. We "start the world" once we are the only + // thread state left to undo the stop the world call in `PyOS_BeforeFork`. + // That needs to happen before `_PyThreadState_DeleteList`, because that + // may call destructors. + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_DeleteList(list); + status = _PyImport_ReInitLock(tstate->interp); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 78c13d619e6ee0..d88ac65c5cf300 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -579,9 +579,8 @@ PyEval_ReleaseThread(PyThreadState *tstate) } #ifdef HAVE_FORK -/* This function is called from PyOS_AfterFork_Child to destroy all threads - which are not running in the child process, and clear internal locks - which might be held by those threads. */ +/* This function is called from PyOS_AfterFork_Child to re-initialize the + GIL and pending calls lock. */ PyStatus _PyEval_ReInitThreads(PyThreadState *tstate) { @@ -598,8 +597,6 @@ _PyEval_ReInitThreads(PyThreadState *tstate) struct _pending_calls *pending = &tstate->interp->ceval.pending; _PyMutex_at_fork_reinit(&pending->mutex); - /* Destroy all threads except the current one */ - _PyThreadState_DeleteExcept(tstate); return _PyStatus_OK(); } #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 683534d342f437..1d315b80d88ce0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1934,8 +1934,11 @@ Py_FinalizeEx(void) will be called in the current Python thread. Since _PyRuntimeState_SetFinalizing() has been called, no other Python thread can take the GIL at this point: if they try, they will exit - immediately. */ - _PyThreadState_DeleteExcept(tstate); + immediately. We start the world once we are the only thread state left, + before we call destructors. */ + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + _PyEval_StartTheWorldAll(runtime); + _PyThreadState_DeleteList(list); /* At this point no Python code should be running at all. The only thread state left should be the main thread of the main diff --git a/Python/pystate.c b/Python/pystate.c index 3ef405105a8d46..47d327ae28933b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1763,15 +1763,17 @@ PyThreadState_DeleteCurrent(void) } -/* - * Delete all thread states except the one passed as argument. - * Note that, if there is a current thread state, it *must* be the one - * passed as argument. Also, this won't touch any other interpreters - * than the current one, since we don't know which thread state should - * be kept in those other interpreters. - */ -void -_PyThreadState_DeleteExcept(PyThreadState *tstate) +// Unlinks and removes all thread states from `tstate->interp`, with the +// exception of the one passed as an argument. However, it does not delete +// these thread states. Instead, it returns the removed thread states as a +// linked list. +// +// Note that if there is a current thread state, it *must* be the one +// passed as argument. Also, this won't touch any interpreters other +// than the current one, since we don't know which thread state should +// be kept in those other interpreters. +PyThreadState * +_PyThreadState_RemoveExcept(PyThreadState *tstate) { assert(tstate != NULL); PyInterpreterState *interp = tstate->interp; @@ -1783,8 +1785,7 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) HEAD_LOCK(runtime); /* Remove all thread states, except tstate, from the linked list of - thread states. This will allow calling PyThreadState_Clear() - without holding the lock. */ + thread states. */ PyThreadState *list = interp->threads.head; if (list == tstate) { list = tstate->next; @@ -1799,11 +1800,19 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) interp->threads.head = tstate; HEAD_UNLOCK(runtime); - _PyEval_StartTheWorldAll(runtime); + return list; +} + +// Deletes the thread states in the linked list `list`. +// +// This is intended to be used in conjunction with _PyThreadState_RemoveExcept. +void +_PyThreadState_DeleteList(PyThreadState *list) +{ + // The world can't be stopped because we PyThreadState_Clear() can + // call destructors. + assert(!_PyRuntime.stoptheworld.world_stopped); - /* Clear and deallocate all stale thread states. Even if this - executes Python code, we should be safe since it executes - in the current thread, not one of the stale threads. */ PyThreadState *p, *next; for (p = list; p; p = next) { next = p->next; From c85d84166a84a5cb2d724012726bad34229ad24e Mon Sep 17 00:00:00 2001 From: Will Childs-Klein Date: Thu, 21 Mar 2024 14:16:36 -0500 Subject: [PATCH 72/82] gh-116333: Relax error string text expectations in SSL-related tests (GH-116334) * Relax error string text expectations in SSL-related tests As suggested [here][1], this change relaxes the OpenSSL error string text expectations in a number of tests. This was specifically done in support of more easily building CPython [AWS-LC][2], but because AWS-LC is a fork of [BoringSSL][3], it should increase compatibility with that library as well. In addition to the error string relaxations, we also add some guards around the `tls-unique` channel binding being used with TLSv1.3, as that feature (described in [RFC 6929][4]) is [not defined][5] for TLSv1.3. [1]: https://discuss.python.org/t/support-building-ssl-and-hashlib-modules-against-aws-lc/44505/4 [2]: https://github.com/aws/aws-lc [3]: https://github.com/google/boringssl [4]: https://datatracker.ietf.org/doc/html/rfc5929#section-3 [5]: https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 --- Lib/test/test_asyncio/test_events.py | 12 ++- Lib/test/test_imaplib.py | 22 +++-- Lib/test/test_ssl.py | 97 +++++++++++++------ ...-03-21-11-32-29.gh-issue-116333.F-9Ram.rst | 3 + 4 files changed, 96 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 5b9c871e1d1b5a..ae0bff06479093 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1125,12 +1125,16 @@ def test_create_server_ssl_match_failed(self): # incorrect server_hostname f_c = self.loop.create_connection(MyProto, host, port, ssl=sslcontext_client) + + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with mock.patch.object(self.loop, 'call_exception_handler'): with test_utils.disable_logger(): - with self.assertRaisesRegex( - ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + with self.assertRaisesRegex(ssl.CertificateError, regex): self.loop.run_until_complete(f_c) # close connection diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index b97474acca370f..79bf7dbdbb81a0 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -8,6 +8,7 @@ import time import calendar import threading +import re import socket from test.support import verbose, run_with_tz, run_with_locale, cpython_only, requires_resource @@ -558,9 +559,13 @@ def test_ssl_raises(self): self.assertEqual(ssl_context.check_hostname, True) ssl_context.load_verify_locations(CAFILE) - with self.assertRaisesRegex(ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): _, server = self._setup(SimpleIMAPHandler) client = self.imap_class(*server.server_address, ssl_context=ssl_context) @@ -954,10 +959,13 @@ def test_ssl_verified(self): ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ssl_context.load_verify_locations(CAFILE) - with self.assertRaisesRegex( - ssl.CertificateError, - "IP address mismatch, certificate is not valid for " - "'127.0.0.1'"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + IP address mismatch, certificate is not valid for '127.0.0.1' # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.CertificateError, regex): with self.reaped_server(SimpleIMAPHandler) as server: client = self.imap_class(*server.server_address, ssl_context=ssl_context) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 489cb5e23ba57e..3fa806ddc2cde7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -551,7 +551,7 @@ def test_openssl_version(self): else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" self.assertTrue( - s.startswith((openssl_ver, libressl_ver)), + s.startswith((openssl_ver, libressl_ver, "AWS-LC")), (s, t, hex(n)) ) @@ -1169,24 +1169,30 @@ def test_load_cert_chain(self): with self.assertRaises(OSError) as cm: ctx.load_cert_chain(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(BADCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(EMPTYCERT) # Separate key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_cert_chain(ONLYCERT, ONLYKEY) ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYCERT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(ONLYKEY) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) # Mismatching key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) @@ -1254,7 +1260,7 @@ def test_load_verify_locations(self): with self.assertRaises(OSError) as cm: ctx.load_verify_locations(NONEXISTINGCERT) self.assertEqual(cm.exception.errno, errno.ENOENT) - with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): ctx.load_verify_locations(BADCERT) ctx.load_verify_locations(CERTFILE, CAPATH) ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) @@ -1662,9 +1668,10 @@ def test_lib_reason(self): with self.assertRaises(ssl.SSLError) as cm: ctx.load_dh_params(CERTFILE) self.assertEqual(cm.exception.library, 'PEM') - self.assertEqual(cm.exception.reason, 'NO_START_LINE') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue(s.startswith("[PEM: NO_START_LINE] no start line"), s) + self.assertTrue("NO_START_LINE" in s, s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1844,7 +1851,13 @@ def test_connect_fail(self): s = test_wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_ex(self): @@ -1912,7 +1925,13 @@ def test_connect_with_context_fail(self): server_hostname=SIGNED_CERTFILE_HOSTNAME ) self.addCleanup(s.close) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, s.connect, self.server_addr) def test_connect_capath(self): @@ -2129,14 +2148,16 @@ def test_bio_handshake(self): self.assertIsNone(sslobj.version()) self.assertIsNone(sslobj.shared_ciphers()) self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertIsNone(sslobj.get_channel_binding('tls-unique')) self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) self.assertTrue(sslobj.cipher()) self.assertIsNone(sslobj.shared_ciphers()) self.assertIsNotNone(sslobj.version()) self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": self.assertTrue(sslobj.get_channel_binding('tls-unique')) try: self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) @@ -2861,11 +2882,16 @@ def test_crl_check(self): client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname=hostname) as s: - with self.assertRaisesRegex(ssl.SSLError, - "certificate verify failed"): + with self.assertRaisesRegex(ssl.SSLError, regex): s.connect((HOST, server.port)) # now load a CRL file. The CRL file is signed by the CA. @@ -2896,12 +2922,16 @@ def test_check_hostname(self): # incorrect hostname should raise an exception server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) with server: with client_context.wrap_socket(socket.socket(), server_hostname="invalid") as s: - with self.assertRaisesRegex( - ssl.CertificateError, - "Hostname mismatch, certificate is not valid for 'invalid'."): + with self.assertRaisesRegex(ssl.CertificateError, regex): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too @@ -3137,7 +3167,7 @@ def test_wrong_cert_tls13(self): s.connect((HOST, server.port)) with self.assertRaisesRegex( ssl.SSLError, - 'alert unknown ca|EOF occurred' + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') @@ -3201,13 +3231,21 @@ def test_ssl_cert_verify_error(self): server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: try: s.connect((HOST, server.port)) + self.fail("Expected connection failure") except ssl.SSLError as e: msg = 'unable to get local issuer certificate' self.assertIsInstance(e, ssl.SSLCertVerificationError) self.assertEqual(e.verify_code, 20) self.assertEqual(e.verify_message, msg) - self.assertIn(msg, repr(e)) - self.assertIn('certificate verify failed', repr(e)) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) def test_PROTOCOL_TLS(self): """Connecting to an SSLv23 server with various client options""" @@ -3739,7 +3777,7 @@ def test_no_shared_ciphers(self): server_hostname=hostname) as s: with self.assertRaises(OSError): s.connect((HOST, server.port)) - self.assertIn("no shared cipher", server.conn_errors[0]) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) def test_version_basic(self): """ @@ -3827,7 +3865,7 @@ def test_min_max_version_mismatch(self): server_hostname=hostname) as s: with self.assertRaises(ssl.SSLError) as e: s.connect((HOST, server.port)) - self.assertIn("alert", str(e.exception)) + self.assertRegex("(alert|ALERT)", str(e.exception)) @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): @@ -3869,6 +3907,10 @@ def test_tls_unique_channel_binding(self): client_context, server_context, hostname = testing_context() + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server = ThreadedEchoServer(context=server_context, chatty=True, connectionchatty=False) @@ -3969,7 +4011,7 @@ def test_dh_params(self): cipher = stats["cipher"][0] parts = cipher.split("-") if "ADH" not in parts and "EDH" not in parts and "DHE" not in parts: - self.fail("Non-DH cipher: " + cipher[0]) + self.fail("Non-DH key exchange: " + cipher[0]) def test_ecdh_curve(self): # server secp384r1, client auto @@ -4136,8 +4178,9 @@ def cb_raising(ssl_sock, server_name, initial_context): chatty=False, sni_name='supermessage') - self.assertEqual(cm.exception.reason, - 'SSLV3_ALERT_HANDSHAKE_FAILURE') + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(regex, cm.exception.reason) self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) def test_sni_callback_wrong_return_type(self): diff --git a/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst b/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst new file mode 100644 index 00000000000000..3fdb6bb3bd7af7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-03-21-11-32-29.gh-issue-116333.F-9Ram.rst @@ -0,0 +1,3 @@ +Tests of TLS related things (error codes, etc) were updated to be more +lenient about specific error message strings and behaviors as seen in the +BoringSSL and AWS-LC forks of OpenSSL. From 570a82d46abfebb9976961113fb0f8bb400ad182 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 21 Mar 2024 12:37:41 -0700 Subject: [PATCH 73/82] gh-117045: Add code object to function version cache (#117028) Changes to the function version cache: - In addition to the function object, also store the code object, and allow the latter to be retrieved even if the function has been evicted. - Stop assigning new function versions after a critical attribute (e.g. `__code__`) has been modified; the version is permanently reset to zero in this case. - Changes to `__annotations__` are no longer considered critical. (This fixes gh-109998.) Changes to the Tier 2 optimization machinery: - If we cannot map a function version to a function, but it is still mapped to a code object, we continue projecting the trace. The operand of the `_PUSH_FRAME` and `_POP_FRAME` opcodes can be either NULL, a function object, or a code object with the lowest bit set. This allows us to trace through code that calls an ephemeral function, i.e., a function that may not be alive when we are constructing the executor, e.g. a generator expression or certain nested functions. We will lose globals removal inside such functions, but we can still do other peephole operations (and even possibly [call inlining](https://github.com/python/cpython/pull/116290), if we decide to do it), which only need the code object. As before, if we cannot retrieve the code object from the cache, we stop projecting. --- Include/internal/pycore_frame.h | 2 +- Include/internal/pycore_function.h | 15 ++- Objects/codeobject.c | 1 + Objects/funcobject.c | 143 ++++++++++++++++++----------- Python/optimizer.c | 62 +++++++++---- Python/optimizer_analysis.c | 30 ++++-- Python/optimizer_bytecodes.c | 27 ++++-- Python/optimizer_cases.c.h | 25 +++-- 8 files changed, 209 insertions(+), 96 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0f9e7333cf1e1c..74d9e4cac72c0e 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -55,7 +55,7 @@ enum _frameowner { }; typedef struct _PyInterpreterFrame { - PyObject *f_executable; /* Strong reference */ + PyObject *f_executable; /* Strong reference (code object or None) */ struct _PyInterpreterFrame *previous; PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index dad6a89af77dec..24fbb3ddbee602 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -17,20 +17,27 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 #define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + +struct _func_version_cache_item { + PyFunctionObject *func; + PyObject *code; +}; + struct _py_func_state { uint32_t next_version; - // Borrowed references to function objects whose + // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE // once was equal to the index in the table. - // They are cleared when the function is deallocated. - PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; + // They are cleared when the function or code object is deallocated. + struct _func_version_cache_item func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); +void _PyFunction_ClearCodeByVersion(uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3df733eb4ee578..bdde12d77caf07 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1710,6 +1710,7 @@ code_dealloc(PyCodeObject *co) } Py_SET_REFCNT(co, 0); + _PyFunction_ClearCodeByVersion(co->co_version); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index a506166916de48..a3c0800e7891d3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -218,43 +218,61 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname } /* -Function versions ------------------ +(This is purely internal documentation. There are no public APIs here.) -Function versions are used to detect when a function object has been -updated, invalidating inline cache data used by the `CALL` bytecode -(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). +Function (and code) versions +---------------------------- -They are also used by the Tier 2 superblock creation code to find -the function being called (and from there the code object). +The Tier 1 specializer generates CALL variants that can be invalidated +by changes to critical function attributes: -How does a function's `func_version` field get initialized? +- __code__ +- __defaults__ +- __kwdefaults__ +- __closure__ -- `PyFunction_New` and friends initialize it to 0. -- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. -- It is reset to 0 when various attributes like `__code__` are set. -- A new version is allocated by `_PyFunction_GetVersionForCurrentState` - when the specializer needs a version and the version is 0. +For this purpose function objects have a 32-bit func_version member +that the specializer writes to the specialized instruction's inline +cache and which is checked by a guard on the specialized instructions. -The latter allocates versions using a counter in the interpreter state, -`interp->func_state.next_version`. -When the counter wraps around to 0, no more versions are allocated. -There is one other special case: functions with a non-standard -`vectorcall` field are not given a version. +The MAKE_FUNCTION bytecode sets func_version from the code object's +co_version field. The latter is initialized from a counter in the +interpreter state (interp->func_state.next_version) and never changes. +When this counter overflows, it remains zero and the specializer loses +the ability to specialize calls to new functions. -When the function version is 0, the `CALL` bytecode is not specialized. +The func_version is reset to zero when any of the critical attributes +is modified; after this point the specializer will no longer specialize +calls to this function, and the guard will always fail. -Code object versions --------------------- +The function and code version cache +----------------------------------- -So where to code objects get their `co_version`? -They share the same counter, `interp->func_state.next_version`. +The Tier 2 optimizer now has a problem, since it needs to find the +function and code objects given only the version number from the inline +cache. Our solution is to maintain a cache mapping version numbers to +function and code objects. To limit the cache size we could hash +the version number, but for now we simply use it modulo the table size. + +There are some corner cases (e.g. generator expressions) where we will +be unable to find the function object in the cache but we can still +find the code object. For this reason the cache stores both the +function object and the code object. + +The cache doesn't contain strong references; cache entries are +invalidated whenever the function or code object is deallocated. + +Invariants +---------- + +These should hold at any time except when one of the cache-mutating +functions is running. + +- For any slot s at index i: + - s->func == NULL or s->func->func_version % FUNC_VERSION_CACHE_SIZE == i + - s->code == NULL or s->code->co_version % FUNC_VERSION_CACHE_SIZE == i + if s->func != NULL, then s->func->func_code == s->code -Code objects get a new `co_version` allocated from this counter upon -creation. Since code objects are nominally immutable, `co_version` can -not be invalidated. The only way it can be 0 is when 2**32 or more -code objects have been created during the process's lifetime. -(The counter isn't reset by `fork()`, extending the lifetime.) */ void @@ -262,28 +280,61 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (func->func_version != 0) { - PyFunctionObject **slot = + struct _func_version_cache_item *slot = interp->func_state.func_version_cache + (func->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == func) { - *slot = NULL; + if (slot->func == func) { + slot->func = NULL; + // Leave slot->code alone, there may be use for it. } } func->func_version = version; if (version != 0) { - interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE] = func; + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + slot->func = func; + slot->code = func->func_code; + } +} + +void +_PyFunction_ClearCodeByVersion(uint32_t version) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + slot->code = NULL; + slot->func = NULL; + } } } PyFunctionObject * -_PyFunction_LookupByVersion(uint32_t version) +_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject *func = interp->func_state.func_version_cache[ - version % FUNC_VERSION_CACHE_SIZE]; - if (func != NULL && func->func_version == version) { - return func; + struct _func_version_cache_item *slot = + interp->func_state.func_version_cache + + (version % FUNC_VERSION_CACHE_SIZE); + if (slot->code) { + assert(PyCode_Check(slot->code)); + PyCodeObject *code = (PyCodeObject *)slot->code; + if (code->co_version == version) { + *p_code = slot->code; + } + } + else { + *p_code = NULL; + } + if (slot->func && slot->func->func_version == version) { + assert(slot->func->func_code == slot->code); + return slot->func; } return NULL; } @@ -291,19 +342,7 @@ _PyFunction_LookupByVersion(uint32_t version) uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { - if (func->func_version != 0) { - return func->func_version; - } - if (func->vectorcall != _PyFunction_Vectorcall) { - return 0; - } - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->func_state.next_version == 0) { - return 0; - } - uint32_t v = interp->func_state.next_version++; - _PyFunction_SetVersion(func, v); - return v; + return func->func_version; } PyObject * @@ -507,7 +546,6 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); return 0; } @@ -731,7 +769,6 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); return 0; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 4a3cd46ce80a26..177ad343618c37 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -211,7 +211,7 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, PyObject **stack_pointer, _PyExecutorObject **executor_ptr) { - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame); assert(PyCode_Check(code)); PyInterpreterState *interp = _PyInterpreterState_GET(); if (!has_space_for_executor(code, start)) { @@ -479,8 +479,9 @@ BRANCH_TO_GUARD[4][2] = { ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \ goto done; \ } \ - assert(func->func_code == (PyObject *)code); \ + assert(func == NULL || func->func_code == (PyObject *)code); \ trace_stack[trace_stack_depth].func = func; \ + trace_stack[trace_stack_depth].code = code; \ trace_stack[trace_stack_depth].instr = instr; \ trace_stack_depth++; #define TRACE_STACK_POP() \ @@ -489,7 +490,8 @@ BRANCH_TO_GUARD[4][2] = { } \ trace_stack_depth--; \ func = trace_stack[trace_stack_depth].func; \ - code = (PyCodeObject *)trace_stack[trace_stack_depth].func->func_code; \ + code = trace_stack[trace_stack_depth].code; \ + assert(func == NULL || func->func_code == (PyObject *)code); \ instr = trace_stack[trace_stack_depth].instr; /* Returns 1 on success, @@ -505,7 +507,7 @@ translate_bytecode_to_trace( _PyBloomFilter *dependencies) { bool progress_needed = true; - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; @@ -515,6 +517,7 @@ translate_bytecode_to_trace( int max_length = buffer_size; struct { PyFunctionObject *func; + PyCodeObject *code; _Py_CODEUNIT *instr; } trace_stack[TRACE_STACK_SIZE]; int trace_stack_depth = 0; @@ -719,9 +722,19 @@ translate_bytecode_to_trace( if (uop == _POP_FRAME) { TRACE_STACK_POP(); - /* Set the operand to the function object returned to, - * to assist optimization passes */ - ADD_TO_TRACE(uop, oparg, (uintptr_t)func, target); + /* Set the operand to the function or code object returned to, + * to assist optimization passes. (See _PUSH_FRAME below.) + */ + if (func != NULL) { + operand = (uintptr_t)func; + } + else if (code != NULL) { + operand = (uintptr_t)code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); DPRINTF(2, "Returning to %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -738,10 +751,12 @@ translate_bytecode_to_trace( // Add one to account for the actual opcode/oparg pair: + 1; uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyFunctionObject *new_func = _PyFunction_LookupByVersion(func_version); - DPRINTF(2, "Function: version=%#x; object=%p\n", (int)func_version, new_func); - if (new_func != NULL) { - PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(new_func); + PyCodeObject *new_code = NULL; + PyFunctionObject *new_func = + _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); + DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", + (int)func_version, new_func, new_code); + if (new_code != NULL) { if (new_code == code) { // Recursive call, bail (we could be here forever). DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", @@ -766,9 +781,22 @@ translate_bytecode_to_trace( instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; TRACE_STACK_PUSH(); _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function object, - * to assist optimization passes */ - ADD_TO_TRACE(uop, oparg, (uintptr_t)new_func, target); + /* Set the operand to the callee's function or code object, + * to assist optimization passes. + * We prefer setting it to the function (for remove_globals()) + * but if that's not available but the code is available, + * use the code, setting the low bit so the optimizer knows. + */ + if (new_func != NULL) { + operand = (uintptr_t)new_func; + } + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; + } + else { + operand = 0; + } + ADD_TO_TRACE(uop, oparg, operand, target); code = new_code; func = new_func; instr = _PyCode_CODE(code); @@ -780,8 +808,8 @@ translate_bytecode_to_trace( 2 * INSTR_IP(instr, code)); goto top; } - DPRINTF(2, "Bail, new_func == NULL\n"); - ADD_TO_TRACE(uop, oparg, operand, target); + DPRINTF(2, "Bail, new_code == NULL\n"); + ADD_TO_TRACE(uop, oparg, 0, target); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); goto done; } @@ -1116,7 +1144,7 @@ counter_optimize( int Py_UNUSED(curr_stackentries) ) { - PyCodeObject *code = (PyCodeObject *)frame->f_executable; + PyCodeObject *code = _PyFrame_GetCode(frame); int oparg = instr->op.arg; while (instr->op.code == EXTENDED_ARG) { instr++; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6c460c5359d71e..95924a57cfdaf4 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -228,7 +228,12 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched <<= 1; globals_watched <<= 1; function_checked <<= 1; - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + uint64_t operand = buffer[pc].operand; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; if (func == NULL) { return 1; } @@ -251,7 +256,15 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, builtins_watched >>= 1; globals_watched >>= 1; function_checked >>= 1; - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + uint64_t operand = buffer[pc].operand; + if (operand == 0 || (operand & 1)) { + // It's either a code object or NULL, so bail + return 1; + } + PyFunctionObject *func = (PyFunctionObject *)operand; + if (func == NULL) { + return 1; + } assert(PyFunction_Check(func)); function_version = func->func_version; globals = func->func_globals; @@ -522,7 +535,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) static void peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) { - PyCodeObject *co = (PyCodeObject *)frame->f_executable; + PyCodeObject *co = _PyFrame_GetCode(frame); for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch(opcode) { @@ -545,11 +558,16 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s case _PUSH_FRAME: case _POP_FRAME: { - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; - if (func == NULL) { + uint64_t operand = buffer[pc].operand; + if (operand & 1) { + co = (PyCodeObject *)(operand & ~1); + assert(PyCode_Check(co)); + } + else if (operand == 0) { co = NULL; } else { + PyFunctionObject *func = (PyFunctionObject *)operand; assert(PyFunction_Check(func)); co = (PyCodeObject *)func->func_code; } @@ -587,7 +605,7 @@ _Py_uop_analyze_and_optimize( peephole_opt(frame, buffer, buffer_size); err = optimize_uops( - (PyCodeObject *)frame->f_executable, buffer, + _PyFrame_GetCode(frame), buffer, buffer_size, curr_stacklen, dependencies); if (err == 0) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a1ef644e4621e2..e761b1b3433f04 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -543,14 +543,25 @@ dummy_func(void) { (void)callable; - PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; - DPRINTF(3, "func: %p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - goto done; - } - PyCodeObject *co = (PyCodeObject *)func->func_code; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uintptr_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + goto done; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); + } assert(self_or_null != NULL); assert(args != NULL); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a0ecf58905f287..2908a26c20973e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1596,14 +1596,25 @@ callable = stack_pointer[-2 - oparg]; int argcount = oparg; (void)callable; - PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; - DPRINTF(3, "func: %p ", func); - if (func == NULL) { - DPRINTF(3, "\n"); - DPRINTF(1, "Missing function\n"); - goto done; + PyCodeObject *co = NULL; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + uintptr_t push_operand = (this_instr + 2)->operand; + if (push_operand & 1) { + co = (PyCodeObject *)(push_operand & ~1); + DPRINTF(3, "code=%p ", co); + assert(PyCode_Check(co)); + } + else { + PyFunctionObject *func = (PyFunctionObject *)push_operand; + DPRINTF(3, "func=%p ", func); + if (func == NULL) { + DPRINTF(3, "\n"); + DPRINTF(1, "Missing function\n"); + goto done; + } + co = (PyCodeObject *)func->func_code; + DPRINTF(3, "code=%p ", co); } - PyCodeObject *co = (PyCodeObject *)func->func_code; assert(self_or_null != NULL); assert(args != NULL); if (sym_is_not_null(self_or_null)) { From 0907871d43bffb613cbd560224e1a9db13d06c06 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Thu, 21 Mar 2024 15:47:09 -0400 Subject: [PATCH 74/82] docs: fix over-linking in dataclasses.rst (#117005) --- Doc/library/dataclasses.rst | 158 ++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 78 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index c612c138fc6ea8..61b2263339da71 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -1,5 +1,5 @@ -:mod:`dataclasses` --- Data Classes -=================================== +:mod:`!dataclasses` --- Data Classes +==================================== .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. @@ -31,7 +31,7 @@ using :pep:`526` type annotations. For example, this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -will add, among other things, a :meth:`~object.__init__` that looks like:: +will add, among other things, a :meth:`!__init__` that looks like:: def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name @@ -49,26 +49,26 @@ Module contents .. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. + :term:`special methods ` to classes, as described below. - The :func:`dataclass` decorator examines the class to find + The ``@dataclass`` decorator examines the class to find ``field``\s. A ``field`` is defined as a class variable that has a :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` + exceptions described below, nothing in ``@dataclass`` examines the type specified in the variable annotation. The order of the fields in all of the generated methods is the order in which they appear in the class definition. - The :func:`dataclass` decorator will add various "dunder" methods to + The ``@dataclass`` decorator will add various "dunder" methods to the class, described below. If any of the added methods already exist in the class, the behavior depends on the parameter, as documented below. The decorator returns the same class that it is called on; no new class is created. - If :func:`dataclass` is used just as a simple decorator with no parameters, + If ``@dataclass`` is used just as a simple decorator with no parameters, it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are + signature. That is, these three uses of ``@dataclass`` are equivalent:: @dataclass @@ -84,12 +84,12 @@ Module contents class C: ... - The parameters to :func:`dataclass` are: + The parameters to ``@dataclass`` are: - ``init``: If true (the default), a :meth:`~object.__init__` method will be generated. - If the class already defines :meth:`~object.__init__`, this parameter is + If the class already defines :meth:`!__init__`, this parameter is ignored. - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be @@ -99,7 +99,7 @@ Module contents are not included. For example: ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - If the class already defines :meth:`~object.__repr__`, this parameter is + If the class already defines :meth:`!__repr__`, this parameter is ignored. - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be @@ -107,7 +107,7 @@ Module contents of its fields, in order. Both instances in the comparison must be of the identical type. - If the class already defines :meth:`~object.__eq__`, this parameter is + If the class already defines :meth:`!__eq__`, this parameter is ignored. - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, @@ -117,43 +117,43 @@ Module contents identical type. If ``order`` is true and ``eq`` is false, a :exc:`ValueError` is raised. - If the class already defines any of :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then + If the class already defines any of :meth:`!__lt__`, + :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method is generated according to how ``eq`` and ``frozen`` are set. - :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are + :meth:`!__hash__` is used by built-in :meth:`hash()`, and when objects are added to hashed collections such as dictionaries and sets. Having a - :meth:`~object.__hash__` implies that instances of the class are immutable. + :meth:`!__hash__` implies that instances of the class are immutable. Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`~object.__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + intent, the existence and behavior of :meth:`!__eq__`, and the values of + the ``eq`` and ``frozen`` flags in the ``@dataclass`` decorator. - By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` + By default, ``@dataclass`` will not implicitly add a :meth:`~object.__hash__` method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`~object.__hash__` method. Setting the class + existing explicitly defined :meth:`!__hash__` method. Setting the class attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`~object.__hash__` documentation. + described in the :meth:`!__hash__` documentation. - If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case + If :meth:`!__hash__` is not explicitly defined, or if it is set to ``None``, + then ``@dataclass`` *may* add an implicit :meth:`!__hash__` method. + Although not recommended, you can force ``@dataclass`` to create a + :meth:`!__hash__` method with ``unsafe_hash=True``. This might be the case if your class is logically immutable but can still be mutated. This is a specialized use case and should be considered carefully. - Here are the rules governing implicit creation of a :meth:`~object.__hash__` - method. Note that you cannot both have an explicit :meth:`~object.__hash__` + Here are the rules governing implicit creation of a :meth:`!__hash__` + method. Note that you cannot both have an explicit :meth:`!__hash__` method in your dataclass and set ``unsafe_hash=True``; this will result in a :exc:`TypeError`. - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it + If ``eq`` and ``frozen`` are both true, by default ``@dataclass`` will + generate a :meth:`!__hash__` method for you. If ``eq`` is true and + ``frozen`` is false, :meth:`!__hash__` will be set to ``None``, marking it unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` + :meth:`!__hash__` will be left untouched meaning the :meth:`!__hash__` method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). @@ -165,7 +165,7 @@ Module contents - ``match_args``: If true (the default is ``True``), the ``__match_args__`` tuple will be created from the list of parameters to the generated :meth:`~object.__init__` method (even if - :meth:`~object.__init__` is not generated, see above). If false, or if + :meth:`!__init__` is not generated, see above). If false, or if ``__match_args__`` is already defined in the class, then ``__match_args__`` will not be generated. @@ -175,7 +175,7 @@ Module contents fields will be marked as keyword-only. If a field is marked as keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`~object.__init__` is called. There is no + with a keyword when :meth:`!__init__` is called. There is no effect on any other aspect of dataclasses. See the :term:`parameter` glossary entry for details. Also see the :const:`KW_ONLY` section. @@ -184,7 +184,7 @@ Module contents - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. - If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` + If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` is raised. .. versionadded:: 3.10 @@ -229,7 +229,7 @@ Module contents required. There are, however, some dataclass features that require additional per-field information. To satisfy this need for additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: + with a call to the provided :func:`!field` function. For example:: @dataclass class C: @@ -243,10 +243,10 @@ Module contents used because ``None`` is a valid value for some parameters with a distinct meaning. No code should directly use the :const:`MISSING` value. - The parameters to :func:`field` are: + The parameters to :func:`!field` are: - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself + field. This is needed because the :func:`!field` call itself replaces the normal position of the default value. - ``default_factory``: If provided, it must be a zero-argument @@ -293,10 +293,10 @@ Module contents .. versionadded:: 3.10 If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be + :func:`!field`, then the class attribute for this field will be replaced by the specified ``default`` value. If no ``default`` is provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class + that after the :func:`@dataclass ` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -314,10 +314,10 @@ Module contents .. class:: Field - :class:`Field` objects describe each defined field. These objects + :class:`!Field` objects describe each defined field. These objects are created internally, and are returned by the :func:`fields` module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: + :class:`!Field` object directly. Its documented attributes are: - ``name``: The name of the field. - ``type``: The type of the field. @@ -343,7 +343,7 @@ Module contents lists, and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. - Example of using :func:`asdict` on nested dataclasses:: + Example of using :func:`!asdict` on nested dataclasses:: @dataclass class Point: @@ -364,7 +364,7 @@ Module contents dict((field.name, getattr(obj, field.name)) for field in fields(obj)) - :func:`asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass instance. .. function:: astuple(obj, *, tuple_factory=tuple) @@ -384,7 +384,7 @@ Module contents tuple(getattr(obj, field.name) for field in dataclasses.fields(obj)) - :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass + :func:`!astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass instance. .. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None) @@ -397,7 +397,7 @@ Module contents ``typing.Any`` is used for ``type``. The values of ``init``, ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have - the same meaning as they do in :func:`dataclass`. + the same meaning as they do in :func:`@dataclass `. If ``module`` is defined, the ``__module__`` attribute of the dataclass is set to that value. @@ -405,7 +405,7 @@ Module contents This function is not strictly required, because any Python mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to + then apply the ``@dataclass`` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -438,15 +438,15 @@ Module contents :meth:`__post_init__`, if present, is also called. Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :meth:`__post_init__`. + specified on the call to :func:`!replace` so that they can be passed to + :meth:`!__init__` and :meth:`__post_init__`. It is an error for ``changes`` to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised in this case. Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but + :func:`!replace`. They are not copied from the source object, but rather are initialized in :meth:`__post_init__`, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise @@ -475,11 +475,11 @@ Module contents .. data:: KW_ONLY A sentinel value used as a type annotation. Any fields after a - pseudo-field with the type of :const:`KW_ONLY` are marked as + pseudo-field with the type of :const:`!KW_ONLY` are marked as keyword-only fields. Note that a pseudo-field of type - :const:`KW_ONLY` is otherwise completely ignored. This includes the + :const:`!KW_ONLY` is otherwise completely ignored. This includes the name of such a field. By convention, a name of ``_`` is used for a - :const:`KW_ONLY` field. Keyword-only fields signify + :const:`!KW_ONLY` field. Keyword-only fields signify :meth:`~object.__init__` parameters that must be specified as keywords when the class is instantiated. @@ -495,7 +495,7 @@ Module contents p = Point(0, y=1.5, z=2.0) In a single dataclass, it is an error to specify more than one - field whose type is :const:`KW_ONLY`. + field whose type is :const:`!KW_ONLY`. .. versionadded:: 3.10 @@ -515,9 +515,9 @@ Post-init processing When defined on the class, it will be called by the generated :meth:`~object.__init__`, normally as ``self.__post_init__()``. However, if any ``InitVar`` fields are defined, they will also be - passed to :meth:`__post_init__` in the order they were defined in the - class. If no :meth:`~object.__init__` method is generated, then - :meth:`__post_init__` will not automatically be called. + passed to :meth:`!__post_init__` in the order they were defined in the + class. If no :meth:`!__init__` method is generated, then + :meth:`!__post_init__` will not automatically be called. Among other uses, this allows for initializing field values that depend on one or more other fields. For example:: @@ -531,8 +531,8 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base -class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method +The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -548,18 +548,18 @@ that has to be called, it is common to call this method in a def __post_init__(self): super().__init__(self.side, self.side) -Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods +Note, however, that in general the dataclass-generated :meth:`!__init__` methods don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how +parameters to :meth:`!__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. Class variables --------------- -One of the few places where :func:`dataclass` actually inspects the type +One of the few places where :func:`@dataclass ` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is ``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded @@ -570,7 +570,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`dataclass` inspects a type annotation is to +Another place where :func:`@dataclass ` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type ``dataclasses.InitVar``. If a field is an ``InitVar``, it is considered a pseudo-field called an init-only @@ -602,19 +602,19 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`!object.__setattr__`. +must use :meth:`!__setattr__`. Inheritance ----------- -When the dataclass is being created by the :meth:`dataclass` decorator, +When the dataclass is being created by the :func:`@dataclass ` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -641,8 +641,8 @@ The generated :meth:`~object.__init__` method for ``C`` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): -Re-ordering of keyword-only parameters in :meth:`~object.__init__` ------------------------------------------------------------------- +Re-ordering of keyword-only parameters in :meth:`!__init__` +----------------------------------------------------------- After the parameters needed for :meth:`~object.__init__` are computed, any keyword-only parameters are moved to come after all regular @@ -665,7 +665,7 @@ fields, and ``Base.x`` and ``D.z`` are regular fields:: z: int = 10 t: int = field(kw_only=True, default=0) -The generated :meth:`~object.__init__` method for ``D`` will look like:: +The generated :meth:`!__init__` method for ``D`` will look like:: def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): @@ -674,7 +674,7 @@ the list of fields: parameters derived from regular fields are followed by parameters derived from keyword-only fields. The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`~object.__init__` parameter list. +re-ordered :meth:`!__init__` parameter list. Default factory functions @@ -689,7 +689,7 @@ example, to create a new instance of a list, use:: If a field is excluded from :meth:`~object.__init__` (using ``init=False``) and the field also specifies ``default_factory``, then the default factory function will always be called from the generated -:meth:`~object.__init__` function. This happens because there is no other +:meth:`!__init__` function. This happens because there is no other way to give the field an initial value. Mutable default values @@ -738,7 +738,7 @@ for ``x`` when creating a class instance will share the same copy of ``x``. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`ValueError` if it +:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -764,15 +764,17 @@ Descriptor-typed fields Fields that are assigned :ref:`descriptor objects ` as their default value have the following special behaviors: -* The value for the field passed to the dataclass's ``__init__`` method is - passed to the descriptor's ``__set__`` method rather than overwriting the +* The value for the field passed to the dataclass's :meth:`~object.__init__` method is + passed to the descriptor's :meth:`~object.__set__` method rather than overwriting the descriptor object. + * Similarly, when getting or setting the field, the descriptor's - ``__get__`` or ``__set__`` method is called rather than returning or + :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, ``dataclasses`` - will call the descriptor's ``__get__`` method using its class access - form (i.e. ``descriptor.__get__(obj=None, type=cls)``. If the + +* To determine whether a field contains a default value, :func:`@dataclass ` + will call the descriptor's :meth:`!__get__` method using its class access + form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the field's default. On the other hand, if the descriptor raises :exc:`AttributeError` in this situation, no default value will be From 50f9b0b1e0fb181875751cef951351ed007b6397 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Mar 2024 23:17:09 +0100 Subject: [PATCH 75/82] gh-117061: Fix test_posix.test_sched_setaffinity() on RHEL9 (#117126) On RHEL9, sched_setaffinity(0, []) does not fail. --- Lib/test/test_posix.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 2706d5eb6d9830..1d22869046fd12 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1335,12 +1335,21 @@ def test_sched_getaffinity(self): def test_sched_setaffinity(self): mask = posix.sched_getaffinity(0) self.addCleanup(posix.sched_setaffinity, 0, list(mask)) + if len(mask) > 1: # Empty masks are forbidden mask.pop() posix.sched_setaffinity(0, mask) self.assertEqual(posix.sched_getaffinity(0), mask) - self.assertRaises(OSError, posix.sched_setaffinity, 0, []) + + try: + posix.sched_setaffinity(0, []) + # gh-117061: On RHEL9, sched_setaffinity(0, []) does not fail + except OSError: + # sched_setaffinity() manual page documents EINVAL error + # when the mask is empty. + pass + self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) From 3ec57307e70ee6f42410e844d3399bbd598917ba Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Thu, 21 Mar 2024 23:52:29 +0000 Subject: [PATCH 76/82] gh-71052: Add Android build script and instructions (#116426) --- Android/README.md | 64 ++++++ Android/android-env.sh | 87 ++++++++ Android/android.py | 202 ++++++++++++++++++ Include/cpython/pystate.h | 4 + ...4-03-06-17-26-55.gh-issue-71052.vLbu9u.rst | 1 + configure | 29 ++- configure.ac | 35 ++- 7 files changed, 403 insertions(+), 19 deletions(-) create mode 100644 Android/README.md create mode 100644 Android/android-env.sh create mode 100755 Android/android.py create mode 100644 Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst diff --git a/Android/README.md b/Android/README.md new file mode 100644 index 00000000000000..5ed186e06e3951 --- /dev/null +++ b/Android/README.md @@ -0,0 +1,64 @@ +# Python for Android + +These instructions are only needed if you're planning to compile Python for +Android yourself. Most users should *not* need to do this. If you're looking to +use Python on Android, one of the following tools will provide a much more +approachable user experience: + +* [Briefcase](https://briefcase.readthedocs.io), from the BeeWare project +* [Buildozer](https://buildozer.readthedocs.io), from the Kivy project +* [Chaquopy](https://chaquo.com/chaquopy/) + + +## Prerequisites + +Export the `ANDROID_HOME` environment variable to point at your Android SDK. If +you don't already have the SDK, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + + +## Building + +Building for Android requires doing a cross-build where you have a "build" +Python to help produce an Android build of CPython. This procedure has been +tested on Linux and macOS. + +The easiest way to do a build is to use the `android.py` script. You can either +have it perform the entire build process from start to finish in one step, or +you can do it in discrete steps that mirror running `configure` and `make` for +each of the two builds of Python you end up producing. + +The discrete steps for building via `android.py` are: + +```sh +./android.py configure-build +./android.py make-build +./android.py configure-host HOST +./android.py make-host HOST +``` + +To see the possible values of HOST, run `./android.py configure-host --help`. + +Or to do it all in a single command, run: + +```sh +./android.py build HOST +``` + +In the end you should have a build Python in `cross-build/build`, and an Android +build in `cross-build/HOST`. + +You can use `--` as a separator for any of the `configure`-related commands – +including `build` itself – to pass arguments to the underlying `configure` +call. For example, if you want a pydebug build that also caches the results from +`configure`, you can do: + +```sh +./android.py build HOST -- -C --with-pydebug +``` diff --git a/Android/android-env.sh b/Android/android-env.sh new file mode 100644 index 00000000000000..3ce3e035cfb8fe --- /dev/null +++ b/Android/android-env.sh @@ -0,0 +1,87 @@ +# This script must be sourced with the following variables already set: +: ${ANDROID_HOME:?} # Path to Android SDK +: ${HOST:?} # GNU target triplet + +# You may also override the following: +: ${api_level:=21} # Minimum Android API level the build will run on +: ${PREFIX:-} # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=26.2.11394342 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e $ndk ]; then + log "Installing NDK: this may take several minutes" + yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version" +fi + +if [ $HOST = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet=$HOST +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ $HOST = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix=$(realpath $PREFIX) + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# Use the same variable name as conda-build +if [ $(uname) = "Darwin" ]; then + export CPU_COUNT=$(sysctl -n hw.ncpu) +else + export CPU_COUNT=$(nproc) +fi diff --git a/Android/android.py b/Android/android.py new file mode 100755 index 00000000000000..5c57e53c415d2b --- /dev/null +++ b/Android/android.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import shutil +import subprocess +import sys +import sysconfig +from os.path import relpath +from pathlib import Path + +SCRIPT_NAME = Path(__file__).name +CHECKOUT = Path(__file__).resolve().parent.parent +CROSS_BUILD_DIR = CHECKOUT / "cross-build" + + +def delete_if_exists(path): + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def subdir(name, *, clean=None): + path = CROSS_BUILD_DIR / name + if clean: + delete_if_exists(path) + if not path.exists(): + if clean is None: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}.") + else: + path.mkdir(parents=True) + return path + + +def run(command, *, host=None, **kwargs): + env = os.environ.copy() + if host: + env_script = CHECKOUT / "Android/android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={subdir(host)}/prefix; " + f". {env_script}; " + f"export", + check=True, shell=True, text=True, stdout=subprocess.PIPE + ).stdout + + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if env.get(key) != value: + print(line) + env[key] = value + + if env == os.environ: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + print(">", " ".join(map(str, command))) + try: + subprocess.run(command, check=True, env=env, **kwargs) + except subprocess.CalledProcessError as e: + sys.exit(e) + + +def build_python_path(): + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{build_dir}") + + return binary + + +def configure_build_python(context): + os.chdir(subdir("build", clean=context.clean)) + + command = [relpath(CHECKOUT / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context): + os.chdir(subdir("build")) + run(["make", "-j", str(os.cpu_count())]) + + +def unpack_deps(host): + deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1", + "sqlite-3.45.1-0", "xz-5.4.6-0"]: + filename = f"{name_ver}-{host}.tar.gz" + run(["wget", f"{deps_url}/{name_ver}/{filename}"]) + run(["tar", "-xf", filename]) + os.remove(filename) + + +def configure_host_python(context): + host_dir = subdir(context.host, clean=context.clean) + + prefix_dir = host_dir / "prefix" + if not prefix_dir.exists(): + prefix_dir.mkdir() + os.chdir(prefix_dir) + unpack_deps(context.host) + + build_dir = host_dir / "build" + build_dir.mkdir(exist_ok=True) + os.chdir(build_dir) + + command = [ + # Basic cross-compiling configuration + relpath(CHECKOUT / "configure"), + f"--host={context.host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--without-ensurepip", + + # Android always uses a shared libpython. + "--enable-shared", + "--without-static-libpython", + + # Dependent libraries. The others are found using pkg-config: see + # android-env.sh. + f"--with-openssl={prefix_dir}", + ] + + if context.args: + command.extend(context.args) + run(command, host=context.host) + + +def make_host_python(context): + host_dir = subdir(context.host) + os.chdir(host_dir / "build") + run(["make", "-j", str(os.cpu_count())], host=context.host) + run(["make", "install", f"prefix={host_dir}/prefix"], host=context.host) + + +def build_all(context): + steps = [configure_build_python, make_build_python, configure_host_python, + make_host_python] + for step in steps: + step(context) + + +def clean_all(context): + delete_if_exists(CROSS_BUILD_DIR) + + +def main(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for Android") + make_host = subcommands.add_parser("make-host", + help="Run `make` for Android") + clean = subcommands.add_parser("clean", help="Delete files and directories " + "created by this script") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--clean", action="store_true", default=False, dest="clean", + help="Delete any relevant directories before building") + for subcommand in build, configure_host, make_host: + subcommand.add_argument( + "host", metavar="HOST", + choices=["aarch64-linux-android", "x86_64-linux-android"], + help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + + context = parser.parse_args() + dispatch = {"configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all} + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 38d0897ea13161..f1540fde925921 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -211,6 +211,10 @@ struct _ts { # define Py_C_RECURSION_LIMIT 800 #elif defined(_WIN32) # define Py_C_RECURSION_LIMIT 3000 +#elif defined(__ANDROID__) + // On an ARM64 emulator, API level 34 was OK with 10000, but API level 21 + // crashed in test_compiler_recursion_limit. +# define Py_C_RECURSION_LIMIT 3000 #elif defined(_Py_ADDRESS_SANITIZER) # define Py_C_RECURSION_LIMIT 4000 #else diff --git a/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst b/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst new file mode 100644 index 00000000000000..53776c0216f553 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-06-17-26-55.gh-issue-71052.vLbu9u.rst @@ -0,0 +1 @@ +Add Android build script and instructions. diff --git a/configure b/configure index 229f0d32d322dd..0493d382f48481 100755 --- a/configure +++ b/configure @@ -17595,13 +17595,21 @@ fi if test "$ac_sys_system" = "Linux-android"; then # When these functions are used in an unprivileged process, they crash rather # than returning an error. - privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid - setresuid setreuid setuid" - - # These functions are unimplemented and always return an error. - unimplemented_funcs="sem_open sem_unlink" + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi - for name in $privileged_funcs $unimplemented_funcs; do + for name in $blocked_funcs; do as_func_var=`printf "%s\n" "ac_cv_func_$name" | $as_tr_sh` eval "$as_func_var=no" @@ -22156,6 +22164,10 @@ fi done fi +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then for ac_func in clock_nanosleep do : @@ -22166,7 +22178,7 @@ then : else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for clock_nanosleep in -lrt" >&5 printf %s "checking for clock_nanosleep in -lrt... " >&6; } if test ${ac_cv_lib_rt_clock_nanosleep+y} then : @@ -22204,7 +22216,7 @@ printf "%s\n" "$ac_cv_lib_rt_clock_nanosleep" >&6; } if test "x$ac_cv_lib_rt_clock_nanosleep" = xyes then : - printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h + printf "%s\n" "#define HAVE_CLOCK_NANOSLEEP 1" >>confdefs.h fi @@ -22213,6 +22225,7 @@ fi fi done +fi for ac_func in nanosleep diff --git a/configure.ac b/configure.ac index cd17977738482d..cdfafc2d7272f1 100644 --- a/configure.ac +++ b/configure.ac @@ -4934,13 +4934,21 @@ fi if test "$ac_sys_system" = "Linux-android"; then # When these functions are used in an unprivileged process, they crash rather # than returning an error. - privileged_funcs="chroot initgroups setegid seteuid setgid setregid setresgid - setresuid setreuid setuid" - - # These functions are unimplemented and always return an error. - unimplemented_funcs="sem_open sem_unlink" + blocked_funcs="chroot initgroups setegid seteuid setgid sethostname + setregid setresgid setresuid setreuid setuid" + + # These functions are unimplemented and always return an error + # (https://android.googlesource.com/platform/system/sepolicy/+/refs/heads/android13-release/public/domain.te#1044) + blocked_funcs="$blocked_funcs sem_open sem_unlink" + + # Before API level 23, when fchmodat is called with the unimplemented flag + # AT_SYMLINK_NOFOLLOW, instead of returning ENOTSUP as it should, it actually + # follows the symlink. + if test "$ANDROID_API_LEVEL" -lt 23; then + blocked_funcs="$blocked_funcs fchmodat" + fi - for name in $privileged_funcs $unimplemented_funcs; do + for name in $blocked_funcs; do AS_VAR_PUSHDEF([func_var], [ac_cv_func_$name]) AS_VAR_SET([func_var], [no]) AS_VAR_POPDEF([func_var]) @@ -5303,11 +5311,16 @@ then ]) fi -AC_CHECK_FUNCS([clock_nanosleep], [], [ - AC_CHECK_LIB([rt], [clock_nanosleep], [ - AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) - ]) -]) +# On Android before API level 23, clock_nanosleep returns the wrong value when +# interrupted by a signal (https://issuetracker.google.com/issues/216495770). +if ! { test "$ac_sys_system" = "Linux-android" && + test "$ANDROID_API_LEVEL" -lt 23; }; then + AC_CHECK_FUNCS([clock_nanosleep], [], [ + AC_CHECK_LIB([rt], [clock_nanosleep], [ + AC_DEFINE([HAVE_CLOCK_NANOSLEEP], [1]) + ]) + ]) +fi AC_CHECK_FUNCS([nanosleep], [], [ AC_CHECK_LIB([rt], [nanosleep], [ From c4bf58a14f162557038a1535ca22c52b49d81d7b Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Thu, 21 Mar 2024 19:54:50 -0400 Subject: [PATCH 77/82] gh-116745: Remove all internal usage of @LIBPYTHON@ (#116746) Replace with MODULE_LDFLAGS. --- Misc/python-config.sh.in | 2 +- Misc/python.pc.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index eb02223ddcd2c3..c3c0b34fc1451d 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -46,7 +46,7 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" -LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" +LIBS="@MODULE_LDFLAGS@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" LDLIBRARY="@LDLIBRARY@" diff --git a/Misc/python.pc.in b/Misc/python.pc.in index 027dba38585a89..c2c740e82b1fde 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: -L${libdir} @LIBPYTHON@ +Libs: -L${libdir} @MODULE_LDFLAGS@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ From b3d25df8d38b79310587da54dbd88b06a16d4904 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 21 Mar 2024 18:20:20 -0600 Subject: [PATCH 78/82] gh-105716: Fix _PyInterpreterState_IsRunningMain() For Embedders (gh-117140) When I added _PyInterpreterState_IsRunningMain() and friends last year, I tried to accommodate applications that embed Python but don't call _PyInterpreterState_SetRunningMain() (not that they're expected to). That mostly worked fine until my recent changes in gh-117049, where the subtleties with the fallback code led to failures; the change ended up breaking test_tools.test_freeze, which exercises a basic embedding situation. The simplest fix is to drop the fallback code I originally added to _PyInterpreterState_IsRunningMain() (and later to _PyThreadState_IsRunningMain()). I've kept the fallback in the _xxsubinterpreters module though. I've also updated Py_FrozenMain() to call _PyInterpreterState_SetRunningMain(). --- Modules/_xxsubinterpretersmodule.c | 22 ++++++++++++++++++++-- Python/frozenmain.c | 9 +++++++++ Python/pystate.c | 30 ++++++++++-------------------- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index befa225c9183c5..5e5b3c10201867 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -56,6 +56,24 @@ _get_current_module(void) } +static int +is_running_main(PyInterpreterState *interp) +{ + if (_PyInterpreterState_IsRunningMain(interp)) { + return 1; + } + // Unlike with the general C-API, we can be confident that someone + // using this module for the main interpreter is doing so through + // the main program. Thus we can make this extra check. This benefits + // applications that embed Python but haven't been updated yet + // to call_PyInterpreterState_SetRunningMain(). + if (_Py_IsMainInterpreter(interp)) { + return 1; + } + return 0; +} + + /* Cross-interpreter Buffer Views *******************************************/ // XXX Release when the original interpreter is destroyed. @@ -509,7 +527,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_PyInterpreterState_IsRunningMain(interp)) { + if (is_running_main(interp)) { PyErr_Format(PyExc_RuntimeError, "interpreter running"); return NULL; } @@ -977,7 +995,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - if (_PyInterpreterState_IsRunningMain(interp)) { + if (is_running_main(interp)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 3ce9476c9ad46c..ec4566bd4f84bc 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -54,6 +54,12 @@ Py_FrozenMain(int argc, char **argv) Py_ExitStatusException(status); } + PyInterpreterState *interp = PyInterpreterState_Get(); + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + PyErr_Print(); + exit(1); + } + #ifdef MS_WINDOWS PyWinFreeze_ExeInit(); #endif @@ -83,6 +89,9 @@ Py_FrozenMain(int argc, char **argv) #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif + + _PyInterpreterState_SetNotRunningMain(interp); + if (Py_FinalizeEx() < 0) { sts = 120; } diff --git a/Python/pystate.c b/Python/pystate.c index 47d327ae28933b..921e74ed5a9826 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1042,24 +1042,15 @@ _PyInterpreterState_IsRunningMain(PyInterpreterState *interp) if (interp->threads.main != NULL) { return 1; } - // For now, we assume the main interpreter is always running. - if (_Py_IsMainInterpreter(interp)) { - return 1; - } + // Embedders might not know to call _PyInterpreterState_SetRunningMain(), + // so their main thread wouldn't show it is running the main interpreter's + // program. (Py_Main() doesn't have this problem.) For now this isn't + // critical. If it were, we would need to infer "running main" from other + // information, like if it's the main interpreter. We used to do that + // but the naive approach led to some inconsistencies that caused problems. return 0; } -#ifndef NDEBUG -static int -is_running_main(PyThreadState *tstate) -{ - if (tstate->interp->threads.main != NULL) { - return tstate == tstate->interp->threads.main; - } - return 0; -} -#endif - int _PyThreadState_IsRunningMain(PyThreadState *tstate) { @@ -1067,9 +1058,8 @@ _PyThreadState_IsRunningMain(PyThreadState *tstate) if (interp->threads.main != NULL) { return tstate == interp->threads.main; } - if (_Py_IsMainInterpreter(interp)) { - return tstate->thread_id == interp->runtime->main_thread; - } + // See the note in _PyInterpreterState_IsRunningMain() about + // possible false negatives here for embedders. return 0; } @@ -1571,7 +1561,7 @@ PyThreadState_Clear(PyThreadState *tstate) { assert(tstate->_status.initialized && !tstate->_status.cleared); assert(current_fast_get()->interp == tstate->interp); - assert(!is_running_main(tstate)); + assert(!_PyThreadState_IsRunningMain(tstate)); // XXX assert(!tstate->_status.bound || tstate->_status.unbound); tstate->_status.finalizing = 1; // just in case @@ -1670,7 +1660,7 @@ tstate_delete_common(PyThreadState *tstate) assert(tstate->_status.cleared && !tstate->_status.finalized); assert(tstate->state != _Py_THREAD_ATTACHED); tstate_verify_not_active(tstate); - assert(!is_running_main(tstate)); + assert(!_PyThreadState_IsRunningMain(tstate)); PyInterpreterState *interp = tstate->interp; if (interp == NULL) { From 97ba910e47ad298114800587979ce7beb0a705a3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 21 Mar 2024 18:27:48 -0700 Subject: [PATCH 79/82] gh-108716:: Remove _PyStaticCode_Init/Fini (#117141) More deepfreeze cleanup. --- Objects/codeobject.c | 46 -------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index bdde12d77caf07..f14ff73394b168 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2349,49 +2349,3 @@ _PyCode_ConstantKey(PyObject *op) } return key; } - -void -_PyStaticCode_Fini(PyCodeObject *co) -{ - if (co->co_executors != NULL) { - clear_executors(co); - } - deopt_code(co, _PyCode_CODE(co)); - PyMem_Free(co->co_extra); - if (co->_co_cached != NULL) { - Py_CLEAR(co->_co_cached->_co_code); - Py_CLEAR(co->_co_cached->_co_cellvars); - Py_CLEAR(co->_co_cached->_co_freevars); - Py_CLEAR(co->_co_cached->_co_varnames); - PyMem_Free(co->_co_cached); - co->_co_cached = NULL; - } - co->co_extra = NULL; - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)co); - co->co_weakreflist = NULL; - } - free_monitoring_data(co->_co_monitoring); - co->_co_monitoring = NULL; -} - -int -_PyStaticCode_Init(PyCodeObject *co) -{ - int res = intern_strings(co->co_names); - if (res < 0) { - return -1; - } - res = intern_string_constants(co->co_consts, NULL); - if (res < 0) { - return -1; - } - res = intern_strings(co->co_localsplusnames); - if (res < 0) { - return -1; - } - _PyCode_Quicken(co); - return 0; -} - -#define MAX_CODE_UNITS_PER_LOC_ENTRY 8 From 8383915031942f441f435a5ae800790116047b80 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Thu, 21 Mar 2024 22:27:25 -0500 Subject: [PATCH 80/82] GH-116939: Rewrite binarysort() (#116940) Rewrote binarysort() for clarity. Also changed the signature to be more coherent (it was mixing sortslice with raw pointers). No change in method or functionality. However, I left some experiments in, disabled for now via `#if` tricks. Since this code was first written, some kinds of comparisons have gotten enormously faster (like for lists of floats), which changes the tradeoffs. For example, plain insertion sort's simpler innermost loop and highly predictable branches leave it very competitive (even beating, by a bit) binary insertion when comparisons are very cheap, despite that it can do many more compares. And it wins big on runs that are already sorted (moving the next one in takes only 1 compare then). So I left code for a plain insertion sort, to make future experimenting easier. Also made the maximum value of minrun a `#define` (``MAX_MINRUN`) to make experimenting with that easier too. And another bit of `#if``-disabled code rewrites binary insertion's innermost loop to remove its unpredictable branch. Surprisingly, this doesn't really seem to help overall. I'm unclear on why not. It certainly adds more instructions, but they're very simple, and it's hard to be believe they cost as much as a branch miss. --- Objects/listobject.c | 188 ++++++++++++++++++++++++++++++------------- Objects/listsort.txt | 19 ++--- 2 files changed, 141 insertions(+), 66 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index fc20a9bff3af47..470ad8eb8135db 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1628,6 +1628,15 @@ sortslice_advance(sortslice *slice, Py_ssize_t n) /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 +/* The largest value of minrun. This must be a power of 2, and >= 1, so that + * the compute_minrun() algorithm guarantees to return a result no larger than + * this, + */ +#define MAX_MINRUN 64 +#if ((MAX_MINRUN) < 1) || ((MAX_MINRUN) & ((MAX_MINRUN) - 1)) +#error "MAX_MINRUN must be a power of 2, and >= 1" +#endif + /* One MergeState exists on the stack per invocation of mergesort. It's just * a convenient way to pass state around among the helper functions. */ @@ -1685,68 +1694,133 @@ struct s_MergeState { int (*tuple_elem_compare)(PyObject *, PyObject *, MergeState *); }; -/* binarysort is the best method for sorting small arrays: it does - few compares, but can do data movement quadratic in the number of - elements. - [lo.keys, hi) is a contiguous slice of a list of keys, and is sorted via - binary insertion. This sort is stable. - On entry, must have lo.keys <= start <= hi, and that - [lo.keys, start) is already sorted (pass start == lo.keys if you don't - know!). - If islt() complains return -1, else 0. +/* binarysort is the best method for sorting small arrays: it does few + compares, but can do data movement quadratic in the number of elements. + ss->keys is viewed as an array of n kays, a[:n]. a[:ok] is already sorted. + Pass ok = 0 (or 1) if you don't know. + It's sorted in-place, by a stable binary insertion sort. If ss->values + isn't NULL, it's permuted in lockstap with ss->keys. + On entry, must have n >= 1, and 0 <= ok <= n <= MAX_MINRUN. + Return -1 if comparison raises an exception, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). */ static int -binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) +binarysort(MergeState *ms, const sortslice *ss, Py_ssize_t n, Py_ssize_t ok) { - Py_ssize_t k; - PyObject **l, **p, **r; + Py_ssize_t k; /* for IFLT macro expansion */ + PyObject ** const a = ss->keys; + PyObject ** const v = ss->values; + const bool has_values = v != NULL; PyObject *pivot; - - assert(lo.keys <= start && start <= hi); - /* assert [lo.keys, start) is sorted */ - if (lo.keys == start) - ++start; - for (; start < hi; ++start) { - /* set l to where *start belongs */ - l = lo.keys; - r = start; - pivot = *r; - /* Invariants: - * pivot >= all in [lo.keys, l). - * pivot < all in [r, start). - * These are vacuously true at the start. + Py_ssize_t M; + + assert(0 <= ok && ok <= n && 1 <= n && n <= MAX_MINRUN); + /* assert a[:ok] is sorted */ + if (! ok) + ++ok; + /* Regular insertion sort has average- and worst-case O(n**2) cost + for both # of comparisons and number of bytes moved. But its branches + are highly predictable, and it loves sorted input (n-1 compares and no + data movement). This is significant in cases like sortperf.py's %sort, + where an out-of-order element near the start of a run is moved into + place slowly but then the remaining elements up to length minrun are + generally at worst one slot away from their correct position (so only + need 1 or 2 commpares to resolve). If comparisons are very fast (such + as for a list of Python floats), the simple inner loop leaves it + very competitive with binary insertion, despite that it does + significantly more compares overall on random data. + + Binary insertion sort has worst, average, and best case O(n log n) + cost for # of comparisons, but worst and average case O(n**2) cost + for data movement. The more expensive comparisons, the more important + the comparison advantage. But its branches are less predictable the + more "randomish" the data, and that's so significant its worst case + in real life is random input rather than reverse-ordered (which does + about twice the data movement than random input does). + + Note that the number of bytes moved doesn't seem to matter. MAX_MINRUN + of 64 is so small that the key and value pointers all fit in a corner + of L1 cache, and moving things around in that is very fast. */ +#if 0 // ordinary insertion sort. + PyObject * vpivot = NULL; + for (; ok < n; ++ok) { + pivot = a[ok]; + if (has_values) + vpivot = v[ok]; + for (M = ok - 1; M >= 0; --M) { + k = ISLT(pivot, a[M]); + if (k < 0) { + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + goto fail; + } + else if (k) { + a[M + 1] = a[M]; + if (has_values) + v[M + 1] = v[M]; + } + else + break; + } + a[M + 1] = pivot; + if (has_values) + v[M + 1] = vpivot; + } +#else // binary insertion sort + Py_ssize_t L, R; + for (; ok < n; ++ok) { + /* set L to where a[ok] belongs */ + L = 0; + R = ok; + pivot = a[ok]; + /* Slice invariants. vacuously true at the start: + * all a[0:L] <= pivot + * all a[L:R] unknown + * all a[R:ok] > pivot */ - assert(l < r); + assert(L < R); do { - p = l + ((r - l) >> 1); - IFLT(pivot, *p) - r = p; + /* don't do silly ;-) things to prevent overflow when finding + the midpoint; L and R are very far from filling a Py_ssize_t */ + M = (L + R) >> 1; +#if 1 // straightforward, but highly unpredictable branch on random data + IFLT(pivot, a[M]) + R = M; else - l = p+1; - } while (l < r); - assert(l == r); - /* The invariants still hold, so pivot >= all in [lo.keys, l) and - pivot < all in [l, start), so pivot belongs at l. Note - that if there are elements equal to pivot, l points to the - first slot after them -- that's why this sort is stable. - Slide over to make room. - Caution: using memmove is much slower under MSVC 5; - we're not usually moving many slots. */ - for (p = start; p > l; --p) - *p = *(p-1); - *l = pivot; - if (lo.values != NULL) { - Py_ssize_t offset = lo.values - lo.keys; - p = start + offset; - pivot = *p; - l += offset; - for ( ; p > l; --p) - *p = *(p-1); - *l = pivot; + L = M + 1; +#else + /* Try to get compiler to generate conditional move instructions + instead. Works fine, but leaving it disabled for now because + it's not yielding consistently faster sorts. Needs more + investigation. More computation in the inner loop adds its own + costs, which can be significant when compares are fast. */ + k = ISLT(pivot, a[M]); + if (k < 0) + goto fail; + Py_ssize_t Mp1 = M + 1; + R = k ? M : R; + L = k ? L : Mp1; +#endif + } while (L < R); + assert(L == R); + /* a[:L] holds all elements from a[:ok] <= pivot now, so pivot belongs + at index L. Slide a[L:ok] to the right a slot to make room for it. + Caution: using memmove is much slower under MSVC 5; we're not + usually moving many slots. Years later: under Visual Studio 2022, + memmove seems just slightly slower than doing it "by hand". */ + for (M = ok; M > L; --M) + a[M] = a[M - 1]; + a[L] = pivot; + if (has_values) { + pivot = v[ok]; + for (M = ok; M > L; --M) + v[M] = v[M - 1]; + v[L] = pivot; } } +#endif // pick binary or regular insertion sort return 0; fail: @@ -2559,10 +2633,10 @@ merge_force_collapse(MergeState *ms) /* Compute a good value for the minimum run length; natural runs shorter * than this are boosted artificially via binary insertion. * - * If n < 64, return n (it's too small to bother with fancy stuff). - * Else if n is an exact power of 2, return 32. - * Else return an int k, 32 <= k <= 64, such that n/k is close to, but - * strictly less than, an exact power of 2. + * If n < MAX_MINRUN return n (it's too small to bother with fancy stuff). + * Else if n is an exact power of 2, return MAX_MINRUN / 2. + * Else return an int k, MAX_MINRUN / 2 <= k <= MAX_MINRUN, such that n/k is + * close to, but strictly less than, an exact power of 2. * * See listsort.txt for more info. */ @@ -2572,7 +2646,7 @@ merge_compute_minrun(Py_ssize_t n) Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ assert(n >= 0); - while (n >= 64) { + while (n >= MAX_MINRUN) { r |= n & 1; n >>= 1; } @@ -2956,7 +3030,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; - if (binarysort(&ms, lo, lo.keys + force, lo.keys + n) < 0) + if (binarysort(&ms, &lo, force, n) < 0) goto fail; n = force; } diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 4f84e2c87da7f1..f387d9c116e502 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -270,9 +270,9 @@ result. This has two primary good effects: Computing minrun ---------------- -If N < 64, minrun is N. IOW, binary insertion sort is used for the whole -array then; it's hard to beat that given the overheads of trying something -fancier (see note BINSORT). +If N < MAX_MINRUN, minrun is N. IOW, binary insertion sort is used for the +whole array then; it's hard to beat that given the overheads of trying +something fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -310,12 +310,13 @@ place, and r < minrun is small compared to N), or q a little larger than a power of 2 regardless of r (then we've got a case similar to "2112", again leaving too little work for the last merge to do). -Instead we pick a minrun in range(32, 65) such that N/minrun is exactly a -power of 2, or if that isn't possible, is close to, but strictly less than, -a power of 2. This is easier to do than it may sound: take the first 6 -bits of N, and add 1 if any of the remaining bits are set. In fact, that -rule covers every case in this section, including small N and exact powers -of 2; merge_compute_minrun() is a deceptively simple function. +Instead we pick a minrun in range(MAX_MINRUN / 2, MAX_MINRUN + 1) such that +N/minrun is exactly a power of 2, or if that isn't possible, is close to, but +strictly less than, a power of 2. This is easier to do than it may sound: +take the first log2(MAX_MINRUN) bits of N, and add 1 if any of the remaining +bits are set. In fact, that rule covers every case in this section, including +small N and exact powers of 2; merge_compute_minrun() is a deceptively simple +function. The Merge Pattern From 42ae924d278c48a719fb0ab86357f3235a9f7ab9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 22 Mar 2024 10:42:18 +0100 Subject: [PATCH 81/82] gh-117127: glob tests: Reopen dir_fd to pick up directory changes (GH-117128) --- Lib/test/test_glob.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 2de997501039ad..6719bdbb0cc9b1 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -41,6 +41,11 @@ def setUp(self): os.symlink(self.norm('broken'), self.norm('sym1')) os.symlink('broken', self.norm('sym2')) os.symlink(os.path.join('a', 'bcd'), self.norm('sym3')) + self.open_dirfd() + + def open_dirfd(self): + if self.dir_fd is not None: + os.close(self.dir_fd) if {os.open, os.stat} <= os.supports_dir_fd and os.scandir in os.supports_fd: self.dir_fd = os.open(self.tempdir, os.O_RDONLY | os.O_DIRECTORY) else: @@ -350,6 +355,10 @@ def test_glob_non_directory(self): def test_glob_named_pipe(self): path = os.path.join(self.tempdir, 'mypipe') os.mkfifo(path) + + # gh-117127: Reopen self.dir_fd to pick up directory changes + self.open_dirfd() + self.assertEqual(self.rglob('mypipe'), [path]) self.assertEqual(self.rglob('mypipe*'), [path]) self.assertEqual(self.rglob('mypipe', ''), []) From 63d6f2623ef2aa90f51c6a928b96845b9b380d89 Mon Sep 17 00:00:00 2001 From: NGRsoftlab <78017794+NGRsoftlab@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:25:38 +0300 Subject: [PATCH 82/82] gh-117068: Remove useless code in bytesio.c:resize_buffer() (GH-117069) Co-authored-by: i.khabibulin --- Modules/_io/bytesio.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 4a15c8e841f25f..fb66d3db0f7a1f 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -155,9 +155,6 @@ resize_buffer(bytesio *self, size_t size) alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(char)) - goto overflow; - if (SHARED_BUF(self)) { if (unshare_buffer(self, alloc) < 0) return -1;