From cdeb1a6caad5e3067f01d6058238803b8517f9de Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Sat, 22 Jul 2023 01:24:26 +0100 Subject: [PATCH 01/73] gh-96663: Add a better error message for __dict__-less classes setattr (#103232) --- Lib/test/test_class.py | 18 +++++++++++++++++- Lib/test/test_descrtut.py | 2 +- ...23-04-04-00-40-04.gh-issue-96663.PdR9hK.rst | 1 + Objects/object.c | 16 +++++++++++++--- 4 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-04-00-40-04.gh-issue-96663.PdR9hK.rst diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 894e0ca67deabc..bb35baea508c76 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -641,6 +641,14 @@ class A: class B: y = 0 __slots__ = ('z',) + class C: + __slots__ = ("y",) + + def __setattr__(self, name, value) -> None: + if name == "z": + super().__setattr__("y", 1) + else: + super().__setattr__(name, value) error_msg = "'A' object has no attribute 'x'" with self.assertRaisesRegex(AttributeError, error_msg): @@ -653,8 +661,16 @@ class B: B().x with self.assertRaisesRegex(AttributeError, error_msg): del B().x - with self.assertRaisesRegex(AttributeError, error_msg): + with self.assertRaisesRegex( + AttributeError, + "'B' object has no attribute 'x' and no __dict__ for setting new attributes" + ): B().x = 0 + with self.assertRaisesRegex( + AttributeError, + "'C' object has no attribute 'x'" + ): + C().x = 0 error_msg = "'B' object attribute 'y' is read-only" with self.assertRaisesRegex(AttributeError, error_msg): diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 7796031ed0602f..13e3ea41bdb76c 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -139,7 +139,7 @@ def merge(self, other): >>> a.x1 = 1 Traceback (most recent call last): File "", line 1, in ? - AttributeError: 'defaultdict2' object has no attribute 'x1' + AttributeError: 'defaultdict2' object has no attribute 'x1' and no __dict__ for setting new attributes >>> """ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-04-00-40-04.gh-issue-96663.PdR9hK.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-04-00-40-04.gh-issue-96663.PdR9hK.rst new file mode 100644 index 00000000000000..cb806b5ea7a9f3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-04-00-40-04.gh-issue-96663.PdR9hK.rst @@ -0,0 +1 @@ +Add a better, more introspect-able error message when setting attributes on classes without a ``__dict__`` and no slot member for the attribute. diff --git a/Objects/object.c b/Objects/object.c index d30e048335ab63..bfbc87198f5b3c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1576,9 +1576,18 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } if (dictptr == NULL) { if (descr == NULL) { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); + if (tp->tp_setattro == PyObject_GenericSetAttr) { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U' and no " + "__dict__ for setting new attributes", + tp->tp_name, name); + } + else { + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + } + set_attribute_error_context(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1611,6 +1620,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } + set_attribute_error_context(obj, name); } done: Py_XDECREF(descr); From 3372bcba9893030e4063a9264ec0b4d1b6166883 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 22 Jul 2023 09:43:13 +0200 Subject: [PATCH 02/73] gh-106970: Fix Argument Clinic 'destination clear' command (#106972) Add test for the 'destination clear' command, and the 'destination' directive in general. Fix two bugs in 'destination clear' command: 1. The text attribute of the allocator is called 'text', not '_text' 2. Return after processing the 'clear' command, instead of proceeding directly to the fail(). --- Lib/test/clinic.test.c | 56 +++++++++++++++++++ Lib/test/test_clinic.py | 9 +++ ...-07-21-23-16-05.gh-issue-106970.NLRnml.rst | 4 ++ Tools/clinic/clinic.py | 16 +++--- 4 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index da99e58c77f021..a660ccf8876e2d 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4948,3 +4948,59 @@ Test_meth_coexist(TestObj *self, PyObject *Py_UNUSED(ignored)) static PyObject * Test_meth_coexist_impl(TestObj *self) /*[clinic end generated code: output=808a293d0cd27439 input=2a1d75b5e6fec6dd]*/ + + +/*[clinic input] +output push +output preset buffer +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=5bff3376ee0df0b5]*/ + +/*[clinic input] +buffer_clear + a: int +We'll call 'destination buffer clear' after this. + +Argument Clinic's buffer preset puts most generated code into the +'buffer' destination, except from 'impl_definition', which is put into +the 'block' destination, so we should expect everything but +'impl_definition' to be cleared. +[clinic start generated code]*/ + +static PyObject * +buffer_clear_impl(PyObject *module, int a) +/*[clinic end generated code: output=f14bba74677e1846 input=a4c308a6fdab043c]*/ + +/*[clinic input] +destination buffer clear +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f20d06adb8252084]*/ + + +/*[clinic input] +output push +destination test1 new buffer +output everything suppress +output docstring_definition test1 +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=5a77c454970992fc]*/ + +/*[clinic input] +new_dest + a: int +Only this docstring should be outputted to test1. +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=da5af421ed8996ed]*/ + +/*[clinic input] +dump test1 +output pop +[clinic start generated code]*/ + +PyDoc_STRVAR(new_dest__doc__, +"new_dest($module, /, a)\n" +"--\n" +"\n" +"Only this docstring should be outputted to test1."); +/*[clinic end generated code: output=9cac703f51d90e84 input=090db8df4945576d]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 7c725e33f53928..e40579b0041cf3 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -249,6 +249,15 @@ def test_cpp_monitor_fail_no_matching_if(self): out = self.expect_failure(raw) self.assertEqual(out, msg) + def test_unknown_destination_command(self): + out = self.expect_failure(""" + /*[clinic input] + destination buffer nosuchcommand + [clinic start generated code]*/ + """) + msg = "unknown destination command 'nosuchcommand'" + self.assertIn(msg, out) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst new file mode 100644 index 00000000000000..194e3351b0470c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst @@ -0,0 +1,4 @@ +Fix bugs in the Argument Clinic ``destination clear`` command; the +destination buffers would never be cleared, and the ``destination`` +directive parser would simply continue to the fault handler after processing +the command. Patch by Erlend E. Aasland. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index a204c1dd0dc2d6..8acf513a6c1fc6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1983,7 +1983,7 @@ def __getitem__(self, i): def clear(self): for ta in self._array: - ta._text.clear() + ta.text.clear() def dump(self): texts = [ta.output() for ta in self._array] @@ -4487,13 +4487,13 @@ def directive_destination( command: str, *args ) -> None: - if command == 'new': - self.clinic.add_destination(name, *args) - return - - if command == 'clear': - self.clinic.get_destination(name).clear() - fail("unknown destination command", repr(command)) + match command: + case "new": + self.clinic.add_destination(name, *args) + case "clear": + self.clinic.get_destination(name).clear() + case _: + fail("unknown destination command", repr(command)) def directive_output( From 806d7c98a5da5c1fd2e52a5b666f36ca4f545092 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 22 Jul 2023 10:12:43 +0200 Subject: [PATCH 03/73] gh-101100: Docs: Check Sphinx warnings and fail if improved (#106460) --- .github/workflows/reusable-docs.yml | 27 ++--- Doc/tools/.nitignore | 11 +- Doc/tools/check-warnings.py | 149 ++++++++++++++++++++++++++++ Doc/tools/touch-clean-files.py | 65 ------------ Doc/tools/warnings-to-gh-actions.py | 25 ----- 5 files changed, 161 insertions(+), 116 deletions(-) create mode 100644 Doc/tools/check-warnings.py delete mode 100644 Doc/tools/touch-clean-files.py delete mode 100644 Doc/tools/warnings-to-gh-actions.py diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index b39d8cea6421ea..56932c4860573c 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -28,10 +28,8 @@ jobs: cache-dependency-path: 'Doc/requirements.txt' - name: 'Install build dependencies' run: make -C Doc/ venv - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - # Add pull request annotations for Sphinx nitpicks (missing references) + # To annotate PRs with Sphinx nitpicks (missing references) - name: 'Get list of changed files' if: github.event_name == 'pull_request' id: changed_files @@ -39,24 +37,19 @@ jobs: with: filter: "Doc/**" format: csv # works for paths with spaces - - name: 'Build changed files in nit-picky mode' - if: github.event_name == 'pull_request' + - name: 'Build HTML documentation' continue-on-error: true run: | set -Eeuo pipefail - # Mark files the pull request modified - python Doc/tools/touch-clean-files.py --clean '${{ steps.changed_files.outputs.added_modified }}' - # Build docs with the '-n' (nit-picky) option; convert warnings to annotations - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n --keep-going" html 2>&1 | - python Doc/tools/warnings-to-gh-actions.py - - # Ensure some files always pass Sphinx nit-picky mode (no missing references) - - name: 'Build known-good files in nit-picky mode' + # Build docs with the '-n' (nit-picky) option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + - name: 'Check warnings' + if: github.event_name == 'pull_request' run: | - # Mark files that must pass nit-picky - python Doc/tools/touch-clean-files.py - # Build docs with the '-n' (nit-picky) option, convert warnings to errors (-W) - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going" html 2>&1 + python Doc/tools/check-warnings.py \ + --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --fail-if-regression \ + --fail-if-improved # This build doesn't use problem matchers or check annotations build_doc_oldest_supported_sphinx: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 23aa30c956b3bd..ca6f32c55acd69 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -1,7 +1,6 @@ # All RST files under Doc/ -- except these -- must pass Sphinx nit-picky mode, -# as tested on the CI via touch-clean-files.py in doc.yml. -# Add blank lines between files and keep them sorted lexicographically -# to help avoid merge conflicts. +# as tested on the CI via check-warnings.py in reusable-docs.yml. +# Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/allocation.rst Doc/c-api/apiabiversion.rst @@ -18,7 +17,6 @@ Doc/c-api/complex.rst Doc/c-api/conversion.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst -Doc/c-api/dict.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst Doc/c-api/float.rst @@ -48,7 +46,6 @@ Doc/c-api/typehints.rst Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/c-api/veryhigh.rst -Doc/c-api/weakref.rst Doc/extending/embedding.rst Doc/extending/extending.rst Doc/extending/newtypes.rst @@ -88,8 +85,6 @@ Doc/library/bdb.rst Doc/library/bisect.rst Doc/library/bz2.rst Doc/library/calendar.rst -Doc/library/cgi.rst -Doc/library/cmath.rst Doc/library/cmd.rst Doc/library/code.rst Doc/library/codecs.rst @@ -105,7 +100,6 @@ Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst Doc/library/ctypes.rst -Doc/library/curses.ascii.rst Doc/library/curses.rst Doc/library/datetime.rst Doc/library/dbm.rst @@ -134,7 +128,6 @@ Doc/library/fractions.rst Doc/library/ftplib.rst Doc/library/functions.rst Doc/library/functools.rst -Doc/library/getopt.rst Doc/library/getpass.rst Doc/library/gettext.rst Doc/library/graphlib.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py new file mode 100644 index 00000000000000..c17d0f51cd1272 --- /dev/null +++ b/Doc/tools/check-warnings.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Check the output of running Sphinx in nit-picky mode (missing references). +""" +import argparse +import csv +import os +import re +import sys +from pathlib import Path + +# Exclude these whether they're dirty or clean, +# because they trigger a rebuild of dirty files. +EXCLUDE_FILES = { + "Doc/whatsnew/changelog.rst", +} + +# Subdirectories of Doc/ to exclude. +EXCLUDE_SUBDIRS = { + ".env", + ".venv", + "env", + "includes", + "venv", +} + +PATTERN = re.compile(r"(?P[^:]+):(?P\d+): WARNING: (?P.+)") + + +def check_and_annotate(warnings: list[str], files_to_check: str) -> None: + """ + Convert Sphinx warning messages to GitHub Actions. + + Converts lines like: + .../Doc/library/cgi.rst:98: WARNING: reference target not found + to: + ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found + + Non-matching lines are echoed unchanged. + + see: + https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message + """ + files_to_check = next(csv.reader([files_to_check])) + for warning in warnings: + if any(filename in warning for filename in files_to_check): + if match := PATTERN.fullmatch(warning): + print("::warning file={file},line={line}::{msg}".format_map(match)) + + +def fail_if_regression( + warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + Ensure some files always pass Sphinx nit-picky mode (no missing references). + These are files which are *not* in .nitignore. + """ + all_rst = { + str(rst) + for rst in Path("Doc/").rglob("*.rst") + if rst.parts[1] not in EXCLUDE_SUBDIRS + } + should_be_clean = all_rst - files_with_expected_nits - EXCLUDE_FILES + problem_files = sorted(should_be_clean & files_with_nits) + if problem_files: + print("\nError: must not contain warnings:\n") + for filename in problem_files: + print(filename) + for warning in warnings: + if filename in warning: + if match := PATTERN.fullmatch(warning): + print(" {line}: {msg}".format_map(match)) + return -1 + return 0 + + +def fail_if_improved( + files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + We may have fixed warnings in some files so that the files are now completely clean. + Good news! Let's add them to .nitignore to prevent regression. + """ + files_with_no_nits = files_with_expected_nits - files_with_nits + if files_with_no_nits: + print("\nCongratulations! You improved:\n") + for filename in sorted(files_with_no_nits): + print(filename) + print("\nPlease remove from Doc/tools/.nitignore\n") + return -1 + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check-and-annotate", + help="Comma-separated list of files to check, " + "and annotate those with warnings on GitHub Actions", + ) + parser.add_argument( + "--fail-if-regression", + action="store_true", + help="Fail if known-good files have warnings", + ) + parser.add_argument( + "--fail-if-improved", + action="store_true", + help="Fail if new files with no nits are found", + ) + args = parser.parse_args() + exit_code = 0 + + wrong_directory_msg = "Must run this script from the repo root" + assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + + with Path("Doc/sphinx-warnings.txt").open() as f: + warnings = f.read().splitlines() + + cwd = str(Path.cwd()) + os.path.sep + files_with_nits = { + warning.removeprefix(cwd).split(":")[0] + for warning in warnings + if "Doc/" in warning + } + + with Path("Doc/tools/.nitignore").open() as clean_files: + files_with_expected_nits = { + filename.strip() + for filename in clean_files + if filename.strip() and not filename.startswith("#") + } + + if args.check_and_annotate: + check_and_annotate(warnings, args.check_and_annotate) + + if args.fail_if_regression: + exit_code += fail_if_regression( + warnings, files_with_expected_nits, files_with_nits + ) + + if args.fail_if_improved: + exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Doc/tools/touch-clean-files.py b/Doc/tools/touch-clean-files.py deleted file mode 100644 index 2b045bd68a0cf0..00000000000000 --- a/Doc/tools/touch-clean-files.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -""" -Touch files that must pass Sphinx nit-picky mode -so they are rebuilt and we can catch regressions. -""" -import argparse -import csv -import sys -from pathlib import Path - -wrong_directory_msg = "Must run this script from the repo root" -assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - -# Exclude these whether they're dirty or clean, -# because they trigger a rebuild of dirty files. -EXCLUDE_FILES = { - Path("Doc/whatsnew/changelog.rst"), -} - -# Subdirectories of Doc/ to exclude. -EXCLUDE_SUBDIRS = { - ".env", - ".venv", - "env", - "includes", - "venv", -} - -ALL_RST = { - rst for rst in Path("Doc/").rglob("*.rst") if rst.parts[1] not in EXCLUDE_SUBDIRS -} - - -parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter -) -parser.add_argument("-c", "--clean", help="Comma-separated list of clean files") -args = parser.parse_args() - -if args.clean: - clean_files = next(csv.reader([args.clean])) - CLEAN = { - Path(filename.strip()) - for filename in clean_files - if Path(filename.strip()).is_file() - } -elif args.clean is not None: - print( - "Not touching any files: an empty string `--clean` arg value passed.", - ) - sys.exit(0) -else: - with Path("Doc/tools/.nitignore").open() as ignored_files: - IGNORED = { - Path(filename.strip()) - for filename in ignored_files - if filename.strip() and not filename.startswith("#") - } - CLEAN = ALL_RST - IGNORED - EXCLUDE_FILES - -print("Touching:") -for filename in sorted(CLEAN): - print(filename) - filename.touch() -print(f"Touched {len(CLEAN)} files") diff --git a/Doc/tools/warnings-to-gh-actions.py b/Doc/tools/warnings-to-gh-actions.py deleted file mode 100644 index da33a4ede07abc..00000000000000 --- a/Doc/tools/warnings-to-gh-actions.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -""" -Convert Sphinx warning messages to GitHub Actions. - -Converts lines like: - .../Doc/library/cgi.rst:98: WARNING: reference target not found -to: - ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - -Non-matching lines are echoed unchanged. - -see: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message -""" - -import re -import sys - -pattern = re.compile(r'(?P[^:]+):(?P\d+): WARNING: (?P.+)') - -for line in sys.stdin: - if match := pattern.fullmatch(line.strip()): - print('::warning file={file},line={line}::{msg}'.format_map(match)) - else: - print(line) From d55b4da10c56f3299998b5b8fee3a8a744fac76c Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Sat, 22 Jul 2023 10:27:48 +0200 Subject: [PATCH 04/73] gh-106973: Change non-integral to non-integer in "3.12 What's New" (#106984) --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 1e35427c497c8b..45093122eb7b72 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1456,7 +1456,7 @@ Changes in the Python API * Removed ``randrange()`` functionality deprecated since Python 3.10. Formerly, ``randrange(10.0)`` losslessly converted to ``randrange(10)``. Now, it raises a - :exc:`TypeError`. Also, the exception raised for non-integral values such as + :exc:`TypeError`. Also, the exception raised for non-integer values such as ``randrange(10.5)`` or ``randrange('10')`` has been changed from :exc:`ValueError` to :exc:`TypeError`. This also prevents bugs where ``randrange(1e25)`` would silently select from a larger range than ``randrange(10**25)``. From 443d9b3033bc6189e7f1ae936806779c02a46c43 Mon Sep 17 00:00:00 2001 From: littlebutt's workshop Date: Sat, 22 Jul 2023 08:34:48 +0000 Subject: [PATCH 05/73] gh-106976:alphabetise bullets by module name task1 (#106982) --- Doc/whatsnew/3.13.rst | 53 +++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0203e3d18738d4..a778e12aba1bce 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -150,30 +150,7 @@ Optimizations Deprecated ========== -* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` - methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. - They will be removed in Python 3.15. - (Contributed by Victor Stinner in :gh:`105096`.) -* :mod:`typing`: Creating a :class:`typing.NamedTuple` class using keyword arguments to denote - the fields (``NT = NamedTuple("NT", x=int, y=int)``) is deprecated, and will - be disallowed in Python 3.15. Use the class-based syntax or the functional - syntax instead. (Contributed by Alex Waygood in :gh:`105566`.) -* :mod:`typing`: When using the functional syntax to create a :class:`typing.NamedTuple` - class or a :class:`typing.TypedDict` class, failing to pass a value to the - 'fields' parameter (``NT = NamedTuple("NT")`` or ``TD = TypedDict("TD")``) is - deprecated. Passing ``None`` to the 'fields' parameter - (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also - deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple - class with 0 fields, use ``class NT(NamedTuple): pass`` or - ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use - ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. - (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) -* :func:`typing.no_type_check_decorator` is deprecated, and scheduled for - removal in Python 3.15. After eight years in the :mod:`typing` module, it - has yet to be supported by any major type checkers. - (Contributed by Alex Waygood in :gh:`106309`.) - -* :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, +* :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, emits :exc:`DeprecationWarning` since 3.13 and will be removed in Python 3.16. Use the ``'w'`` format code instead. @@ -184,13 +161,39 @@ Deprecated Replace ``ctypes.ARRAY(item_type, size)`` with ``item_type * size``. (Contributed by Victor Stinner in :gh:`105733`.) -* The :mod:`getopt` and :mod:`optparse` modules are now +* :mod:`getopt` and :mod:`optparse` modules: They are now :term:`soft deprecated`: the :mod:`argparse` should be used for new projects. Previously, the :mod:`optparse` module was already deprecated, its removal was not scheduled, and no warnings was emitted: so there is no change in practice. (Contributed by Victor Stinner in :gh:`106535`.) +* :mod:`typing`: Creating a :class:`typing.NamedTuple` class using keyword arguments to denote + the fields (``NT = NamedTuple("NT", x=int, y=int)``) is deprecated, and will + be disallowed in Python 3.15. Use the class-based syntax or the functional + syntax instead. (Contributed by Alex Waygood in :gh:`105566`.) + + * When using the functional syntax to create a :class:`typing.NamedTuple` + class or a :class:`typing.TypedDict` class, failing to pass a value to the + 'fields' parameter (``NT = NamedTuple("NT")`` or ``TD = TypedDict("TD")``) is + deprecated. Passing ``None`` to the 'fields' parameter + (``NT = NamedTuple("NT", None)`` or ``TD = TypedDict("TD", None)``) is also + deprecated. Both will be disallowed in Python 3.15. To create a NamedTuple + class with 0 fields, use ``class NT(NamedTuple): pass`` or + ``NT = NamedTuple("NT", [])``. To create a TypedDict class with 0 fields, use + ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. + (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) + + * :func:`typing.no_type_check_decorator` is deprecated, and scheduled for + removal in Python 3.15. After eight years in the :mod:`typing` module, it + has yet to be supported by any major type checkers. + (Contributed by Alex Waygood in :gh:`106309`.) + +* :mod:`wave`: Deprecate the ``getmark()``, ``setmark()`` and ``getmarkers()`` + methods of the :class:`wave.Wave_read` and :class:`wave.Wave_write` classes. + They will be removed in Python 3.15. + (Contributed by Victor Stinner in :gh:`105096`.) + Pending Removal in Python 3.14 ------------------------------ From ee5c01b473eeadb007b9f330db3143e34e46038b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 22 Jul 2023 11:32:10 +0200 Subject: [PATCH 06/73] gh-106368: Increase coverage for Argument Clinic output directive (#106979) --- Lib/test/test_clinic.py | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e40579b0041cf3..c24b8cd3899f28 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -249,6 +249,59 @@ def test_cpp_monitor_fail_no_matching_if(self): out = self.expect_failure(raw) self.assertEqual(out, msg) + def test_directive_output_unknown_preset(self): + out = self.expect_failure(""" + /*[clinic input] + output preset nosuchpreset + [clinic start generated code]*/ + """) + msg = "Unknown preset 'nosuchpreset'" + self.assertIn(msg, out) + + def test_directive_output_cant_pop(self): + out = self.expect_failure(""" + /*[clinic input] + output pop + [clinic start generated code]*/ + """) + msg = "Can't 'output pop', stack is empty" + self.assertIn(msg, out) + + def test_directive_output_print(self): + raw = dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """) + out = self.clinic.parse(raw) + # The generated output will differ for every run, but we can check that + # it starts with the clinic block, we check that it contains all the + # expected fields, and we check that it contains the checksum line. + self.assertTrue(out.startswith(dedent(""" + /*[clinic input] + output print 'I told you once.' + [clinic start generated code]*/ + """))) + fields = { + "cpp_endif", + "cpp_if", + "docstring_definition", + "docstring_prototype", + "impl_definition", + "impl_prototype", + "methoddef_define", + "methoddef_ifndef", + "parser_definition", + "parser_prototype", + } + for field in fields: + with self.subTest(field=field): + self.assertIn(field, out) + last_line = out.rstrip().split("\n")[-1] + self.assertTrue( + last_line.startswith("/*[clinic end generated code: output=") + ) + def test_unknown_destination_command(self): out = self.expect_failure(""" /*[clinic input] From 6acd85d91063380d1afb86c116a66e0aadd4f909 Mon Sep 17 00:00:00 2001 From: Sven Arends <88426829+picnic-sven@users.noreply.github.com> Date: Sat, 22 Jul 2023 11:38:47 +0200 Subject: [PATCH 07/73] gh-106978: Bump sphinx-lint to 0.6.8 (#106990) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d62c57c044728f..85a6de4abe0146 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: types_or: [c, python, rst] - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.6.7 + rev: v0.6.8 hooks: - id: sphinx-lint args: [--enable=default-role] From f8f16d0cfcbe287deaaf1a6eb6461b11b7837a1c Mon Sep 17 00:00:00 2001 From: Daniele Procida Date: Sat, 22 Jul 2023 12:31:44 +0200 Subject: [PATCH 08/73] gh-106996: Amend the introduction to the turtle graphics documentation (#106997) Co-authored-by: Hugo van Kemenade --- Doc/library/turtle.rst | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index c9ce955a6d2ba4..f14a677b7dd88c 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -19,9 +19,27 @@ Introduction ============ -Turtle graphics is a popular way for introducing programming to kids. It was -part of the original Logo programming language developed by Wally Feurzeig, -Seymour Papert and Cynthia Solomon in 1967. +Turtle graphics is an implementation of `the popular geometric drawing tools +introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon +in 1967. + +In Python, turtle graphics provides a representation of a physical "turtle" +(a little robot with a pen) that draws on a sheet of paper on the floor. + +It's an effective and well-proven way for learners to encounter +programming concepts and interaction with software, as it provides instant, +visible feedback. It also provides convenient access to graphical output +in general. + +Turtle drawing was originally created as an educational tool, to be used by +teachers in the classroom. For the programmer who needs to produce some +graphical output it can be a way to do that without the overhead of +introducing more complex or external libraries into their work. + + +Get started +=========== Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the From c5adf26b1867767781af30ada6f05a29449fc650 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 22 Jul 2023 12:46:42 +0200 Subject: [PATCH 09/73] gh-104050: Argument Clinic: Annotate the BufferSeries class (#106935) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 8acf513a6c1fc6..7a4c9c4cacf55b 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1965,12 +1965,12 @@ class BufferSeries: e.g. o[-1] is an element immediately preceding o[0]. """ - def __init__(self): + def __init__(self) -> None: self._start = 0 - self._array = [] + self._array: list[_TextAccumulator] = [] self._constructor = _text_accumulator - def __getitem__(self, i): + def __getitem__(self, i: int) -> _TextAccumulator: i -= self._start if i < 0: self._start += i @@ -1981,11 +1981,11 @@ def __getitem__(self, i): self._array.append(self._constructor()) return self._array[i] - def clear(self): + def clear(self) -> None: for ta in self._array: ta.text.clear() - def dump(self): + def dump(self) -> str: texts = [ta.output() for ta in self._array] return "".join(texts) From 6dbffaed17d59079d6a2788d686009f762a3278f Mon Sep 17 00:00:00 2001 From: Sebastiaan Zeeff <33516116+SebastiaanZ@users.noreply.github.com> Date: Sat, 22 Jul 2023 14:13:44 +0200 Subject: [PATCH 10/73] gh-106969: Indicate no modules were added in 3.10 & 3.12 (#106988) The "New Modules" section was left in place to ensure that the anchor link for new modules will still exist: /whatsnew/3.12.html#new-modules /whatsnew/3.10.html#new-modules This means that existing links to this section don't break. --- Doc/whatsnew/3.10.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index d1d2193de4a95d..e4ca3c4c81ba58 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -887,7 +887,7 @@ Other Language Changes New Modules =========== -* None yet. +* None. Improved Modules diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 45093122eb7b72..a46621071b3d8b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -486,7 +486,7 @@ Other Language Changes New Modules =========== -* None yet. +* None. Improved Modules From 4a1026d7647c084b0dc80dd49163d16ba12a2e55 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 14:17:25 +0200 Subject: [PATCH 11/73] gh-106714: Fix test_capi to not write a coredump (#107007) test_capi: Fix test_no_FatalError_infinite_loop() to no longer write a coredump, by using test.support.SuppressCrashReport. --- Lib/test/test_capi/test_misc.py | 12 +++++++++--- .../2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e40ffdfd121519..dc155cc99961c4 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -85,9 +85,15 @@ def test_instancemethod(self): @support.requires_subprocess() def test_no_FatalError_infinite_loop(self): - run_result, _cmd_line = run_python_until_end( - '-c', 'import _testcapi; _testcapi.crash_no_current_thread()', - ) + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.crash_no_current_thread() + """) + + run_result, _cmd_line = run_python_until_end('-c', code) _rc, out, err = run_result self.assertEqual(out, b'') # This used to cause an infinite loop. diff --git a/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst b/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst new file mode 100644 index 00000000000000..955620521c8f68 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst @@ -0,0 +1,3 @@ +test_capi: Fix test_no_FatalError_infinite_loop() to no longer write a +coredump, by using test.support.SuppressCrashReport. Patch by Victor +Stinner. From ed491d9f782480fb00535abcf667027e0e323287 Mon Sep 17 00:00:00 2001 From: Joe Kaufeld Date: Sat, 22 Jul 2023 08:19:30 -0400 Subject: [PATCH 12/73] Reformat code block to make it easier to read (#106965) --- Doc/tutorial/errors.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 6419ff621f1b31..8a207c385c6ab7 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -535,11 +535,20 @@ of a certain type while letting all other exceptions propagate to other clauses and eventually to be reraised. :: >>> def f(): - ... raise ExceptionGroup("group1", - ... [OSError(1), - ... SystemError(2), - ... ExceptionGroup("group2", - ... [OSError(3), RecursionError(4)])]) + ... raise ExceptionGroup( + ... "group1", + ... [ + ... OSError(1), + ... SystemError(2), + ... ExceptionGroup( + ... "group2", + ... [ + ... OSError(3), + ... RecursionError(4) + ... ] + ... ) + ... ] + ... ) ... >>> try: ... f() From adda43dc0bcea853cbfa33126e5549c584cef8be Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 22 Jul 2023 06:21:55 -0600 Subject: [PATCH 13/73] gh-105699: Add some stress tests for subinterpreter creation (#106966) --- Lib/test/test_interpreters.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index d1bebe47158322..5981d96de8de06 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -7,6 +7,7 @@ from test import support from test.support import import_helper +from test.support import threading_helper _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters @@ -463,6 +464,27 @@ def test_bytes_for_script(self): # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. +class StressTests(TestBase): + + # In these tests we generally want a lot of interpreters, + # but not so many that any test takes too long. + + def test_create_many_sequential(self): + alive = [] + for _ in range(100): + interp = interpreters.create() + alive.append(interp) + + def test_create_many_threaded(self): + alive = [] + def task(): + interp = interpreters.create() + alive.append(interp) + threads = (threading.Thread(target=task) for _ in range(200)) + with threading_helper.start_threads(threads): + pass + + class TestIsShareable(TestBase): def test_default_shareables(self): From 76e20c361c8d6bc20d939d436a1c3d4077a58186 Mon Sep 17 00:00:00 2001 From: Menelaos Kotoglou Date: Sat, 22 Jul 2023 15:23:23 +0300 Subject: [PATCH 14/73] gh-106989: Remove tok report warnings (#106993) --- Parser/tokenizer.c | 5 ----- Parser/tokenizer.h | 1 - 2 files changed, 6 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f19198600fa018..ccff16045233de 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -116,7 +116,6 @@ tok_new(void) tok->implicit_newline = 0; tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0}; tok->tok_mode_stack_index = 0; - tok->tok_report_warnings = 1; #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; #endif @@ -1545,10 +1544,6 @@ static int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { - if (!tok->tok_report_warnings) { - return 0; - } - PyObject *msg = PyUnicode_FromFormat( "invalid escape sequence '\\%c'", (char) first_invalid_escape_char diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index cb44845c1d306e..1e1daa3648f5d0 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -128,7 +128,6 @@ struct tok_state { // TODO: Factor this into its own thing tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; - int tok_report_warnings; int tok_extra_tokens; int comment_newline; int implicit_newline; From 463b56da1265a17e4207f651d6d5835a8d6de2dd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 15:15:05 +0200 Subject: [PATCH 15/73] gh-106320: Remove private _PyUnicode_AsString() alias (#107021) Remove private _PyUnicode_AsString() alias to PyUnicode_AsUTF8(). It was kept for backward compatibility with Python 3.0 - 3.2. The PyUnicode_AsUTF8() is available since Python 3.3. The PyUnicode_AsUTF8String() function can be used to keep compatibility with Python 3.2 and older. --- Include/cpython/unicodeobject.h | 5 ----- .../C API/2023-07-22-14-40-48.gh-issue-106320.H3u7x4.rst | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-07-22-14-40-48.gh-issue-106320.H3u7x4.rst diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index e75b5e154943dc..51eb998db4ab6a 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -446,17 +446,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation in the unicodeobject. - _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to - support the previous internal function with the same behaviour. - Use of this API is DEPRECATED since no size information can be extracted from the returned data. */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); -#define _PyUnicode_AsString PyUnicode_AsUTF8 - /* === Characters Type APIs =============================================== */ /* These should not be used directly. Use the Py_UNICODE_IS* and diff --git a/Misc/NEWS.d/next/C API/2023-07-22-14-40-48.gh-issue-106320.H3u7x4.rst b/Misc/NEWS.d/next/C API/2023-07-22-14-40-48.gh-issue-106320.H3u7x4.rst new file mode 100644 index 00000000000000..1e0ba0d71e7555 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-07-22-14-40-48.gh-issue-106320.H3u7x4.rst @@ -0,0 +1,5 @@ +Remove private ``_PyUnicode_AsString()`` alias to +:c:func:`PyUnicode_AsUTF8`. It was kept for backward compatibility with +Python 3.0 - 3.2. The :c:func:`PyUnicode_AsUTF8` is available since Python +3.3. The :c:func:`PyUnicode_AsUTF8String` function can be used to keep +compatibility with Python 3.2 and older. Patch by Victor Stinner. From 2aba047f0a2be57cc33a57756707452fdd6a1b3f Mon Sep 17 00:00:00 2001 From: Bart Skowron Date: Sat, 22 Jul 2023 15:20:40 +0200 Subject: [PATCH 16/73] gh-69714: Make `calendar` module fully tested (#93655) There are 3 paths to use `locale` argument in `calendar.Locale{Text|HTML}Calendar.__init__(..., locale=None)`: (1) `locale=None` -- denotes the "default locale"[1] (2) `locale=""` -- denotes the native environment (3) `locale=other_valid_locale` -- denotes a custom locale So far case (2) is covered and case (1) is in 78935daf5a (same branch). This commit adds a remaining case (3). [1] In the current implementation, this translates into the following approach: GET current locale IF current locale == "C" THEN SET current locale TO "" GET current locale ENDIF * Remove unreachable code (and increase test coverage) This condition cannot be true. `_locale.setlocale()` from the C module raises `locale.Error` instead of returning `None` for `different_locale.__enter__` (where `self.oldlocale` is set). * Expand the try clause to calls to `LocaleTextCalendar.formatmonthname()`. This method temporarily changes the current locale to the given locale, so `_locale.setlocale()` may raise `local.Error`. Co-authored-by: Rohit Mediratta Co-authored-by: Jessica McKellar Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/calendar.py | 16 +- Lib/test/test_calendar.py | 240 ++++++++++++++---- Misc/ACKS | 1 + ...2-06-09-21-27-38.gh-issue-69714.49tyHW.rst | 1 + 4 files changed, 204 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-06-09-21-27-38.gh-issue-69714.49tyHW.rst diff --git a/Lib/calendar.py b/Lib/calendar.py index ea56f12ccc41d0..e43ba4a078bcac 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -585,8 +585,6 @@ def __enter__(self): _locale.setlocale(_locale.LC_TIME, self.locale) def __exit__(self, *args): - if self.oldlocale is None: - return _locale.setlocale(_locale.LC_TIME, self.oldlocale) @@ -690,7 +688,7 @@ def timegm(tuple): return seconds -def main(args): +def main(args=None): import argparse parser = argparse.ArgumentParser() textgroup = parser.add_argument_group('text only arguments') @@ -747,7 +745,7 @@ def main(args): help="month number (1-12, text only)" ) - options = parser.parse_args(args[1:]) + options = parser.parse_args(args) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -756,6 +754,9 @@ def main(args): locale = options.locale, options.encoding if options.type == "html": + if options.month: + parser.error("incorrect number of arguments") + sys.exit(1) if options.locale: cal = LocaleHTMLCalendar(locale=locale) else: @@ -767,11 +768,8 @@ def main(args): write = sys.stdout.buffer.write if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif options.month is None: - write(cal.formatyearpage(options.year, **optdict)) else: - parser.error("incorrect number of arguments") - sys.exit(1) + write(cal.formatyearpage(options.year, **optdict)) else: if options.locale: cal = LocaleTextCalendar(locale=locale) @@ -795,4 +793,4 @@ def main(args): if __name__ == "__main__": - main(sys.argv) + main() diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 0df084e17a3c8e..1f9ffc5e9a5c33 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -3,11 +3,13 @@ from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -import time -import locale -import sys +import contextlib import datetime +import io +import locale import os +import sys +import time # From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday result_0_02_text = """\ @@ -549,26 +551,92 @@ def test_months(self): # verify it "acts like a sequence" in two forms of iteration self.assertEqual(value[::-1], list(reversed(value))) - def test_locale_calendars(self): + def test_locale_text_calendar(self): + try: + cal = calendar.LocaleTextCalendar(locale='') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale=None) + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + cal = calendar.LocaleTextCalendar(locale='C') + local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) + local_month = cal.formatmonthname(2010, 10, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) + self.assertIsInstance(local_month, str) + self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) + self.assertGreaterEqual(len(local_month), 10) + + def test_locale_html_calendar(self): + try: + cal = calendar.LocaleHTMLCalendar(locale='') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + except locale.Error: + # cannot set the system default locale -- skip rest of test + raise unittest.SkipTest('cannot set the system default locale') + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale=None) + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + cal = calendar.LocaleHTMLCalendar(locale='C') + local_weekday = cal.formatweekday(1) + local_month = cal.formatmonthname(2010, 10) + self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_month, str) + + def test_locale_calendars_reset_locale_properly(self): # ensure that Locale{Text,HTML}Calendar resets the locale properly # (it is still not thread-safe though) old_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) try: cal = calendar.LocaleTextCalendar(locale='') local_weekday = cal.formatweekday(1, 10) + local_weekday_abbr = cal.formatweekday(1, 3) local_month = cal.formatmonthname(2010, 10, 10) except locale.Error: # cannot set the system default locale -- skip rest of test raise unittest.SkipTest('cannot set the system default locale') self.assertIsInstance(local_weekday, str) + self.assertIsInstance(local_weekday_abbr, str) self.assertIsInstance(local_month, str) self.assertEqual(len(local_weekday), 10) + self.assertEqual(len(local_weekday_abbr), 3) self.assertGreaterEqual(len(local_month), 10) + cal = calendar.LocaleHTMLCalendar(locale='') local_weekday = cal.formatweekday(1) local_month = cal.formatmonthname(2010, 10) self.assertIsInstance(local_weekday, str) self.assertIsInstance(local_month, str) + new_october = calendar.TextCalendar().formatmonthname(2010, 10, 10) self.assertEqual(old_october, new_october) @@ -589,6 +657,21 @@ def test_locale_calendar_formatweekday(self): except locale.Error: raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_calendar_formatmonthname(self): + try: + # formatmonthname uses the same month names regardless of the width argument. + cal = calendar.LocaleTextCalendar(locale='en_US') + # For too short widths, a full name (with year) is used. + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 2, withyear=True), "June 2022") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=False), "June") + self.assertEqual(cal.formatmonthname(2022, 6, 3, withyear=True), "June 2022") + # For long widths, a centered name is used. + self.assertEqual(cal.formatmonthname(2022, 6, 10, withyear=False), " June ") + self.assertEqual(cal.formatmonthname(2022, 6, 15, withyear=True), " June 2022 ") + except locale.Error: + raise unittest.SkipTest('cannot set the en_US locale') + def test_locale_html_calendar_custom_css_class_month_name(self): try: cal = calendar.LocaleHTMLCalendar(locale='') @@ -847,46 +930,104 @@ def conv(s): return s.replace('\n', os.linesep).encode() class CommandLineTestCase(unittest.TestCase): - def run_ok(self, *args): + def setUp(self): + self.runners = [self.run_cli_ok, self.run_cmd_ok] + + @contextlib.contextmanager + def captured_stdout_with_buffer(self): + orig_stdout = sys.stdout + buffer = io.BytesIO() + sys.stdout = io.TextIOWrapper(buffer) + try: + yield sys.stdout + finally: + sys.stdout.flush() + sys.stdout.buffer.seek(0) + sys.stdout = orig_stdout + + @contextlib.contextmanager + def captured_stderr_with_buffer(self): + orig_stderr = sys.stderr + buffer = io.BytesIO() + sys.stderr = io.TextIOWrapper(buffer) + try: + yield sys.stderr + finally: + sys.stderr.flush() + sys.stderr.buffer.seek(0) + sys.stderr = orig_stderr + + def run_cli_ok(self, *args): + with self.captured_stdout_with_buffer() as stdout: + calendar.main(args) + return stdout.buffer.read() + + def run_cmd_ok(self, *args): return assert_python_ok('-m', 'calendar', *args)[1] - def assertFailure(self, *args): + def assertCLIFails(self, *args): + with self.captured_stderr_with_buffer() as stderr: + self.assertRaises(SystemExit, calendar.main, args) + stderr = stderr.buffer.read() + self.assertIn(b'usage:', stderr) + return stderr + + def assertCmdFails(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) + return rc, stdout, stderr + + def assertFailure(self, *args): + self.assertCLIFails(*args) + self.assertCmdFails(*args) def test_help(self): - stdout = self.run_ok('-h') + stdout = self.run_cmd_ok('-h') self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) + # special case: stdout but sys.exit() + with self.captured_stdout_with_buffer() as output: + self.assertRaises(SystemExit, calendar.main, ['-h']) + output = output.buffer.read() + self.assertIn(b'usage:', output) + self.assertIn(b'--help', output) + def test_illegal_arguments(self): self.assertFailure('-z') self.assertFailure('spam') self.assertFailure('2004', 'spam') + self.assertFailure('2004', '1', 'spam') + self.assertFailure('2004', '1', '1') + self.assertFailure('2004', '1', '1', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): - stdout = self.run_ok() - year = datetime.datetime.now().year - self.assertIn((' %s' % year).encode(), stdout) - self.assertIn(b'January', stdout) - self.assertIn(b'Mo Tu We Th Fr Sa Su', stdout) + for run in self.runners: + output = run() + year = datetime.datetime.now().year + self.assertIn(conv(' %s' % year), output) + self.assertIn(b'January', output) + self.assertIn(b'Mo Tu We Th Fr Sa Su', output) def test_output_year(self): - stdout = self.run_ok('2004') - self.assertEqual(stdout, conv(result_2004_text)) + for run in self.runners: + output = run('2004') + self.assertEqual(output, conv(result_2004_text)) def test_output_month(self): - stdout = self.run_ok('2004', '1') - self.assertEqual(stdout, conv(result_2004_01_text)) + for run in self.runners: + output = run('2004', '1') + self.assertEqual(output, conv(result_2004_01_text)) def test_option_encoding(self): self.assertFailure('-e') self.assertFailure('--encoding') - stdout = self.run_ok('--encoding', 'utf-16-le', '2004') - self.assertEqual(stdout, result_2004_text.encode('utf-16-le')) + for run in self.runners: + output = run('--encoding', 'utf-16-le', '2004') + self.assertEqual(output, result_2004_text.encode('utf-16-le')) def test_option_locale(self): self.assertFailure('-L') @@ -904,66 +1045,75 @@ def test_option_locale(self): locale.setlocale(locale.LC_TIME, oldlocale) except (locale.Error, ValueError): self.skipTest('cannot set the system default locale') - stdout = self.run_ok('--locale', lang, '--encoding', enc, '2004') - self.assertIn('2004'.encode(enc), stdout) + for run in self.runners: + for type in ('text', 'html'): + output = run( + '--type', type, '--locale', lang, '--encoding', enc, '2004' + ) + self.assertIn('2004'.encode(enc), output) def test_option_width(self): self.assertFailure('-w') self.assertFailure('--width') self.assertFailure('-w', 'spam') - stdout = self.run_ok('--width', '3', '2004') - self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', stdout) + for run in self.runners: + output = run('--width', '3', '2004') + self.assertIn(b'Mon Tue Wed Thu Fri Sat Sun', output) def test_option_lines(self): self.assertFailure('-l') self.assertFailure('--lines') self.assertFailure('-l', 'spam') - stdout = self.run_ok('--lines', '2', '2004') - self.assertIn(conv('December\n\nMo Tu We'), stdout) + for run in self.runners: + output = run('--lines', '2', '2004') + self.assertIn(conv('December\n\nMo Tu We'), output) def test_option_spacing(self): self.assertFailure('-s') self.assertFailure('--spacing') self.assertFailure('-s', 'spam') - stdout = self.run_ok('--spacing', '8', '2004') - self.assertIn(b'Su Mo', stdout) + for run in self.runners: + output = run('--spacing', '8', '2004') + self.assertIn(b'Su Mo', output) def test_option_months(self): self.assertFailure('-m') self.assertFailure('--month') self.assertFailure('-m', 'spam') - stdout = self.run_ok('--months', '1', '2004') - self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), stdout) + for run in self.runners: + output = run('--months', '1', '2004') + self.assertIn(conv('\nMo Tu We Th Fr Sa Su\n'), output) def test_option_type(self): self.assertFailure('-t') self.assertFailure('--type') self.assertFailure('-t', 'spam') - stdout = self.run_ok('--type', 'text', '2004') - self.assertEqual(stdout, conv(result_2004_text)) - stdout = self.run_ok('--type', 'html', '2004') - self.assertEqual(stdout[:6], b'Calendar for 2004', stdout) + for run in self.runners: + output = run('--type', 'text', '2004') + self.assertEqual(output, conv(result_2004_text)) + output = run('--type', 'html', '2004') + self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): - stdout = self.run_ok('--type', 'html') - year = datetime.datetime.now().year - self.assertIn(('Calendar for %s' % year).encode(), - stdout) - self.assertIn(b'January', - stdout) + for run in self.runners: + output = run('--type', 'html') + year = datetime.datetime.now().year + self.assertIn(('Calendar for %s' % year).encode(), output) + self.assertIn(b'January', output) def test_html_output_year_encoding(self): - stdout = self.run_ok('-t', 'html', '--encoding', 'ascii', '2004') - self.assertEqual(stdout, - result_2004_html.format(**default_format).encode('ascii')) + for run in self.runners: + output = run('-t', 'html', '--encoding', 'ascii', '2004') + self.assertEqual(output, result_2004_html.format(**default_format).encode('ascii')) def test_html_output_year_css(self): self.assertFailure('-t', 'html', '-c') self.assertFailure('-t', 'html', '--css') - stdout = self.run_ok('-t', 'html', '--css', 'custom.css', '2004') - self.assertIn(b'', stdout) + for run in self.runners: + output = run('-t', 'html', '--css', 'custom.css', '2004') + self.assertIn(b'', output) class MiscTestCase(unittest.TestCase): diff --git a/Misc/ACKS b/Misc/ACKS index 645ad5b700baaa..fadf488888aa8b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1700,6 +1700,7 @@ Ngalim Siregar Kragen Sitaker Kaartic Sivaraam Stanisław Skonieczny +Bart Skowron Roman Skurikhin Ville Skyttä Michael Sloan diff --git a/Misc/NEWS.d/next/Tests/2022-06-09-21-27-38.gh-issue-69714.49tyHW.rst b/Misc/NEWS.d/next/Tests/2022-06-09-21-27-38.gh-issue-69714.49tyHW.rst new file mode 100644 index 00000000000000..e28b94a171c40e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-06-09-21-27-38.gh-issue-69714.49tyHW.rst @@ -0,0 +1 @@ +Add additional tests to :mod:`calendar` to achieve full test coverage. From 756add081ead8fa9aeb11ac178e581ba2450f296 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 15:30:13 +0200 Subject: [PATCH 17/73] gh-106320: Document private C API removal in What's New 3.13 (#107027) --- Doc/whatsnew/3.13.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a778e12aba1bce..85f0ca1ae1a678 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -838,6 +838,13 @@ Deprecated Removed ------- +* Remove many APIs (functions, macros, variables) with names prefixed by + ``_Py`` or ``_PY`` (considered as private API). If your project is affected + by one of these removals and you consider that the removed API should remain + available, please open a new issue to request a public C API and + add ``cc @vstinner`` to the issue to notify Victor Stinner. + (Contributed by Victor Stinner in :gh:`106320`.) + * Remove functions deprecated in Python 3.9. * ``PyEval_CallObject()``, ``PyEval_CallObjectWithKeywords()``: use From 89f987544860d1360e6232b6f52b8ced92a4d690 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 15:49:37 +0200 Subject: [PATCH 18/73] gh-106320: Move private _PyHash API to the internal C API (#107026) * No longer export most private _PyHash symbols, only export the ones which are needed by shared extensions. * Modules/_xxtestfuzz/fuzzer.c now uses the internal C API. --- Include/internal/pycore_pyhash.h | 85 ++++++++++++++++++++++++++++++-- Include/pyhash.h | 84 ++----------------------------- Modules/_elementtree.c | 1 + Modules/_hashopenssl.c | 3 +- Modules/_xxtestfuzz/fuzzer.c | 5 ++ Modules/pyexpat.c | 1 + Python/hashtable.c | 1 + Python/pyhash.c | 1 + 8 files changed, 94 insertions(+), 87 deletions(-) diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 34dfa53771288e..59e416ca18a729 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -1,10 +1,86 @@ -#ifndef Py_INTERNAL_HASH_H -#define Py_INTERNAL_HASH_H +#ifndef Py_INTERNAL_PYHASH_H +#define Py_INTERNAL_PYHASH_H #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif +/* Helpers for hash functions */ +extern Py_hash_t _Py_HashDouble(PyObject *, double); +// _decimal shared extensions uses _Py_HashPointer() +PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); +// Similar to _Py_HashPointer(), but don't replace -1 with -2 +extern Py_hash_t _Py_HashPointerRaw(const void*); +// _datetime shared extension uses _Py_HashBytes() +PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); + +/* Prime multiplier used in string and various other hashes. */ +#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ + +/* Parameters used for the numeric hash implementation. See notes for + _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on + reduction modulo the prime 2**_PyHASH_BITS - 1. */ + +#if SIZEOF_VOID_P >= 8 +# define _PyHASH_BITS 61 +#else +# define _PyHASH_BITS 31 +#endif + +#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) +#define _PyHASH_INF 314159 +#define _PyHASH_IMAG _PyHASH_MULTIPLIER + +/* Hash secret + * + * memory layout on 64 bit systems + * cccccccc cccccccc cccccccc uc -- unsigned char[24] + * pppppppp ssssssss ........ fnv -- two Py_hash_t + * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t + * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t + * ........ ........ eeeeeeee pyexpat XML hash salt + * + * memory layout on 32 bit systems + * cccccccc cccccccc cccccccc uc + * ppppssss ........ ........ fnv -- two Py_hash_t + * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) + * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t + * ........ ........ eeee.... pyexpat XML hash salt + * + * (*) The siphash member may not be available on 32 bit platforms without + * an unsigned int64 data type. + */ +typedef union { + /* ensure 24 bytes */ + unsigned char uc[24]; + /* two Py_hash_t for FNV */ + struct { + Py_hash_t prefix; + Py_hash_t suffix; + } fnv; + /* two uint64 for SipHash24 */ + struct { + uint64_t k0; + uint64_t k1; + } siphash; + /* a different (!) Py_hash_t for small string optimization */ + struct { + unsigned char padding[16]; + Py_hash_t suffix; + } djbx33a; + struct { + unsigned char padding[16]; + Py_hash_t hashsalt; + } expat; +} _Py_HashSecret_t; + +// _elementtree shared extension uses _Py_HashSecret.expat +PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; + +#ifdef Py_DEBUG +extern int _Py_HashSecret_Initialized; +#endif + struct pyhash_runtime_state { struct { @@ -34,7 +110,6 @@ struct pyhash_runtime_state { } -uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t); - +extern uint64_t _Py_KeyedHash(uint64_t key, const void *src, Py_ssize_t src_sz); -#endif // Py_INTERNAL_HASH_H +#endif // !Py_INTERNAL_PYHASH_H diff --git a/Include/pyhash.h b/Include/pyhash.h index 182d223fab1cac..6e969f86fa2625 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -1,87 +1,10 @@ #ifndef Py_HASH_H - #define Py_HASH_H #ifdef __cplusplus extern "C" { #endif -/* Helpers for hash functions */ #ifndef Py_LIMITED_API -PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); -PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); -// Similar to _Py_HashPointer(), but don't replace -1 with -2 -PyAPI_FUNC(Py_hash_t) _Py_HashPointerRaw(const void*); -PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); -#endif - -/* Prime multiplier used in string and various other hashes. */ -#define _PyHASH_MULTIPLIER 1000003UL /* 0xf4243 */ - -/* Parameters used for the numeric hash implementation. See notes for - _Py_HashDouble in Python/pyhash.c. Numeric hashes are based on - reduction modulo the prime 2**_PyHASH_BITS - 1. */ - -#if SIZEOF_VOID_P >= 8 -# define _PyHASH_BITS 61 -#else -# define _PyHASH_BITS 31 -#endif - -#define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) -#define _PyHASH_INF 314159 -#define _PyHASH_IMAG _PyHASH_MULTIPLIER - - -/* hash secret - * - * memory layout on 64 bit systems - * cccccccc cccccccc cccccccc uc -- unsigned char[24] - * pppppppp ssssssss ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t - * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt - * - * memory layout on 32 bit systems - * cccccccc cccccccc cccccccc uc - * ppppssss ........ ........ fnv -- two Py_hash_t - * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) - * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt - * - * (*) The siphash member may not be available on 32 bit platforms without - * an unsigned int64 data type. - */ -#ifndef Py_LIMITED_API -typedef union { - /* ensure 24 bytes */ - unsigned char uc[24]; - /* two Py_hash_t for FNV */ - struct { - Py_hash_t prefix; - Py_hash_t suffix; - } fnv; - /* two uint64 for SipHash24 */ - struct { - uint64_t k0; - uint64_t k1; - } siphash; - /* a different (!) Py_hash_t for small string optimization */ - struct { - unsigned char padding[16]; - Py_hash_t suffix; - } djbx33a; - struct { - unsigned char padding[16]; - Py_hash_t hashsalt; - } expat; -} _Py_HashSecret_t; -PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; - -#ifdef Py_DEBUG -PyAPI_DATA(int) _Py_HashSecret_Initialized; -#endif - - /* hash function definition */ typedef struct { Py_hash_t (*const hash)(const void *, Py_ssize_t); @@ -94,7 +17,7 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #endif -/* cutoff for small string DJBX33A optimization in range [1, cutoff). +/* Cutoff for small string DJBX33A optimization in range [1, cutoff). * * About 50% of the strings in a typical Python application are smaller than * 6 to 7 chars. However DJBX33A is vulnerable to hash collision attacks. @@ -112,7 +35,7 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #endif /* Py_HASH_CUTOFF */ -/* hash algorithm selection +/* Hash algorithm selection * * The values for Py_HASH_* are hard-coded in the * configure script. @@ -140,5 +63,4 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #ifdef __cplusplus } #endif - -#endif /* !Py_HASH_H */ +#endif // !Py_HASH_H diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index a8d68d68420d36..30327e0b6fc31f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -17,6 +17,7 @@ #include "Python.h" #include "pycore_import.h" // _PyImport_GetModuleAttrString() +#include "pycore_pyhash.h" // _Py_HashSecret #include "structmember.h" // PyMemberDef #include "expat.h" #include "pyexpat.h" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 246eea74098820..6d60037cd2054e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -24,8 +24,9 @@ #include "Python.h" #include "pycore_hashtable.h" -#include "hashlib.h" +#include "pycore_pyhash.h" // _Py_HashBytes() #include "pycore_strhex.h" // _Py_strhex() +#include "hashlib.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index 37d402824853f0..54f8a42273401f 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -10,7 +10,12 @@ See the source code for LLVMFuzzerTestOneInput for details. */ +#ifndef Py_BUILD_CORE +# define Py_BUILD_CORE 1 +#endif + #include +#include "pycore_pyhash.h" // _Py_HashBytes() #include #include diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index bd8a98a46579a3..33a56c57a876ec 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_import.h" // _PyImport_SetModule() +#include "pycore_pyhash.h" // _Py_HashSecret #include #include "structmember.h" // PyMemberDef diff --git a/Python/hashtable.c b/Python/hashtable.c index 09501de199b0e6..0f07cd20b1a95e 100644 --- a/Python/hashtable.c +++ b/Python/hashtable.c @@ -46,6 +46,7 @@ #include "Python.h" #include "pycore_hashtable.h" +#include "pycore_pyhash.h" // _Py_HashPointerRaw() #define HASHTABLE_MIN_SIZE 16 #define HASHTABLE_HIGH 0.50 diff --git a/Python/pyhash.c b/Python/pyhash.c index d5ac9f83be61cc..b2bdab5099d86a 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -4,6 +4,7 @@ All the utility functions (_Py_Hash*()) return "-1" to signify an error. */ #include "Python.h" +#include "pycore_pyhash.h" // _Py_HashSecret_t #ifdef __APPLE__ # include From eda9ce148714947c164ff72b720e2645ca2cd5fb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 16:12:17 +0200 Subject: [PATCH 19/73] gh-106320: Move _PyNone_Type to the internal C API (#107030) Move private types _PyNone_Type and _PyNotImplemented_Type to internal C API. --- Include/cpython/object.h | 3 --- Include/internal/pycore_object.h | 4 ++++ Modules/_pickle.c | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index cd421b4f7e0d49..0e5b6acb43b416 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -377,9 +377,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -PyAPI_DATA(PyTypeObject) _PyNone_Type; -PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; - /* Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. * Defined in object.c. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2358f48738a905..90588daa64cc3b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -438,6 +438,10 @@ extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( (meth)((self), (args), (kw)) #endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE +// _pickle shared extension uses _PyNone_Type and _PyNotImplemented_Type +PyAPI_DATA(PyTypeObject) _PyNone_Type; +PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; + #ifdef __cplusplus } #endif diff --git a/Modules/_pickle.c b/Modules/_pickle.c index ea44b494cdd7cd..f2e98c10ef7188 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -12,6 +12,7 @@ #include "pycore_bytesobject.h" // _PyBytesWriter #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyNone_Type #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "structmember.h" // PyMemberDef From ae8b114c5b9d211f47bf521fcfdbf40ef72a1bac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 16:21:16 +0200 Subject: [PATCH 20/73] gh-106320: Move private _PyGen API to the internal C API (#107032) Move private _PyGen API to internal C API: * _PyAsyncGenAThrow_Type * _PyAsyncGenWrappedValue_Type * _PyCoroWrapper_Type * _PyGen_FetchStopIterationValue() * _PyGen_Finalize() * _PyGen_SetStopIterationValue() No longer these symbols, except of the ones by the _asyncio shared extensions. --- Include/cpython/genobject.h | 6 ------ Include/internal/pycore_genobject.h | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 7856481b5db300..49e46c277d75ae 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -41,9 +41,6 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); -PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); -PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); -PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -54,7 +51,6 @@ typedef struct { } PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; -PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, @@ -69,8 +65,6 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenWrappedValue_Type; -PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type; PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index dc60b4ca705112..fb63d9c3537a67 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -9,9 +9,20 @@ extern "C" { #endif extern PyObject *_PyGen_yf(PyGenObject *); +extern void _PyGen_Finalize(PyObject *self); + +// _asyncio shared extensions uses _PyGen_SetStopIterationValue() and +// _PyGen_FetchStopIterationValue() +PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); +PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); + extern PyObject *_PyCoro_GetAwaitableIter(PyObject *o); extern PyObject *_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +extern PyTypeObject _PyCoroWrapper_Type; +extern PyTypeObject _PyAsyncGenWrappedValue_Type; +extern PyTypeObject _PyAsyncGenAThrow_Type; + /* runtime lifecycle */ extern void _PyAsyncGen_Fini(PyInterpreterState *); From e5252c6127ad788b39221982964f935bd1513028 Mon Sep 17 00:00:00 2001 From: Jocelyn Castellano Date: Sat, 22 Jul 2023 10:29:08 -0400 Subject: [PATCH 21/73] gh-105090: Replace incorrect TLSv1.2 with TLSv1.3 (#105404) --- Doc/library/ssl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 7f095526327685..1f8bbe1e3de3ee 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2592,7 +2592,7 @@ disabled by default. >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.2 and later (if +The SSL context created above will only allow TLSv1.3 and later (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. From d228825e08883fc13f35eb91435f95d32524931c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 16:45:56 +0200 Subject: [PATCH 22/73] gh-106320: Remove _PyOS_ReadlineTState API (#107034) Remove _PyOS_ReadlineTState variable from the public C API. The symbol is still exported for the readline shared extension. --- Include/cpython/pythonrun.h | 1 - Modules/readline.c | 3 +++ Parser/myreadline.c | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index fb617655374026..3b2537e01b83b1 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -117,5 +117,4 @@ PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject /* Stuff with no proper home (yet) */ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); -PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); diff --git a/Modules/readline.c b/Modules/readline.c index a592919692cb83..6729a09cb0da5e 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1313,6 +1313,9 @@ rlhandler(char *text) static char * readline_until_enter_or_signal(const char *prompt, int *signal) { + // Defined in Parser/myreadline.c + extern PyThreadState *_PyOS_ReadlineTState; + char * not_done_reading = ""; fd_set selectset; diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 7074aba74b728c..815387388218c6 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -20,7 +20,9 @@ #endif /* MS_WINDOWS */ -PyThreadState* _PyOS_ReadlineTState = NULL; +// Export the symbol since it's used by the readline shared extension +PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; +PyThreadState *_PyOS_ReadlineTState = NULL; static PyThread_type_lock _PyOS_ReadlineLock = NULL; From 3782def5a2b4d24ab5d356f89da181e99a9a59b2 Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Sat, 22 Jul 2023 16:46:59 +0200 Subject: [PATCH 23/73] gh-65495: Use lowercase `mail from` and `rcpt to` in `smtplib.SMTP` (#107019) Use lowercase `mail from` and `rcpt to` in `smtplib.SMTP` SMTP commands are case-insensitive. `smtplib` uses lowercase commands, however it writes `mail FROM` and `rcpt TO`, lacking consistency. --- Lib/smtplib.py | 4 ++-- Lib/test/test_smtplib.py | 14 ++++++++++++++ .../2023-07-22-14-29-34.gh-issue-65495.fw84qM.rst | 1 + 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-14-29-34.gh-issue-65495.fw84qM.rst diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 18c91746fd7bf2..b3cc68a789a7d8 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -542,7 +542,7 @@ def mail(self, sender, options=()): raise SMTPNotSupportedError( 'SMTPUTF8 not supported by server') optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) + self.putcmd("mail", "from:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() def rcpt(self, recip, options=()): @@ -550,7 +550,7 @@ def rcpt(self, recip, options=()): optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) + self.putcmd("rcpt", "to:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() def data(self, msg): diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index b6d5b8c3d82580..f2e02dab1c3ca5 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -831,6 +831,7 @@ class SimSMTPChannel(smtpd.SMTPChannel): def __init__(self, extra_features, *args, **kw): self._extrafeatures = ''.join( [ "250-{0}\r\n".format(x) for x in extra_features ]) + self.all_received_lines = [] super(SimSMTPChannel, self).__init__(*args, **kw) # AUTH related stuff. It would be nice if support for this were in smtpd. @@ -845,6 +846,7 @@ def found_terminator(self): self.smtp_state = self.COMMAND self.push('%s %s' % (e.smtp_code, e.smtp_error)) return + self.all_received_lines.append(self.received_lines) super().found_terminator() @@ -1349,6 +1351,18 @@ def test_name_field_not_included_in_envelop_addresses(self): self.assertEqual(self.serv._addresses['from'], 'michael@example.com') self.assertEqual(self.serv._addresses['tos'], ['rene@example.com']) + def test_lowercase_mail_from_rcpt_to(self): + m = 'A test message' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + + smtp.sendmail('John', 'Sally', m) + + self.assertIn(['mail from: size=14'], self.serv._SMTPchannel.all_received_lines) + self.assertIn(['rcpt to:'], self.serv._SMTPchannel.all_received_lines) + class SimSMTPUTF8Server(SimSMTPServer): diff --git a/Misc/NEWS.d/next/Library/2023-07-22-14-29-34.gh-issue-65495.fw84qM.rst b/Misc/NEWS.d/next/Library/2023-07-22-14-29-34.gh-issue-65495.fw84qM.rst new file mode 100644 index 00000000000000..e75b6c0f1d6759 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-14-29-34.gh-issue-65495.fw84qM.rst @@ -0,0 +1 @@ +Use lowercase ``mail from`` and ``rcpt to`` in :class:`smptlib.SMTP`. From c92ef6fe0e1384c090b94143cdc01e5e114a8747 Mon Sep 17 00:00:00 2001 From: Oliver Rew <12968411+oliver-rew@users.noreply.github.com> Date: Sat, 22 Jul 2023 16:57:51 +0200 Subject: [PATCH 24/73] gh-106967: remove Release and Date fields from whatsnew for 3.12 and 3.13 (#107000) --- Doc/whatsnew/3.12.rst | 3 +-- Doc/whatsnew/3.13.rst | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a46621071b3d8b..ac03af52db0716 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -3,8 +3,7 @@ What's New In Python 3.12 **************************** -:Release: |release| -:Date: |today| +:Editor: TBD .. Rules for maintenance: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 85f0ca1ae1a678..5d4fa44aab695c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,8 +3,7 @@ What's New In Python 3.13 **************************** -:Release: |release| -:Date: |today| +:Editor: TBD .. Rules for maintenance: From 5e4af2a3e90d78fdf6a4cc42e0472576b1f2b975 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 17:04:34 +0200 Subject: [PATCH 25/73] gh-106320: Move private _PySet API to the internal API (#107041) * Add pycore_setobject.h header file. * Move the following API to the internal C API: * _PySet_Dummy * _PySet_NextEntry() * _PySet_Update() --- Include/cpython/setobject.h | 5 ----- Include/internal/pycore_pymem.h | 2 +- Include/internal/pycore_setobject.h | 25 +++++++++++++++++++++++++ Makefile.pre.in | 1 + Modules/_abc.c | 1 + Modules/_pickle.c | 1 + Objects/codeobject.c | 1 + Objects/dictobject.c | 1 + Objects/setobject.c | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 +++ Python/ast_opt.c | 5 +++-- Python/bytecodes.c | 7 ++++--- Python/ceval.c | 7 ++++--- Python/compile.c | 3 ++- Python/executor.c | 1 + Python/marshal.c | 3 ++- Python/pylifecycle.c | 1 + 18 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 Include/internal/pycore_setobject.h diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index 20fd63eaae56e2..1778c778a05324 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -65,8 +65,3 @@ static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { return _PySet_CAST(so)->used; } #define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) - -PyAPI_DATA(PyObject *) _PySet_Dummy; - -PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash); -PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 81a707a0a5ddf3..c2f03254bb8760 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -95,4 +95,4 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_PYMEM_H */ +#endif // !Py_INTERNAL_PYMEM_H diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h new file mode 100644 index 00000000000000..96f9aea7be9fa7 --- /dev/null +++ b/Include/internal/pycore_setobject.h @@ -0,0 +1,25 @@ +#ifndef Py_INTERNAL_SETOBJECT_H +#define Py_INTERNAL_SETOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// _pickle shared extension uses _PySet_NextEntry() and _PySet_Update() +PyAPI_FUNC(int) _PySet_NextEntry( + PyObject *set, + Py_ssize_t *pos, + PyObject **key, + Py_hash_t *hash); +PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); + +// Export _PySet_Dummy for the gdb plugin's benefit +PyAPI_DATA(PyObject *) _PySet_Dummy; + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_SETOBJECT_H diff --git a/Makefile.pre.in b/Makefile.pre.in index f3a0db163ed48c..a4f76eaaaba525 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1798,6 +1798,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_runtime.h \ $(srcdir)/Include/internal/pycore_runtime_init_generated.h \ $(srcdir)/Include/internal/pycore_runtime_init.h \ + $(srcdir)/Include/internal/pycore_setobject.h \ $(srcdir)/Include/internal/pycore_signal.h \ $(srcdir)/Include/internal/pycore_sliceobject.h \ $(srcdir)/Include/internal/pycore_strhex.h \ diff --git a/Modules/_abc.c b/Modules/_abc.c index 8a3aa9cb88880f..9473905243d438 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyType_GetMRO() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "clinic/_abc.c.h" diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f2e98c10ef7188..d9395e35daf8b4 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -15,6 +15,7 @@ #include "pycore_object.h" // _PyNone_Type #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "structmember.h" // PyMemberDef #include // strtol() diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d2670c71caa44a..977e5b05b3d2a5 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -8,6 +8,7 @@ #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs #include "pycore_opcode.h" // _PyOpcode_Deopt #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "clinic/codeobject.c.h" diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 3f9297c8a411fc..26e2dc31e3664a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -121,6 +121,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "stringlib/eq.h" // unicode_eq() #include diff --git a/Objects/setobject.c b/Objects/setobject.c index 4ac541b9509752..2f12320254ec94 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -34,6 +34,7 @@ #include "Python.h" #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_setobject.h" // _PySet_NextEntry() definition #include // offsetof() /* Object used as dummy key to fill deleted entries */ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 0fe24082ca4ccd..bfe59acf12a69d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -261,6 +261,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 2b3793cb0f76e2..0a8b0c3faf51e1 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -687,6 +687,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 276e910089a277..ad1e312b084b74 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1,9 +1,10 @@ /* AST Optimizer */ #include "Python.h" #include "pycore_ast.h" // _PyAST_GetDocString() -#include "pycore_long.h" // _PyLong -#include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_format.h" // F_LJUST +#include "pycore_long.h" // _PyLong +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_setobject.h" // _PySet_NextEntry() typedef struct { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3a191aca730e91..7ce8978e998b97 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -10,23 +10,24 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" +#include "pycore_instruments.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_instruments.h" -#include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode.h" // EXTRA_CASES #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() -#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_dict.h" #include "dictobject.h" diff --git a/Python/ceval.c b/Python/ceval.c index 9f2855f964ace1..e9f082f85bd1b4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -7,24 +7,25 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SignalAsyncExc() #include "pycore_code.h" +#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" +#include "pycore_instruments.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() -#include "pycore_instruments.h" -#include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode.h" // EXTRA_CASES #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject +#include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uops.h" // _PyUOpExecutorObject -#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_dict.h" #include "dictobject.h" diff --git a/Python/compile.c b/Python/compile.c index 9a13ab525d8a47..600118768e6392 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -28,12 +28,13 @@ #define NEED_OPCODE_TABLES #include "pycore_opcode_utils.h" #undef NEED_OPCODE_TABLES -#include "pycore_flowgraph.h" #include "pycore_code.h" // _PyCode_New() #include "pycore_compile.h" +#include "pycore_flowgraph.h" #include "pycore_intrinsics.h" #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_pystate.h" // _Py_GetConfig() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_symtable.h" // PySTEntryObject, _PyFuture_FromAST() #define NEED_OPCODE_METADATA diff --git a/Python/executor.c b/Python/executor.c index 17322526a7bd7c..57525df202d861 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -11,6 +11,7 @@ #include "pycore_opcode_utils.h" #include "pycore_pyerrors.h" #include "pycore_range.h" +#include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" #include "pycore_uops.h" diff --git a/Python/marshal.c b/Python/marshal.c index 517220a4463cf3..8940582c7f5328 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -9,8 +9,9 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_code.h" // _PyCode_New() -#include "pycore_long.h" // _PyLong_DigitCount #include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_long.h" // _PyLong_DigitCount +#include "pycore_setobject.h" // _PySet_NextEntry() #include "marshal.h" // Py_MARSHAL_VERSION /*[clinic input] diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f91fac9379ec51..ceca7776a276ed 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -24,6 +24,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_sliceobject.h" // _PySlice_Fini() #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() From 149748ea4f552e6fe43a1d6d69bd65910a7c4813 Mon Sep 17 00:00:00 2001 From: wulmer Date: Sat, 22 Jul 2023 17:44:44 +0200 Subject: [PATCH 26/73] Fix Sphinx warnings in `re` module docs (#107044) --- Doc/library/re.rst | 23 ++++++++++++++++++----- Doc/tools/.nitignore | 1 - 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index b7510b93d75427..629ee472cca681 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -501,6 +501,8 @@ The special characters are: in the ASCII range (``b'\x00'``-``b'\x7f'``). +.. _re-special-sequences: + The special sequences consist of ``'\'`` and a character from the list below. If the ordinary character is not an ASCII digit or an ASCII letter, then the resulting RE will match the second character. For example, ``\$`` matches the @@ -779,6 +781,17 @@ Flags Corresponds to the inline flag ``(?s)``. +.. data:: U + UNICODE + + In Python 2, this flag made :ref:`special sequences ` + include Unicode characters in matches. Since Python 3, Unicode characters + are matched by default. + + See :const:`A` for restricting matching on ASCII characters instead. + + This flag is only kept for backward compatibility. + .. data:: X VERBOSE @@ -1520,14 +1533,14 @@ Simulating scanf() .. index:: single: scanf() -Python does not currently have an equivalent to :c:func:`scanf`. Regular +Python does not currently have an equivalent to :c:func:`!scanf`. Regular expressions are generally more powerful, though also more verbose, than -:c:func:`scanf` format strings. The table below offers some more-or-less -equivalent mappings between :c:func:`scanf` format tokens and regular +:c:func:`!scanf` format strings. The table below offers some more-or-less +equivalent mappings between :c:func:`!scanf` format tokens and regular expressions. +--------------------------------+---------------------------------------------+ -| :c:func:`scanf` Token | Regular Expression | +| :c:func:`!scanf` Token | Regular Expression | +================================+=============================================+ | ``%c`` | ``.`` | +--------------------------------+---------------------------------------------+ @@ -1552,7 +1565,7 @@ To extract the filename and numbers from a string like :: /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`scanf` format like :: +you would use a :c:func:`!scanf` format like :: %s - %d errors, %d warnings diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ca6f32c55acd69..87157f0eff4c4d 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -174,7 +174,6 @@ Doc/library/pyclbr.rst Doc/library/pydoc.rst Doc/library/pyexpat.rst Doc/library/random.rst -Doc/library/re.rst Doc/library/readline.rst Doc/library/reprlib.rst Doc/library/resource.rst From 6e5f2235f3754307292c7d8d3698958136b5e311 Mon Sep 17 00:00:00 2001 From: Matthieu Caneill Date: Sat, 22 Jul 2023 17:58:06 +0200 Subject: [PATCH 27/73] gh-83006: Document behavior of `shutil.disk_usage` for non-mounted filesystems on Unix (#107031) --- Doc/library/shutil.rst | 6 ++++++ .../Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 5bd80b10a65806..7699d22a72aaff 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -399,6 +399,12 @@ Directory and files operations total, used and free space, in bytes. *path* may be a file or a directory. + .. note:: + + On Unix filesystems, *path* must point to a path within a **mounted** + filesystem partition. On those platforms, CPython doesn't attempt to + retrieve disk usage information from non-mounted filesystems. + .. versionadded:: 3.3 .. versionchanged:: 3.8 diff --git a/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst b/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst new file mode 100644 index 00000000000000..e64d1860828430 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst @@ -0,0 +1,2 @@ +Document behavior of :func:`shutil.disk_usage` for non-mounted filesystems +on Unix. From 9c38206925246ab919cf558ac069ae9458720ba7 Mon Sep 17 00:00:00 2001 From: Moritz Neeb Date: Sat, 22 Jul 2023 17:59:11 +0200 Subject: [PATCH 28/73] gh-75371: reformat Makefile.pre.in to accommodate for empty FRAMEWORKALTINSTALLLAST (#107035) in the case of an empty FRAMEWORKALTINSTALLLAST, this patch prevents leaving an astray linebreak and two tabs in the resulting Makefile. Before change: ``` .PHONY: commoninstall commoninstall: check-clean-src \ altbininstall libinstall inclinstall libainstall \ sharedinstall altmaninstall \ ``` After change (with empty FRAMEWORKALTINSTALLLAST): ``` .PHONY: commoninstall commoninstall: check-clean-src \ altbininstall libinstall inclinstall libainstall \ sharedinstall altmaninstall ``` --- Makefile.pre.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a4f76eaaaba525..3725feaca66ce3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1940,8 +1940,7 @@ altinstall: commoninstall .PHONY: commoninstall commoninstall: check-clean-src @FRAMEWORKALTINSTALLFIRST@ \ altbininstall libinstall inclinstall libainstall \ - sharedinstall altmaninstall \ - @FRAMEWORKALTINSTALLLAST@ + sharedinstall altmaninstall @FRAMEWORKALTINSTALLLAST@ # Install shared libraries enabled by Setup DESTDIRS= $(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED) From 6fbc717214210e06313a283b2f3ec8ce67209609 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 18:07:07 +0200 Subject: [PATCH 29/73] gh-106320: Remove _Py_SwappedOp from the C API (#107036) Move _Py_SwappedOp to the internal C API (pycore_object.h). --- Include/cpython/object.h | 5 ----- Include/internal/pycore_object.h | 3 +++ Objects/object.c | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 0e5b6acb43b416..49101c18756860 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -377,11 +377,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -/* Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. - * Defined in object.c. - */ -PyAPI_DATA(int) _Py_SwappedOp[]; - PyAPI_FUNC(void) _PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, size_t sizeof_block); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 90588daa64cc3b..a9c996557430ad 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -442,6 +442,9 @@ extern PyObject* _PyCFunctionWithKeywords_TrampolineCall( PyAPI_DATA(PyTypeObject) _PyNone_Type; PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; +/* Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. Defined in Objects/object.c. */ +PyAPI_DATA(int) _Py_SwappedOp[]; + #ifdef __cplusplus } #endif diff --git a/Objects/object.c b/Objects/object.c index bfbc87198f5b3c..3b8839b8772614 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -9,7 +9,7 @@ #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_namespace.h" // _PyNamespace_Type -#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError() +#include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() From 26e08dfdd7ac1b3d567d30cd35e4898121580390 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 22 Jul 2023 19:54:50 +0300 Subject: [PATCH 30/73] gh-107008: Document the curses module variables LINES and COLS (GH-107011) LINES and COLS referred in curses.update_lines_cols() documentations are the module variables, not the environment variables. --- Doc/library/curses.rst | 20 ++++++++++++++++++- Doc/whatsnew/3.5.rst | 4 ++-- ...-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst | 2 ++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index cf208f3ba0db36..391c81a844d3e0 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -641,7 +641,8 @@ The module :mod:`curses` defines the following functions: .. function:: update_lines_cols() - Update :envvar:`LINES` and :envvar:`COLS`. Useful for detecting manual screen resize. + Update the :const:`LINES` and :const:`COLS` module variables. + Useful for detecting manual screen resize. .. versionadded:: 3.5 @@ -1342,10 +1343,27 @@ The :mod:`curses` module defines the following data members: .. data:: COLORS The maximum number of colors the terminal can support. + It is defined only after the call to :func:`start_color`. .. data:: COLOR_PAIRS The maximum number of color pairs the terminal can support. + It is defined only after the call to :func:`start_color`. + +.. data:: COLS + + The width of the screen, i.e., the number of columns. + It is defined only after the call to :func:`initscr`. + Updated by :func:`update_lines_cols`, :func:`resizeterm` and + :func:`resize_term`. + +.. data:: LINES + + The height of the screen, i.e., the number of lines. + It is defined only after the call to :func:`initscr`. + Updated by :func:`update_lines_cols`, :func:`resizeterm` and + :func:`resize_term`. + Some constants are available to specify character cell attributes. The exact constants available are system dependent. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 262e4ed32c0a3b..66610fa70efe9e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -1045,8 +1045,8 @@ not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) curses ------ -The new :func:`~curses.update_lines_cols` function updates the :envvar:`LINES` -and :envvar:`COLS` environment variables. This is useful for detecting +The new :func:`~curses.update_lines_cols` function updates the :data:`LINES` +and :data:`COLS` module variables. This is useful for detecting manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) diff --git a/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst b/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst new file mode 100644 index 00000000000000..a0fa27ec10303e --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst @@ -0,0 +1,2 @@ +Document the :mod:`curses` module variables :const:`~curses.LINES` and +:const:`~curses.COLS`. From f8b7fe2f2647813ae8249675a80e59c117d30fe1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 22 Jul 2023 21:35:22 +0300 Subject: [PATCH 31/73] gh-106948: Add standard external names to nitpick_ignore (GH-106949) It includes standard C types, macros and variables like "size_t", "LONG_MAX" and "errno", and standard environment variables like "PATH". --- Doc/c-api/arg.rst | 2 +- Doc/c-api/memory.rst | 2 +- Doc/c-api/unicode.rst | 22 ++++---- Doc/c-api/veryhigh.rst | 2 +- Doc/conf.py | 52 +++++++++++++++++++ Doc/library/array.rst | 4 +- Doc/library/ctypes.rst | 12 ++--- Doc/library/os.rst | 2 +- Doc/library/struct.rst | 4 +- Doc/library/venv.rst | 6 +-- Doc/library/webbrowser.rst | 6 +-- Doc/tools/.nitignore | 7 --- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.13.rst | 10 ++-- Doc/whatsnew/3.3.rst | 2 +- Doc/whatsnew/3.5.rst | 2 +- Doc/whatsnew/3.9.rst | 4 +- Misc/NEWS.d/3.10.0a5.rst | 2 +- Misc/NEWS.d/3.12.0a1.rst | 2 +- Misc/NEWS.d/3.12.0b1.rst | 2 +- ...-05-31-18-37-57.gh-issue-105156.R4El5V.rst | 4 +- ...-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst | 1 + 22 files changed, 99 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index e8d141f1739331..9d744a12e374be 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -555,7 +555,7 @@ Building values Same as ``s#``. ``u`` (:class:`str`) [const wchar_t \*] - Convert a null-terminated :c:expr:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) + Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``, ``None`` is returned. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 35c356f25c6c70..4ca3b8804427f8 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -581,7 +581,7 @@ that the treatment of negative indices differs from a Python slice): default). A serial number, incremented by 1 on each call to a malloc-like or - realloc-like function. Big-endian ``size_t``. If "bad memory" is detected + realloc-like function. Big-endian :c:type:`size_t`. If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. The static function bumpserialno() in obmalloc.c is the only place the serial diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index beb4f42a208fa2..cf97b11bdbbf41 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -44,7 +44,7 @@ Python: .. c:type:: Py_UNICODE - This is a typedef of :c:expr:`wchar_t`, which is a 16-bit type or 32-bit type + This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type depending on the platform. .. versionchanged:: 3.3 @@ -437,11 +437,11 @@ APIs: +----------+-----------------------------------------------------+ | ``ll`` | :c:expr:`long long` or :c:expr:`unsigned long long` | +----------+-----------------------------------------------------+ - | ``j`` | :c:expr:`intmax_t` or :c:expr:`uintmax_t` | + | ``j`` | :c:type:`intmax_t` or :c:type:`uintmax_t` | +----------+-----------------------------------------------------+ - | ``z`` | :c:expr:`size_t` or :c:expr:`ssize_t` | + | ``z`` | :c:type:`size_t` or :c:type:`ssize_t` | +----------+-----------------------------------------------------+ - | ``t`` | :c:expr:`ptrdiff_t` | + | ``t`` | :c:type:`ptrdiff_t` | +----------+-----------------------------------------------------+ The length modifier ``l`` for following conversions ``s`` or ``V`` specify @@ -520,7 +520,7 @@ APIs: .. note:: The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes or :c:expr:`wchar_t` + The precision formatter unit is number of bytes or :c:type:`wchar_t` items (if the length modifier ``l`` is used) for ``"%s"`` and ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` @@ -839,11 +839,11 @@ conversion function: wchar_t Support """"""""""""""" -:c:expr:`wchar_t` support for platforms which support it: +:c:type:`wchar_t` support for platforms which support it: .. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) - Create a Unicode object from the :c:expr:`wchar_t` buffer *w* of the given *size*. + Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, using wcslen. Return ``NULL`` on failure. @@ -851,9 +851,9 @@ wchar_t Support .. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) - Copy the Unicode object contents into the :c:expr:`wchar_t` buffer *w*. At most - *size* :c:expr:`wchar_t` characters are copied (excluding a possibly trailing - null termination character). Return the number of :c:expr:`wchar_t` characters + Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most + *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing + null termination character). Return the number of :c:type:`wchar_t` characters copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is @@ -867,7 +867,7 @@ wchar_t Support Convert the Unicode object to a wide character string. The output string always ends with a null character. If *size* is not ``NULL``, write the number of wide characters (excluding the trailing null termination character) into - *\*size*. Note that the resulting :c:expr:`wchar_t` string might contain + *\*size*. Note that the resulting :c:type:`wchar_t` string might contain null characters, which would cause the string to be truncated when used with most C functions. If *size* is ``NULL`` and the :c:expr:`wchar_t*` string contains null characters a :exc:`ValueError` is raised. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 1e8a945512d78c..56fa2d6abd9139 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -17,7 +17,7 @@ parameter. The available start symbols are :c:data:`Py_eval_input`, following the functions which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One -particular issue which needs to be handled carefully is that the :c:expr:`FILE` +particular issue which needs to be handled carefully is that the :c:type:`FILE` structure for different C libraries can be different and incompatible. Under Windows (at least), it is possible for dynamically linked extensions to actually use different libraries, so care should be taken that :c:expr:`FILE*` parameters diff --git a/Doc/conf.py b/Doc/conf.py index 09e12e245891d2..bd01850589b273 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -77,6 +77,58 @@ exclude_patterns.append(venvdir + '/*') nitpick_ignore = [ + # Standard C types + ('c:type', 'FILE'), + ('c:type', '__int'), + ('c:type', 'intmax_t'), + ('c:type', 'off_t'), + ('c:type', 'ptrdiff_t'), + ('c:type', 'siginfo_t'), + ('c:type', 'size_t'), + ('c:type', 'ssize_t'), + ('c:type', 'time_t'), + ('c:type', 'uintmax_t'), + ('c:type', 'va_list'), + ('c:type', 'wchar_t'), + # Standard C macros + ('c:macro', 'LLONG_MAX'), + ('c:macro', 'LLONG_MIN'), + ('c:macro', 'LONG_MAX'), + ('c:macro', 'LONG_MIN'), + # Standard C variables + ('c:data', 'errno'), + # Standard environment variables + ('envvar', 'BROWSER'), + ('envvar', 'COLUMNS'), + ('envvar', 'COMSPEC'), + ('envvar', 'DISPLAY'), + ('envvar', 'HOME'), + ('envvar', 'HOMEDRIVE'), + ('envvar', 'HOMEPATH'), + ('envvar', 'IDLESTARTUP'), + ('envvar', 'LANG'), + ('envvar', 'LANGUAGE'), + ('envvar', 'LC_ALL'), + ('envvar', 'LC_CTYPE'), + ('envvar', 'LC_COLLATE'), + ('envvar', 'LC_MESSAGES'), + ('envvar', 'LC_MONETARY'), + ('envvar', 'LC_NUMERIC'), + ('envvar', 'LC_TIME'), + ('envvar', 'LINES'), + ('envvar', 'LOGNAME'), + ('envvar', 'PAGER'), + ('envvar', 'PATH'), + ('envvar', 'PATHEXT'), + ('envvar', 'SOURCE_DATE_EPOCH'), + ('envvar', 'TEMP'), + ('envvar', 'TERM'), + ('envvar', 'TMP'), + ('envvar', 'TMPDIR'), + ('envvar', 'TZ'), + ('envvar', 'USER'), + ('envvar', 'USERNAME'), + ('envvar', 'USERPROFILE'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 0afc217642a756..ad622627724217 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -53,9 +53,9 @@ Notes: It can be 16 bits or 32 bits depending on the platform. .. versionchanged:: 3.9 - ``array('u')`` now uses ``wchar_t`` as C type instead of deprecated + ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of ``wchar_t`` since Python 3.3. + ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. .. deprecated-removed:: 3.3 3.16 Please migrate to ``'w'`` typecode. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 81509c0920bb6e..c253a45e1a8b54 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -220,7 +220,7 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_char` | :c:expr:`char` | 1-character bytes object | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:expr:`wchar_t` | 1-character string | +| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | +----------------------+------------------------------------------+----------------------------+ | :class:`c_byte` | :c:expr:`char` | int | +----------------------+------------------------------------------+----------------------------+ @@ -243,9 +243,9 @@ Fundamental data types | :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | | | :c:expr:`unsigned long long` | | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:expr:`size_t` | int | +| :class:`c_size_t` | :c:type:`size_t` | int | +----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:expr:`ssize_t` or | int | +| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | | | :c:expr:`Py_ssize_t` | | +----------------------+------------------------------------------+----------------------------+ | :class:`c_time_t` | :c:type:`time_t` | int | @@ -335,7 +335,7 @@ property:: The :func:`create_string_buffer` function replaces the old :func:`c_buffer` function (which is still available as an alias). To create a mutable memory -block containing unicode characters of the C type :c:expr:`wchar_t`, use the +block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -478,7 +478,7 @@ By default functions are assumed to return the C :c:expr:`int` type. Other return types can be specified by setting the :attr:`restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because ``time_t`` +The C prototype of ``time()`` is ``time_t time(time_t *)``. Because :c:type:`time_t` might be of a different type than the default return type ``int``, you should specify the ``restype``:: @@ -2407,7 +2407,7 @@ These are the fundamental ctypes data types: .. class:: c_wchar - Represents the C :c:expr:`wchar_t` datatype, and interprets the value as a + Represents the C :c:type:`wchar_t` datatype, and interprets the value as a single character unicode string. The constructor accepts an optional string initializer, the length of the string must be exactly one character. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 127d1616388b3e..57405916ed7b31 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4650,7 +4650,7 @@ written in Python, such as a mail server's external command delivery program. :data:`WNOHANG` and :data:`WNOWAIT` are additional optional flags. The return value is an object representing the data contained in the - :c:type:`!siginfo_t` structure with the following attributes: + :c:type:`siginfo_t` structure with the following attributes: * :attr:`!si_pid` (process ID) * :attr:`!si_uid` (real user ID of the child) diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 6d2739b4557fbf..c94dfde4d55763 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -231,9 +231,9 @@ platform-dependent. | ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:expr:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:expr:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | integer | | \(3) | +--------+--------------------------+--------------------+----------------+------------+ | ``e`` | \(6) | float | 2 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 9e5672545dea35..2482441d649790 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -60,7 +60,7 @@ running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory (``bin`` on POSIX; ``Scripts`` on Windows). -This will prepend that directory to your :envvar:`!PATH`, so that running +This will prepend that directory to your :envvar:`PATH`, so that running :program:`python` will invoke the environment's Python interpreter and you can run installed scripts without having to use their full path. The invocation of the activation script is platform-specific @@ -100,10 +100,10 @@ In order to achieve this, scripts installed into virtual environments have a "shebang" line which points to the environment's Python interpreter, i.e. :samp:`#!/{}/bin/python`. This means that the script will run with that interpreter regardless of the -value of :envvar:`!PATH`. On Windows, "shebang" line processing is supported if +value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if you have the :ref:`launcher` installed. Thus, double-clicking an installed script in a Windows Explorer window should run it with the correct interpreter -without the environment needing to be activated or on the :envvar:`!PATH`. +without the environment needing to be activated or on the :envvar:`PATH`. When a virtual environment has been activated, the :envvar:`!VIRTUAL_ENV` environment variable is set to the path of the environment. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index b6762f78830a5f..4667b81e38ada2 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -20,7 +20,7 @@ will be used if graphical browsers are not available or an X11 display isn't available. If text-mode browsers are used, the calling process will block until the user exits the browser. -If the environment variable :envvar:`!BROWSER` exists, it is interpreted as the +If the environment variable :envvar:`BROWSER` exists, it is interpreted as the :data:`os.pathsep`-separated list of browsers to try ahead of the platform defaults. When the value of a list part contains the string ``%s``, then it is interpreted as a literal browser command line to be used with the argument URL @@ -97,7 +97,7 @@ The following functions are defined: Setting *preferred* to ``True`` makes this browser a preferred result for a :func:`get` call with no argument. Otherwise, this entry point is only - useful if you plan to either set the :envvar:`!BROWSER` variable or call + useful if you plan to either set the :envvar:`BROWSER` variable or call :func:`get` with a nonempty argument matching the name of a handler you declare. @@ -224,4 +224,4 @@ module-level convenience functions: .. rubric:: Footnotes .. [1] Executables named here without a full path will be searched in the - directories given in the :envvar:`!PATH` environment variable. + directories given in the :envvar:`PATH` environment variable. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 87157f0eff4c4d..14d9b2e121d3b6 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -91,7 +91,6 @@ Doc/library/codecs.rst Doc/library/codeop.rst Doc/library/collections.abc.rst Doc/library/collections.rst -Doc/library/compileall.rst Doc/library/concurrent.futures.rst Doc/library/concurrent.rst Doc/library/configparser.rst @@ -100,7 +99,6 @@ Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst Doc/library/ctypes.rst -Doc/library/curses.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst @@ -137,7 +135,6 @@ Doc/library/http.client.rst Doc/library/http.cookiejar.rst Doc/library/http.cookies.rst Doc/library/http.server.rst -Doc/library/idle.rst Doc/library/importlib.resources.abc.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst @@ -165,11 +162,9 @@ Doc/library/pickletools.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/poplib.rst -Doc/library/posix.rst Doc/library/pprint.rst Doc/library/profile.rst Doc/library/pty.rst -Doc/library/py_compile.rst Doc/library/pyclbr.rst Doc/library/pydoc.rst Doc/library/pyexpat.rst @@ -192,7 +187,6 @@ Doc/library/ssl.rst Doc/library/stat.rst Doc/library/stdtypes.rst Doc/library/string.rst -Doc/library/struct.rst Doc/library/subprocess.rst Doc/library/sys.rst Doc/library/sys_path_init.rst @@ -254,7 +248,6 @@ Doc/tutorial/modules.rst Doc/tutorial/stdlib2.rst Doc/using/cmdline.rst Doc/using/configure.rst -Doc/using/unix.rst Doc/using/windows.rst Doc/whatsnew/2.0.rst Doc/whatsnew/2.1.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ac03af52db0716..ea9806987f6872 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1768,7 +1768,7 @@ Porting to Python 3.12 for example). * Add support of more formatting options (left aligning, octals, uppercase - hexadecimals, ``intmax_t``, ``ptrdiff_t``, ``wchar_t`` C + hexadecimals, :c:type:`intmax_t`, :c:type:`ptrdiff_t`, :c:type:`wchar_t` C strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV`. (Contributed by Serhiy Storchaka in :gh:`98836`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5d4fa44aab695c..1d34d8a0fa463c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -329,7 +329,7 @@ Pending Removal in Python 3.15 Pending Removal in Python 3.16 ------------------------------ -* :class:`array.array` ``'u'`` type (``wchar_t``): +* :class:`array.array` ``'u'`` type (:c:type:`wchar_t`): use the ``'w'`` type instead (``Py_UCS4``). Pending Removal in Future Versions @@ -802,8 +802,8 @@ Deprecated ---------- * Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the ``wchar_t`` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to ``wchar_t``. + the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and + ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. (Contributed by Victor Stinner in :gh:`105156`.) * Deprecate old Python initialization functions: @@ -1013,8 +1013,8 @@ Pending Removal in Python 3.15 * :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. * :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. * :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. -* :c:type:`!Py_UNICODE_WIDE` type: use ``wchar_t`` instead. -* :c:type:`Py_UNICODE` type: use ``wchar_t`` instead. +* :c:type:`!Py_UNICODE_WIDE` type: use :c:type:`wchar_t` instead. +* :c:type:`Py_UNICODE` type: use :c:type:`wchar_t` instead. * Python initialization functions: * :c:func:`PySys_ResetWarnOptions`: clear :data:`sys.warnoptions` and diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index dbac74167a1268..911e5ba47df41b 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1984,7 +1984,7 @@ the form '-rwxrwxrwx'. struct ------ -The :mod:`struct` module now supports ``ssize_t`` and ``size_t`` via the +The :mod:`struct` module now supports :c:type:`ssize_t` and :c:type:`size_t` via the new codes ``n`` and ``N``, respectively. (Contributed by Antoine Pitrou in :issue:`3163`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 66610fa70efe9e..1e30569c559b98 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2192,7 +2192,7 @@ encode error with ``\N{...}`` escapes. (Contributed by Serhiy Storchaka in :issue:`19676`.) A new :c:func:`PyErr_FormatV` function similar to :c:func:`PyErr_Format`, -but accepts a ``va_list`` argument. +but accepts a :c:type:`va_list` argument. (Contributed by Antoine Pitrou in :issue:`18711`.) A new :c:data:`PyExc_RecursionError` exception. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 9280f947400051..bf889b7ffbadff 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1115,9 +1115,9 @@ Changes in the Python API ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` was clashing with ``CO_FUTURE_DIVISION``. (Contributed by Batuhan Taskaya in :issue:`39562`) -* ``array('u')`` now uses ``wchar_t`` as C type instead of ``Py_UNICODE``. +* ``array('u')`` now uses :c:type:`wchar_t` as C type instead of ``Py_UNICODE``. This change doesn't affect to its behavior because ``Py_UNICODE`` is alias - of ``wchar_t`` since Python 3.3. + of :c:type:`wchar_t` since Python 3.3. (Contributed by Inada Naoki in :issue:`34538`.) * The :func:`logging.getLogger` API now returns the root logger when passed diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index 497e3849171831..dc95e8ce072fd9 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -667,4 +667,4 @@ exception (if an exception is set). Patch by Victor Stinner. .. section: C API Fixed a compiler warning in :c:func:`Py_UNICODE_ISSPACE()` on platforms with -signed ``wchar_t``. +signed :c:type:`wchar_t`. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 9109db537396ed..209d079feb5144 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -5308,7 +5308,7 @@ parameter. Patch by Kumar Aditya. .. section: Build Python now always use the ``%zu`` and ``%zd`` printf formats to format a -``size_t`` or ``Py_ssize_t`` number. Building Python 3.12 requires a C11 +:c:type:`size_t` or ``Py_ssize_t`` number. Building Python 3.12 requires a C11 compiler, so these printf formats are now always supported. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 51d80711d91ede..89af6efdae048f 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -2382,7 +2382,7 @@ Patch by Dong-hee Na. .. section: C API Add support of more formatting options (left aligning, octals, uppercase -hexadecimals, :c:expr:`intmax_t`, :c:expr:`ptrdiff_t`, :c:expr:`wchar_t` C +hexadecimals, :c:type:`intmax_t`, :c:type:`ptrdiff_t`, :c:type:`wchar_t` C strings, variable width and precision) in :c:func:`PyUnicode_FromFormat` and :c:func:`PyUnicode_FromFormatV`. diff --git a/Misc/NEWS.d/next/C API/2023-05-31-18-37-57.gh-issue-105156.R4El5V.rst b/Misc/NEWS.d/next/C API/2023-05-31-18-37-57.gh-issue-105156.R4El5V.rst index cbdb8379f24ccd..536e484116690d 100644 --- a/Misc/NEWS.d/next/C API/2023-05-31-18-37-57.gh-issue-105156.R4El5V.rst +++ b/Misc/NEWS.d/next/C API/2023-05-31-18-37-57.gh-issue-105156.R4El5V.rst @@ -1,4 +1,4 @@ Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly -the ``wchar_t`` type instead. Since Python 3.3, ``Py_UNICODE`` and -``PY_UNICODE_TYPE`` are just aliases to ``wchar_t``. Patch by Victor +the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and +``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst b/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst new file mode 100644 index 00000000000000..42b6348153b56a --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst @@ -0,0 +1 @@ +Add a number of standard external names to ``nitpick_ignore``. From 22422e9d1a50b4970bc6957a88569618d579c47f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 21:31:55 +0200 Subject: [PATCH 32/73] gh-106320: Remove private _PyInterpreterID C API (#107053) Move the private _PyInterpreterID C API to the internal C API: add a new pycore_interp_id.h header file. Remove Include/interpreteridobject.h and Include/cpython/interpreteridobject.h header files. --- Include/cpython/interpreteridobject.h | 11 ----------- Include/internal/pycore_interp_id.h | 28 +++++++++++++++++++++++++++ Include/interpreteridobject.h | 17 ---------------- Makefile.pre.in | 3 +-- Modules/_testinternalcapi.c | 4 ++-- Modules/_xxinterpchannelsmodule.c | 2 +- Modules/_xxsubinterpretersmodule.c | 2 +- Objects/interpreteridobject.c | 2 +- Objects/object.c | 4 ++-- PCbuild/pythoncore.vcxproj | 3 +-- PCbuild/pythoncore.vcxproj.filters | 9 +++------ Tools/c-analyzer/cpython/_parser.py | 1 - 12 files changed, 40 insertions(+), 46 deletions(-) delete mode 100644 Include/cpython/interpreteridobject.h create mode 100644 Include/internal/pycore_interp_id.h delete mode 100644 Include/interpreteridobject.h diff --git a/Include/cpython/interpreteridobject.h b/Include/cpython/interpreteridobject.h deleted file mode 100644 index 5076584209b90b..00000000000000 --- a/Include/cpython/interpreteridobject.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" -#endif - -/* Interpreter ID Object */ - -PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; - -PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); -PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); -PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); diff --git a/Include/internal/pycore_interp_id.h b/Include/internal/pycore_interp_id.h new file mode 100644 index 00000000000000..34c96df98ef730 --- /dev/null +++ b/Include/internal/pycore_interp_id.h @@ -0,0 +1,28 @@ +/* Interpreter ID Object */ + +#ifndef Py_INTERNAL_INTERPRETERIDOBJECT_H +#define Py_INTERNAL_INTERPRETERIDOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Export for the _xxsubinterpreters shared extension +PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; + +// Export for the _xxsubinterpreters shared extension +PyAPI_FUNC(PyObject*) _PyInterpreterID_New(int64_t); + +// Export for the _xxinterpchannels shared extension +PyAPI_FUNC(PyObject*) _PyInterpreterState_GetIDObject(PyInterpreterState *); + +// Export for the _testinternalcapi shared extension +PyAPI_FUNC(PyInterpreterState*) _PyInterpreterID_LookUp(PyObject *); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_INTERPRETERIDOBJECT_H diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index 8432632f339e92..00000000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 3725feaca66ce3..5f1988b0d17213 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1631,7 +1631,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/floatobject.h \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ - $(srcdir)/Include/interpreteridobject.h \ $(srcdir)/Include/intrcheck.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ @@ -1701,7 +1700,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/genobject.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ - $(srcdir)/Include/cpython/interpreteridobject.h \ $(srcdir)/Include/cpython/listobject.h \ $(srcdir)/Include/cpython/longintrepr.h \ $(srcdir)/Include/cpython/longobject.h \ @@ -1773,6 +1771,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_import.h \ $(srcdir)/Include/internal/pycore_initconfig.h \ $(srcdir)/Include/internal/pycore_interp.h \ + $(srcdir)/Include/internal/pycore_interp_id.h \ $(srcdir)/Include/internal/pycore_intrinsics.h \ $(srcdir)/Include/internal/pycore_list.h \ $(srcdir)/Include/internal/pycore_long.h \ diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 74c932fa921cd0..ecc2721a4988f9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -11,18 +11,18 @@ #include "Python.h" #include "frameobject.h" -#include "interpreteridobject.h" // _PyInterpreterID_LookUp() #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() +#include "pycore_interp_id.h" // _PyInterpreterID_LookUp() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 82472555ec7d62..bdffdba52aa02e 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -6,8 +6,8 @@ #endif #include "Python.h" -#include "interpreteridobject.h" #include "pycore_atexit.h" // _Py_AtExit() +#include "pycore_interp_id.h" // _PyInterpreterState_GetIDObject() /* diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index d2e0593872c5f0..bf01aaf6ed0409 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -7,7 +7,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState_GetMainModule() -#include "interpreteridobject.h" +#include "pycore_interp_id.h" // _PyInterpreterState_GetIDObject() #define MODULE_NAME "_xxsubinterpreters" diff --git a/Objects/interpreteridobject.c b/Objects/interpreteridobject.c index 46239100dcb7b7..7a3e245ce3357e 100644 --- a/Objects/interpreteridobject.c +++ b/Objects/interpreteridobject.c @@ -3,7 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_interp.h" // _PyInterpreterState_LookUpID() -#include "interpreteridobject.h" +#include "pycore_interp_id.h" // _PyInterpreterID_Type typedef struct interpid { diff --git a/Objects/object.c b/Objects/object.c index 3b8839b8772614..93396bf5491d35 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -8,16 +8,16 @@ #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() +#include "pycore_interp_id.h" // _PyInterpreterID_Type #include "pycore_namespace.h" // _PyNamespace_Type #include "pycore_object.h" // PyAPI_DATA() _Py_SwappedOp definition #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type -#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_typeobject.h" // _PyBufferWrapper_Type +#include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_initialize_generic #include "pycore_unionobject.h" // _PyUnion_Type -#include "interpreteridobject.h" // _PyInterpreterID_Type #ifdef Py_LIMITED_API // Prevent recursive call _Py_IncRef() <=> Py_INCREF() diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index bfe59acf12a69d..5ccc8958330650 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -152,7 +152,6 @@ - @@ -238,6 +237,7 @@ + @@ -281,7 +281,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0a8b0c3faf51e1..54a77f81a9a1ab 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -312,9 +312,6 @@ Include - - Include - Modules @@ -462,9 +459,6 @@ Include - - Include\cpython - Include\cpython @@ -618,6 +612,9 @@ Include\internal + + Include\internal + Include\cpython diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 9bc7285e18b2fb..64ce4cda7d6ed7 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -227,7 +227,6 @@ def clean_lines(text): Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 From 5e5a34ac3a827e040cd89426b1774fec2123336a Mon Sep 17 00:00:00 2001 From: Jochem Boersma Date: Sat, 22 Jul 2023 22:43:18 +0200 Subject: [PATCH 33/73] gh-107028: tiny textual changes in logging docs and docstrings (GH-107029) --- Doc/library/logging.handlers.rst | 9 +++++---- Lib/logging/handlers.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 47bfe4ff7f90ed..72e5ffb3a1218e 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -917,8 +917,9 @@ should, then :meth:`flush` is expected to do the flushing. .. method:: flush() - You can override this to implement custom flushing behavior. This version - just zaps the buffer to empty. + For a :class:`BufferingHandler` instance, flushing means that it sets the + buffer to an empty list. This method can be overwritten to implement more useful + flushing behavior. .. method:: shouldFlush(record) @@ -950,9 +951,9 @@ should, then :meth:`flush` is expected to do the flushing. .. method:: flush() - For a :class:`MemoryHandler`, flushing means just sending the buffered + For a :class:`MemoryHandler` instance, flushing means just sending the buffered records to the target, if there is one. The buffer is also cleared when - this happens. Override if you want different behavior. + buffered records are sent to the target. Override if you want different behavior. .. method:: setTarget(target) diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 9847104446eaf6..671cc9596b02dd 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1399,7 +1399,7 @@ def flush(self): records to the target, if there is one. Override if you want different behaviour. - The record buffer is also cleared by this operation. + The record buffer is only cleared if a target has been set. """ self.acquire() try: From ee15844db88fab3a282d14325662bcfd026272ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 22:57:59 +0200 Subject: [PATCH 34/73] gh-106320: Move _PyMethodWrapper_Type to internal C API (#107064) --- Include/cpython/descrobject.h | 2 -- Include/internal/pycore_descrobject.h | 2 ++ Objects/descrobject.c | 2 +- Objects/object.c | 1 + Python/specialize.c | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Include/cpython/descrobject.h b/Include/cpython/descrobject.h index e2ea1b9a2d3058..bbad8b59c225ab 100644 --- a/Include/cpython/descrobject.h +++ b/Include/cpython/descrobject.h @@ -57,8 +57,6 @@ typedef struct { void *d_wrapped; /* This can be any function pointer */ } PyWrapperDescrObject; -PyAPI_DATA(PyTypeObject) _PyMethodWrapper_Type; - PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); PyAPI_FUNC(int) PyDescr_IsData(PyObject *); diff --git a/Include/internal/pycore_descrobject.h b/Include/internal/pycore_descrobject.h index 76378569df90e3..3cec59a68a3d2b 100644 --- a/Include/internal/pycore_descrobject.h +++ b/Include/internal/pycore_descrobject.h @@ -20,6 +20,8 @@ typedef struct { typedef propertyobject _PyPropertyObject; +extern PyTypeObject _PyMethodWrapper_Type; + #ifdef __cplusplus } #endif diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 810bd196e8f7e7..74aa70bed7de98 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -4,11 +4,11 @@ #include "pycore_abstract.h" // _PyObject_RealIsSubclass() #include "pycore_call.h" // _PyStack_AsDict() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() +#include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "structmember.h" // PyMemberDef -#include "pycore_descrobject.h" /*[clinic input] class mappingproxy "mappingproxyobject *" "&PyDictProxy_Type" diff --git a/Objects/object.c b/Objects/object.c index 93396bf5491d35..740d4a22c103b2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -5,6 +5,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_context.h" // _PyContextTokenMissing_Type +#include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MakeDictFromInstanceAttributes() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() diff --git a/Python/specialize.c b/Python/specialize.c index dcf4be712db20d..5892f5441c98e2 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_code.h" +#include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState() #include "pycore_global_strings.h" // _Py_ID() @@ -7,9 +8,8 @@ #include "pycore_moduleobject.h" #include "pycore_object.h" #include "pycore_opcode.h" // _PyOpcode_Caches -#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX -#include "pycore_descrobject.h" #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() +#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX #include // rand() From b7dc795dfd175c0d25a479cfaf94a13c368a5a7b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 22 Jul 2023 16:07:40 -0500 Subject: [PATCH 35/73] gh-106527: asyncio: optimize to add/remove readers and writers (#106528) --- Lib/asyncio/selector_events.py | 64 +++++++++---------- Lib/test/test_asyncio/test_selector_events.py | 36 +++++------ ...-07-07-18-22-07.gh-issue-106527.spHQ0W.rst | 1 + 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-07-18-22-07.gh-issue-106527.spHQ0W.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index f895750e3cf959..d521b4e2e255a9 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -274,9 +274,8 @@ def _ensure_fd_no_transport(self, fd): def _add_reader(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_READ, (handle, None)) else: @@ -290,30 +289,27 @@ def _add_reader(self, fd, callback, *args): def _remove_reader(self, fd): if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + mask &= ~selectors.EVENT_READ + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - mask &= ~selectors.EVENT_READ - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (None, writer)) + self._selector.modify(fd, mask, (None, writer)) - if reader is not None: - reader.cancel() - return True - else: - return False + if reader is not None: + reader.cancel() + return True + else: + return False def _add_writer(self, fd, callback, *args): self._check_closed() handle = events.Handle(callback, args, self, None) - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: self._selector.register(fd, selectors.EVENT_WRITE, (None, handle)) else: @@ -328,24 +324,22 @@ def _remove_writer(self, fd): """Remove a writer callback.""" if self.is_closed(): return False - try: - key = self._selector.get_key(fd) - except KeyError: + key = self._selector.get_map().get(fd) + if key is None: return False + mask, (reader, writer) = key.events, key.data + # Remove both writer and connector. + mask &= ~selectors.EVENT_WRITE + if not mask: + self._selector.unregister(fd) else: - mask, (reader, writer) = key.events, key.data - # Remove both writer and connector. - mask &= ~selectors.EVENT_WRITE - if not mask: - self._selector.unregister(fd) - else: - self._selector.modify(fd, mask, (reader, None)) + self._selector.modify(fd, mask, (reader, None)) - if writer is not None: - writer.cancel() - return True - else: - return False + if writer is not None: + writer.cancel() + return True + else: + return False def add_reader(self, fd, callback, *args): """Add a reader callback.""" diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 47693ea4d3ce2e..c22b780b5edcb8 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -178,7 +178,7 @@ def test_sock_connect_resolve_using_socket_params(self, m_gai): sock.connect.assert_called_with(('127.0.0.1', 0)) def test_add_reader(self): - self.loop._selector.get_key.side_effect = KeyError + self.loop._selector.get_map.return_value = {} cb = lambda: True self.loop.add_reader(1, cb) @@ -192,8 +192,8 @@ def test_add_reader(self): def test_add_reader_existing(self): reader = mock.Mock() writer = mock.Mock() - self.loop._selector.get_key.return_value = selectors.SelectorKey( - 1, 1, selectors.EVENT_WRITE, (reader, writer)) + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (reader, writer))} cb = lambda: True self.loop.add_reader(1, cb) @@ -208,8 +208,8 @@ def test_add_reader_existing(self): def test_add_reader_existing_writer(self): writer = mock.Mock() - self.loop._selector.get_key.return_value = selectors.SelectorKey( - 1, 1, selectors.EVENT_WRITE, (None, writer)) + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, writer))} cb = lambda: True self.loop.add_reader(1, cb) @@ -222,8 +222,8 @@ def test_add_reader_existing_writer(self): self.assertEqual(writer, w) def test_remove_reader(self): - self.loop._selector.get_key.return_value = selectors.SelectorKey( - 1, 1, selectors.EVENT_READ, (None, None)) + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (None, None))} self.assertFalse(self.loop.remove_reader(1)) self.assertTrue(self.loop._selector.unregister.called) @@ -231,9 +231,9 @@ def test_remove_reader(self): def test_remove_reader_read_write(self): reader = mock.Mock() writer = mock.Mock() - self.loop._selector.get_key.return_value = selectors.SelectorKey( + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, - (reader, writer)) + (reader, writer))} self.assertTrue( self.loop.remove_reader(1)) @@ -243,12 +243,12 @@ def test_remove_reader_read_write(self): self.loop._selector.modify.call_args[0]) def test_remove_reader_unknown(self): - self.loop._selector.get_key.side_effect = KeyError + self.loop._selector.get_map.return_value = {} self.assertFalse( self.loop.remove_reader(1)) def test_add_writer(self): - self.loop._selector.get_key.side_effect = KeyError + self.loop._selector.get_map.return_value = {} cb = lambda: True self.loop.add_writer(1, cb) @@ -262,8 +262,8 @@ def test_add_writer(self): def test_add_writer_existing(self): reader = mock.Mock() writer = mock.Mock() - self.loop._selector.get_key.return_value = selectors.SelectorKey( - 1, 1, selectors.EVENT_READ, (reader, writer)) + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_READ, (reader, writer))} cb = lambda: True self.loop.add_writer(1, cb) @@ -277,8 +277,8 @@ def test_add_writer_existing(self): self.assertEqual(cb, w._callback) def test_remove_writer(self): - self.loop._selector.get_key.return_value = selectors.SelectorKey( - 1, 1, selectors.EVENT_WRITE, (None, None)) + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( + 1, 1, selectors.EVENT_WRITE, (None, None))} self.assertFalse(self.loop.remove_writer(1)) self.assertTrue(self.loop._selector.unregister.called) @@ -286,9 +286,9 @@ def test_remove_writer(self): def test_remove_writer_read_write(self): reader = mock.Mock() writer = mock.Mock() - self.loop._selector.get_key.return_value = selectors.SelectorKey( + self.loop._selector.get_map.return_value = {1: selectors.SelectorKey( 1, 1, selectors.EVENT_READ | selectors.EVENT_WRITE, - (reader, writer)) + (reader, writer))} self.assertTrue( self.loop.remove_writer(1)) @@ -298,7 +298,7 @@ def test_remove_writer_read_write(self): self.loop._selector.modify.call_args[0]) def test_remove_writer_unknown(self): - self.loop._selector.get_key.side_effect = KeyError + self.loop._selector.get_map.return_value = {} self.assertFalse( self.loop.remove_writer(1)) diff --git a/Misc/NEWS.d/next/Library/2023-07-07-18-22-07.gh-issue-106527.spHQ0W.rst b/Misc/NEWS.d/next/Library/2023-07-07-18-22-07.gh-issue-106527.spHQ0W.rst new file mode 100644 index 00000000000000..204bda1c73eb36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-18-22-07.gh-issue-106527.spHQ0W.rst @@ -0,0 +1 @@ +Reduce overhead to add and remove :mod:`asyncio` readers and writers. From 0927a2b25c059988e237108605ed8ab0c5459c53 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 23:35:27 +0200 Subject: [PATCH 36/73] GH-103082: Rename PY_MONITORING_EVENTS to _PY_MONITORING_EVENTS (#107069) Rename private C API constants: * Rename PY_MONITORING_UNGROUPED_EVENTS to _PY_MONITORING_UNGROUPED_EVENTS * Rename PY_MONITORING_EVENTS to _PY_MONITORING_EVENTS --- Include/cpython/code.h | 6 ++--- Include/internal/pycore_interp.h | 4 ++-- Python/ceval.c | 2 +- Python/instrumentation.c | 38 ++++++++++++++++---------------- Python/pystate.c | 8 +++---- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 1b65b0d01d89f8..b8953e26ecddcb 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -10,13 +10,13 @@ extern "C" { /* Count of all "real" monitoring events (not derived from other events) */ -#define PY_MONITORING_UNGROUPED_EVENTS 14 +#define _PY_MONITORING_UNGROUPED_EVENTS 14 /* Count of all monitoring events */ -#define PY_MONITORING_EVENTS 16 +#define _PY_MONITORING_EVENTS 16 /* Table of which tools are active for each monitored event. */ typedef struct _Py_Monitors { - uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_Monitors; /* Each instruction in a code object is a fixed-width value, diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index bb37cafe6286a9..8d1b8ac6671f27 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -25,7 +25,7 @@ extern "C" { #include "pycore_gc.h" // struct _gc_runtime_state #include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_import.h" // struct _import_state -#include "pycore_instruments.h" // PY_MONITORING_EVENTS +#include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state #include "pycore_object_state.h" // struct _py_object_state #include "pycore_obmalloc.h" // struct obmalloc_state @@ -190,7 +190,7 @@ struct _is { bool sys_trace_initialized; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ - PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; struct _Py_interp_cached_objects cached_objects; diff --git a/Python/ceval.c b/Python/ceval.c index e9f082f85bd1b4..a8987efc21a981 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1914,7 +1914,7 @@ static int do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, int event) { - assert(event < PY_MONITORING_UNGROUPED_EVENTS); + assert(event < _PY_MONITORING_UNGROUPED_EVENTS); PyObject *exc = PyErr_GetRaisedException(); assert(exc != NULL); int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index e29748f0ad9872..e1b62bb32ec8bd 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -137,7 +137,7 @@ is_instrumented(int opcode) static inline bool monitors_equals(_Py_Monitors a, _Py_Monitors b) { - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (a.tools[i] != b.tools[i]) { return false; } @@ -150,7 +150,7 @@ static inline _Py_Monitors monitors_sub(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & ~b.tools[i]; } return res; @@ -161,7 +161,7 @@ static inline _Py_Monitors monitors_and(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] & b.tools[i]; } return res; @@ -172,7 +172,7 @@ static inline _Py_Monitors monitors_or(_Py_Monitors a, _Py_Monitors b) { _Py_Monitors res; - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { res.tools[i] = a.tools[i] | b.tools[i]; } return res; @@ -181,7 +181,7 @@ monitors_or(_Py_Monitors a, _Py_Monitors b) static inline bool monitors_are_empty(_Py_Monitors m) { - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (m.tools[i]) { return false; } @@ -192,7 +192,7 @@ monitors_are_empty(_Py_Monitors m) static inline bool multiple_tools(_Py_Monitors *m) { - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { if (_Py_popcount32(m->tools[i]) > 1) { return true; } @@ -204,7 +204,7 @@ static inline _PyMonitoringEventSet get_events(_Py_Monitors *m, int tool_id) { _PyMonitoringEventSet result = 0; - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((m->tools[e] >> tool_id) & 1) { result |= (1 << e); } @@ -339,7 +339,7 @@ static void dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) { fprintf(out, "%s monitors:\n", prefix); - for (int event = 0; event < PY_MONITORING_UNGROUPED_EVENTS; event++) { + for (int event = 0; event < _PY_MONITORING_UNGROUPED_EVENTS; event++) { fprintf(out, " Event %d: Tools %x\n", event, monitors.tools[event]); } } @@ -907,7 +907,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, uint8_t tools; assert(event != PY_MONITORING_EVENT_LINE); assert(event != PY_MONITORING_EVENT_INSTRUCTION); - if (event >= PY_MONITORING_UNGROUPED_EVENTS) { + if (event >= _PY_MONITORING_UNGROUPED_EVENTS) { assert(event == PY_MONITORING_EVENT_C_RAISE || event == PY_MONITORING_EVENT_C_RETURN); event = PY_MONITORING_EVENT_CALL; @@ -1220,7 +1220,7 @@ _PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj) { PyInterpreterState *is = _PyInterpreterState_GET(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - assert(0 <= event_id && event_id < PY_MONITORING_EVENTS); + assert(0 <= event_id && event_id < _PY_MONITORING_EVENTS); PyObject *callback = is->monitoring_callables[tool_id][event_id]; is->monitoring_callables[tool_id][event_id] = Py_XNewRef(obj); return callback; @@ -1653,7 +1653,7 @@ static void set_events(_Py_Monitors *m, int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { uint8_t *tools = &m->tools[e]; int val = (events >> e) & 1; *tools &= ~(1 << tool_id); @@ -1678,7 +1678,7 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; } @@ -1696,7 +1696,7 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent { assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(events < (1 << PY_MONITORING_UNGROUPED_EVENTS)); + assert(events < (1 << _PY_MONITORING_UNGROUPED_EVENTS)); if (check_tool(interp, tool_id)) { return -1; } @@ -1835,7 +1835,7 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, return NULL; } int event_id = _Py_bit_length(event)-1; - if (event_id < 0 || event_id >= PY_MONITORING_EVENTS) { + if (event_id < 0 || event_id >= _PY_MONITORING_EVENTS) { PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } @@ -1885,7 +1885,7 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) if (check_valid_tool(tool_id)) { return NULL; } - if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); return NULL; } @@ -1927,7 +1927,7 @@ monitoring_get_local_events_impl(PyObject *module, int tool_id, _PyMonitoringEventSet event_set = 0; _PyCoMonitoringData *data = ((PyCodeObject *)code)->_co_monitoring; if (data != NULL) { - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { if ((data->local_monitors.tools[e] >> tool_id) & 1) { event_set |= (1 << e); } @@ -1961,7 +1961,7 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, if (check_valid_tool(tool_id)) { return NULL; } - if (event_set < 0 || event_set >= (1 << PY_MONITORING_EVENTS)) { + if (event_set < 0 || event_set >= (1 << _PY_MONITORING_EVENTS)) { PyErr_Format(PyExc_ValueError, "invalid event set 0x%x", event_set); return NULL; } @@ -2042,7 +2042,7 @@ monitoring__all_events_impl(PyObject *module) if (res == NULL) { return NULL; } - for (int e = 0; e < PY_MONITORING_UNGROUPED_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_UNGROUPED_EVENTS; e++) { uint8_t tools = interp->monitors.tools[e]; if (tools == 0) { continue; @@ -2101,7 +2101,7 @@ PyObject *_Py_CreateMonitoringObject(void) if (err) { goto error; } - for (int i = 0; i < PY_MONITORING_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_EVENTS; i++) { if (add_power2_constant(events, event_names[i], i)) { goto error; } diff --git a/Python/pystate.c b/Python/pystate.c index a9b404bd5c93e3..cdd975f3672b9b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -679,11 +679,11 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_EVENTS; e++) { interp->monitoring_callables[t][e] = NULL; } @@ -841,11 +841,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - for (int e = 0; e < PY_MONITORING_EVENTS; e++) { + for (int e = 0; e < _PY_MONITORING_EVENTS; e++) { Py_CLEAR(interp->monitoring_callables[t][e]); } } From c1331ad50891b1727896afe62258bda73a459d40 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 23:41:11 +0200 Subject: [PATCH 37/73] gh-106320: Remove private _PyModule API (#107070) Move private _PyModule API to the internal C API (pycore_moduleobject.h): * _PyModule_Clear() * _PyModule_ClearDict() * _PyModuleSpec_IsInitializing() * _PyModule_IsExtension() No longer export these functions. --- Include/internal/pycore_moduleobject.h | 6 ++++++ Include/moduleobject.h | 11 ----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 31a31e724d0b21..5644bbe5e0552b 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern void _PyModule_Clear(PyObject *); +extern void _PyModule_ClearDict(PyObject *); +extern int _PyModuleSpec_IsInitializing(PyObject *); + +extern int _PyModule_IsExtension(PyObject *obj); + typedef struct { PyObject_HEAD PyObject *md_dict; diff --git a/Include/moduleobject.h b/Include/moduleobject.h index b8bdfe29d80406..ea08145381cee6 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -27,11 +27,6 @@ PyAPI_FUNC(PyObject *) PyModule_GetNameObject(PyObject *); PyAPI_FUNC(const char *) PyModule_GetName(PyObject *); Py_DEPRECATED(3.2) PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyModule_Clear(PyObject *); -PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *); -PyAPI_FUNC(int) _PyModuleSpec_IsInitializing(PyObject *); -#endif PyAPI_FUNC(PyModuleDef*) PyModule_GetDef(PyObject*); PyAPI_FUNC(void*) PyModule_GetState(PyObject*); @@ -103,12 +98,6 @@ struct PyModuleDef { freefunc m_free; }; - -// Internal C API -#ifdef Py_BUILD_CORE -extern int _PyModule_IsExtension(PyObject *obj); -#endif - #ifdef __cplusplus } #endif From 889851ecc355cad3d79f5692a6dbd3ae3127647b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 22 Jul 2023 23:44:33 +0200 Subject: [PATCH 38/73] gh-106320: Remove _PyFunction_Vectorcall() API (#107071) Move _PyFunction_Vectorcall() API to the internal C API. No longer export the function. --- Include/cpython/funcobject.h | 6 ------ Include/internal/pycore_function.h | 6 ++++++ Objects/call.c | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 6f78f5868d0166..de2013323d2c72 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -79,12 +79,6 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( - PyObject *func, - PyObject *const *stack, - size_t nargsf, - PyObject *kwnames); - #define _PyFunction_CAST(func) \ (assert(PyFunction_Check(func)), _Py_CAST(PyFunctionObject*, func)) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index ecbb7001e7d840..e844d323ec7927 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -8,6 +8,12 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyObject* _PyFunction_Vectorcall( + PyObject *func, + PyObject *const *stack, + size_t nargsf, + PyObject *kwnames); + #define FUNC_MAX_WATCHERS 8 struct _py_func_state { diff --git a/Objects/call.c b/Objects/call.c index 396552d85cfca0..b1610dababd466 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -2,6 +2,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgsTstate() #include "pycore_ceval.h" // _Py_EnterRecursiveCallTstate() #include "pycore_dict.h" // _PyDict_FromItems() +#include "pycore_function.h" // _PyFunction_Vectorcall() definition #include "pycore_modsupport.h" // _Py_VaBuildStack() #include "pycore_object.h" // _PyCFunctionWithKeywords_TrampolineCall() #include "pycore_pyerrors.h" // _PyErr_Occurred() From 3aeffc0d8f28655186f99d013ee9653c65b92f84 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 23 Jul 2023 00:30:42 +0200 Subject: [PATCH 39/73] gh-104050: Argument Clinic: Increase typing coverage (#107074) Co-authored-by: Alex Waygood --- Tools/clinic/clinic.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 7a4c9c4cacf55b..eecb81dcad5881 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2950,12 +2950,18 @@ def parse_argument(self, args: list[str]) -> None: # All the functions after here are intended as extension points. # - def simple_declaration(self, by_reference=False, *, in_parser=False): + def simple_declaration( + self, + by_reference: bool = False, + *, + in_parser: bool = False + ) -> str: """ Computes the basic declaration of the variable. Used in computing the prototype declaration and the variable declaration. """ + assert isinstance(self.type, str) prototype = [self.type] if by_reference or not self.type.endswith('*'): prototype.append(" ") @@ -2970,7 +2976,7 @@ def simple_declaration(self, by_reference=False, *, in_parser=False): prototype.append(name) return "".join(prototype) - def declaration(self, *, in_parser=False) -> str: + def declaration(self, *, in_parser: bool = False) -> str: """ The C statement to declare this variable. """ @@ -3579,9 +3585,9 @@ class object_converter(CConverter): def converter_init( self, *, - converter=None, - type=None, - subclass_of=None + converter: str | None = None, + type: str | None = None, + subclass_of: str | None = None ) -> None: if converter: if subclass_of: @@ -3973,7 +3979,7 @@ class self_converter(CConverter): type = None format_unit = '' - def converter_init(self, *, type=None) -> None: + def converter_init(self, *, type: str | None = None) -> None: self.specified_type = type def pre_render(self): @@ -4047,7 +4053,7 @@ def render(self, parameter, data): assert data.impl_arguments[0] == self.name data.impl_arguments[0] = '(' + self.type + ")" + data.impl_arguments[0] - def set_template_dict(self, template_dict): + def set_template_dict(self, template_dict: TemplateDict) -> None: template_dict['self_name'] = self.name template_dict['self_type'] = self.parser_type kind = self.function.kind @@ -4066,7 +4072,7 @@ def set_template_dict(self, template_dict): line = f'{type_check} &&\n ' template_dict['self_type_check'] = line - type_object = self.function.cls.type_object + type_object = cls.type_object type_ptr = f'PyTypeObject *base_tp = {type_object};' template_dict['base_type_ptr'] = type_ptr @@ -4276,11 +4282,11 @@ def eval_ast_expr( class IndentStack: - def __init__(self): - self.indents = [] - self.margin = None + def __init__(self) -> None: + self.indents: list[int] = [] + self.margin: str | None = None - def _ensure(self): + def _ensure(self) -> None: if not self.indents: fail('IndentStack expected indents, but none are defined.') @@ -4341,6 +4347,7 @@ def indent(self, line: str) -> str: """ Indents a line by the currently defined margin. """ + assert self.margin is not None, "Cannot call .indent() before calling .infer()" return self.margin + line def dedent(self, line: str) -> str: @@ -4348,6 +4355,7 @@ def dedent(self, line: str) -> str: Dedents a line by the currently defined margin. (The inverse of 'indent'.) """ + assert self.margin is not None, "Cannot call .indent() before calling .infer()" margin = self.margin indent = self.indents[-1] if not line.startswith(margin): @@ -5232,6 +5240,7 @@ def parse_slash(self, function: Function) -> None: p.kind = inspect.Parameter.POSITIONAL_ONLY def state_parameter_docstring_start(self, line: str | None) -> None: + assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" self.parameter_docstring_indent = len(self.indent.margin) assert self.indent.depth == 3 return self.next(self.state_parameter_docstring, line) @@ -5534,7 +5543,7 @@ def add_parameter(text): return docstring - def state_terminal(self, line): + def state_terminal(self, line: str | None) -> None: """ Called when processing the block is done. """ From 9a6b278769b9f24e0650283f6c347db8ae52b7b3 Mon Sep 17 00:00:00 2001 From: Lukas van de Wiel <30800501+LukasvdWiel@users.noreply.github.com> Date: Sun, 23 Jul 2023 01:20:03 +0200 Subject: [PATCH 40/73] gh-106962: Detect mpicc in configure.ac (#106961) Don't let autoconf mistake MPI compilers for Intel compilers; filter out the MPI case to prevent Intel specific options from being applied. --- .../next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst | 1 + configure | 3 +++ configure.ac | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst diff --git a/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst b/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst new file mode 100644 index 00000000000000..32e196fe26d3b7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst @@ -0,0 +1 @@ +Detect MPI compilers in :file:`configure`. diff --git a/configure b/configure index e6fb5e3c2b0c2f..18d404c8dc0605 100755 --- a/configure +++ b/configure @@ -10154,6 +10154,9 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ esac case "$CC" in +*mpicc*) + CFLAGS_NODIST="$CFLAGS_NODIST" + ;; *icc*) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" diff --git a/configure.ac b/configure.ac index a1ee78047692fd..cdb88a3071976c 100644 --- a/configure.ac +++ b/configure.ac @@ -2656,6 +2656,9 @@ yes) esac case "$CC" in +*mpicc*) + CFLAGS_NODIST="$CFLAGS_NODIST" + ;; *icc*) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" From 7fc9be350af055538e70ece8d7de78414bad431e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 22 Jul 2023 18:22:33 -0700 Subject: [PATCH 41/73] gh-107082: Remove debug hack from get_first_executor in test_capi.test_misc (#107085) (Even though it doesn't look like it fixes gh-107082 -- see discussion there -- it still removes debug code that should never have been committed.) --- Lib/test/test_capi/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index dc155cc99961c4..0d3f93941e8205 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2436,7 +2436,7 @@ def get_first_executor(func): co_code = code.co_code JUMP_BACKWARD = opcode.opmap["JUMP_BACKWARD"] for i in range(0, len(co_code), 2): - if co_code[i] == JUMP_BACKWARD or 1: + if co_code[i] == JUMP_BACKWARD: try: return _testinternalcapi.get_executor(code, i) except ValueError: From 9eeb4b485f4f9e13e5cb638e43a0baecb3ff4e86 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 22 Jul 2023 23:07:14 -0500 Subject: [PATCH 42/73] gh-82500: Fix asyncio sendfile overflow on 32bit (#107056) --- Lib/asyncio/unix_events.py | 3 +++ .../next/Library/2023-07-22-16-44-58.gh-issue-82500.cQYoPj.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-16-44-58.gh-issue-82500.cQYoPj.rst diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 17fb4d5f7646ce..a2680865ed968f 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -394,6 +394,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fut.set_result(total_sent) return + # On 32-bit architectures truncate to 1GiB to avoid OverflowError + blocksize = min(blocksize, sys.maxsize//2 + 1) + try: sent = os.sendfile(fd, fileno, offset, blocksize) except (BlockingIOError, InterruptedError): diff --git a/Misc/NEWS.d/next/Library/2023-07-22-16-44-58.gh-issue-82500.cQYoPj.rst b/Misc/NEWS.d/next/Library/2023-07-22-16-44-58.gh-issue-82500.cQYoPj.rst new file mode 100644 index 00000000000000..065394fd6ee712 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-16-44-58.gh-issue-82500.cQYoPj.rst @@ -0,0 +1 @@ +Fix overflow on 32-bit systems with :mod:`asyncio` :func:`os.sendfile` implemention. From 0af247da0932692592ed85ba8b4a1520627ab4ac Mon Sep 17 00:00:00 2001 From: wulmer Date: Sun, 23 Jul 2023 10:50:38 +0200 Subject: [PATCH 43/73] gh-102111: Add link to string escape sequences in re module (#106995) Co-authored-by: Alex Waygood --- Doc/library/re.rst | 4 ++-- Doc/reference/lexical_analysis.rst | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 629ee472cca681..3f03f0341d8166 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -634,8 +634,8 @@ character ``'$'``. single: \x; in regular expressions single: \\; in regular expressions -Most of the standard escapes supported by Python string literals are also -accepted by the regular expression parser:: +Most of the :ref:`escape sequences ` supported by Python +string literals are also accepted by the regular expression parser:: \a \b \f \n \N \r \t \u diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 47062f86810e91..dde7ba1d941dce 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -549,6 +549,10 @@ retained), except that three unescaped quotes in a row terminate the literal. ( .. _escape-sequences: + +Escape sequences +^^^^^^^^^^^^^^^^ + Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: From 9629d4442e24681e79c1ba93cb92ed2e3b967224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8Cervinka?= Date: Sun, 23 Jul 2023 10:57:52 +0200 Subject: [PATCH 44/73] gh-107017: removed mention that C does it the same way (#107020) --- Doc/tutorial/controlflow.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index e140f51f1dda78..138d87f892e891 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -4,8 +4,8 @@ More Control Flow Tools *********************** -Besides the :keyword:`while` statement just introduced, Python uses the usual -flow control statements known from other languages, with some twists. +As well as the :keyword:`while` statement just introduced, Python uses a few more +that we will encounter in this chapter. .. _tut-if: @@ -163,14 +163,21 @@ arguments. In chapter :ref:`tut-structures`, we will discuss in more detail abo :keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops ============================================================================================ -The :keyword:`break` statement, like in C, breaks out of the innermost enclosing +The :keyword:`break` statement breaks out of the innermost enclosing :keyword:`for` or :keyword:`while` loop. -Loop statements may have an :keyword:`!else` clause; it is executed when the loop -terminates through exhaustion of the iterable (with :keyword:`for`) or when the -condition becomes false (with :keyword:`while`), but not when the loop is -terminated by a :keyword:`break` statement. This is exemplified by the -following loop, which searches for prime numbers:: +A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. + +In a :keyword:`for` loop, the :keyword:`!else` clause is executed +after the loop reaches its final iteration. + +In a :keyword:`while` loop, it's executed after the loop's condition becomes false. + +In either kind of loop, the :keyword:`!else` clause is **not** executed +if the loop was terminated by a :keyword:`break`. + +This is exemplified in the following :keyword:`!for` loop, +which searches for prime numbers:: >>> for n in range(2, 10): ... for x in range(2, n): From 680f3e1591e406deb04ba44adbef9a5a02395f80 Mon Sep 17 00:00:00 2001 From: wulmer Date: Sun, 23 Jul 2023 11:00:42 +0200 Subject: [PATCH 45/73] gh-71261: Add paragraph on shadowing submodules with star imports (#107004) --- Doc/tutorial/modules.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 3bd034bcc9703f..734dd1cfe6871a 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -512,6 +512,22 @@ code:: This would mean that ``from sound.effects import *`` would import the three named submodules of the :mod:`sound.effects` package. +Be aware that submodules might become shadowed by locally defined names. For +example, if you added a ``reverse`` function to the +:file:`sound/effects/__init__.py` file, the ``from sound.effects import *`` +would only import the two submodules ``echo`` and ``surround``, but *not* the +``reverse`` submodule, because it is shadowed by the locally defined +``reverse`` function:: + + __all__ = [ + "echo", # refers to the 'echo.py' file + "surround", # refers to the 'surround.py' file + "reverse", # !!! refers to the 'reverse' function now !!! + ] + + def reverse(msg: str): # <-- this name shadows the 'reverse.py' submodule + return msg[::-1] # in the case of a 'from sound.effects import *' + If ``__all__`` is not defined, the statement ``from sound.effects import *`` does *not* import all submodules from the package :mod:`sound.effects` into the current namespace; it only ensures that the package :mod:`sound.effects` has From dcd7acb04a719d8d30c8d03b80d3d48b6c035e14 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Sun, 23 Jul 2023 11:10:38 +0200 Subject: [PATCH 46/73] gh-54738: Add argparse i18n howto (#104562) --- Doc/howto/argparse.rst | 53 +++++++++++++++++++ Doc/library/gettext.rst | 1 + ...3-05-16-22-08-24.gh-issue-54738.mJvCnj.rst | 1 + 3 files changed, 55 insertions(+) create mode 100644 Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 52e98fa9620194..ae5bab90bf8131 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -788,6 +788,59 @@ but not both at the same time: -q, --quiet +How to translate the argparse output +==================================== + +The output of the :mod:`argparse` module such as its help text and error +messages are all made translatable using the :mod:`gettext` module. This +allows applications to easily localize messages produced by +:mod:`argparse`. See also :ref:`i18n-howto`. + +For instance, in this :mod:`argparse` output: + +.. code-block:: shell-session + + $ python prog.py --help + usage: prog.py [-h] [-v | -q] x y + + calculate X to the power of Y + + positional arguments: + x the base + y the exponent + + options: + -h, --help show this help message and exit + -v, --verbose + -q, --quiet + +The strings ``usage:``, ``positional arguments:``, ``options:`` and +``show this help message and exit`` are all translatable. + +In order to translate these strings, they must first be extracted +into a ``.po`` file. For example, using `Babel `__, +run this command: + +.. code-block:: shell-session + + $ pybabel extract -o messages.po /usr/lib/python3.12/argparse.py + +This command will extract all translatable strings from the :mod:`argparse` +module and output them into a file named ``messages.po``. This command assumes +that your Python installation is in ``/usr/lib``. + +You can find out the location of the :mod:`argparse` module on your system +using this script:: + + import argparse + print(argparse.__file__) + +Once the messages in the ``.po`` file are translated and the translations are +installed using :mod:`gettext`, :mod:`argparse` will be able to display the +translated messages. + +To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`. + Conclusion ========== diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 747f8703b750ec..88a65b980d310f 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -411,6 +411,7 @@ One difference between this module and Henstridge's: his catalog objects supported access through a mapping API, but this appears to be unused and so is not currently supported. +.. _i18n-howto: Internationalizing your programs and modules -------------------------------------------- diff --git a/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst b/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst new file mode 100644 index 00000000000000..4da58fc982b6d7 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst @@ -0,0 +1 @@ +Add documentation on how to localize the :mod:`argparse` module. From f5147c0cfbd7943ff10917225448c36a53f9828d Mon Sep 17 00:00:00 2001 From: wulmer Date: Sun, 23 Jul 2023 11:23:44 +0200 Subject: [PATCH 47/73] gh-101100: Fix some broken sphinx references (#107095) --- Doc/c-api/iterator.rst | 2 +- Doc/c-api/mapping.rst | 6 +++--- Doc/c-api/refcounting.rst | 2 +- Doc/c-api/sequence.rst | 2 +- Doc/howto/functional.rst | 4 ++-- Doc/howto/regex.rst | 2 ++ Doc/howto/sorting.rst | 6 +++--- Doc/howto/unicode.rst | 8 ++++---- Doc/library/_thread.rst | 9 +++++---- Doc/library/codeop.rst | 4 ++-- Doc/library/constants.rst | 6 +++--- Doc/tools/.nitignore | 10 ---------- 12 files changed, 27 insertions(+), 34 deletions(-) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 3fcf099134d4dd..95952237ca746f 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -6,7 +6,7 @@ Iterator Objects ---------------- Python provides two general-purpose iterator objects. The first, a sequence -iterator, works with an arbitrary sequence supporting the :meth:`__getitem__` +iterator, works with an arbitrary sequence supporting the :meth:`~object.__getitem__` method. The second works with a callable object and a sentinel value, calling the callable for each item in the sequence, and ending the iteration when the sentinel value is returned. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 9176a4652cbf29..82011d9d36a524 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -13,7 +13,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and Return ``1`` if the object provides the mapping protocol or supports slicing, and ``0`` otherwise. Note that it returns ``1`` for Python classes with - a :meth:`__getitem__` method, since in general it is impossible to + a :meth:`~object.__getitem__` method, since in general it is impossible to determine what type of keys the class supports. This function always succeeds. @@ -90,7 +90,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and This is equivalent to the Python expression ``key in o``. This function always succeeds. - Note that exceptions which occur while calling the :meth:`__getitem__` + Note that exceptions which occur while calling the :meth:`~object.__getitem__` method will get suppressed. To get error reporting use :c:func:`PyObject_GetItem()` instead. @@ -101,7 +101,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and This is equivalent to the Python expression ``key in o``. This function always succeeds. - Note that exceptions which occur while calling the :meth:`__getitem__` + Note that exceptions which occur while calling the :meth:`~object.__getitem__` method and creating a temporary string object will get suppressed. To get error reporting use :c:func:`PyMapping_GetItemString()` instead. diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index d8e9c2da6f3ff3..640c5c610899f8 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -101,7 +101,7 @@ of Python objects. .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. - when a class instance with a :meth:`__del__` method is deallocated). While + when a class instance with a :meth:`~object.__del__` method is deallocated). While exceptions in such code are not propagated, the executed code has free access to all Python global variables. This means that any object that is reachable from a global variable should be in a consistent state before :c:func:`Py_DECREF` is diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index 402a3e5e09ff56..ce28839f5ba739 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -9,7 +9,7 @@ Sequence Protocol .. c:function:: int PySequence_Check(PyObject *o) Return ``1`` if the object provides the sequence protocol, and ``0`` otherwise. - Note that it returns ``1`` for Python classes with a :meth:`__getitem__` + Note that it returns ``1`` for Python classes with a :meth:`~object.__getitem__` method, unless they are :class:`dict` subclasses, since in general it is impossible to determine what type of keys the class supports. This function always succeeds. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 5cf12cc52bde4e..b0f9d22d74f0e3 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1072,8 +1072,8 @@ write the obvious :keyword:`for` loop:: A related function is :func:`itertools.accumulate(iterable, func=operator.add) `. It performs the same calculation, but instead of -returning only the final result, :func:`accumulate` returns an iterator that -also yields each partial result:: +returning only the final result, :func:`~itertools.accumulate` returns an iterator +that also yields each partial result:: itertools.accumulate([1, 2, 3, 4, 5]) => 1, 3, 6, 10, 15 diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 655df59e27b641..c19c48301f5848 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -518,6 +518,8 @@ cache. Compilation Flags ----------------- +.. currentmodule:: re + Compilation flags let you modify some aspects of how regular expressions work. Flags are available in the :mod:`re` module under two names, a long name such as :const:`IGNORECASE` and a short, one-letter form such as :const:`I`. (If you're diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index decce12bf3faf6..38dd09f0a721d2 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -273,7 +273,7 @@ Odds and Ends * The sort routines use ``<`` when making comparisons between two objects. So, it is easy to add a standard sort order to a class by - defining an :meth:`__lt__` method: + defining an :meth:`~object.__lt__` method: .. doctest:: @@ -281,8 +281,8 @@ Odds and Ends >>> sorted(student_objects) [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - However, note that ``<`` can fall back to using :meth:`__gt__` if - :meth:`__lt__` is not implemented (see :func:`object.__lt__`). + However, note that ``<`` can fall back to using :meth:`~object.__gt__` if + :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). * Key functions need not depend directly on the objects being sorted. A key function can also access external resources. For instance, if the student grades diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index b0faa68d240896..254fe729355353 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -424,8 +424,8 @@ lowercase letters 'ss'. A second tool is the :mod:`unicodedata` module's :func:`~unicodedata.normalize` function that converts strings to one -of several normal forms, where letters followed by a combining -character are replaced with single characters. :func:`normalize` can +of several normal forms, where letters followed by a combining character are +replaced with single characters. :func:`~unicodedata.normalize` can be used to perform string comparisons that won't falsely report inequality if two strings use combining characters differently: @@ -474,8 +474,8 @@ The Unicode Standard also specifies how to do caseless comparisons:: print(compare_caseless(single_char, multiple_chars)) -This will print ``True``. (Why is :func:`NFD` invoked twice? Because -there are a few characters that make :meth:`casefold` return a +This will print ``True``. (Why is :func:`!NFD` invoked twice? Because +there are a few characters that make :meth:`~str.casefold` return a non-normalized string, so the result needs to be normalized again. See section 3.13 of the Unicode Standard for a discussion and an example.) diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 3ff6b48b083b51..0442c298c137ba 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -150,8 +150,8 @@ This module defines the following constants and functions: .. data:: TIMEOUT_MAX The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. + :meth:`Lock.acquire `. Specifying a timeout greater + than this value will raise an :exc:`OverflowError`. .. versionadded:: 3.2 @@ -217,8 +217,9 @@ In addition to these methods, lock objects can also be used via the * Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is equivalent to calling :func:`_thread.exit`. -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. +* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on + a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock + has been acquired. * When the main thread exits, it is system defined whether the other threads survive. On most systems, they are killed without executing diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 90df499f8207b7..55606e1c5f09ac 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -58,7 +58,7 @@ To do just the former: .. class:: Compile() - Instances of this class have :meth:`__call__` methods identical in signature to + Instances of this class have :meth:`~object.__call__` methods identical in signature to the built-in function :func:`compile`, but with the difference that if the instance compiles program text containing a :mod:`__future__` statement, the instance 'remembers' and compiles all subsequent program texts with the @@ -67,7 +67,7 @@ To do just the former: .. class:: CommandCompiler() - Instances of this class have :meth:`__call__` methods identical in signature to + Instances of this class have :meth:`~object.__call__` methods identical in signature to :func:`compile_command`; the difference is that if the instance compiles program text containing a :mod:`__future__` statement, the instance 'remembers' and compiles all subsequent program texts with the statement in force. diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 38dd552a0363ac..401dc9a320c5e0 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,16 +22,16 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`NoneType` type. + ``None`` is the sole instance of the :data:`~types.NoneType` type. .. data:: NotImplemented A special value which should be returned by the binary special methods - (e.g. :meth:`__eq__`, :meth:`__lt__`, :meth:`__add__`, :meth:`__rsub__`, + (e.g. :meth:`~object.__eq__`, :meth:`~object.__lt__`, :meth:`~object.__add__`, :meth:`~object.__rsub__`, etc.) to indicate that the operation is not implemented with respect to the other type; may be returned by the in-place binary special methods - (e.g. :meth:`__imul__`, :meth:`__iand__`, etc.) for the same purpose. + (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 14d9b2e121d3b6..818276a81111e8 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -27,15 +27,12 @@ Doc/c-api/init_config.rst Doc/c-api/intro.rst Doc/c-api/iterator.rst Doc/c-api/long.rst -Doc/c-api/mapping.rst Doc/c-api/marshal.rst Doc/c-api/memory.rst Doc/c-api/memoryview.rst Doc/c-api/module.rst Doc/c-api/none.rst Doc/c-api/object.rst -Doc/c-api/refcounting.rst -Doc/c-api/sequence.rst Doc/c-api/set.rst Doc/c-api/stable.rst Doc/c-api/structures.rst @@ -59,18 +56,13 @@ Doc/glossary.rst Doc/howto/curses.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/functional.rst Doc/howto/instrumentation.rst Doc/howto/isolating-extensions.rst Doc/howto/logging-cookbook.rst Doc/howto/logging.rst -Doc/howto/regex.rst -Doc/howto/sorting.rst -Doc/howto/unicode.rst Doc/howto/urllib2.rst Doc/install/index.rst Doc/library/__future__.rst -Doc/library/_thread.rst Doc/library/abc.rst Doc/library/ast.rst Doc/library/asyncio-dev.rst @@ -88,13 +80,11 @@ Doc/library/calendar.rst Doc/library/cmd.rst Doc/library/code.rst Doc/library/codecs.rst -Doc/library/codeop.rst Doc/library/collections.abc.rst Doc/library/collections.rst Doc/library/concurrent.futures.rst Doc/library/concurrent.rst Doc/library/configparser.rst -Doc/library/constants.rst Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst From 102a773716981c59ee7469e65e4528f7ab5e9c47 Mon Sep 17 00:00:00 2001 From: littlebutt's workshop Date: Sun, 23 Jul 2023 09:41:34 +0000 Subject: [PATCH 48/73] gh-106976: alphabetise bullets by module name task2-3 (#107005) Co-authored-by: Hugo van Kemenade --- Doc/whatsnew/3.12.rst | 474 +++++++++++++++++++++--------------------- 1 file changed, 237 insertions(+), 237 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ea9806987f6872..f043ec764ac554 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -966,67 +966,76 @@ Demos and Tools Deprecated ========== -* :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` - and :class:`collections.abc.Sized`. (:gh:`94309`.) - -* The :mod:`sqlite3` :ref:`default adapters and converters - ` are now deprecated. - Instead, use the :ref:`sqlite3-adapter-converter-recipes` - and tailor them to your needs. - (Contributed by Erlend E. Aasland in :gh:`90016`.) - -* In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted - when :ref:`named placeholders ` are used together with - parameters supplied as a :term:`sequence` instead of as a :class:`dict`. - Starting from Python 3.14, using named placeholders with parameters supplied - as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. - (Contributed by Erlend E. Aasland in :gh:`101698`.) - -* The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, - :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and - may be removed in a future version of Python. Use the single-arg versions - of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) - -* :exc:`DeprecationWarning` is now raised when ``__package__`` on a - module differs from ``__spec__.parent`` (previously it was - :exc:`ImportWarning`). - (Contributed by Brett Cannon in :gh:`65961`.) - -* The :meth:`~asyncio.get_event_loop` method of the +* :mod:`asyncio`: The :meth:`~asyncio.get_event_loop` method of the default event loop policy now emits a :exc:`DeprecationWarning` if there is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* The :mod:`xml.etree.ElementTree` module now emits :exc:`DeprecationWarning` - when testing the truth value of an :class:`xml.etree.ElementTree.Element`. - Before, the Python implementation emitted :exc:`FutureWarning`, and the C - implementation emitted nothing. +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are deprecated and + replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) -* In accordance with :pep:`699`, the ``ma_version_tag`` field in :c:type:`PyDictObject` - is deprecated for extension modules. Accessing this field will generate a compiler - warning at compile time. This field will be removed in Python 3.14. - (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) +* :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and + :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be + removed in a future version. Instead, use timezone-aware objects to represent + datetimes in UTC: respectively, call :meth:`~datetime.datetime.now` and + :meth:`~datetime.datetime.fromtimestamp` with the *tz* parameter set to + :const:`datetime.UTC`. + (Contributed by Paul Ganssle in :gh:`103857`.) -* The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on +* :mod:`os`: The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on Windows are deprecated. In a future release, they will contain the last metadata change time, consistent with other platforms. For now, they still contain the creation time, which is also available in the new ``st_birthtime`` field. (Contributed by Steve Dower in :gh:`99726`.) -* The :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated as will be removed + in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) + +* :mod:`sqlite3`: + * :ref:`default adapters and converters + ` are now deprecated. + Instead, use the :ref:`sqlite3-adapter-converter-recipes` + and tailor them to your needs. + (Contributed by Erlend E. Aasland in :gh:`90016`.) + + * In :meth:`~sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted + when :ref:`named placeholders ` are used together with + parameters supplied as a :term:`sequence` instead of as a :class:`dict`. + Starting from Python 3.14, using named placeholders with parameters supplied + as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. + (Contributed by Erlend E. Aasland in :gh:`101698`.) + +* :mod:`sys`: The :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` fields are deprecated. Use :data:`sys.last_exc` instead. (Contributed by Irit Katriel in :gh:`102778`.) -* The *onerror* argument of :func:`shutil.rmtree` is deprecated as will be removed - in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) - -* Extracting tar archives without specifying *filter* is deprecated until +* :mod:`tarfile`: Extracting tar archives without specifying *filter* is deprecated until Python 3.14, when ``'data'`` filter will become the default. See :ref:`tarfile-extraction-filter` for details. -* ``calendar.January`` and ``calendar.February`` constants are deprecated and - replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. - (Contributed by Prince Roshan in :gh:`103636`.) +* :mod:`typing`: :class:`typing.Hashable` and :class:`typing.Sized` aliases for :class:`collections.abc.Hashable` + and :class:`collections.abc.Sized`. (:gh:`94309`.) + +* :mod:`xml.etree.ElementTree`: The module now emits :exc:`DeprecationWarning` + when testing the truth value of an :class:`xml.etree.ElementTree.Element`. + Before, the Python implementation emitted :exc:`FutureWarning`, and the C + implementation emitted nothing. + +* The 3-arg signatures (type, value, traceback) of :meth:`~coroutine.throw`, + :meth:`~generator.throw` and :meth:`~agen.athrow` are deprecated and + may be removed in a future version of Python. Use the single-arg versions + of these functions instead. (Contributed by Ofey Chan in :gh:`89874`.) + +* :exc:`DeprecationWarning` is now raised when ``__package__`` on a + module differs from ``__spec__.parent`` (previously it was + :exc:`ImportWarning`). + (Contributed by Brett Cannon in :gh:`65961`.) + +* In accordance with :pep:`699`, the ``ma_version_tag`` field in :c:type:`PyDictObject` + is deprecated for extension modules. Accessing this field will generate a compiler + warning at compile time. This field will be removed in Python 3.14. + (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) * The bitwise inversion operator (``~``) on bool is deprecated. It will throw an error in Python 3.14. Use ``not`` for logical negation of bools instead. @@ -1034,16 +1043,6 @@ Deprecated ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* :class:`datetime.datetime`'s - :meth:`~datetime.datetime.utcnow` and - :meth:`~datetime.datetime.utcfromtimestamp` are deprecated and will be - removed in a future version. Instead, use timezone-aware objects to represent - datetimes in UTC: respectively, call - :meth:`~datetime.datetime.now` and - :meth:`~datetime.datetime.fromtimestamp` with the *tz* parameter set to - :const:`datetime.UTC`. - (Contributed by Paul Ganssle in :gh:`103857`.) - Pending Removal in Python 3.13 ------------------------------ @@ -1088,7 +1087,33 @@ APIs: Pending Removal in Python 3.14 ------------------------------ -* Deprecated the following :mod:`importlib.abc` classes, scheduled for removal in +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following :mod:`ast` features have been deprecated in documentation since + Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime + when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc`: Deprecated the following classes, scheduled for removal in Python 3.14: * :class:`!importlib.abc.ResourceReader` @@ -1102,27 +1127,13 @@ Pending Removal in Python 3.14 (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) -* Deprecated :class:`collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - -* :class:`typing.ByteString`, deprecated since Python 3.9, now causes a - :exc:`DeprecationWarning` to be emitted when it is used. - -* Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. - -* Deprecated the *isdst* parameter in :func:`email.utils.localtime`. - (Contributed by Alan Williams in :gh:`72346`.) - -* ``__package__`` and ``__cached__`` will cease to be set or taken - into consideration by the import system (:gh:`97879`). - -* Testing the truth value of an :class:`xml.etree.ElementTree.Element` - is deprecated and will raise an exception in Python 3.14. +* :mod:`itertools`: The module had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) -* The default :mod:`multiprocessing` start method will change to a safer one on +* :mod:`multiprocessing`: The default :mod:`multiprocessing` start method will change to a safer one on Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently the default (:gh:`84559`). Adding a runtime warning about this was deemed too disruptive as the majority of code is not expected to care. Use the @@ -1130,47 +1141,35 @@ Pending Removal in Python 3.14 :func:`~multiprocessing.set_start_method` APIs to explicitly specify when your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. -* :mod:`pty` has two undocumented ``master_open()`` and ``slave_open()`` +* :mod:`pkgutil`: :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: The module has two undocumented ``master_open()`` and ``slave_open()`` functions that have been deprecated since Python 2 but only gained a proper :exc:`DeprecationWarning` in 3.12. Remove them in 3.14. -* :mod:`itertools` had undocumented, inefficient, historically buggy, - and inconsistent support for copy, deepcopy, and pickle operations. - This will be removed in 3.14 for a significant reduction in code - volume and maintenance burden. - (Contributed by Raymond Hettinger in :gh:`101588`.) - -* Accessing ``co_lnotab`` was deprecated in :pep:`626` since 3.10 - and was planned to be removed in 3.12 - but it only got a proper :exc:`DeprecationWarning` in 3.12. - May be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`101866`.) - -* The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, +* :mod:`shutil`: The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, and will be removed in 3.14. -* The *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction` are deprecated - and will be removed in 3.14. - (Contributed by Nikita Sobolev in :gh:`92248`.) +* :mod:`typing`: :class:`typing.ByteString`, deprecated since Python 3.9, now causes a + :exc:`DeprecationWarning` to be emitted when it is used. -* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` - now raise :exc:`DeprecationWarning`; - use :func:`importlib.util.find_spec` instead. - (Contributed by Nikita Sobolev in :gh:`97850`.) +* :mod:`xml.etree.ElementTree`: Testing the truth value of an :class:`xml.etree.ElementTree.Element` + is deprecated and will raise an exception in Python 3.14. -* The following :mod:`ast` features have been deprecated in documentation since - Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at runtime - when they are accessed or used, and will be removed in Python 3.14: +* Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable + bases using the C API. - * :class:`!ast.Num` - * :class:`!ast.Str` - * :class:`!ast.Bytes` - * :class:`!ast.NameConstant` - * :class:`!ast.Ellipsis` +* ``__package__`` and ``__cached__`` will cease to be set or taken + into consideration by the import system (:gh:`97879`). - Use :class:`ast.Constant` instead. - (Contributed by Serhiy Storchaka in :gh:`90953`.) +* Accessing ``co_lnotab`` was deprecated in :pep:`626` since 3.10 + and was planned to be removed in 3.12 + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`101866`.) Pending Removal in Future Versions ---------------------------------- @@ -1193,13 +1192,29 @@ although there is currently no date scheduled for their removal. Removed ======= -* Remove the ``distutils`` package. It was deprecated in Python 3.10 by +* ``asynchat`` and ``asyncore``: These two modules have been removed + according to the schedule in :pep:`594`, + having been deprecated in Python 3.6. + Use :mod:`asyncio` instead. + (Contributed by Nikita Sobolev in :gh:`96580`.) + +* :mod:`configparser`: Several names deprecated in the :mod:`configparser` way back in 3.2 have + been removed per :gh:`89336`: + + * :class:`configparser.ParsingError` no longer has a ``filename`` attribute + or argument. Use the ``source`` attribute and argument instead. + * :mod:`configparser` no longer has a ``SafeConfigParser`` class. Use the + shorter :class:`~configparser.ConfigParser` name instead. + * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. + Use :meth:`~configparser.ConfigParser.read_file` instead. + +* ``distutils``: Remove the ``distutils`` package. It was deprecated in Python 3.10 by :pep:`632` "Deprecate distutils module". For projects still using ``distutils`` and cannot be updated to something else, the ``setuptools`` project can be installed: it still provides ``distutils``. (Contributed by Victor Stinner in :gh:`92584`.) -* Remove the bundled setuptools wheel from :mod:`ensurepip`, +* :mod:`ensurepip`: Remove the bundled setuptools wheel from :mod:`ensurepip`, and stop installing setuptools in environments created by :mod:`venv`. ``pip (>= 22.1)`` does not require setuptools to be installed in the @@ -1217,94 +1232,9 @@ Removed (Contributed by Pradyun Gedam in :gh:`95299`.) -* Removed many old deprecated :mod:`unittest` features: - - - A number of :class:`~unittest.TestCase` method aliases: - - ============================ =============================== =============== - Deprecated alias Method Name Deprecated in - ============================ =============================== =============== - ``failUnless`` :meth:`.assertTrue` 3.1 - ``failIf`` :meth:`.assertFalse` 3.1 - ``failUnlessEqual`` :meth:`.assertEqual` 3.1 - ``failIfEqual`` :meth:`.assertNotEqual` 3.1 - ``failUnlessAlmostEqual`` :meth:`.assertAlmostEqual` 3.1 - ``failIfAlmostEqual`` :meth:`.assertNotAlmostEqual` 3.1 - ``failUnlessRaises`` :meth:`.assertRaises` 3.1 - ``assert_`` :meth:`.assertTrue` 3.2 - ``assertEquals`` :meth:`.assertEqual` 3.2 - ``assertNotEquals`` :meth:`.assertNotEqual` 3.2 - ``assertAlmostEquals`` :meth:`.assertAlmostEqual` 3.2 - ``assertNotAlmostEquals`` :meth:`.assertNotAlmostEqual` 3.2 - ``assertRegexpMatches`` :meth:`.assertRegex` 3.2 - ``assertRaisesRegexp`` :meth:`.assertRaisesRegex` 3.2 - ``assertNotRegexpMatches`` :meth:`.assertNotRegex` 3.5 - ============================ =============================== =============== - - You can use https://github.com/isidentical/teyit to automatically modernise - your unit tests. - - - Undocumented and broken :class:`~unittest.TestCase` method - ``assertDictContainsSubset`` (deprecated in Python 3.2). - - - Undocumented :meth:`TestLoader.loadTestsFromModule - ` parameter *use_load_tests* - (deprecated and ignored since Python 3.2). - - - An alias of the :class:`~unittest.TextTestResult` class: - ``_TextTestResult`` (deprecated in Python 3.2). - - (Contributed by Serhiy Storchaka in :issue:`45162`.) - -* Several names deprecated in the :mod:`configparser` way back in 3.2 have - been removed per :gh:`89336`: - - * :class:`configparser.ParsingError` no longer has a ``filename`` attribute - or argument. Use the ``source`` attribute and argument instead. - * :mod:`configparser` no longer has a ``SafeConfigParser`` class. Use the - shorter :class:`~configparser.ConfigParser` name instead. - * :class:`configparser.ConfigParser` no longer has a ``readfp`` method. - Use :meth:`~configparser.ConfigParser.read_file` instead. - -* The following undocumented :mod:`sqlite3` features, deprecated in Python - 3.10, are now removed: - - * ``sqlite3.enable_shared_cache()`` - * ``sqlite3.OptimizedUnicode`` - - If a shared cache must be used, open the database in URI mode using the - ``cache=shared`` query parameter. - - The ``sqlite3.OptimizedUnicode`` text factory has been an alias for - :class:`str` since Python 3.3. Code that previously set the text factory to - ``OptimizedUnicode`` can either use ``str`` explicitly, or rely on the - default value which is also ``str``. - - (Contributed by Erlend E. Aasland in :gh:`92548`.) - -* ``smtpd`` has been removed according to the schedule in :pep:`594`, - having been deprecated in Python 3.4.7 and 3.5.4. - Use aiosmtpd_ PyPI module or any other - :mod:`asyncio`-based server instead. - (Contributed by Oleg Iarygin in :gh:`93243`.) - -.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ - -* ``asynchat`` and ``asyncore`` have been removed - according to the schedule in :pep:`594`, - having been deprecated in Python 3.6. - Use :mod:`asyncio` instead. - (Contributed by Nikita Sobolev in :gh:`96580`.) - -* Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python - 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) - function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is - also a static method. - (Contributed by Victor Stinner in :gh:`94169`.) - -* Remove the :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: - use :func:`os.urandom` or :func:`ssl.RAND_bytes` instead. - (Contributed by Victor Stinner in :gh:`94199`.) +* :mod:`ftplib`: Remove the ``FTP_TLS.ssl_version`` class attribute: use the + *context* parameter instead. + (Contributed by Victor Stinner in :gh:`94172`.) * :mod:`gzip`: Remove the ``filename`` attribute of :class:`gzip.GzipFile`, deprecated since Python 2.6, use the :attr:`~gzip.GzipFile.name` attribute @@ -1312,43 +1242,13 @@ Removed extension if it was not present. (Contributed by Victor Stinner in :gh:`94196`.) -* Remove the :func:`!ssl.match_hostname` function. - It was deprecated in Python 3.7. OpenSSL performs - hostname matching since Python 3.7, Python no longer uses the - :func:`!ssl.match_hostname` function. - (Contributed by Victor Stinner in :gh:`94199`.) - -* Remove the :func:`!locale.format` function, deprecated in Python 3.7: - use :func:`locale.format_string` instead. - (Contributed by Victor Stinner in :gh:`94226`.) - * :mod:`hashlib`: Remove the pure Python implementation of :func:`hashlib.pbkdf2_hmac()`, deprecated in Python 3.10. Python 3.10 and newer requires OpenSSL 1.1.1 (:pep:`644`): this OpenSSL version provides a C implementation of :func:`~hashlib.pbkdf2_hmac()` which is faster. (Contributed by Victor Stinner in :gh:`94199`.) -* :mod:`xml.etree.ElementTree`: Remove the ``ElementTree.Element.copy()`` method of the - pure Python implementation, deprecated in Python 3.10, use the - :func:`copy.copy` function instead. The C implementation of :mod:`xml.etree.ElementTree` - has no ``copy()`` method, only a ``__copy__()`` method. - (Contributed by Victor Stinner in :gh:`94383`.) - -* :mod:`zipimport`: Remove ``find_loader()`` and ``find_module()`` methods, - deprecated in Python 3.10: use the ``find_spec()`` method instead. See - :pep:`451` for the rationale. - (Contributed by Victor Stinner in :gh:`94379`.) - -* Remove the :func:`!ssl.wrap_socket` function, deprecated in Python 3.7: - instead, create a :class:`ssl.SSLContext` object and call its - :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses - :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a - SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 - `_: Improper Certificate - Validation. - (Contributed by Victor Stinner in :gh:`94199`.) - -* Many previously deprecated cleanups in :mod:`importlib` have now been +* :mod:`importlib`: Many previously deprecated cleanups in :mod:`importlib` have now been completed: * References to, and support for :meth:`!module_repr()` has been removed. @@ -1413,6 +1313,115 @@ Removed ``PY_COMPILED``, ``C_EXTENSION``, ``PY_RESOURCE``, ``PKG_DIRECTORY``, ``C_BUILTIN``, ``PY_FROZEN``, ``PY_CODERESOURCE``, ``IMP_HOOK``. +* :mod:`io`: Remove ``io.OpenWrapper`` and ``_pyio.OpenWrapper``, deprecated in Python + 3.10: just use :func:`open` instead. The :func:`open` (:func:`io.open`) + function is a built-in function. Since Python 3.10, :func:`!_pyio.open` is + also a static method. + (Contributed by Victor Stinner in :gh:`94169`.) + +* :mod:`locale`: Remove the :func:`!locale.format` function, deprecated in Python 3.7: + use :func:`locale.format_string` instead. + (Contributed by Victor Stinner in :gh:`94226`.) + +* ``smtpd``: The module has been removed according to the schedule in :pep:`594`, + having been deprecated in Python 3.4.7 and 3.5.4. + Use aiosmtpd_ PyPI module or any other + :mod:`asyncio`-based server instead. + (Contributed by Oleg Iarygin in :gh:`93243`.) + +.. _aiosmtpd: https://pypi.org/project/aiosmtpd/ + +* :mod:`sqlite3`: The following undocumented :mod:`sqlite3` features, deprecated in Python + 3.10, are now removed: + + * ``sqlite3.enable_shared_cache()`` + * ``sqlite3.OptimizedUnicode`` + + If a shared cache must be used, open the database in URI mode using the + ``cache=shared`` query parameter. + + The ``sqlite3.OptimizedUnicode`` text factory has been an alias for + :class:`str` since Python 3.3. Code that previously set the text factory to + ``OptimizedUnicode`` can either use ``str`` explicitly, or rely on the + default value which is also ``str``. + + (Contributed by Erlend E. Aasland in :gh:`92548`.) + +* :mod:`ssl`: + + * Remove the :func:`!ssl.RAND_pseudo_bytes` function, deprecated in Python 3.6: + use :func:`os.urandom` or :func:`ssl.RAND_bytes` instead. + (Contributed by Victor Stinner in :gh:`94199`.) + + * Remove the :func:`!ssl.match_hostname` function. + It was deprecated in Python 3.7. OpenSSL performs + hostname matching since Python 3.7, Python no longer uses the + :func:`!ssl.match_hostname` function. + (Contributed by Victor Stinner in :gh:`94199`.) + + * Remove the :func:`!ssl.wrap_socket` function, deprecated in Python 3.7: + instead, create a :class:`ssl.SSLContext` object and call its + :class:`ssl.SSLContext.wrap_socket` method. Any package that still uses + :func:`!ssl.wrap_socket` is broken and insecure. The function neither sends a + SNI TLS extension nor validates server hostname. Code is subject to `CWE-295 + `_: Improper Certificate + Validation. + (Contributed by Victor Stinner in :gh:`94199`.) + +* :mod:`unittest`: Removed many old deprecated :mod:`unittest` features: + + - A number of :class:`~unittest.TestCase` method aliases: + + ============================ =============================== =============== + Deprecated alias Method Name Deprecated in + ============================ =============================== =============== + ``failUnless`` :meth:`.assertTrue` 3.1 + ``failIf`` :meth:`.assertFalse` 3.1 + ``failUnlessEqual`` :meth:`.assertEqual` 3.1 + ``failIfEqual`` :meth:`.assertNotEqual` 3.1 + ``failUnlessAlmostEqual`` :meth:`.assertAlmostEqual` 3.1 + ``failIfAlmostEqual`` :meth:`.assertNotAlmostEqual` 3.1 + ``failUnlessRaises`` :meth:`.assertRaises` 3.1 + ``assert_`` :meth:`.assertTrue` 3.2 + ``assertEquals`` :meth:`.assertEqual` 3.2 + ``assertNotEquals`` :meth:`.assertNotEqual` 3.2 + ``assertAlmostEquals`` :meth:`.assertAlmostEqual` 3.2 + ``assertNotAlmostEquals`` :meth:`.assertNotAlmostEqual` 3.2 + ``assertRegexpMatches`` :meth:`.assertRegex` 3.2 + ``assertRaisesRegexp`` :meth:`.assertRaisesRegex` 3.2 + ``assertNotRegexpMatches`` :meth:`.assertNotRegex` 3.5 + ============================ =============================== =============== + + You can use https://github.com/isidentical/teyit to automatically modernise + your unit tests. + + - Undocumented and broken :class:`~unittest.TestCase` method + ``assertDictContainsSubset`` (deprecated in Python 3.2). + + - Undocumented :meth:`TestLoader.loadTestsFromModule + ` parameter *use_load_tests* + (deprecated and ignored since Python 3.2). + + - An alias of the :class:`~unittest.TextTestResult` class: + ``_TextTestResult`` (deprecated in Python 3.2). + + (Contributed by Serhiy Storchaka in :issue:`45162`.) + +* :mod:`webbrowser`: Remove support for obsolete browsers from :mod:`webbrowser`. + Removed browsers include: Grail, Mosaic, Netscape, Galeon, Skipstone, + Iceape, Firebird, and Firefox versions 35 and below (:gh:`102871`). + +* :mod:`xml.etree.ElementTree`: Remove the ``ElementTree.Element.copy()`` method of the + pure Python implementation, deprecated in Python 3.10, use the + :func:`copy.copy` function instead. The C implementation of :mod:`xml.etree.ElementTree` + has no ``copy()`` method, only a ``__copy__()`` method. + (Contributed by Victor Stinner in :gh:`94383`.) + +* :mod:`zipimport`: Remove ``find_loader()`` and ``find_module()`` methods, + deprecated in Python 3.10: use the ``find_spec()`` method instead. See + :pep:`451` for the rationale. + (Contributed by Victor Stinner in :gh:`94379`.) + * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint `_. @@ -1426,15 +1435,6 @@ Removed (*ssl_context* in :mod:`imaplib`) instead. (Contributed by Victor Stinner in :gh:`94172`.) -* :mod:`ftplib`: Remove the ``FTP_TLS.ssl_version`` class attribute: use the - *context* parameter instead. - (Contributed by Victor Stinner in :gh:`94172`.) - -* Remove support for obsolete browsers from :mod:`webbrowser`. - Removed browsers include: Grail, Mosaic, Netscape, Galeon, Skipstone, - Iceape, Firebird, and Firefox versions 35 and below (:gh:`102871`). - - .. _whatsnew312-porting-to-python312: Porting to Python 3.12 From a2a0e51400685d8d44324bc1135f63b281a5a71d Mon Sep 17 00:00:00 2001 From: Daniele Procida Date: Sun, 23 Jul 2023 11:43:27 +0200 Subject: [PATCH 49/73] gh-106996: Add the basics of a turtle graphics tutorial (#107072) Co-authored-by: Hugo van Kemenade --- Doc/includes/turtle-star.py | 10 --- Doc/library/turtle.rst | 150 ++++++++++++++++++++++++++++++++---- 2 files changed, 134 insertions(+), 26 deletions(-) delete mode 100644 Doc/includes/turtle-star.py diff --git a/Doc/includes/turtle-star.py b/Doc/includes/turtle-star.py deleted file mode 100644 index 1a5db761b32385..00000000000000 --- a/Doc/includes/turtle-star.py +++ /dev/null @@ -1,10 +0,0 @@ -from turtle import * -color('red', 'yellow') -begin_fill() -while True: - forward(200) - left(170) - if abs(pos()) < 1: - break -end_fill() -done() diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index f14a677b7dd88c..58ac546e75860b 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -24,6 +24,23 @@ introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon in 1967. + +Get started +=========== + +Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the +command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the +direction it is facing, drawing a line as it moves. Give it the command +``turtle.right(25)``, and it rotates in-place 25 degrees clockwise. + +.. sidebar:: Turtle star + + Turtle can draw intricate shapes using programs that repeat simple + moves. + + .. image:: turtle-star.* + :align: center + In Python, turtle graphics provides a representation of a physical "turtle" (a little robot with a pen) that draws on a sheet of paper on the floor. @@ -38,26 +55,127 @@ graphical output it can be a way to do that without the overhead of introducing more complex or external libraries into their work. -Get started -=========== +Tutorial +======== -Imagine a robotic turtle starting at (0, 0) in the x-y plane. After an ``import turtle``, give it the -command ``turtle.forward(15)``, and it moves (on-screen!) 15 pixels in the -direction it is facing, drawing a line as it moves. Give it the command -``turtle.right(25)``, and it rotates in-place 25 degrees clockwise. +New users should start here. In this tutorial we'll explore some of the +basics of turtle drawing. -.. sidebar:: Turtle star - Turtle can draw intricate shapes using programs that repeat simple - moves. +Starting a turtle environment +----------------------------- - .. image:: turtle-star.* - :align: center +In a Python shell, import all the objects of the ``turtle`` module:: + + from turtle import * + +If you run into a ``No module named '_tkinter'`` error, you'll have to +install the :mod:`Tk interface package ` on your system. + + +Basic drawing +------------- + +Send the turtle forward 100 steps:: + + forward(100) + +You should see (most likely, in a new window on your display) a line +drawn by the turtle, heading East. Change the direction of the turtle, +so that it turns 120 degrees left (anti-clockwise):: - .. literalinclude:: ../includes/turtle-star.py + left(120) -By combining together these and similar commands, intricate shapes and pictures -can easily be drawn. +Let's continue by drawing a triangle:: + + forward(100) + left(120) + forward(100) + +Notice how the turtle, represented by an arrow, points in different +directions as you steer it. + +Experiment with those commands, and also with ``backward()`` and +``right()``. + + +Pen control +~~~~~~~~~~~ + +Try changing the color - for example, ``color('blue')`` - and +width of the line - for example, ``width(3)`` - and then drawing again. + +You can also move the turtle around without drawing, by lifting up the pen: +``up()`` before moving. To start drawing again, use ``down()``. + + +The turtle's position +~~~~~~~~~~~~~~~~~~~~~ + +Send your turtle back to its starting-point (useful if it has disappeared +off-screen):: + + home() + +The home position is at the center of the turtle's screen. If you ever need to +know them, get the turtle's x-y co-ordinates with:: + + pos() + +Home is at ``(0, 0)``. + +And after a while, it will probably help to clear the window so we can start +anew:: + + clearscreen() + + +Making algorithmic patterns +--------------------------- + +Using loops, it's possible to build up geometric patterns:: + + for steps in range(100): + for c in ('blue', 'red', 'green'): + color(c) + forward(steps) + right(30) + + +\ - which of course, are limited only by the imagination! + +Let's draw the star shape at the top of this page. We want red lines, +filled in with yellow:: + + color('red') + fillcolor('yellow') + +Just as ``up()`` and ``down()`` determine whether lines will be drawn, +filling can be turned on and off:: + + begin_fill() + +Next we'll create a loop:: + + while True: + forward(200) + left(170) + if abs(pos()) < 1: + break + +``abs(pos()) < 1`` is a good way to know when the turtle is back at its +home position. + +Finally, complete the filling:: + + end_fill() + +(Note that filling only actually takes place when you give the +``end_fill()`` command.) + + +Explanation +=========== The :mod:`turtle` module is an extended reimplementation of the same-named module from the Python standard distribution up to version Python 2.5. @@ -112,8 +230,8 @@ To use multiple turtles on a screen one has to use the object-oriented interface omitted here. -Overview of available Turtle and Screen methods -================================================= +Turtle graphics reference +========================= Turtle methods -------------- From 54632528eeba841e4a8cc95ecbd84c9aca8eef57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gilles=20Bassi=C3=A8re?= Date: Sun, 23 Jul 2023 11:59:19 +0200 Subject: [PATCH 50/73] bpo-18319: gettext() can retrieve a message even if a plural form exists (#19869) --- Lib/gettext.py | 10 ++++++---- Lib/test/test_gettext.py | 4 ++++ .../Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst diff --git a/Lib/gettext.py b/Lib/gettext.py index 6c5ec4e517f637..cc938e40028c24 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -422,10 +422,12 @@ def gettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: - if self._fallback: - return self._fallback.gettext(message) - return message - return tmsg + tmsg = self._catalog.get((message, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.gettext(message) + return message def ngettext(self, msgid1, msgid2, n): try: diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 1608d1b18e98fb..aa3520d2c142e4 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -320,6 +320,8 @@ def test_plural_forms1(self): eq(x, 'Hay %s fichero') x = gettext.ngettext('There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros') + x = gettext.gettext('There is %s file') + eq(x, 'Hay %s fichero') def test_plural_context_forms1(self): eq = self.assertEqual @@ -338,6 +340,8 @@ def test_plural_forms2(self): eq(x, 'Hay %s fichero') x = t.ngettext('There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros') + x = t.gettext('There is %s file') + eq(x, 'Hay %s fichero') def test_plural_context_forms2(self): eq = self.assertEqual diff --git a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst new file mode 100644 index 00000000000000..a1a4cf6d63725a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst @@ -0,0 +1,2 @@ +Ensure `gettext(msg)` retrieve translations even if a plural form exists. In +other words: `gettext(msg) == ngettext(msg, '', 1)`. From c65592c4d6d7552fb6284442906a96a6874cb266 Mon Sep 17 00:00:00 2001 From: htsedebenham <31847376+htsedebenham@users.noreply.github.com> Date: Sun, 23 Jul 2023 11:25:18 +0100 Subject: [PATCH 51/73] gh-106186: Don't report MultipartInvariantViolationDefect for valid multipart emails when parsing header only (#107016) --- Lib/email/feedparser.py | 2 +- Lib/test/test_email/data/msg_47.txt | 14 ++++++++++++++ Lib/test/test_email/test_email.py | 10 ++++++++++ .../2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst | 3 +++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_email/data/msg_47.txt create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 53d71f50225152..06d6b4a3afcd07 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -189,7 +189,7 @@ def close(self): assert not self._msgstack # Look for final set of defects if root.get_content_maintype() == 'multipart' \ - and not root.is_multipart(): + and not root.is_multipart() and not self._headersonly: defect = errors.MultipartInvariantViolationDefect() self.policy.handle_defect(root, defect) return root diff --git a/Lib/test/test_email/data/msg_47.txt b/Lib/test/test_email/data/msg_47.txt new file mode 100644 index 00000000000000..bb48b47d96baf8 --- /dev/null +++ b/Lib/test/test_email/data/msg_47.txt @@ -0,0 +1,14 @@ +Date: 01 Jan 2001 00:01+0000 +From: arthur@example.example +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=foo + +--foo +Content-Type: text/plain +bar + +--foo +Content-Type: text/html +

baz

+ +--foo-- \ No newline at end of file diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index b4f3a2481976e8..cdb6ef1275e520 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -3712,6 +3712,16 @@ def test_bytes_header_parser(self): self.assertIsInstance(msg.get_payload(), str) self.assertIsInstance(msg.get_payload(decode=True), bytes) + def test_header_parser_multipart_is_valid(self): + # Don't flag valid multipart emails as having defects + with openfile('msg_47.txt', encoding="utf-8") as fp: + msgdata = fp.read() + + parser = email.parser.Parser(policy=email.policy.default) + parsed_msg = parser.parsestr(msgdata, headersonly=True) + + self.assertEqual(parsed_msg.defects, []) + def test_bytes_parser_does_not_close_file(self): with openfile('msg_02.txt', 'rb') as fp: email.parser.BytesParser().parse(fp) diff --git a/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst b/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst new file mode 100644 index 00000000000000..07fdcc96fa38a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst @@ -0,0 +1,3 @@ +Do not report ``MultipartInvariantViolationDefect`` defect +when the :class:`email.parser.Parser` class is used +to parse emails with ``headersonly=True``. From 08a228da05a7aec937b65eea21f4091fa3c6b5cf Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 23 Jul 2023 13:27:05 +0300 Subject: [PATCH 52/73] gh-107091: Fix the use of some C domain roles (#107092) --- Doc/c-api/buffer.rst | 4 ++-- Doc/c-api/method.rst | 4 ++-- Doc/c-api/module.rst | 2 +- Doc/c-api/type.rst | 2 +- Doc/c-api/typeobj.rst | 4 ++-- Doc/extending/extending.rst | 10 +++++----- Doc/howto/isolating-extensions.rst | 4 ++-- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.9.rst | 2 +- Misc/NEWS.d/3.10.0a2.rst | 4 ++-- Misc/NEWS.d/3.12.0a1.rst | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 91d1edd9b2ec46..6e5443f0d6cdc5 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -225,7 +225,7 @@ object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical structure of the memory can vary drastically, the consumer uses the *flags* argument to specify the exact buffer type it can handle. -All :c:data:`Py_buffer` fields are unambiguously defined by the request +All :c:type:`Py_buffer` fields are unambiguously defined by the request type. request-independent fields @@ -464,7 +464,7 @@ Buffer-related functions .. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *format) - Return the implied :c:data:`~Py_buffer.itemsize` from :c:data:`~Py_buffer.format`. + Return the implied :c:member:`~Py_buffer.itemsize` from :c:member:`~Py_buffer.format`. On error, raise an exception and return -1. .. versionadded:: 3.9 diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 93ad30cd4f7a8d..0d75ab8e1af111 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -7,8 +7,8 @@ Instance Method Objects .. index:: pair: object; instancemethod -An instance method is a wrapper for a :c:data:`PyCFunction` and the new way -to bind a :c:data:`PyCFunction` to a class object. It replaces the former call +An instance method is a wrapper for a :c:type:`PyCFunction` and the new way +to bind a :c:type:`PyCFunction` to a class object. It replaces the former call ``PyMethod_New(func, NULL, class)``. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index a4f79f8989e96f..e85baa03a2247a 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -145,7 +145,7 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: PyModuleDef_Base m_base - Always initialize this member to :c:data:`PyModuleDef_HEAD_INIT`. + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. .. c:member:: const char *m_name diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index d9dcd22a24839e..553d86aa3d9975 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -215,7 +215,7 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when ``PyCMethod`` cannot + See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot be used. .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 33591ed14eaf13..9aa076baaa977f 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2255,8 +2255,8 @@ Number Object Structures .. note:: - The :c:data:`nb_reserved` field should always be ``NULL``. It - was previously called :c:data:`nb_long`, and was renamed in + The :c:member:`~PyNumberMethods.nb_reserved` field should always be ``NULL``. It + was previously called :c:member:`!nb_long`, and was renamed in Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 0163ba1a5b3135..0c5da49fe61344 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -237,10 +237,10 @@ Note that the Python name for the exception object is :exc:`spam.error`. The being :exc:`Exception` (unless another class is passed in instead of ``NULL``), described in :ref:`bltin-exceptions`. -Note also that the :c:data:`SpamError` variable retains a reference to the newly +Note also that the :c:data:`!SpamError` variable retains a reference to the newly created exception class; this is intentional! Since the exception could be removed from the module by external code, an owned reference to the class is -needed to ensure that it will not be discarded, causing :c:data:`SpamError` to +needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. @@ -281,9 +281,9 @@ statement:: It returns ``NULL`` (the error indicator for functions returning object pointers) if an error is detected in the argument list, relying on the exception set by :c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`command`. This is a pointer assignment and +copied to the local variable :c:data:`!command`. This is a pointer assignment and you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`command` should properly be declared as ``const char +the variable :c:data:`!command` should properly be declared as ``const char *command``). The next statement is a call to the Unix function :c:func:`system`, passing it @@ -291,7 +291,7 @@ the string we just got from :c:func:`PyArg_ParseTuple`:: sts = system(command); -Our :func:`spam.system` function must return the value of :c:data:`sts` as a +Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a Python object. This is done using the function :c:func:`PyLong_FromLong`. :: return PyLong_FromLong(sts); diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index f01801ea47212c..cc4a908febfb7f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -483,14 +483,14 @@ to get the state:: return NULL; } -``PyType_GetModuleByDef`` works by searching the +:c:func:`!PyType_GetModuleByDef` works by searching the :term:`method resolution order` (i.e. all superclasses) for the first superclass that has a corresponding module. .. note:: In very exotic cases (inheritance chains spanning multiple modules - created from the same definition), ``PyType_GetModuleByDef`` might not + created from the same definition), :c:func:`!PyType_GetModuleByDef` might not return the module of the true defining class. However, it will always return a module with the same definition, ensuring a compatible C memory layout. diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index bae78c1d9c205f..ec5263ec35c52f 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2227,7 +2227,7 @@ New Features (Contributed by Christian Heimes in :issue:`45459`.) -* Added the :c:data:`PyType_GetModuleByDef` function, used to get the module +* Added the :c:func:`PyType_GetModuleByDef` function, used to get the module in which a method was defined, in cases where this information is not available directly (via :c:type:`PyCMethod`). (Contributed by Petr Viktorin in :issue:`46613`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index bf889b7ffbadff..e48cc13eff617a 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1276,7 +1276,7 @@ New Features * :pep:`573`: Added :c:func:`PyType_FromModuleAndSpec` to associate a module with a class; :c:func:`PyType_GetModule` and :c:func:`PyType_GetModuleState` to retrieve the module and its state; and - :c:data:`PyCMethod` and :c:macro:`METH_METHOD` to allow a method to + :c:type:`PyCMethod` and :c:macro:`METH_METHOD` to allow a method to access the class it was defined in. (Contributed by Marcel Plch and Petr Viktorin in :issue:`38787`.) diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index eeb179a980bb8c..78b25779802d6e 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -847,8 +847,8 @@ Victor Stinner. .. section: C API Fix potential crash in deallocating method objects when dynamically -allocated `PyMethodDef`'s lifetime is managed through the ``self`` argument -of a `PyCFunction`. +allocated :c:type:`PyMethodDef`'s lifetime is managed through the ``self`` argument +of a :c:type:`PyCFunction`. .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 209d079feb5144..8ae78ebb1b629c 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -5934,7 +5934,7 @@ not override :c:member:`~PyTypeObject.tp_call` now inherit the .. nonce: aiRSgr .. section: C API -Creating :c:data:`immutable types ` with mutable +Creating :c:macro:`immutable types ` with mutable bases is deprecated and is planned to be disabled in Python 3.14. .. From e59da0c4f283b966ccb175fb94460f58211a9704 Mon Sep 17 00:00:00 2001 From: TommyUnreal <45427816+TommyUnreal@users.noreply.github.com> Date: Sun, 23 Jul 2023 14:07:24 +0200 Subject: [PATCH 53/73] gh-107017: Analolgy to Pascal and C replaced. (#107025) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/tutorial/introduction.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index ebc2e9187534b4..4b886f604aca23 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -52,8 +52,8 @@ Numbers The interpreter acts as a simple calculator: you can type an expression at it and it will write the value. Expression syntax is straightforward: the -operators ``+``, ``-``, ``*`` and ``/`` work just like in most other languages -(for example, Pascal or C); parentheses (``()``) can be used for grouping. +operators ``+``, ``-``, ``*`` and ``/`` can be used to perform +arithmetic; parentheses (``()``) can be used for grouping. For example:: >>> 2 + 2 From b273837fea129df8477da557ad3cb23a0ed12c20 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 23 Jul 2023 22:51:12 +0900 Subject: [PATCH 54/73] gh-107122: Add clear method to dbm.gdbm.module (gh-107127) --- Doc/library/dbm.rst | 7 ++++ Lib/test/test_dbm_gnu.py | 14 ++++++++ ...-07-23-13-07-34.gh-issue-107122.9HFUyb.rst | 1 + Modules/_gdbmmodule.c | 32 +++++++++++++++++++ Modules/clinic/_gdbmmodule.c.h | 24 +++++++++++++- 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 2be499337a2a15..d226fa9f962cb4 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -245,6 +245,13 @@ supported. Close the ``gdbm`` database. + .. method:: gdbm.clear() + + Remove all items from the ``gdbm`` database. + + .. versionadded:: 3.13 + + :mod:`dbm.ndbm` --- Interface based on ndbm ------------------------------------------- diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 73602cab5180fc..e20addf1f04f1b 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -192,6 +192,20 @@ def test_open_with_bytes_path(self): def test_open_with_pathlib_bytes_path(self): gdbm.open(FakePath(os.fsencode(filename)), "c").close() + def test_clear(self): + kvs = [('foo', 'bar'), ('1234', '5678')] + with gdbm.open(filename, 'c') as db: + for k, v in kvs: + db[k] = v + self.assertIn(k, db) + self.assertEqual(len(db), len(kvs)) + + db.clear() + for k, v in kvs: + self.assertNotIn(k, db) + self.assertEqual(len(db), 0) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst new file mode 100644 index 00000000000000..c2e1c58c1b42c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst @@ -0,0 +1 @@ +Add :meth:`dbm.gdbm.clear` to :mod:`dbm.gdbm`. Patch By Dong-hee Na. diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index bedbdc081425c2..47f2da8e0e3d04 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -561,6 +561,37 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls) Py_RETURN_NONE; } +/*[clinic input] +_gdbm.gdbm.clear + cls: defining_class + / +Remove all items from the database. + +[clinic start generated code]*/ + +static PyObject * +_gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls) +/*[clinic end generated code: output=673577c573318661 input=34136d52fcdd4210]*/ +{ + _gdbm_state *state = PyType_GetModuleState(cls); + assert(state != NULL); + check_gdbmobject_open(self, state->gdbm_error); + datum key; + // Invalidate cache + self->di_size = -1; + while (1) { + key = gdbm_firstkey(self->di_dbm); + if (key.dptr == NULL) { + break; + } + if (gdbm_delete(self->di_dbm, key) < 0) { + PyErr_SetString(state->gdbm_error, "cannot delete item from database"); + return NULL; + } + } + Py_RETURN_NONE; +} + static PyObject * gdbm__enter__(PyObject *self, PyObject *args) { @@ -582,6 +613,7 @@ static PyMethodDef gdbm_methods[] = { _GDBM_GDBM_SYNC_METHODDEF _GDBM_GDBM_GET_METHODDEF _GDBM_GDBM_SETDEFAULT_METHODDEF + _GDBM_GDBM_CLEAR_METHODDEF {"__enter__", gdbm__enter__, METH_NOARGS, NULL}, {"__exit__", gdbm__exit__, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 5c6aeeee7789f7..76f6db318f8cc5 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -247,6 +247,28 @@ _gdbm_gdbm_sync(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_s return _gdbm_gdbm_sync_impl(self, cls); } +PyDoc_STRVAR(_gdbm_gdbm_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the database."); + +#define _GDBM_GDBM_CLEAR_METHODDEF \ + {"clear", _PyCFunction_CAST(_gdbm_gdbm_clear), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_clear__doc__}, + +static PyObject * +_gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); + +static PyObject * +_gdbm_gdbm_clear(gdbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); + return NULL; + } + return _gdbm_gdbm_clear_impl(self, cls); +} + PyDoc_STRVAR(dbmopen__doc__, "open($module, filename, flags=\'r\', mode=0o666, /)\n" "--\n" @@ -322,4 +344,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c6e721d82335adb3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c613cbd88e57480 input=a9049054013a1b77]*/ From b3c34e55c053846beb35f5e4253ef237b3494bd0 Mon Sep 17 00:00:00 2001 From: Tomas R Date: Sun, 23 Jul 2023 16:08:28 +0200 Subject: [PATCH 55/73] gh-62519: Make pgettext search plurals when translation is not found (#107118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/gettext.py | 10 ++++++---- Lib/test/test_gettext.py | 4 ++++ .../2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst diff --git a/Lib/gettext.py b/Lib/gettext.py index cc938e40028c24..b72b15f82d4355 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -446,10 +446,12 @@ def pgettext(self, context, message): missing = object() tmsg = self._catalog.get(ctxt_msg_id, missing) if tmsg is missing: - if self._fallback: - return self._fallback.pgettext(context, message) - return message - return tmsg + tmsg = self._catalog.get((ctxt_msg_id, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.pgettext(context, message) + return message def npgettext(self, context, msgid1, msgid2, n): ctxt_msg_id = self.CONTEXT % (context, msgid1) diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index aa3520d2c142e4..8430fc234d00ee 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -331,6 +331,8 @@ def test_plural_context_forms1(self): x = gettext.npgettext('With context', 'There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros (context)') + x = gettext.pgettext('With context', 'There is %s file') + eq(x, 'Hay %s fichero (context)') def test_plural_forms2(self): eq = self.assertEqual @@ -353,6 +355,8 @@ def test_plural_context_forms2(self): x = t.npgettext('With context', 'There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros (context)') + x = gettext.pgettext('With context', 'There is %s file') + eq(x, 'Hay %s fichero (context)') # Examples from http://www.gnu.org/software/gettext/manual/gettext.html diff --git a/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst b/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst new file mode 100644 index 00000000000000..96e2a3dcc24fb0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst @@ -0,0 +1,2 @@ +Make :func:`gettext.pgettext` search plural definitions when +translation is not found. From 0ae4870d09de82ed5063b6998c172cc63628437e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 23 Jul 2023 23:26:23 +0900 Subject: [PATCH 56/73] gh-107122: Add clear method to dbm.ndbm module (gh-107126) --- Doc/library/dbm.rst | 6 ++++ Lib/test/test_dbm.py | 15 +++++++++ Lib/test/test_dbm_ndbm.py | 13 ++++++++ ...-07-23-21-16-54.gh-issue-107122.VNuNcq.rst | 1 + Modules/_dbmmodule.c | 33 +++++++++++++++++++ Modules/clinic/_dbmmodule.c.h | 24 +++++++++++++- 6 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index d226fa9f962cb4..766847b971b645 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -320,6 +320,12 @@ to locate the appropriate header file to simplify building this module. Close the ``ndbm`` database. + .. method:: ndbm.clear() + + Remove all items from the ``ndbm`` database. + + .. versionadded:: 3.13 + :mod:`dbm.dumb` --- Portable DBM implementation ----------------------------------------------- diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index f21eebc6530c76..e3924d8ec8b5c1 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -155,6 +155,21 @@ def test_keys(self): self.assertNotIn(b'xxx', d) self.assertRaises(KeyError, lambda: d[b'xxx']) + def test_clear(self): + with dbm.open(_fname, 'c') as d: + self.assertEqual(d.keys(), []) + a = [(b'a', b'b'), (b'12345678910', b'019237410982340912840198242')] + for k, v in a: + d[k] = v + for k, _ in a: + self.assertIn(k, d) + self.assertEqual(len(d), len(a)) + + d.clear() + self.assertEqual(len(d), 0) + for k, _ in a: + self.assertNotIn(k, d) + def setUp(self): self.addCleanup(setattr, dbm, '_defaultmod', dbm._defaultmod) dbm._defaultmod = self.module diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 8f37e3cc624e2e..e0f31c9a9a337d 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -147,6 +147,19 @@ def test_bool_on_closed_db_raises(self): db['a'] = 'b' self.assertRaises(dbm.ndbm.error, bool, db) + def test_clear(self): + kvs = [('foo', 'bar'), ('1234', '5678')] + with dbm.ndbm.open(self.filename, 'c') as db: + for k, v in kvs: + db[k] = v + self.assertIn(k, db) + self.assertEqual(len(db), len(kvs)) + + db.clear() + for k, v in kvs: + self.assertNotIn(k, db) + self.assertEqual(len(db), 0) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst new file mode 100644 index 00000000000000..d63aeb54ae1454 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst @@ -0,0 +1 @@ +Add :meth:`dbm.ndbm.clear` to :mod:`dbm.ndbm`. Patch By Dong-hee Na. diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 5be444d53e8da3..bd807698927e86 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -414,6 +414,38 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, return default_value; } +/*[clinic input] +_dbm.dbm.clear + cls: defining_class + / +Remove all items from the database. + +[clinic start generated code]*/ + +static PyObject * +_dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls) +/*[clinic end generated code: output=8d126b9e1d01a434 input=43aa6ca1acb7f5f5]*/ +{ + _dbm_state *state = PyType_GetModuleState(cls); + assert(state != NULL); + check_dbmobject_open(self, state->dbm_error); + datum key; + // Invalidate cache + self->di_size = -1; + while (1) { + key = dbm_firstkey(self->di_dbm); + if (key.dptr == NULL) { + break; + } + if (dbm_delete(self->di_dbm, key) < 0) { + dbm_clearerr(self->di_dbm); + PyErr_SetString(state->dbm_error, "cannot delete item from database"); + return NULL; + } + } + Py_RETURN_NONE; +} + static PyObject * dbm__enter__(PyObject *self, PyObject *args) { @@ -431,6 +463,7 @@ static PyMethodDef dbm_methods[] = { _DBM_DBM_KEYS_METHODDEF _DBM_DBM_GET_METHODDEF _DBM_DBM_SETDEFAULT_METHODDEF + _DBM_DBM_CLEAR_METHODDEF {"__enter__", dbm__enter__, METH_NOARGS, NULL}, {"__exit__", dbm__exit__, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 172dc4b9d5793e..98aac07423c8ab 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -138,6 +138,28 @@ _dbm_dbm_setdefault(dbmobject *self, PyTypeObject *cls, PyObject *const *args, P return return_value; } +PyDoc_STRVAR(_dbm_dbm_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from the database."); + +#define _DBM_DBM_CLEAR_METHODDEF \ + {"clear", _PyCFunction_CAST(_dbm_dbm_clear), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _dbm_dbm_clear__doc__}, + +static PyObject * +_dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); + +static PyObject * +_dbm_dbm_clear(dbmobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (nargs) { + PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); + return NULL; + } + return _dbm_dbm_clear_impl(self, cls); +} + PyDoc_STRVAR(dbmopen__doc__, "open($module, filename, flags=\'r\', mode=0o666, /)\n" "--\n" @@ -200,4 +222,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=28dcf736654137c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b3053c67ecfcc29c input=a9049054013a1b77]*/ From d1b839b4538fbfc5a3f28d9b0b0422ec026e334c Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Mon, 24 Jul 2023 00:08:50 +0900 Subject: [PATCH 57/73] gh-107122: Update what's news for dbm.*dbm.clear() method (gh-107135) --- Doc/whatsnew/3.13.rst | 7 +++++++ .../2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst | 2 +- .../2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 1d34d8a0fa463c..8ca0abf5525763 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -101,6 +101,13 @@ array It can be used instead of ``'u'`` type code, which is deprecated. (Contributed by Inada Naoki in :gh:`80480`.) +dbm +--- + +* Add :meth:`dbm.gnu.gdbm.clear` and :meth:`dbm.ndbm.ndbm.clear` methods that remove all items + from the database. + (Contributed by Dong-hee Na in :gh:`107122`.) + io -- diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst index c2e1c58c1b42c5..64ac8ac6df09b8 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-13-07-34.gh-issue-107122.9HFUyb.rst @@ -1 +1 @@ -Add :meth:`dbm.gdbm.clear` to :mod:`dbm.gdbm`. Patch By Dong-hee Na. +Add :meth:`dbm.gnu.gdbm.clear` to :mod:`dbm.gnu`. Patch By Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst index d63aeb54ae1454..5b7cc98ddc6414 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-23-21-16-54.gh-issue-107122.VNuNcq.rst @@ -1 +1 @@ -Add :meth:`dbm.ndbm.clear` to :mod:`dbm.ndbm`. Patch By Dong-hee Na. +Add :meth:`dbm.ndbm.ndbm.clear` to :mod:`dbm.ndbm`. Patch By Dong-hee Na. From 956b3de816f4b18f6b947a11c8eec6c39f60ea88 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 23 Jul 2023 17:18:50 +0200 Subject: [PATCH 58/73] Remove superflous whitespaces in `layout.html`. (GH-107067) --- Doc/tools/templates/layout.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 18a49271df5f20..9832feba141675 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -4,16 +4,16 @@ {%- if outdated %}
{% trans %}This document is for an old version of Python that is no longer supported. - You should upgrade, and read the {% endtrans %} - {% trans %} Python documentation for the current stable release{% endtrans %}. + You should upgrade, and read the{% endtrans %} + {% trans %}Python documentation for the current stable release{% endtrans %}.
{%- endif %} {%- if is_deployment_preview %}
{% trans %}This is a deploy preview created from a pull request. - For authoritative documentation, see {% endtrans %} - {% trans %} the current stable release{% endtrans %}. + For authoritative documentation, see{% endtrans %} + {% trans %}the current stable release{% endtrans %}.
{%- endif %} {% endblock %} From adb27ea2d5e2f74f4be8f15b199889161c33e801 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jul 2023 21:07:12 +0200 Subject: [PATCH 59/73] gh-106320: Remove _PyIsSelectable_fd() C API (#107142) Move _PyIsSelectable_fd() macro to the internal C API (pycore_fileutils.h). --- Include/fileobject.h | 8 -------- Include/internal/pycore_fileutils.h | 7 +++++++ Modules/_ssl.c | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/fileobject.h b/Include/fileobject.h index 2deef544d667a5..6a6d11409497fa 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -29,14 +29,6 @@ Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; Py_DEPRECATED(3.12) PyAPI_DATA(int) Py_UTF8Mode; #endif -/* A routine to check if a file descriptor can be select()-ed. */ -#ifdef _MSC_VER - /* On Windows, any socket fd can be select()-ed, no matter how high */ - #define _PyIsSelectable_fd(FD) (1) -#else - #define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE) -#endif - #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEOBJECT_H # include "cpython/fileobject.h" diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ef6642d00f1b54..7ba9b3ee58be41 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -10,6 +10,13 @@ extern "C" { #include /* struct lconv */ +/* A routine to check if a file descriptor can be select()-ed. */ +#ifdef _MSC_VER + /* On Windows, any socket fd can be select()-ed, no matter how high */ + #define _PyIsSelectable_fd(FD) (1) +#else + #define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE) +#endif struct _fileutils_state { int force_ascii; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index ed720b4295f8ec..699431c248a071 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -26,6 +26,7 @@ #define OPENSSL_NO_DEPRECATED 1 #include "Python.h" +#include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_weakref.h" // _PyWeakref_GET_REF() /* Include symbols from _socket module */ From 0810b0c435415c09c1907c6f418585bed558a2c1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jul 2023 21:16:21 +0200 Subject: [PATCH 60/73] gh-106320: Remove _PyTuple_MaybeUntrack() C API (#107143) Move _PyTuple_MaybeUntrack() and _PyTuple_DebugMallocStats() functions to the internal C API (pycore_tuple.h). No longer export these functions. --- Include/cpython/tupleobject.h | 3 --- Include/internal/pycore_tuple.h | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 370da1612a61ed..e530c8beda44ab 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -11,7 +11,6 @@ typedef struct { } PyTupleObject; PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); /* Cast argument to PyTupleObject* type. */ #define _PyTuple_CAST(op) \ @@ -37,5 +36,3 @@ PyTuple_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { } #define PyTuple_SET_ITEM(op, index, value) \ PyTuple_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value)) - -PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 335edad89792c3..4fa7a12206bcb2 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "tupleobject.h" /* _PyTuple_CAST() */ - +extern void _PyTuple_MaybeUntrack(PyObject *); +extern void _PyTuple_DebugMallocStats(FILE *out); /* runtime lifecycle */ From 0d6dfd68d22762c115d202515fe3310bfb42e327 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jul 2023 22:09:08 +0200 Subject: [PATCH 61/73] gh-106320: Remove private _PyObject C API (#107147) Move private debug _PyObject functions to the internal C API (pycore_object.h): * _PyDebugAllocatorStats() * _PyObject_CheckConsistency() * _PyObject_DebugTypeStats() * _PyObject_IsFreed() No longer export most of these functions, except of _PyObject_IsFreed(). Move test functions using _PyObject_IsFreed() from _testcapi to _testinternalcapi. check_pyobject_is_freed() test no longer catch _testcapi.error: the tested function cannot raise _testcapi.error. --- Include/cpython/object.h | 22 --------- Include/internal/pycore_object.h | 21 +++++++++ Lib/test/test_capi/test_mem.py | 17 ++++--- Modules/_testcapi/mem.c | 75 ----------------------------- Modules/_testclinic.c | 1 + Modules/_testinternalcapi.c | 81 +++++++++++++++++++++++++++++++- Objects/dictobject.c | 2 +- Objects/floatobject.c | 2 +- Objects/listobject.c | 2 +- Objects/obmalloc.c | 1 + Objects/tupleobject.c | 2 +- Python/sysmodule.c | 2 +- 12 files changed, 116 insertions(+), 112 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 49101c18756860..90e45f65e298e1 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -289,7 +289,6 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -377,12 +376,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -PyAPI_FUNC(void) -_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, - size_t sizeof_block); -PyAPI_FUNC(void) -_PyObject_DebugTypeStats(FILE *out); - /* Define a pair of assertion macros: _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT(). @@ -431,21 +424,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( int line, const char *function); -/* Check if an object is consistent. For example, ensure that the reference - counter is greater than or equal to 1, and ensure that ob_type is not NULL. - - Call _PyObject_AssertFailed() if the object is inconsistent. - - If check_content is zero, only check header fields: reduce the overhead. - - The function always return 1. The return value is just here to be able to - write: - - assert(_PyObject_CheckConsistency(obj, 1)); */ -PyAPI_FUNC(int) _PyObject_CheckConsistency( - PyObject *op, - int check_content); - /* Trashcan mechanism, thanks to Christian Tismer. diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a9c996557430ad..661820c894dc0e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,6 +14,27 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime +/* Check if an object is consistent. For example, ensure that the reference + counter is greater than or equal to 1, and ensure that ob_type is not NULL. + + Call _PyObject_AssertFailed() if the object is inconsistent. + + If check_content is zero, only check header fields: reduce the overhead. + + The function always return 1. The return value is just here to be able to + write: + + assert(_PyObject_CheckConsistency(obj, 1)); */ +extern int _PyObject_CheckConsistency(PyObject *op, int check_content); + +extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, + int num_blocks, size_t sizeof_block); + +extern void _PyObject_DebugTypeStats(FILE *out); + +// Export for shared _testinternalcapi extension +PyAPI_DATA(int) _PyObject_IsFreed(PyObject *); + /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid designated initializer conflicts in C++20. If we use the deinition in object.h, we will be mixing designated and non-designated initializers in diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index a9ff410cb93ab8..527000875b7241 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -8,8 +8,10 @@ from test.support.script_helper import assert_python_failure, assert_python_ok -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi and _testinternalcapi extensions are not +# available. _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') @requires_subprocess() class PyMemDebugTests(unittest.TestCase): @@ -84,16 +86,13 @@ def test_pyobject_malloc_without_gil(self): def check_pyobject_is_freed(self, func_name): code = textwrap.dedent(f''' - import gc, os, sys, _testcapi + import gc, os, sys, _testinternalcapi # Disable the GC to avoid crash on GC collection gc.disable() - try: - _testcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - except _testcapi.error: - os._exit(1) + _testinternalcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) ''') assert_python_ok( '-c', code, diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 979b3a4b2b1af6..0b2b1cd31fb0a1 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -526,75 +526,6 @@ pymem_malloc_without_gil(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -test_pyobject_is_freed(const char *test_name, PyObject *op) -{ - if (!_PyObject_IsFreed(op)) { - PyErr_SetString(PyExc_AssertionError, - "object is not seen as freed"); - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyObject *op = NULL; - return test_pyobject_is_freed("check_pyobject_null_is_freed", op); -} - - -static PyObject * -check_pyobject_uninitialized_is_freed(PyObject *self, - PyObject *Py_UNUSED(args)) -{ - PyObject *op = (PyObject *)PyObject_Malloc(sizeof(PyObject)); - if (op == NULL) { - return NULL; - } - /* Initialize reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* object fields like ob_type are uninitialized! */ - return test_pyobject_is_freed("check_pyobject_uninitialized_is_freed", op); -} - - -static PyObject * -check_pyobject_forbidden_bytes_is_freed(PyObject *self, - PyObject *Py_UNUSED(args)) -{ - /* Allocate an incomplete PyObject structure: truncate 'ob_type' field */ - PyObject *op = (PyObject *)PyObject_Malloc(offsetof(PyObject, ob_type)); - if (op == NULL) { - return NULL; - } - /* Initialize reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* ob_type field is after the memory block: part of "forbidden bytes" - when using debug hooks on memory allocators! */ - return test_pyobject_is_freed("check_pyobject_forbidden_bytes_is_freed", op); -} - - -static PyObject * -check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) -{ - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER - Py_RETURN_NONE; -#else - PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); - if (op == NULL) { - return NULL; - } - Py_TYPE(op)->tp_dealloc(op); - /* Reset reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* object memory is freed! */ - return test_pyobject_is_freed("check_pyobject_freed_is_freed", op); -#endif -} // Tracemalloc tests static PyObject * @@ -656,12 +587,6 @@ tracemalloc_untrack(PyObject *self, PyObject *args) } static PyMethodDef test_methods[] = { - {"check_pyobject_forbidden_bytes_is_freed", - check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, - {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, - {"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS}, - {"check_pyobject_uninitialized_is_freed", - check_pyobject_uninitialized_is_freed, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 26cdb4371ca24c..8ba8f8ecadec7a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -6,6 +6,7 @@ #undef NDEBUG #include "Python.h" +#include "pycore_object.h" // _PyObject_IsFreed() // Used for clone_with_conv_f1 and clone_with_conv_v2 diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ecc2721a4988f9..e2be9a130217d6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,7 +10,6 @@ #undef NDEBUG #include "Python.h" -#include "frameobject.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() @@ -23,9 +22,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_interp_id.h" // _PyInterpreterID_LookUp() +#include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() + +#include "frameobject.h" #include "osdefs.h" // MAXPATHLEN #include "clinic/_testinternalcapi.c.h" @@ -1446,6 +1448,77 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject * +test_pyobject_is_freed(const char *test_name, PyObject *op) +{ + if (!_PyObject_IsFreed(op)) { + PyErr_SetString(PyExc_AssertionError, + "object is not seen as freed"); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *op = NULL; + return test_pyobject_is_freed("check_pyobject_null_is_freed", op); +} + + +static PyObject * +check_pyobject_uninitialized_is_freed(PyObject *self, + PyObject *Py_UNUSED(args)) +{ + PyObject *op = (PyObject *)PyObject_Malloc(sizeof(PyObject)); + if (op == NULL) { + return NULL; + } + /* Initialize reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* object fields like ob_type are uninitialized! */ + return test_pyobject_is_freed("check_pyobject_uninitialized_is_freed", op); +} + + +static PyObject * +check_pyobject_forbidden_bytes_is_freed(PyObject *self, + PyObject *Py_UNUSED(args)) +{ + /* Allocate an incomplete PyObject structure: truncate 'ob_type' field */ + PyObject *op = (PyObject *)PyObject_Malloc(offsetof(PyObject, ob_type)); + if (op == NULL) { + return NULL; + } + /* Initialize reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* ob_type field is after the memory block: part of "forbidden bytes" + when using debug hooks on memory allocators! */ + return test_pyobject_is_freed("check_pyobject_forbidden_bytes_is_freed", op); +} + + +static PyObject * +check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) +{ + /* This test would fail if run with the address sanitizer */ +#ifdef _Py_ADDRESS_SANITIZER + Py_RETURN_NONE; +#else + PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); + if (op == NULL) { + return NULL; + } + Py_TYPE(op)->tp_dealloc(op); + /* Reset reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* object memory is freed! */ + return test_pyobject_is_freed("check_pyobject_freed_is_freed", op); +#endif +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1502,6 +1575,12 @@ static PyMethodDef module_functions[] = { {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, {"test_atexit", test_atexit, METH_NOARGS}, + {"check_pyobject_forbidden_bytes_is_freed", + check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, + {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, + {"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS}, + {"check_pyobject_uninitialized_is_freed", + check_pyobject_uninitialized_is_freed, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 26e2dc31e3664a..41ae1fca90b468 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -118,7 +118,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_code.h" // stats #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() diff --git a/Objects/floatobject.c b/Objects/floatobject.c index fa55481f09dec0..6a0c2e033e3e9a 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -10,7 +10,7 @@ #include "pycore_interp.h" // _PyInterpreterState.float_state #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_Init() +#include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() diff --git a/Objects/listobject.c b/Objects/listobject.c index 144ede6351e03c..c0da9dd916851a 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -6,7 +6,7 @@ #include "pycore_list.h" // struct _Py_list_state, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() #include diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index eb68d7c030d293..7d552ff57c8f1e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_code.h" // stats +#include "pycore_object.h" // _PyDebugAllocatorStats() definition #include "pycore_obmalloc.h" #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e85af2b75e4738..c3ff40fdb14c60 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -6,7 +6,7 @@ #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError(), _PyDebugAllocatorStats() /*[clinic input] class tuple "PyTupleObject *" "&PyTuple_Type" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fea3f61ee01762..fb033a6b5634ac 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -22,7 +22,7 @@ Data members: #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_object.h" // _PyObject_IS_GC() +#include "pycore_object.h" // _PyObject_IS_GC(), _PyObject_DebugTypeStats() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook() From 7d41ead9191a18bc473534f6f8bd1095f2232cc2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 23 Jul 2023 22:10:12 +0200 Subject: [PATCH 62/73] gh-106320: Remove _PyBytes_Join() C API (#107144) Move private _PyBytes functions to the internal C API (pycore_bytesobject.h): * _PyBytes_DecodeEscape() * _PyBytes_FormatEx() * _PyBytes_FromHex() * _PyBytes_Join() No longer export these functions. --- Include/cpython/bytesobject.h | 16 ---------------- Include/internal/pycore_bytesobject.h | 19 +++++++++++++++++++ Modules/_io/bufferedio.c | 3 ++- Parser/string_parser.c | 1 + 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 0af4c83b1e5bc7..816823716e9a6f 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -15,18 +15,6 @@ typedef struct { } PyBytesObject; PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( - const char *format, - Py_ssize_t format_len, - PyObject *args, - int use_bytearray); -PyAPI_FUNC(PyObject*) _PyBytes_FromHex( - PyObject *string, - int use_bytearray); - -/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); /* Macros and static inline functions, trading safety for speed */ #define _PyBytes_CAST(op) \ @@ -43,7 +31,3 @@ static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) { return Py_SIZE(self); } #define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self)) - -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 115c0c52c8f9a9..980065a65f399e 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -8,6 +8,25 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +extern PyObject* _PyBytes_FormatEx( + const char *format, + Py_ssize_t format_len, + PyObject *args, + int use_bytearray); + +extern PyObject* _PyBytes_FromHex( + PyObject *string, + int use_bytearray); + +// Helper for PyBytes_DecodeEscape that detects invalid escape chars. +// Export for test_peg_generator. +PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, + const char *, const char **); + +/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, + x must be an iterable object. */ +extern PyObject* _PyBytes_Join(PyObject *sep, PyObject *x); + /* Substring Search. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index bfc3d2558c9e36..efc8cb93d88512 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -8,8 +8,9 @@ */ #include "Python.h" +#include "pycore_bytesobject.h" // _PyBytes_Join() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_object.h" +#include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "structmember.h" // PyMemberDef diff --git a/Parser/string_parser.c b/Parser/string_parser.c index bc1f99d607ae4d..72898c38b79bde 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -1,6 +1,7 @@ #include #include +#include "pycore_bytesobject.h" // _PyBytes_DecodeEscape() #include "pycore_unicodeobject.h" // _PyUnicode_DecodeUnicodeEscapeInternal() #include "tokenizer.h" From b447e19e720e6781025432a40eb72b1cc93ac944 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 23 Jul 2023 22:56:56 +0200 Subject: [PATCH 63/73] gh-106948: Docs: Disable links for C standard library functions, OS utility functions and system calls (#107062) Co-authored-by: Serhiy Storchaka --- Doc/c-api/exceptions.rst | 2 +- Doc/c-api/sys.rst | 4 ++-- Doc/conf.py | 18 +++++++++++++++ Doc/howto/instrumentation.rst | 6 ++--- Doc/library/mailbox.rst | 8 +++---- Doc/library/os.rst | 6 ++--- Doc/library/select.rst | 42 +++++++++++++++++------------------ Doc/library/signal.rst | 2 +- Doc/tools/.nitignore | 1 - Doc/whatsnew/2.6.rst | 6 ++--- Doc/whatsnew/2.7.rst | 2 +- Misc/NEWS.d/3.10.0a1.rst | 4 ++-- 12 files changed, 59 insertions(+), 42 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 3d4ceb360f5b18..4d81b273638b14 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -163,7 +163,7 @@ For convenience, some of these functions will always return a This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), + second item is the corresponding error message (gotten from :c:func:`!strerror`), and then calls ``PyErr_SetObject(type, object)``. On Unix, when the :c:data:`errno` value is :c:macro:`EINTR`, indicating an interrupted system call, this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 433d69e40f947a..6a717122c8a2f8 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -106,7 +106,7 @@ Operating System Utilities .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around - either :c:func:`sigaction` or :c:func:`signal`. Do not call those functions + either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void (\*)(int)`. @@ -114,7 +114,7 @@ Operating System Utilities .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. - This is a thin wrapper around either :c:func:`sigaction` or :c:func:`signal`. Do + This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void (\*)(int)`. diff --git a/Doc/conf.py b/Doc/conf.py index bd01850589b273..453f4fc63883ad 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -77,6 +77,24 @@ exclude_patterns.append(venvdir + '/*') nitpick_ignore = [ + # Standard C functions + ('c:func', 'calloc'), + ('c:func', 'dlopen'), + ('c:func', 'exec'), + ('c:func', 'fcntl'), + ('c:func', 'fork'), + ('c:func', 'free'), + ('c:func', 'gmtime'), + ('c:func', 'localtime'), + ('c:func', 'main'), + ('c:func', 'malloc'), + ('c:func', 'printf'), + ('c:func', 'realloc'), + ('c:func', 'snprintf'), + ('c:func', 'sprintf'), + ('c:func', 'stat'), + ('c:func', 'system'), + ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), ('c:type', '__int'), diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 4ce15c69dac90b..875f846aad0051 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -292,11 +292,11 @@ Available static markers .. object:: function__return(str filename, str funcname, int lineno) - This marker is the converse of :c:func:`function__entry`, and indicates that + This marker is the converse of :c:func:`!function__entry`, and indicates that execution of a Python function has ended (either via ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. - The arguments are the same as for :c:func:`function__entry` + The arguments are the same as for :c:func:`!function__entry` .. object:: line(str filename, str funcname, int lineno) @@ -304,7 +304,7 @@ Available static markers the equivalent of line-by-line tracing with a Python profiler. It is not triggered within C functions. - The arguments are the same as for :c:func:`function__entry`. + The arguments are the same as for :c:func:`!function__entry`. .. object:: gc__start(int generation) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 56908dedea1b40..91df07d914cae2 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -477,7 +477,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: @@ -588,7 +588,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. For MH mailboxes, locking + :c:func:`!flock` and :c:func:`!lockf` system calls. For MH mailboxes, locking the mailbox means locking the :file:`.mh_sequences` file and, only for the duration of any operations that affect them, locking individual message files. @@ -686,7 +686,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: @@ -737,7 +737,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. unlock() Three locking mechanisms are used---dot locking and, if available, the - :c:func:`flock` and :c:func:`lockf` system calls. + :c:func:`!flock` and :c:func:`!lockf` system calls. .. seealso:: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 57405916ed7b31..bbf227aab649e2 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -714,14 +714,14 @@ process and user. .. function:: getsid(pid, /) - Call the system call :c:func:`getsid`. See the Unix manual for the semantics. + Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. .. function:: setsid() - Call the system call :c:func:`setsid`. See the Unix manual for the semantics. + Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -739,7 +739,7 @@ process and user. .. function:: strerror(code, /) Return the error message corresponding to the error code in *code*. - On platforms where :c:func:`strerror` returns ``NULL`` when given an unknown + On platforms where :c:func:`!strerror` returns ``NULL`` when given an unknown error number, :exc:`ValueError` is raised. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index b0891b0c8f584a..c2941e628d9d78 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -6,10 +6,10 @@ -------------- -This module provides access to the :c:func:`select` and :c:func:`poll` functions -available in most operating systems, :c:func:`devpoll` available on -Solaris and derivatives, :c:func:`epoll` available on Linux 2.5+ and -:c:func:`kqueue` available on most BSD. +This module provides access to the :c:func:`!select` and :c:func:`!poll` functions +available in most operating systems, :c:func:`!devpoll` available on +Solaris and derivatives, :c:func:`!epoll` available on Linux 2.5+ and +:c:func:`!kqueue` available on most BSD. Note that on Windows, it only works for sockets; on other operating systems, it also works for other file types (in particular, on Unix, it works on pipes). It cannot be used on regular files to determine whether a file has grown since @@ -41,10 +41,10 @@ The module defines the following: polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. - :c:func:`devpoll` objects are linked to the number of file + :c:func:`!devpoll` objects are linked to the number of file descriptors allowed at the time of instantiation. If your program - reduces this value, :c:func:`devpoll` will fail. If your program - increases this value, :c:func:`devpoll` may return an + reduces this value, :c:func:`!devpoll` will fail. If your program + increases this value, :c:func:`!devpoll` may return an incomplete list of active file descriptors. The new file descriptor is :ref:`non-inheritable `. @@ -62,7 +62,7 @@ The module defines the following: *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`epoll_create1` is not available; + used on older systems where :c:func:`!epoll_create1` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -117,7 +117,7 @@ The module defines the following: .. function:: select(rlist, wlist, xlist[, timeout]) - This is a straightforward interface to the Unix :c:func:`select` system call. + This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either integers representing file descriptors or objects with a parameterless method named :meth:`~io.IOBase.fileno` returning such an integer: @@ -154,7 +154,7 @@ The module defines the following: .. index:: single: WinSock File objects on Windows are not acceptable, but sockets are. On Windows, - the underlying :c:func:`select` function is provided by the WinSock + the underlying :c:func:`!select` function is provided by the WinSock library, and does not handle file descriptors that don't originate from WinSock. @@ -169,7 +169,7 @@ The module defines the following: The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, - :func:`poll` or another interface in this module. This doesn't apply + :func:`!poll` or another interface in this module. This doesn't apply to other kind of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -184,11 +184,11 @@ The module defines the following: ``/dev/poll`` Polling Objects ----------------------------- -Solaris and derivatives have ``/dev/poll``. While :c:func:`select` is -O(highest file descriptor) and :c:func:`poll` is O(number of file +Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is +O(highest file descriptor) and :c:func:`!poll` is O(number of file descriptors), ``/dev/poll`` is O(active file descriptors). -``/dev/poll`` behaviour is very close to the standard :c:func:`poll` +``/dev/poll`` behaviour is very close to the standard :c:func:`!poll` object. @@ -222,7 +222,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`poll` + check for. The constants are the same that with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -231,7 +231,7 @@ object. Registering a file descriptor that's already registered is not an error, but the result is undefined. The appropriate action is to unregister or modify it first. This is an important difference - compared with :c:func:`poll`. + compared with :c:func:`!poll`. .. method:: devpoll.modify(fd[, eventmask]) @@ -376,13 +376,13 @@ Edge and Level Trigger Polling (epoll) Objects Polling Objects --------------- -The :c:func:`poll` system call, supported on most Unix systems, provides better +The :c:func:`!poll` system call, supported on most Unix systems, provides better scalability for network servers that service many, many clients at the same -time. :c:func:`poll` scales better because the system call only requires listing -the file descriptors of interest, while :c:func:`select` builds a bitmap, turns +time. :c:func:`!poll` scales better because the system call only requires listing +the file descriptors of interest, while :c:func:`!select` builds a bitmap, turns on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`select` is O(highest file descriptor), while -:c:func:`poll` is O(number of file descriptors). +linearly scanned again. :c:func:`!select` is O(highest file descriptor), while +:c:func:`!poll` is O(number of file descriptors). .. method:: poll.register(fd[, eventmask]) diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 523d1ac5001360..e53315bee3ea3e 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -562,7 +562,7 @@ The :mod:`signal` module defines the following functions: Note that installing a signal handler with :func:`signal` will reset the restart behaviour to interruptible by implicitly calling - :c:func:`siginterrupt` with a true *flag* value for the given signal. + :c:func:`!siginterrupt` with a true *flag* value for the given signal. .. function:: signal(signalnum, handler) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 818276a81111e8..fc6de10eb1c2c8 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -56,7 +56,6 @@ Doc/glossary.rst Doc/howto/curses.rst Doc/howto/descriptor.rst Doc/howto/enum.rst -Doc/howto/instrumentation.rst Doc/howto/isolating-extensions.rst Doc/howto/logging-cookbook.rst Doc/howto/logging.rst diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index fb56690ce25117..ad899d53886c59 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -2289,7 +2289,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Raymond Hettinger; :issue:`1861`.) * The :mod:`select` module now has wrapper functions - for the Linux :c:func:`epoll` and BSD :c:func:`kqueue` system calls. + for the Linux :c:func:`!epoll` and BSD :c:func:`!kqueue` system calls. :meth:`modify` method was added to the existing :class:`poll` objects; ``pollobj.modify(fd, eventmask)`` takes a file descriptor or file object and an event mask, modifying the recorded event mask @@ -2328,7 +2328,7 @@ changes, or look through the Subversion logs for all the details. one for reading and one for writing. The writable descriptor will be passed to :func:`set_wakeup_fd`, and the readable descriptor will be added to the list of descriptors monitored by the event loop via - :c:func:`select` or :c:func:`poll`. + :c:func:`!select` or :c:func:`!poll`. On receiving a signal, a byte will be written and the main event loop will be woken up, avoiding the need to poll. @@ -2982,7 +2982,7 @@ Changes to Python's build process and to the C API include: * Python now must be compiled with C89 compilers (after 19 years!). This means that the Python source tree has dropped its - own implementations of :c:func:`memmove` and :c:func:`strerror`, which + own implementations of :c:func:`!memmove` and :c:func:`!strerror`, which are in the C89 standard library. * Python 2.6 can be built with Microsoft Visual Studio 2008 (version diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 7bc68b0cc4e668..4b5a31f8a84810 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -355,7 +355,7 @@ added as a more powerful replacement for the This means Python now supports three different modules for parsing command-line arguments: :mod:`getopt`, :mod:`optparse`, and :mod:`argparse`. The :mod:`getopt` module closely resembles the C -library's :c:func:`getopt` function, so it remains useful if you're writing a +library's :c:func:`!getopt` function, so it remains useful if you're writing a Python prototype that will eventually be rewritten in C. :mod:`optparse` becomes redundant, but there are no plans to remove it because there are many scripts still using it, and there's no diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index 572841db7d89d2..79d85a40df8bbe 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -228,8 +228,8 @@ format string in f-string and :meth:`str.format`. .. section: Core and Builtins The implementation of :func:`signal.siginterrupt` now uses -:c:func:`sigaction` (if it is available in the system) instead of the -deprecated :c:func:`siginterrupt`. Patch by Pablo Galindo. +:c:func:`!sigaction` (if it is available in the system) instead of the +deprecated :c:func:`!siginterrupt`. Patch by Pablo Galindo. .. From 7ca2d8e053e5fdbac4a9cec74701a1676c17a167 Mon Sep 17 00:00:00 2001 From: cLupus Date: Sun, 23 Jul 2023 23:50:01 +0200 Subject: [PATCH 64/73] gh-105291: Add link to migration guide for distutils (#107130) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/whatsnew/3.12.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f043ec764ac554..68e5b0691f00e4 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -83,7 +83,9 @@ Important deprecations, removals or restrictions: * :pep:`623`: Remove wstr from Unicode -* :pep:`632`: Remove the ``distutils`` package +* :pep:`632`: Remove the ``distutils`` package. See + `the migration guide `_ + for advice on its replacement. Improved Error Messages ======================= From 8de8a817ee81e352603380ddd1ae91accc99d4a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Carretero?= Date: Sun, 23 Jul 2023 17:53:25 -0400 Subject: [PATCH 65/73] Docs: fix typo in os.pwrite docstring (#107087) --- Modules/clinic/posixmodule.c.h | 4 ++-- Modules/posixmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index a8f6ce026a331b..81e3162e679d16 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -7366,7 +7366,7 @@ PyDoc_STRVAR(os_pwrite__doc__, "Write bytes to a file descriptor starting at a particular offset.\n" "\n" "Write buffer to fd, starting at offset bytes from the beginning of\n" -"the file. Returns the number of bytes writte. Does not change the\n" +"the file. Returns the number of bytes written. Does not change the\n" "current file offset."); #define OS_PWRITE_METHODDEF \ @@ -11981,4 +11981,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a7e8c3df2db09717 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a85a386b212b0631 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 23bf978d0cdbf1..7e04d12a9e4e7c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11283,13 +11283,13 @@ os.pwrite -> Py_ssize_t Write bytes to a file descriptor starting at a particular offset. Write buffer to fd, starting at offset bytes from the beginning of -the file. Returns the number of bytes writte. Does not change the +the file. Returns the number of bytes written. Does not change the current file offset. [clinic start generated code]*/ static Py_ssize_t os_pwrite_impl(PyObject *module, int fd, Py_buffer *buffer, Py_off_t offset) -/*[clinic end generated code: output=c74da630758ee925 input=19903f1b3dd26377]*/ +/*[clinic end generated code: output=c74da630758ee925 input=614acbc7e5a0339a]*/ { Py_ssize_t size; int async_err = 0; From 1e501122876f7e6abf979fdb756d7edd0d924ba1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 24 Jul 2023 00:25:48 +0200 Subject: [PATCH 66/73] gh-106320: Remove private _PyObject C API (#107159) Move private _PyObject and private _PyType functions to the internal C API (pycore_object.h): * _PyObject_GetMethod() * _PyObject_IsAbstract() * _PyObject_NextNotImplemented() * _PyType_CalculateMetaclass() * _PyType_GetDocFromInternalDoc() * _PyType_GetTextSignatureFromInternalDoc() No longer export these functions. --- Include/cpython/object.h | 12 ------------ Include/internal/pycore_object.h | 11 ++++++++++- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 90e45f65e298e1..fd45fa57e6282b 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -275,14 +275,6 @@ PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyType_LookupId(PyTypeObject *, _Py_Identifier *); PyAPI_FUNC(PyObject *) _PyObject_LookupSpecialId(PyObject *, _Py_Identifier *); -#ifndef Py_BUILD_CORE -// Backward compatibility for 3rd-party extensions -// that may be using the old name. -#define _PyObject_LookupSpecial _PyObject_LookupSpecialId -#endif -PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); -PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) PyType_GetModuleByDef(PyTypeObject *, PyModuleDef *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); @@ -290,14 +282,10 @@ PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, _Py_Identifier *, PyObject *); -PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); - PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_NextNotImplemented(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 661820c894dc0e..96aea477d064a4 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -376,7 +376,11 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { } extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); -PyObject *_PyType_NewManagedObject(PyTypeObject *type); +extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); + +extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); +extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); +extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); extern int _PyObject_InitializeDict(PyObject *obj); int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); @@ -432,6 +436,11 @@ extern int _PyObject_IsInstanceDictEmpty(PyObject *); PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); +extern int _PyObject_IsAbstract(PyObject *); + +extern int _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); +extern PyObject* _PyObject_NextNotImplemented(PyObject *); + /* C function call trampolines to mitigate bad function pointer casts. * * Typical native ABIs ignore additional arguments or fill in missing From 2cf99026d6320f38937257da1ab014fc873a11a6 Mon Sep 17 00:00:00 2001 From: TommyUnreal <45427816+TommyUnreal@users.noreply.github.com> Date: Mon, 24 Jul 2023 07:55:57 +0200 Subject: [PATCH 67/73] gh-107017: Change Chapter Strings to Texts in the Introduction chapter. (#107104) Co-authored-by: Hugo van Kemenade --- Doc/tutorial/introduction.rst | 40 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 4b886f604aca23..0fc75c7d7532e2 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -138,16 +138,25 @@ and uses the ``j`` or ``J`` suffix to indicate the imaginary part .. _tut-strings: -Strings -------- +Text +---- -Besides numbers, Python can also manipulate strings, which can be expressed -in several ways. They can be enclosed in single quotes (``'...'``) or -double quotes (``"..."``) with the same result [#]_. ``\`` can be used -to escape quotes:: +Python can manipulate text (represented by type :class:`str`, so-called +"strings") as well as numbers. This includes characters "``!``", words +"``rabbit``", names "``Paris``", sentences "``Got your back.``", etc. +"``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double +quotes (``"..."``) with the same result [#]_. >>> 'spam eggs' # single quotes 'spam eggs' + >>> "Paris rabbit got your back :)! Yay!" # double quotes + 'Paris rabbit got your back :)! Yay!' + >>> '1975' # digits and numerals enclosed in quotes are also strings + '1975' + +To quote a quote, we need to "escape" it, by preceding it with ``\``. +Alternatively, we can use the other type of quotation marks:: + >>> 'doesn\'t' # use \' to escape the single quote... "doesn't" >>> "doesn't" # ...or use double quotes instead @@ -159,23 +168,14 @@ to escape quotes:: >>> '"Isn\'t," they said.' '"Isn\'t," they said.' -In the interactive interpreter, the output string is enclosed in quotes and -special characters are escaped with backslashes. While this might sometimes -look different from the input (the enclosing quotes could change), the two -strings are equivalent. The string is enclosed in double quotes if -the string contains a single quote and no double quotes, otherwise it is -enclosed in single quotes. The :func:`print` function produces a more -readable output, by omitting the enclosing quotes and by printing escaped -and special characters:: +In the Python shell, the string definition and output string can look +different. The :func:`print` function produces a more readable output, by +omitting the enclosing quotes and by printing escaped and special characters:: - >>> '"Isn\'t," they said.' - '"Isn\'t," they said.' - >>> print('"Isn\'t," they said.') - "Isn't," they said. >>> s = 'First line.\nSecond line.' # \n means newline - >>> s # without print(), \n is included in the output + >>> s # without print(), special characters are included in the string 'First line.\nSecond line.' - >>> print(s) # with print(), \n produces a new line + >>> print(s) # with print(), special characters are interpreted, so \n produces new line First line. Second line. From ebe44a5155e9abc70c4b8914ad26b27c2b84f72b Mon Sep 17 00:00:00 2001 From: Hakan Celik Date: Mon, 24 Jul 2023 14:54:39 +0300 Subject: [PATCH 68/73] Docs: Remove duplicate word in Argument Clinic howto heading (#107169) --- Doc/howto/clinic.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index f6bf1d2234f88d..933fecab9ddd5a 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -582,8 +582,8 @@ How-to guides ============= -How to to rename C functions and variables generated by Argument Clinic ------------------------------------------------------------------------ +How to rename C functions and variables generated by Argument Clinic +-------------------------------------------------------------------- Argument Clinic automatically names the functions it generates for you. Occasionally this may cause a problem, if the generated name collides with From 0a9b339363a59be1249189c767ed6f46fd71e1c7 Mon Sep 17 00:00:00 2001 From: da-woods Date: Mon, 24 Jul 2023 13:54:38 +0100 Subject: [PATCH 69/73] Fix PyVectorcall_Function doc versionadded (#107140) The documentation implies that PyVectorcall_Function() was available in Python 3.8. This is half-true - it was available under a different name. I think it's clearer to set the "version added" to 3.9. --- Doc/c-api/call.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 8be4b5ab80c2de..aed4ae44c76eea 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -152,7 +152,7 @@ Vectorcall Support API This is mostly useful to check whether or not *op* supports vectorcall, which can be done by checking ``PyVectorcall_Function(op) != NULL``. - .. versionadded:: 3.8 + .. versionadded:: 3.9 .. c:function:: PyObject* PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) From fd66baf34a73871c651326a41886240bf6f986d7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 24 Jul 2023 16:02:03 +0200 Subject: [PATCH 70/73] gh-106320: Remove private _PyDict C API (#107145) Move private _PyDict functions to the internal C API (pycore_dict.h): * _PyDict_Contains_KnownHash() * _PyDict_DebugMallocStats() * _PyDict_DelItemIf() * _PyDict_GetItemWithError() * _PyDict_HasOnlyStringKeys() * _PyDict_MaybeUntrack() * _PyDict_MergeEx() No longer export these functions. --- Include/cpython/dictobject.h | 15 ++------------- Include/internal/pycore_dict.h | 20 ++++++++++++++++++++ Modules/_weakref.c | 1 + Modules/gcmodule.c | 7 ++++--- Objects/setobject.c | 1 + Python/_warnings.c | 1 + Python/bltinmodule.c | 1 + Python/getargs.c | 3 ++- Python/sysmodule.c | 1 + 9 files changed, 33 insertions(+), 17 deletions(-) diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index ddada922020aa4..2a42794fdf0c85 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -34,7 +34,6 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -PyAPI_FUNC(PyObject *) _PyDict_GetItemWithError(PyObject *dp, PyObject *key); PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key); PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); @@ -44,8 +43,7 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, PyObject *item, Py_hash_t hash); PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -PyAPI_FUNC(int) _PyDict_DelItemIf(PyObject *mp, PyObject *key, - int (*predicate)(PyObject *value)); + PyAPI_FUNC(int) _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); @@ -58,25 +56,16 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { } #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op)) -PyAPI_FUNC(int) _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); PyAPI_FUNC(int) _PyDict_ContainsId(PyObject *, _Py_Identifier *); + PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); -PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); -PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *, PyObject *, PyObject *); #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) -/* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, - the first occurrence of a key wins, if override is 1, the last occurrence - of a key wins, if override is 2, a KeyError with conflicting key as - argument is raised. -*/ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); PyAPI_FUNC(int) _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); PyAPI_FUNC(int) _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); -PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); /* _PyDictView */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6253e0841ad349..26a913435ef9cf 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -12,6 +12,26 @@ extern "C" { #include "pycore_dict_state.h" #include "pycore_runtime.h" // _PyRuntime +// Unsafe flavor of PyDict_GetItemWithError(): no error checking +extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); + +extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); + +extern int _PyDict_DelItemIf(PyObject *mp, PyObject *key, + int (*predicate)(PyObject *value)); + +extern int _PyDict_HasOnlyStringKeys(PyObject *mp); + +extern void _PyDict_MaybeUntrack(PyObject *mp); + +/* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, + the first occurrence of a key wins, if override is 1, the last occurrence + of a key wins, if override is 2, a KeyError with conflicting key as + argument is raised. +*/ +extern int _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); + +extern void _PyDict_DebugMallocStats(FILE *out); /* runtime lifecycle */ diff --git a/Modules/_weakref.c b/Modules/_weakref.c index b5d80cbd731a28..dcb2984e3e8177 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_dict.h" // _PyDict_DelItemIf() #include "pycore_object.h" // _PyObject_GET_WEAKREFS_LISTPTR #include "pycore_weakref.h" // _PyWeakref_IS_DEAD() diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 97644a7cee774f..246c0a9e160aa9 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -25,12 +25,13 @@ #include "Python.h" #include "pycore_context.h" +#include "pycore_dict.h" // _PyDict_MaybeUntrack() #include "pycore_initconfig.h" -#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_object.h" #include "pycore_pyerrors.h" -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_weakref.h" // _PyWeakref_ClearRef() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" typedef struct _gc_runtime_state GCState; diff --git a/Objects/setobject.c b/Objects/setobject.c index 2f12320254ec94..c96b62e38ec27e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -32,6 +32,7 @@ */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_Contains_KnownHash() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_setobject.h" // _PySet_NextEntry() definition diff --git a/Python/_warnings.c b/Python/_warnings.c index 82e621243a0c15..40ec5f613d5bf4 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_frame.h" #include "pycore_initconfig.h" #include "pycore_interp.h" // PyInterpreterState.warnings diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 7d77fd5c0c328e..9baf233614879e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_Vector() #include "pycore_compile.h" // _PyAST_Compile() +#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_long.h" // _PyLong_CompactValue #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _Py_AddToAllObjects() diff --git a/Python/getargs.c b/Python/getargs.c index 45befab4f8bc37..efcf2e333af8da 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2,8 +2,9 @@ /* New getargs implementation */ #include "Python.h" -#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_dict.h" // _PyDict_HasOnlyStringKeys() #include "pycore_pylifecycle.h" // _PyArg_Fini +#include "pycore_tuple.h" // _PyTuple_ITEMS() #include #include diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fb033a6b5634ac..be026d95ba7e77 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -17,6 +17,7 @@ Data members: #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_SetAsyncGenFinalizer() +#include "pycore_dict.h" // _PyDict_GetItemWithError() #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD From 751695327562462705959e3b1cd4a37dba522178 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 24 Jul 2023 16:14:58 +0200 Subject: [PATCH 71/73] gh-107162: Document errcode.h usage in its comment (#107177) Public and documented PyRun_InteractiveOneFlags() C API uses it. --- Include/errcode.h | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Include/errcode.h b/Include/errcode.h index 54ae929bf25870..8d44e9ae559193 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -1,18 +1,24 @@ +// Error codes passed around between file input, tokenizer, parser and +// interpreter. This is necessary so we can turn them into Python +// exceptions at a higher level. Note that some errors have a +// slightly different meaning when passed from the tokenizer to the +// parser than when passed from the parser to the interpreter; e.g. +// the parser only returns E_EOF when it hits EOF immediately, and it +// never returns E_OK. +// +// The public PyRun_InteractiveOneObjectEx() function can return E_EOF, +// same as its variants: +// +// * PyRun_InteractiveOneObject() +// * PyRun_InteractiveOneFlags() +// * PyRun_InteractiveOne() + #ifndef Py_ERRCODE_H #define Py_ERRCODE_H #ifdef __cplusplus extern "C" { #endif - -/* Error codes passed around between file input, tokenizer, parser and - interpreter. This is necessary so we can turn them into Python - exceptions at a higher level. Note that some errors have a - slightly different meaning when passed from the tokenizer to the - parser than when passed from the parser to the interpreter; e.g. - the parser only returns E_EOF when it hits EOF immediately, and it - never returns E_OK. */ - #define E_OK 10 /* No error */ #define E_EOF 11 /* End Of File */ #define E_INTR 12 /* Interrupted */ From ff5f94b72c8aad8e45c397c263dbe7f19221735f Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 24 Jul 2023 17:22:18 +0200 Subject: [PATCH 72/73] Docs: Add missing markup to Argument Clinic docs (#106876) Co-authored-by: Ezio Melotti Co-authored-by: Serhiy Storchaka --- Doc/howto/clinic.rst | 269 +++++++++++++++++++++++-------------------- 1 file changed, 143 insertions(+), 126 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 933fecab9ddd5a..98d3632ff02325 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -96,7 +96,8 @@ things with all the information you give it. Basic concepts and usage ------------------------ -Argument Clinic ships with CPython; you'll find it in ``Tools/clinic/clinic.py``. +Argument Clinic ships with CPython; you'll find it in +:source:`Tools/clinic/clinic.py`. If you run that script, specifying a C file as an argument: .. code-block:: shell-session @@ -178,9 +179,10 @@ Let's dive in! 1. Find a Python builtin that calls either :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords`, and hasn't been converted to work with Argument Clinic yet. - For my example I'm using ``_pickle.Pickler.dump()``. + For my example I'm using + :py:meth:`_pickle.Pickler.dump `. -2. If the call to the ``PyArg_Parse`` function uses any of the +2. If the call to the :c:func:`!PyArg_Parse*` function uses any of the following format units: .. code-block:: none @@ -197,10 +199,10 @@ Let's dive in! support all of these scenarios. But these are advanced topics—let's do something simpler for your first function. - Also, if the function has multiple calls to :c:func:`PyArg_ParseTuple` + Also, if the function has multiple calls to :c:func:`!PyArg_ParseTuple` or :c:func:`PyArg_ParseTupleAndKeywords` where it supports different types for the same argument, or if the function uses something besides - PyArg_Parse functions to parse its arguments, it probably + :c:func:`!PyArg_Parse*` functions to parse its arguments, it probably isn't suitable for conversion to Argument Clinic. Argument Clinic doesn't support generic functions or polymorphic parameters. @@ -217,7 +219,7 @@ Let's dive in! If the old docstring had a first line that looked like a function signature, throw that line away. (The docstring doesn't need it - anymore—when you use ``help()`` on your builtin in the future, + anymore—when you use :py:func:`help` on your builtin in the future, the first line will be built automatically based on the function's signature.) @@ -264,7 +266,7 @@ Let's dive in! When you declare a class, you must also specify two aspects of its type in C: the type declaration you'd use for a pointer to an instance of - this class, and a pointer to the :c:type:`PyTypeObject` for this class. + this class, and a pointer to the :c:type:`!PyTypeObject` for this class. Sample:: @@ -313,10 +315,10 @@ Let's dive in! Clinic easier. For each parameter, copy the "format unit" for that - parameter from the ``PyArg_Parse()`` format argument and + parameter from the :c:func:`PyArg_Parse` format argument and specify *that* as its converter, as a quoted string. ("format unit" is the formal name for the one-to-three - character substring of the ``format`` parameter that tells + character substring of the *format* parameter that tells the argument parsing function what the type of the variable is and how to convert it. For more on format units please see :ref:`arg-parsing`.) @@ -349,7 +351,7 @@ Let's dive in! itself before the first keyword-only argument, indented the same as the parameter lines. - (``_pickle.Pickler.dump`` has neither, so our sample is unchanged.) + (:py:meth:`!_pickle.Pickler.dump` has neither, so our sample is unchanged.) 10. If the existing C function calls :c:func:`PyArg_ParseTuple` @@ -410,7 +412,7 @@ Let's dive in! 12. Save and close the file, then run ``Tools/clinic/clinic.py`` on it. With luck everything worked---your block now has output, and - a ``.c.h`` file has been generated! Reopen the file in your + a :file:`.c.h` file has been generated! Reopen the file in your text editor to see:: /*[clinic input] @@ -431,8 +433,8 @@ Let's dive in! it found an error in your input. Keep fixing your errors and retrying until Argument Clinic processes your file without complaint. - For readability, most of the glue code has been generated to a ``.c.h`` - file. You'll need to include that in your original ``.c`` file, + For readability, most of the glue code has been generated to a :file:`.c.h` + file. You'll need to include that in your original :file:`.c` file, typically right after the clinic module block:: #include "clinic/_pickle.c.h" @@ -446,8 +448,8 @@ Let's dive in! ensure that the code generated by Argument Clinic calls the *exact* same function. - Second, the format string passed in to :c:func:`PyArg_ParseTuple` or - :c:func:`PyArg_ParseTupleAndKeywords` should be *exactly* the same + Second, the format string passed in to :c:func:`!PyArg_ParseTuple` or + :c:func:`!PyArg_ParseTupleAndKeywords` should be *exactly* the same as the hand-written one in the existing function, up to the colon or semi-colon. @@ -469,7 +471,7 @@ Let's dive in! {"dump", (PyCFunction)__pickle_Pickler_dump, METH_O, __pickle_Pickler_dump__doc__}, This static structure should be *exactly* the same as the existing static - :c:type:`PyMethodDef` structure for this builtin. + :c:type:`!PyMethodDef` structure for this builtin. If any of these items differ in *any way*, adjust your Argument Clinic function specification and rerun @@ -539,14 +541,14 @@ Let's dive in! ... 15. Remember the macro with the :c:type:`PyMethodDef` structure for this - function? Find the existing :c:type:`PyMethodDef` structure for this + function? Find the existing :c:type:`!PyMethodDef` structure for this function and replace it with a reference to the macro. (If the builtin is at module scope, this will probably be very near the end of the file; if the builtin is a class method, this will probably be below but relatively near to the implementation.) Note that the body of the macro contains a trailing comma. So when you - replace the existing static :c:type:`PyMethodDef` structure with the macro, + replace the existing static :c:type:`!PyMethodDef` structure with the macro, *don't* add a comma to the end. Sample:: @@ -562,7 +564,7 @@ Let's dive in! &_Py_ID(new_unique_py_id) - If it does, you'll have to run ``Tools/scripts/generate_global_objects.py`` + If it does, you'll have to run ``make regen-global-objects`` to regenerate the list of precompiled identifiers at this point. @@ -570,7 +572,7 @@ Let's dive in! This change should not introduce any new compile-time warnings or errors, and there should be no externally visible change to Python's behavior. - Well, except for one difference: ``inspect.signature()`` run on your function + Well, except for one difference: :py:func:`inspect.signature` run on your function should now provide a valid signature! Congratulations, you've ported your first function to work with Argument Clinic! @@ -594,15 +596,15 @@ Argument Clinic will use that function name for the base (generated) function, then add ``"_impl"`` to the end and use that for the name of the impl function. For example, if we wanted to rename the C function names generated for -``pickle.Pickler.dump``, it'd look like this:: +:py:meth:`pickle.Pickler.dump`, it'd look like this:: /*[clinic input] pickle.Pickler.dump as pickler_dumper ... -The base function would now be named ``pickler_dumper()``, -and the impl function would now be named ``pickler_dumper_impl()``. +The base function would now be named :c:func:`!pickler_dumper`, +and the impl function would now be named :c:func:`!pickler_dumper_impl`. Similarly, you may have a problem where you want to give a parameter @@ -620,9 +622,9 @@ using the same ``"as"`` syntax:: fix_imports: bool = True Here, the name used in Python (in the signature and the ``keywords`` -array) would be ``file``, but the C variable would be named ``file_obj``. +array) would be *file*, but the C variable would be named ``file_obj``. -You can use this to rename the ``self`` parameter too! +You can use this to rename the *self* parameter too! How to convert functions using ``PyArg_UnpackTuple`` @@ -630,7 +632,7 @@ How to convert functions using ``PyArg_UnpackTuple`` To convert a function parsing its arguments with :c:func:`PyArg_UnpackTuple`, simply write out all the arguments, specifying each as an ``object``. You -may specify the ``type`` argument to cast the type as appropriate. All +may specify the *type* argument to cast the type as appropriate. All arguments should be marked positional-only (add a ``/`` on a line by itself after the last argument). @@ -649,16 +651,16 @@ keyword-only arguments.) This approach was used to simulate optional arguments back before :c:func:`PyArg_ParseTupleAndKeywords` was created. While functions using this approach can often be converted to -use :c:func:`PyArg_ParseTupleAndKeywords`, optional arguments, and default values, +use :c:func:`!PyArg_ParseTupleAndKeywords`, optional arguments, and default values, it's not always possible. Some of these legacy functions have -behaviors :c:func:`PyArg_ParseTupleAndKeywords` doesn't directly support. -The most obvious example is the builtin function ``range()``, which has +behaviors :c:func:`!PyArg_ParseTupleAndKeywords` doesn't directly support. +The most obvious example is the builtin function :py:func:`range`, which has an optional argument on the *left* side of its required argument! -Another example is ``curses.window.addch()``, which has a group of two +Another example is :py:meth:`curses.window.addch`, which has a group of two arguments that must always be specified together. (The arguments are -called ``x`` and ``y``; if you call the function passing in ``x``, -you must also pass in ``y``—and if you don't pass in ``x`` you may not -pass in ``y`` either.) +called *x* and *y*; if you call the function passing in *x*, +you must also pass in *y* — and if you don't pass in *x* you may not +pass in *y* either.) In any case, the goal of Argument Clinic is to support argument parsing for all existing CPython builtins without changing their semantics. @@ -679,7 +681,7 @@ can *only* be used with positional-only parameters. To specify an optional group, add a ``[`` on a line by itself before the parameters you wish to group together, and a ``]`` on a line by itself -after these parameters. As an example, here's how ``curses.window.addch`` +after these parameters. As an example, here's how :py:meth:`curses.window.addch` uses optional groups to make the first two parameters and the last parameter optional:: @@ -765,25 +767,25 @@ the same converters. All arguments to Argument Clinic converters are keyword-only. All Argument Clinic converters accept the following arguments: - ``c_default`` + *c_default* The default value for this parameter when defined in C. Specifically, this will be the initializer for the variable declared in the "parse function". See :ref:`the section on default values ` for how to use this. Specified as a string. - ``annotation`` + *annotation* The annotation value for this parameter. Not currently supported, because :pep:`8` mandates that the Python library may not use annotations. - ``unused`` + *unused* Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature. In addition, some converters accept additional arguments. Here is a list of these arguments, along with their meanings: - ``accept`` + *accept* A set of Python types (and possibly pseudo-types); this restricts the allowable Python argument to values of these types. (This is not a general-purpose facility; as a rule it only supports @@ -791,38 +793,38 @@ of these arguments, along with their meanings: To accept ``None``, add ``NoneType`` to this set. - ``bitwise`` + *bitwise* Only supported for unsigned integers. The native integer value of this Python argument will be written to the parameter without any range checking, even for negative values. - ``converter`` + *converter* Only supported by the ``object`` converter. Specifies the name of a :ref:`C "converter function" ` to use to convert this object to a native type. - ``encoding`` + *encoding* Only supported for strings. Specifies the encoding to use when converting this string from a Python str (Unicode) value into a C ``char *`` value. - ``subclass_of`` + *subclass_of* Only supported for the ``object`` converter. Requires that the Python value be a subclass of a Python type, as expressed in C. - ``type`` + *type* Only supported for the ``object`` and ``self`` converters. Specifies the C type that will be used to declare the variable. Default value is ``"PyObject *"``. - ``zeroes`` + *zeroes* Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are permitted inside the value. The length of the string will be passed in to the impl function, just after the string parameter, as a parameter named ``_length``. Please note, not every possible combination of arguments will work. -Usually these arguments are implemented by specific ``PyArg_ParseTuple`` +Usually these arguments are implemented by specific :c:func:`PyArg_ParseTuple` *format units*, with specific behavior. For example, currently you cannot call ``unsigned_short`` without also specifying ``bitwise=True``. Although it's perfectly reasonable to think this would work, these semantics don't @@ -922,19 +924,19 @@ conversion functions, or types, or strings specifying an encoding. (But "legacy converters" don't support arguments. That's why we skipped them for your first function.) The argument you specified to the format unit is now an argument to the converter; this -argument is either ``converter`` (for ``O&``), ``subclass_of`` (for ``O!``), -or ``encoding`` (for all the format units that start with ``e``). +argument is either *converter* (for ``O&``), *subclass_of* (for ``O!``), +or *encoding* (for all the format units that start with ``e``). -When using ``subclass_of``, you may also want to use the other -custom argument for ``object()``: ``type``, which lets you set the type +When using *subclass_of*, you may also want to use the other +custom argument for ``object()``: *type*, which lets you set the type actually used for the parameter. For example, if you want to ensure -that the object is a subclass of ``PyUnicode_Type``, you probably want +that the object is a subclass of :c:var:`PyUnicode_Type`, you probably want to use the converter ``object(type='PyUnicodeObject *', subclass_of='&PyUnicode_Type')``. One possible problem with using Argument Clinic: it takes away some possible flexibility for the format units starting with ``e``. When writing a -``PyArg_Parse`` call by hand, you could theoretically decide at runtime what -encoding string to pass in to :c:func:`PyArg_ParseTuple`. But now this string must +:c:func:`!PyArg_Parse*` call by hand, you could theoretically decide at runtime what +encoding string to pass to that call. But now this string must be hard-coded at Argument-Clinic-preprocessing-time. This limitation is deliberate; it made supporting this format unit much easier, and may allow for future optimizations. This restriction doesn't seem unreasonable; CPython itself always passes in static @@ -987,7 +989,7 @@ expression. Currently the following are explicitly supported: * Numeric constants (integer and float) * String constants * ``True``, ``False``, and ``None`` -* Simple symbolic constants like ``sys.maxsize``, which must +* Simple symbolic constants like :py:data:`sys.maxsize`, which must start with the name of the module (In the future, this may need to get even more elaborate, @@ -1008,28 +1010,28 @@ Consider the following example: foo: Py_ssize_t = sys.maxsize - 1 -``sys.maxsize`` can have different values on different platforms. Therefore +:py:data:`sys.maxsize` can have different values on different platforms. Therefore Argument Clinic can't simply evaluate that expression locally and hard-code it in C. So it stores the default in such a way that it will get evaluated at runtime, when the user asks for the function's signature. What namespace is available when the expression is evaluated? It's evaluated in the context of the module the builtin came from. So, if your module has an -attribute called "``max_widgets``", you may simply use it: +attribute called :py:attr:`!max_widgets`, you may simply use it: .. code-block:: none foo: Py_ssize_t = max_widgets If the symbol isn't found in the current module, it fails over to looking in -``sys.modules``. That's how it can find ``sys.maxsize`` for example. (Since you -don't know in advance what modules the user will load into their interpreter, +:py:data:`sys.modules`. That's how it can find :py:data:`sys.maxsize` for example. +(Since you don't know in advance what modules the user will load into their interpreter, it's best to restrict yourself to modules that are preloaded by Python itself.) Evaluating default values only at runtime means Argument Clinic can't compute the correct equivalent C default value. So you need to tell it explicitly. When you use an expression, you must also specify the equivalent expression -in C, using the ``c_default`` parameter to the converter: +in C, using the *c_default* parameter to the converter: .. code-block:: none @@ -1095,7 +1097,7 @@ indicate an error has occurred? Normally, a function returns a valid (non-``NUL pointer for success, and ``NULL`` for failure. But if you use an integer return converter, all integers are valid. How can Argument Clinic detect an error? Its solution: each return converter implicitly looks for a special value that indicates an error. If you return -that value, and an error has been set (``PyErr_Occurred()`` returns a true +that value, and an error has been set (c:func:`PyErr_Occurred` returns a true value), then the generated code will propagate the error. Otherwise it will encode the value you return like normal. @@ -1201,9 +1203,9 @@ using a default converter. It automatically sets the ``type`` of this parameter to the "pointer to an instance" you specified when you declared the type. However, you can override Argument Clinic's converter and specify one yourself. -Just add your own ``self`` parameter as the first parameter in a +Just add your own *self* parameter as the first parameter in a block, and ensure that its converter is an instance of -``self_converter`` or a subclass thereof. +:class:`!self_converter` or a subclass thereof. What's the point? This lets you override the type of ``self``, or give it a different default name. @@ -1211,7 +1213,7 @@ or give it a different default name. How do you specify the custom type you want to cast ``self`` to? If you only have one or two functions with the same type for ``self``, you can directly use Argument Clinic's existing ``self`` converter, -passing in the type you want to use as the ``type`` parameter:: +passing in the type you want to use as the *type* parameter:: /*[clinic input] @@ -1226,7 +1228,7 @@ passing in the type you want to use as the ``type`` parameter:: On the other hand, if you have a lot of functions that will use the same type for ``self``, it's best to create your own converter, subclassing -``self_converter`` but overwriting the ``type`` member:: +:class:`!self_converter` but overwriting the :py:attr:`!type` member:: /*[python input] class PicklerObject_converter(self_converter): @@ -1254,8 +1256,8 @@ module level state. Use :c:func:`PyType_FromModuleAndSpec` to associate a new heap type with a module. You can now use :c:func:`PyType_GetModuleState` on the defining class to fetch the module state, for example from a module method. -Example from ``Modules/zlibmodule.c``. First, ``defining_class`` is added to -the clinic input:: +Example from :source:`Modules/zlibmodule.c`. +First, ``defining_class`` is added to the clinic input:: /*[clinic input] zlib.Compress.compress @@ -1285,16 +1287,17 @@ module state:: Each method may only have one argument using this converter, and it must appear after ``self``, or, if ``self`` is not used, as the first argument. The argument will be of type ``PyTypeObject *``. The argument will not appear in the -``__text_signature__``. +:py:attr:`!__text_signature__`. -The ``defining_class`` converter is not compatible with ``__init__`` and ``__new__`` -methods, which cannot use the ``METH_METHOD`` convention. +The ``defining_class`` converter is not compatible with :py:meth:`!__init__` +and :py:meth:`!__new__` methods, which cannot use the :c:macro:`METH_METHOD` +convention. It is not possible to use ``defining_class`` with slot methods. In order to fetch the module state from such methods, use :c:func:`PyType_GetModuleByDef` to look up the module and then :c:func:`PyModule_GetState` to fetch the module state. Example from the ``setattro`` slot method in -``Modules/_threadmodule.c``:: +:source:`Modules/_threadmodule.c`:: static int local_setattro(localobject *self, PyObject *name, PyObject *v) @@ -1312,7 +1315,7 @@ How to write a custom converter ------------------------------- As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from ``CConverter``. +A converter is simply a Python class that inherits from :py:class:`!CConverter`. The main purpose of a custom converter is if you have a parameter using the ``O&`` format unit—parsing this parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". @@ -1323,61 +1326,74 @@ will be automatically registered with Argument Clinic; its name will be the name of your class with the ``_converter`` suffix stripped off. (This is accomplished with a metaclass.) -You shouldn't subclass ``CConverter.__init__``. Instead, you should -write a ``converter_init()`` function. ``converter_init()`` -always accepts a ``self`` parameter; after that, all additional +You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should +write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` +always accepts a *self* parameter; after that, all additional parameters *must* be keyword-only. Any arguments passed in to the converter in Argument Clinic will be passed along to your -``converter_init()``. +:py:meth:`!converter_init`. -There are some additional members of ``CConverter`` you may wish +There are some additional members of :py:class:`!CConverter` you may wish to specify in your subclass. Here's the current list: -``type`` - The C type to use for this variable. - ``type`` should be a Python string specifying the type, e.g. ``int``. - If this is a pointer type, the type string should end with ``' *'``. +.. module:: clinic -``default`` - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. +.. class:: CConverter -``py_default`` - ``default`` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. + .. attribute:: type -``c_default`` - ``default`` as it should appear in C code, - as a string. - Or ``None`` if there is no default. + The C type to use for this variable. + :attr:`!type` should be a Python string specifying the type, + e.g. ``'int'``. + If this is a pointer type, the type string should end with ``' *'``. -``c_ignored_default`` - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups—although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. + .. attribute:: default -``converter`` - The name of the C converter function, as a string. + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. -``impl_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. + .. attribute:: py_default -``parse_by_reference`` - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. + :attr:`!default` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + .. attribute:: c_default -Here's the simplest example of a custom converter, from ``Modules/zlibmodule.c``:: + :attr:`!default` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_ignored_default + + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups—although + properly written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + + .. attribute:: converter + + The name of the C converter function, as a string. + + .. attribute:: impl_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + + .. attribute:: parse_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + +Here's the simplest example of a custom converter, from :source:`Modules/zlibmodule.c`:: /*[python input] @@ -1397,7 +1413,7 @@ automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string ``CConverter``. +source tree; grep the C files for the string :py:class:`!CConverter`. How to write a custom return converter @@ -1407,18 +1423,18 @@ Writing a custom return converter is much like writing a custom converter. Except it's somewhat simpler, because return converters are themselves much simpler. -Return converters must subclass ``CReturnConverter``. +Return converters must subclass :py:class:`!CReturnConverter`. There are no examples yet of custom return converters, because they are not widely used yet. If you wish to -write your own return converter, please read ``Tools/clinic/clinic.py``, -specifically the implementation of ``CReturnConverter`` and +write your own return converter, please read :source:`Tools/clinic/clinic.py`, +specifically the implementation of :py:class:`!CReturnConverter` and all its subclasses. How to convert ``METH_O`` and ``METH_NOARGS`` functions ------------------------------------------------------- -To convert a function using ``METH_O``, make sure the function's +To convert a function using :c:macro:`METH_O`, make sure the function's single argument is using the ``object`` converter, and mark the arguments as positional-only:: @@ -1430,24 +1446,25 @@ arguments as positional-only:: [clinic start generated code]*/ -To convert a function using ``METH_NOARGS``, just don't specify +To convert a function using :c:macro:`METH_NOARGS`, just don't specify any arguments. You can still use a self converter, a return converter, and specify -a ``type`` argument to the object converter for ``METH_O``. +a *type* argument to the object converter for :c:macro:`METH_O`. How to convert ``tp_new`` and ``tp_init`` functions --------------------------------------------------- -You can convert ``tp_new`` and ``tp_init`` functions. Just name -them ``__new__`` or ``__init__`` as appropriate. Notes: +You can convert :c:member:`~PyTypeObject.tp_new` and +:c:member:`~PyTypeObject.tp_init` functions. +Just name them ``__new__`` or ``__init__`` as appropriate. Notes: * The function name generated for ``__new__`` doesn't end in ``__new__`` like it would by default. It's just the name of the class, converted into a valid C identifier. -* No ``PyMethodDef`` ``#define`` is generated for these functions. +* No :c:type:`PyMethodDef` ``#define`` is generated for these functions. * ``__init__`` functions return ``int``, not ``PyObject *``. @@ -1482,7 +1499,7 @@ Let's start with defining some terminology: *field* A field, in this context, is a subsection of Clinic's output. - For example, the ``#define`` for the ``PyMethodDef`` structure + For example, the ``#define`` for the :c:type:`PyMethodDef` structure is a field, called ``methoddef_define``. Clinic has seven different fields it can output per function definition: @@ -1526,8 +1543,8 @@ Let's start with defining some terminology: The filename chosen for the file is ``{basename}.clinic{extension}``, where ``basename`` and ``extension`` were assigned the output from ``os.path.splitext()`` run on the current file. (Example: - the ``file`` destination for ``_pickle.c`` would be written to - ``_pickle.clinic.c``.) + the ``file`` destination for :file:`_pickle.c` would be written to + :file:`_pickle.clinic.c`.) **Important: When using a** ``file`` **destination, you** *must check in* **the generated file!** @@ -1780,7 +1797,7 @@ like so:: } #endif /* HAVE_FUNCTIONNAME */ -Then, remove those three lines from the ``PyMethodDef`` structure, +Then, remove those three lines from the :c:type:`PyMethodDef` structure, replacing them with the macro Argument Clinic generated: .. code-block:: none @@ -1821,7 +1838,7 @@ This may mean that you get a complaint from Argument Clinic: When this happens, just open your file, find the ``dump buffer`` block that Argument Clinic added to your file (it'll be at the very bottom), then -move it above the ``PyMethodDef`` structure where that macro is used. +move it above the :c:type:`PyMethodDef` structure where that macro is used. How to use Argument Clinic in Python files From 032f4809094bf03d92c54e46b305c499ef7e3165 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 24 Jul 2023 09:38:23 -0700 Subject: [PATCH 73/73] Thoroughly refactor the cases generator (#107151) This mostly extracts a whole bunch of stuff out of generate_cases.py into separate files, but there are a few other things going on here. - analysis.py: `Analyzer` etc. - instructions.py: `Instruction` etc. - flags.py: `InstructionFlags`, `variable_used`, `variable_used_unspecialized` - formatting.py: `Formatter` etc. - Rename parser.py to parsing.py, to avoid conflict with stdlib parser.py - Blackify most things - Fix most mypy errors - Remove output filenames from Generator state, add them to `write_instructions()` etc. - Fix unit tests --- Lib/test/test_generated_cases.py | 42 +- Tools/cases_generator/analysis.py | 412 ++++++ Tools/cases_generator/flags.py | 102 ++ Tools/cases_generator/formatting.py | 188 +++ Tools/cases_generator/generate_cases.py | 1287 ++--------------- Tools/cases_generator/instructions.py | 424 ++++++ .../cases_generator/{parser.py => parsing.py} | 18 +- 7 files changed, 1304 insertions(+), 1169 deletions(-) create mode 100644 Tools/cases_generator/analysis.py create mode 100644 Tools/cases_generator/flags.py create mode 100644 Tools/cases_generator/formatting.py create mode 100644 Tools/cases_generator/instructions.py rename Tools/cases_generator/{parser.py => parsing.py} (96%) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ba0e5e8b0f6954..9c23c2e82058c7 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -6,8 +6,10 @@ test_tools.skip_if_missing('cases_generator') with test_tools.imports_under_tool('cases_generator'): + import analysis + import formatting import generate_cases - from parser import StackEffect + from parsing import StackEffect class TestEffects(unittest.TestCase): @@ -27,37 +29,37 @@ def test_effect_sizes(self): StackEffect("q", "", "", ""), StackEffect("r", "", "", ""), ] - self.assertEqual(generate_cases.effect_size(x), (1, "")) - self.assertEqual(generate_cases.effect_size(y), (0, "oparg")) - self.assertEqual(generate_cases.effect_size(z), (0, "oparg*2")) + self.assertEqual(formatting.effect_size(x), (1, "")) + self.assertEqual(formatting.effect_size(y), (0, "oparg")) + self.assertEqual(formatting.effect_size(z), (0, "oparg*2")) self.assertEqual( - generate_cases.list_effect_size(input_effects), + formatting.list_effect_size(input_effects), (1, "oparg + oparg*2"), ) self.assertEqual( - generate_cases.list_effect_size(output_effects), + formatting.list_effect_size(output_effects), (2, "oparg*4"), ) self.assertEqual( - generate_cases.list_effect_size(other_effects), + formatting.list_effect_size(other_effects), (2, "(oparg<<1)"), ) self.assertEqual( - generate_cases.string_effect_size( - generate_cases.list_effect_size(input_effects), + formatting.string_effect_size( + formatting.list_effect_size(input_effects), ), "1 + oparg + oparg*2", ) self.assertEqual( - generate_cases.string_effect_size( - generate_cases.list_effect_size(output_effects), + formatting.string_effect_size( + formatting.list_effect_size(output_effects), ), "2 + oparg*4", ) self.assertEqual( - generate_cases.string_effect_size( - generate_cases.list_effect_size(other_effects), + formatting.string_effect_size( + formatting.list_effect_size(other_effects), ), "2 + (oparg<<1)", ) @@ -90,23 +92,17 @@ def tearDown(self) -> None: def run_cases_test(self, input: str, expected: str): with open(self.temp_input_filename, "w+") as temp_input: - temp_input.write(generate_cases.BEGIN_MARKER) + temp_input.write(analysis.BEGIN_MARKER) temp_input.write(input) - temp_input.write(generate_cases.END_MARKER) + temp_input.write(analysis.END_MARKER) temp_input.flush() - a = generate_cases.Analyzer( - [self.temp_input_filename], - self.temp_output_filename, - self.temp_metadata_filename, - self.temp_pymetadata_filename, - self.temp_executor_filename, - ) + a = generate_cases.Generator([self.temp_input_filename]) a.parse() a.analyze() if a.errors: raise RuntimeError(f"Found {a.errors} errors") - a.write_instructions() + a.write_instructions(self.temp_output_filename, False) with open(self.temp_output_filename) as temp_output: lines = temp_output.readlines() diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py new file mode 100644 index 00000000000000..53ae7360b8ae12 --- /dev/null +++ b/Tools/cases_generator/analysis.py @@ -0,0 +1,412 @@ +import re +import sys +import typing + +from flags import InstructionFlags, variable_used +from formatting import prettify_filename, UNUSED +from instructions import ( + ActiveCacheEffect, + Component, + Instruction, + InstructionOrCacheEffect, + MacroInstruction, + MacroParts, + OverriddenInstructionPlaceHolder, + PseudoInstruction, + StackEffectMapping, +) +import parsing +from parsing import StackEffect + +BEGIN_MARKER = "// BEGIN BYTECODES //" +END_MARKER = "// END BYTECODES //" + +RESERVED_WORDS = { + "co_consts": "Use FRAME_CO_CONSTS.", + "co_names": "Use FRAME_CO_NAMES.", +} + +RE_PREDICTED = r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$" + + +class Analyzer: + """Parse input, analyze it, and write to output.""" + + input_filenames: list[str] + errors: int = 0 + + def __init__(self, input_filenames: list[str]): + self.input_filenames = input_filenames + + def error(self, msg: str, node: parsing.Node) -> None: + lineno = 0 + filename = "" + if context := node.context: + filename = context.owner.filename + # Use line number of first non-comment in the node + for token in context.owner.tokens[context.begin : context.end]: + lineno = token.line + if token.kind != "COMMENT": + break + print(f"{filename}:{lineno}: {msg}", file=sys.stderr) + self.errors += 1 + + everything: list[ + parsing.InstDef + | parsing.Macro + | parsing.Pseudo + | OverriddenInstructionPlaceHolder + ] + instrs: dict[str, Instruction] # Includes ops + macros: dict[str, parsing.Macro] + macro_instrs: dict[str, MacroInstruction] + families: dict[str, parsing.Family] + pseudos: dict[str, parsing.Pseudo] + pseudo_instrs: dict[str, PseudoInstruction] + + def parse(self) -> None: + """Parse the source text. + + We only want the parser to see the stuff between the + begin and end markers. + """ + + self.everything = [] + self.instrs = {} + self.macros = {} + self.families = {} + self.pseudos = {} + + instrs_idx: dict[str, int] = dict() + + for filename in self.input_filenames: + self.parse_file(filename, instrs_idx) + + files = " + ".join(self.input_filenames) + print( + f"Read {len(self.instrs)} instructions/ops, " + f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " + f"and {len(self.families)} families from {files}", + file=sys.stderr, + ) + + def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: + with open(filename) as file: + src = file.read() + + psr = parsing.Parser(src, filename=prettify_filename(filename)) + + # Skip until begin marker + while tkn := psr.next(raw=True): + if tkn.text == BEGIN_MARKER: + break + else: + raise psr.make_syntax_error( + f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" + ) + start = psr.getpos() + + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) + thing: parsing.Node | None + thing_first_token = psr.peek() + while thing := psr.definition(): + thing = typing.cast( + parsing.InstDef | parsing.Macro | parsing.Pseudo | parsing.Family, thing + ) + if ws := [w for w in RESERVED_WORDS if variable_used(thing, w)]: + self.error( + f"'{ws[0]}' is a reserved word. {RESERVED_WORDS[ws[0]]}", thing + ) + + match thing: + case parsing.InstDef(name=name): + if name in self.instrs: + if not thing.override: + raise psr.make_syntax_error( + f"Duplicate definition of '{name}' @ {thing.context} " + f"previous definition @ {self.instrs[name].inst.context}", + thing_first_token, + ) + self.everything[ + instrs_idx[name] + ] = OverriddenInstructionPlaceHolder(name=name) + if name not in self.instrs and thing.override: + raise psr.make_syntax_error( + f"Definition of '{name}' @ {thing.context} is supposed to be " + "an override but no previous definition exists.", + thing_first_token, + ) + self.instrs[name] = Instruction(thing) + instrs_idx[name] = len(self.everything) + self.everything.append(thing) + case parsing.Macro(name): + self.macros[name] = thing + self.everything.append(thing) + case parsing.Family(name): + self.families[name] = thing + case parsing.Pseudo(name): + self.pseudos[name] = thing + self.everything.append(thing) + case _: + typing.assert_never(thing) + if not psr.eof(): + raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") + + def analyze(self) -> None: + """Analyze the inputs. + + Raises SystemExit if there is an error. + """ + self.analyze_macros_and_pseudos() + self.find_predictions() + self.map_families() + self.check_families() + + def find_predictions(self) -> None: + """Find the instructions that need PREDICTED() labels.""" + for instr in self.instrs.values(): + targets: set[str] = set() + for line in instr.block_text: + if m := re.match(RE_PREDICTED, line): + targets.add(m.group(1)) + for target in targets: + if target_instr := self.instrs.get(target): + target_instr.predicted = True + elif target_macro := self.macro_instrs.get(target): + target_macro.predicted = True + else: + self.error( + f"Unknown instruction {target!r} predicted in {instr.name!r}", + instr.inst, # TODO: Use better location + ) + + def map_families(self) -> None: + """Link instruction names back to their family, if they have one.""" + for family in self.families.values(): + for member in [family.name] + family.members: + if member_instr := self.instrs.get(member): + if ( + member_instr.family is not family + and member_instr.family is not None + ): + self.error( + f"Instruction {member} is a member of multiple families " + f"({member_instr.family.name}, {family.name}).", + family, + ) + else: + member_instr.family = family + elif not self.macro_instrs.get(member): + self.error( + f"Unknown instruction {member!r} referenced in family {family.name!r}", + family, + ) + + def check_families(self) -> None: + """Check each family: + + - Must have at least 2 members (including head) + - Head and all members must be known instructions + - Head and all members must have the same cache, input and output effects + """ + for family in self.families.values(): + if family.name not in self.macro_instrs and family.name not in self.instrs: + self.error( + f"Family {family.name!r} has unknown instruction {family.name!r}", + family, + ) + members = [ + member + for member in family.members + if member in self.instrs or member in self.macro_instrs + ] + if members != family.members: + unknown = set(family.members) - set(members) + self.error( + f"Family {family.name!r} has unknown members: {unknown}", family + ) + expected_effects = self.effect_counts(family.name) + for member in members: + member_effects = self.effect_counts(member) + if member_effects != expected_effects: + self.error( + f"Family {family.name!r} has inconsistent " + f"(cache, input, output) effects:\n" + f" {family.name} = {expected_effects}; " + f"{member} = {member_effects}", + family, + ) + + def effect_counts(self, name: str) -> tuple[int, int, int]: + if instr := self.instrs.get(name): + cache = instr.cache_offset + input = len(instr.input_effects) + output = len(instr.output_effects) + elif mac := self.macro_instrs.get(name): + cache = mac.cache_offset + input, output = 0, 0 + for part in mac.parts: + if isinstance(part, Component): + # A component may pop what the previous component pushed, + # so we offset the input/output counts by that. + delta_i = len(part.instr.input_effects) + delta_o = len(part.instr.output_effects) + offset = min(delta_i, output) + input += delta_i - offset + output += delta_o - offset + else: + assert False, f"Unknown instruction {name!r}" + return cache, input, output + + def analyze_macros_and_pseudos(self) -> None: + """Analyze each macro and pseudo instruction.""" + self.macro_instrs = {} + self.pseudo_instrs = {} + for name, macro in self.macros.items(): + self.macro_instrs[name] = self.analyze_macro(macro) + for name, pseudo in self.pseudos.items(): + self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) + + def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: + components = self.check_macro_components(macro) + stack, initial_sp = self.stack_analysis(components) + sp = initial_sp + parts: MacroParts = [] + flags = InstructionFlags.newEmpty() + offset = 0 + for component in components: + match component: + case parsing.CacheEffect() as ceffect: + parts.append(ceffect) + offset += ceffect.size + case Instruction() as instr: + part, sp, offset = self.analyze_instruction( + instr, stack, sp, offset + ) + parts.append(part) + flags.add(instr.instr_flags) + case _: + typing.assert_never(component) + final_sp = sp + format = "IB" + if offset: + format += "C" + "0" * (offset - 1) + return MacroInstruction( + macro.name, stack, initial_sp, final_sp, format, flags, macro, parts, offset + ) + + def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: + targets = [self.instrs[target] for target in pseudo.targets] + assert targets + # Make sure the targets have the same fmt + fmts = list(set([t.instr_fmt for t in targets])) + assert len(fmts) == 1 + assert len(list(set([t.instr_flags.bitmap() for t in targets]))) == 1 + return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) + + def analyze_instruction( + self, instr: Instruction, stack: list[StackEffect], sp: int, offset: int + ) -> tuple[Component, int, int]: + input_mapping: StackEffectMapping = [] + for ieffect in reversed(instr.input_effects): + sp -= 1 + input_mapping.append((stack[sp], ieffect)) + output_mapping: StackEffectMapping = [] + for oeffect in instr.output_effects: + output_mapping.append((stack[sp], oeffect)) + sp += 1 + active_effects: list[ActiveCacheEffect] = [] + for ceffect in instr.cache_effects: + if ceffect.name != UNUSED: + active_effects.append(ActiveCacheEffect(ceffect, offset)) + offset += ceffect.size + return ( + Component(instr, input_mapping, output_mapping, active_effects), + sp, + offset, + ) + + def check_macro_components( + self, macro: parsing.Macro + ) -> list[InstructionOrCacheEffect]: + components: list[InstructionOrCacheEffect] = [] + for uop in macro.uops: + match uop: + case parsing.OpName(name): + if name not in self.instrs: + self.error(f"Unknown instruction {name!r}", macro) + components.append(self.instrs[name]) + case parsing.CacheEffect(): + components.append(uop) + case _: + typing.assert_never(uop) + return components + + def stack_analysis( + self, components: typing.Iterable[InstructionOrCacheEffect] + ) -> tuple[list[StackEffect], int]: + """Analyze a macro. + + Ignore cache effects. + + Return the list of variables (as StackEffects) and the initial stack pointer. + """ + lowest = current = highest = 0 + conditions: dict[int, str] = {} # Indexed by 'current'. + last_instr: Instruction | None = None + for thing in components: + if isinstance(thing, Instruction): + last_instr = thing + for thing in components: + match thing: + case Instruction() as instr: + if any( + eff.size for eff in instr.input_effects + instr.output_effects + ): + # TODO: Eventually this will be needed, at least for macros. + self.error( + f"Instruction {instr.name!r} has variable-sized stack effect, " + "which are not supported in macro instructions", + instr.inst, # TODO: Pass name+location of macro + ) + if any(eff.cond for eff in instr.input_effects): + self.error( + f"Instruction {instr.name!r} has conditional input stack effect, " + "which are not supported in macro instructions", + instr.inst, # TODO: Pass name+location of macro + ) + if ( + any(eff.cond for eff in instr.output_effects) + and instr is not last_instr + ): + self.error( + f"Instruction {instr.name!r} has conditional output stack effect, " + "but is not the last instruction in a macro", + instr.inst, # TODO: Pass name+location of macro + ) + current -= len(instr.input_effects) + lowest = min(lowest, current) + for eff in instr.output_effects: + if eff.cond: + conditions[current] = eff.cond + current += 1 + highest = max(highest, current) + case parsing.CacheEffect(): + pass + case _: + typing.assert_never(thing) + # At this point, 'current' is the net stack effect, + # and 'lowest' and 'highest' are the extremes. + # Note that 'lowest' may be negative. + stack = [ + StackEffect(f"_tmp_{i}", "", conditions.get(highest - i, "")) + for i in reversed(range(1, highest - lowest + 1)) + ] + return stack, -lowest diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py new file mode 100644 index 00000000000000..78acd93b73ac31 --- /dev/null +++ b/Tools/cases_generator/flags.py @@ -0,0 +1,102 @@ +import dataclasses + +from formatting import Formatter +import lexer as lx +import parsing + + +@dataclasses.dataclass +class InstructionFlags: + """Construct and manipulate instruction flags""" + + HAS_ARG_FLAG: bool + HAS_CONST_FLAG: bool + HAS_NAME_FLAG: bool + HAS_JUMP_FLAG: bool + HAS_FREE_FLAG: bool + HAS_LOCAL_FLAG: bool + + def __post_init__(self): + self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} + + @staticmethod + def fromInstruction(instr: parsing.Node): + + has_free = ( + variable_used(instr, "PyCell_New") + or variable_used(instr, "PyCell_GET") + or variable_used(instr, "PyCell_SET") + ) + + return InstructionFlags( + HAS_ARG_FLAG=variable_used(instr, "oparg"), + HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), + HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), + HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), + HAS_FREE_FLAG=has_free, + HAS_LOCAL_FLAG=( + variable_used(instr, "GETLOCAL") or variable_used(instr, "SETLOCAL") + ) + and not has_free, + ) + + @staticmethod + def newEmpty(): + return InstructionFlags(False, False, False, False, False, False) + + def add(self, other: "InstructionFlags") -> None: + for name, value in dataclasses.asdict(other).items(): + if value: + setattr(self, name, value) + + def names(self, value=None): + if value is None: + return dataclasses.asdict(self).keys() + return [n for n, v in dataclasses.asdict(self).items() if v == value] + + def bitmap(self) -> int: + flags = 0 + for name in self.names(): + if getattr(self, name): + flags |= self.bitmask[name] + return flags + + @classmethod + def emit_macros(cls, out: Formatter): + flags = cls.newEmpty() + for name, value in flags.bitmask.items(): + out.emit(f"#define {name} ({value})") + + for name, value in flags.bitmask.items(): + out.emit( + f"#define OPCODE_{name[:-len('_FLAG')]}(OP) " + f"(_PyOpcode_opcode_metadata[OP].flags & ({name}))" + ) + + +def variable_used(node: parsing.Node, name: str) -> bool: + """Determine whether a variable with a given name is used in a node.""" + return any( + token.kind == "IDENTIFIER" and token.text == name for token in node.tokens + ) + + +def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: + """Like variable_used(), but skips #if ENABLE_SPECIALIZATION blocks.""" + tokens: list[lx.Token] = [] + skipping = False + for i, token in enumerate(node.tokens): + if token.kind == "MACRO": + text = "".join(token.text.split()) + # TODO: Handle nested #if + if text == "#if": + if ( + i + 1 < len(node.tokens) + and node.tokens[i + 1].text == "ENABLE_SPECIALIZATION" + ): + skipping = True + elif text in ("#else", "#endif"): + skipping = False + if not skipping: + tokens.append(token) + return any(token.kind == "IDENTIFIER" and token.text == name for token in tokens) diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py new file mode 100644 index 00000000000000..6b5a375af891be --- /dev/null +++ b/Tools/cases_generator/formatting.py @@ -0,0 +1,188 @@ +import contextlib +import re +import typing + +from parsing import StackEffect + +UNUSED = "unused" + + +class Formatter: + """Wraps an output stream with the ability to indent etc.""" + + stream: typing.TextIO + prefix: str + emit_line_directives: bool = False + lineno: int # Next line number, 1-based + filename: str # Slightly improved stream.filename + nominal_lineno: int + nominal_filename: str + + def __init__( + self, stream: typing.TextIO, indent: int, + emit_line_directives: bool = False, comment: str = "//", + ) -> None: + self.stream = stream + self.prefix = " " * indent + self.emit_line_directives = emit_line_directives + self.comment = comment + self.lineno = 1 + self.filename = prettify_filename(self.stream.name) + self.nominal_lineno = 1 + self.nominal_filename = self.filename + + def write_raw(self, s: str) -> None: + self.stream.write(s) + newlines = s.count("\n") + self.lineno += newlines + self.nominal_lineno += newlines + + def emit(self, arg: str) -> None: + if arg: + self.write_raw(f"{self.prefix}{arg}\n") + else: + self.write_raw("\n") + + def set_lineno(self, lineno: int, filename: str) -> None: + if self.emit_line_directives: + if lineno != self.nominal_lineno or filename != self.nominal_filename: + self.emit(f'#line {lineno} "{filename}"') + self.nominal_lineno = lineno + self.nominal_filename = filename + + def reset_lineno(self) -> None: + if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename: + self.set_lineno(self.lineno + 1, self.filename) + + @contextlib.contextmanager + def indent(self): + self.prefix += " " + yield + self.prefix = self.prefix[:-4] + + @contextlib.contextmanager + def block(self, head: str, tail: str = ""): + if head: + self.emit(head + " {") + else: + self.emit("{") + with self.indent(): + yield + self.emit("}" + tail) + + def stack_adjust( + self, + input_effects: list[StackEffect], + output_effects: list[StackEffect], + ): + shrink, isym = list_effect_size(input_effects) + grow, osym = list_effect_size(output_effects) + diff = grow - shrink + if isym and isym != osym: + self.emit(f"STACK_SHRINK({isym});") + if diff < 0: + self.emit(f"STACK_SHRINK({-diff});") + if diff > 0: + self.emit(f"STACK_GROW({diff});") + if osym and osym != isym: + self.emit(f"STACK_GROW({osym});") + + def declare(self, dst: StackEffect, src: StackEffect | None): + if dst.name == UNUSED or dst.cond == "0": + return + typ = f"{dst.type}" if dst.type else "PyObject *" + if src: + cast = self.cast(dst, src) + init = f" = {cast}{src.name}" + elif dst.cond: + init = " = NULL" + else: + init = "" + sepa = "" if typ.endswith("*") else " " + self.emit(f"{typ}{sepa}{dst.name}{init};") + + def assign(self, dst: StackEffect, src: StackEffect): + if src.name == UNUSED: + return + if src.size: + # Don't write sized arrays -- it's up to the user code. + return + cast = self.cast(dst, src) + if re.match(r"^REG\(oparg(\d+)\)$", dst.name): + self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") + else: + stmt = f"{dst.name} = {cast}{src.name};" + if src.cond and src.cond != "1": + if src.cond == "0": + # It will not be executed + return + stmt = f"if ({src.cond}) {{ {stmt} }}" + self.emit(stmt) + + def cast(self, dst: StackEffect, src: StackEffect) -> str: + return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + + +def prettify_filename(filename: str) -> str: + # Make filename more user-friendly and less platform-specific, + # it is only used for error reporting at this point. + filename = filename.replace("\\", "/") + if filename.startswith("./"): + filename = filename[2:] + if filename.endswith(".new"): + filename = filename[:-4] + return filename + + +def list_effect_size(effects: list[StackEffect]) -> tuple[int, str]: + numeric = 0 + symbolic: list[str] = [] + for effect in effects: + diff, sym = effect_size(effect) + numeric += diff + if sym: + symbolic.append(maybe_parenthesize(sym)) + return numeric, " + ".join(symbolic) + + +def effect_size(effect: StackEffect) -> tuple[int, str]: + """Return the 'size' impact of a stack effect. + + Returns a tuple (numeric, symbolic) where: + + - numeric is an int giving the statically analyzable size of the effect + - symbolic is a string representing a variable effect (e.g. 'oparg*2') + + At most one of these will be non-zero / non-empty. + """ + if effect.size: + assert not effect.cond, "Array effects cannot have a condition" + return 0, effect.size + elif effect.cond: + if effect.cond in ("0", "1"): + return int(effect.cond), "" + return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" + else: + return 1, "" + + +def maybe_parenthesize(sym: str) -> str: + """Add parentheses around a string if it contains an operator. + + An exception is made for '*' which is common and harmless + in the context where the symbolic size is used. + """ + if re.match(r"^[\s\w*]+$", sym): + return sym + else: + return f"({sym})" + + +def string_effect_size(arg: tuple[int, str]) -> str: + numeric, symbolic = arg + if numeric and symbolic: + return f"{numeric} + {symbolic}" + elif symbolic: + return symbolic + else: + return str(numeric) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 3a679b2090f19b..967e1e2f5b63bb 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -1,21 +1,31 @@ """Generate the main interpreter switch. - Reads the instruction definitions from bytecodes.c. Writes the cases to generated_cases.c.h, which is #included in ceval.c. """ import argparse import contextlib -import dataclasses import os import posixpath -import re import sys import typing -import lexer as lx -import parser -from parser import StackEffect +from analysis import Analyzer +from formatting import Formatter, list_effect_size, maybe_parenthesize +from flags import InstructionFlags, variable_used +from instructions import ( + AnyInstruction, + Component, + Instruction, + MacroInstruction, + MacroParts, + PseudoInstruction, + StackEffect, + OverriddenInstructionPlaceHolder, + TIER_TWO, +) +import parsing +from parsing import StackEffect HERE = os.path.dirname(__file__) @@ -33,13 +43,6 @@ DEFAULT_EXECUTOR_OUTPUT = os.path.relpath( os.path.join(ROOT, "Python/executor_cases.c.h") ) -BEGIN_MARKER = "// BEGIN BYTECODES //" -END_MARKER = "// END BYTECODES //" -RE_PREDICTED = ( - r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$" -) -UNUSED = "unused" -BITS_PER_CODE_UNIT = 16 # Constants used instead of size for macro expansions. # Note: 1, 2, 4 must match actual cache entry sizes. @@ -52,10 +55,7 @@ "OPARG_BOTTOM": 6, } -RESERVED_WORDS = { - "co_consts" : "Use FRAME_CO_CONSTS.", - "co_names": "Use FRAME_CO_NAMES.", -} +INSTR_FMT_PREFIX = "INSTR_FMT_" arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", @@ -65,10 +65,18 @@ "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) arg_parser.add_argument( - "-m", "--metadata", type=str, help="Generated C metadata", default=DEFAULT_METADATA_OUTPUT + "-m", + "--metadata", + type=str, + help="Generated C metadata", + default=DEFAULT_METADATA_OUTPUT, ) arg_parser.add_argument( - "-p", "--pymetadata", type=str, help="Generated Python metadata", default=DEFAULT_PYMETADATA_OUTPUT + "-p", + "--pymetadata", + type=str, + help="Generated Python metadata", + default=DEFAULT_PYMETADATA_OUTPUT, ) arg_parser.add_argument( "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" @@ -85,966 +93,9 @@ ) -def effect_size(effect: StackEffect) -> tuple[int, str]: - """Return the 'size' impact of a stack effect. - - Returns a tuple (numeric, symbolic) where: - - - numeric is an int giving the statically analyzable size of the effect - - symbolic is a string representing a variable effect (e.g. 'oparg*2') - - At most one of these will be non-zero / non-empty. - """ - if effect.size: - assert not effect.cond, "Array effects cannot have a condition" - return 0, effect.size - elif effect.cond: - if effect.cond in ("0", "1"): - return int(effect.cond), "" - return 0, f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - else: - return 1, "" - - -def maybe_parenthesize(sym: str) -> str: - """Add parentheses around a string if it contains an operator. - - An exception is made for '*' which is common and harmless - in the context where the symbolic size is used. - """ - if re.match(r"^[\s\w*]+$", sym): - return sym - else: - return f"({sym})" - - -def list_effect_size(effects: list[StackEffect]) -> tuple[int, str]: - numeric = 0 - symbolic: list[str] = [] - for effect in effects: - diff, sym = effect_size(effect) - numeric += diff - if sym: - symbolic.append(maybe_parenthesize(sym)) - return numeric, " + ".join(symbolic) - - -def string_effect_size(arg: tuple[int, str]) -> str: - numeric, symbolic = arg - if numeric and symbolic: - return f"{numeric} + {symbolic}" - elif symbolic: - return symbolic - else: - return str(numeric) - - -class Formatter: - """Wraps an output stream with the ability to indent etc.""" - - stream: typing.TextIO - prefix: str - emit_line_directives: bool = False - lineno: int # Next line number, 1-based - filename: str # Slightly improved stream.filename - nominal_lineno: int - nominal_filename: str - - def __init__( - self, stream: typing.TextIO, indent: int, - emit_line_directives: bool = False, comment: str = "//", - ) -> None: - self.stream = stream - self.prefix = " " * indent - self.emit_line_directives = emit_line_directives - self.comment = comment - self.lineno = 1 - self.filename = prettify_filename(self.stream.name) - self.nominal_lineno = 1 - self.nominal_filename = self.filename - - def write_raw(self, s: str) -> None: - self.stream.write(s) - newlines = s.count("\n") - self.lineno += newlines - self.nominal_lineno += newlines - - def emit(self, arg: str) -> None: - if arg: - self.write_raw(f"{self.prefix}{arg}\n") - else: - self.write_raw("\n") - - def set_lineno(self, lineno: int, filename: str) -> None: - if self.emit_line_directives: - if lineno != self.nominal_lineno or filename != self.nominal_filename: - self.emit(f'#line {lineno} "{filename}"') - self.nominal_lineno = lineno - self.nominal_filename = filename - - def reset_lineno(self) -> None: - if self.lineno != self.nominal_lineno or self.filename != self.nominal_filename: - self.set_lineno(self.lineno + 1, self.filename) - - @contextlib.contextmanager - def indent(self): - self.prefix += " " - yield - self.prefix = self.prefix[:-4] - - @contextlib.contextmanager - def block(self, head: str, tail: str = ""): - if head: - self.emit(head + " {") - else: - self.emit("{") - with self.indent(): - yield - self.emit("}" + tail) - - def stack_adjust( - self, - input_effects: list[StackEffect], - output_effects: list[StackEffect], - ): - shrink, isym = list_effect_size(input_effects) - grow, osym = list_effect_size(output_effects) - diff = grow - shrink - if isym and isym != osym: - self.emit(f"STACK_SHRINK({isym});") - if diff < 0: - self.emit(f"STACK_SHRINK({-diff});") - if diff > 0: - self.emit(f"STACK_GROW({diff});") - if osym and osym != isym: - self.emit(f"STACK_GROW({osym});") - - def declare(self, dst: StackEffect, src: StackEffect | None): - if dst.name == UNUSED or dst.cond == "0": - return - typ = f"{dst.type}" if dst.type else "PyObject *" - if src: - cast = self.cast(dst, src) - init = f" = {cast}{src.name}" - elif dst.cond: - init = " = NULL" - else: - init = "" - sepa = "" if typ.endswith("*") else " " - self.emit(f"{typ}{sepa}{dst.name}{init};") - - def assign(self, dst: StackEffect, src: StackEffect): - if src.name == UNUSED: - return - if src.size: - # Don't write sized arrays -- it's up to the user code. - return - cast = self.cast(dst, src) - if re.match(r"^REG\(oparg(\d+)\)$", dst.name): - self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") - else: - stmt = f"{dst.name} = {cast}{src.name};" - if src.cond and src.cond != "1": - if src.cond == "0": - # It will not be executed - return - stmt = f"if ({src.cond}) {{ {stmt} }}" - self.emit(stmt) - - def cast(self, dst: StackEffect, src: StackEffect) -> str: - return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" - -@dataclasses.dataclass -class InstructionFlags: - """Construct and manipulate instruction flags""" - - HAS_ARG_FLAG: bool - HAS_CONST_FLAG: bool - HAS_NAME_FLAG: bool - HAS_JUMP_FLAG: bool - HAS_FREE_FLAG: bool - HAS_LOCAL_FLAG: bool - - def __post_init__(self): - self.bitmask = { - name : (1 << i) for i, name in enumerate(self.names()) - } - - @staticmethod - def fromInstruction(instr: "AnyInstruction"): - - has_free = (variable_used(instr, "PyCell_New") or - variable_used(instr, "PyCell_GET") or - variable_used(instr, "PyCell_SET")) - - return InstructionFlags( - HAS_ARG_FLAG=variable_used(instr, "oparg"), - HAS_CONST_FLAG=variable_used(instr, "FRAME_CO_CONSTS"), - HAS_NAME_FLAG=variable_used(instr, "FRAME_CO_NAMES"), - HAS_JUMP_FLAG=variable_used(instr, "JUMPBY"), - HAS_FREE_FLAG=has_free, - HAS_LOCAL_FLAG=(variable_used(instr, "GETLOCAL") or - variable_used(instr, "SETLOCAL")) and - not has_free, - ) - - @staticmethod - def newEmpty(): - return InstructionFlags(False, False, False, False, False, False) - - def add(self, other: "InstructionFlags") -> None: - for name, value in dataclasses.asdict(other).items(): - if value: - setattr(self, name, value) - - def names(self, value=None): - if value is None: - return dataclasses.asdict(self).keys() - return [n for n, v in dataclasses.asdict(self).items() if v == value] - - def bitmap(self) -> int: - flags = 0 - for name in self.names(): - if getattr(self, name): - flags |= self.bitmask[name] - return flags - - @classmethod - def emit_macros(cls, out: Formatter): - flags = cls.newEmpty() - for name, value in flags.bitmask.items(): - out.emit(f"#define {name} ({value})"); - - for name, value in flags.bitmask.items(): - out.emit( - f"#define OPCODE_{name[:-len('_FLAG')]}(OP) " - f"(_PyOpcode_opcode_metadata[OP].flags & ({name}))") - - -@dataclasses.dataclass -class ActiveCacheEffect: - """Wraps a CacheEffect that is actually used, in context.""" - effect: parser.CacheEffect - offset: int - - -FORBIDDEN_NAMES_IN_UOPS = ( - "resume_with_error", - "kwnames", - "next_instr", - "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST - "JUMPBY", - "DISPATCH", - "INSTRUMENTED_JUMP", - "throwflag", - "exception_unwind", - "import_from", - "import_name", - "_PyObject_CallNoArgs", # Proxy for BEFORE_WITH -) - - -# Interpreter tiers -TIER_ONE = 1 # Specializing adaptive interpreter (PEP 659) -TIER_TWO = 2 # Experimental tracing interpreter -Tiers: typing.TypeAlias = typing.Literal[1, 2] - - -@dataclasses.dataclass -class Instruction: - """An instruction with additional data and code.""" - - # Parts of the underlying instruction definition - inst: parser.InstDef - kind: typing.Literal["inst", "op"] - name: str - block: parser.Block - block_text: list[str] # Block.text, less curlies, less PREDICT() calls - block_line: int # First line of block in original code - - # Computed by constructor - always_exits: bool - cache_offset: int - cache_effects: list[parser.CacheEffect] - input_effects: list[StackEffect] - output_effects: list[StackEffect] - unmoved_names: frozenset[str] - instr_fmt: str - instr_flags: InstructionFlags - active_caches: list[ActiveCacheEffect] - - # Set later - family: parser.Family | None = None - predicted: bool = False - - def __init__(self, inst: parser.InstDef): - self.inst = inst - self.kind = inst.kind - self.name = inst.name - self.block = inst.block - self.block_text, self.check_eval_breaker, self.block_line = \ - extract_block_text(self.block) - self.always_exits = always_exits(self.block_text) - self.cache_effects = [ - effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) - ] - self.cache_offset = sum(c.size for c in self.cache_effects) - self.input_effects = [ - effect for effect in inst.inputs if isinstance(effect, StackEffect) - ] - self.output_effects = inst.outputs # For consistency/completeness - unmoved_names: set[str] = set() - for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: - unmoved_names.add(ieffect.name) - else: - break - self.unmoved_names = frozenset(unmoved_names) - - self.instr_flags = InstructionFlags.fromInstruction(inst) - - self.active_caches = [] - offset = 0 - for effect in self.cache_effects: - if effect.name != UNUSED: - self.active_caches.append(ActiveCacheEffect(effect, offset)) - offset += effect.size - - if self.instr_flags.HAS_ARG_FLAG: - fmt = "IB" - else: - fmt = "IX" - if offset: - fmt += "C" + "0"*(offset-1) - self.instr_fmt = fmt - - def is_viable_uop(self) -> bool: - """Whether this instruction is viable as a uop.""" - dprint: typing.Callable[..., None] = lambda *args, **kwargs: None - # if self.name.startswith("CALL"): - # dprint = print - - if self.name == "EXIT_TRACE": - return True # This has 'return frame' but it's okay - if self.always_exits: - dprint(f"Skipping {self.name} because it always exits") - return False - if len(self.active_caches) > 1: - # print(f"Skipping {self.name} because it has >1 cache entries") - return False - res = True - for forbidden in FORBIDDEN_NAMES_IN_UOPS: - # NOTE: To disallow unspecialized uops, use - # if variable_used(self.inst, forbidden): - if variable_used_unspecialized(self.inst, forbidden): - dprint(f"Skipping {self.name} because it uses {forbidden}") - res = False - return res - - def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: - """Write one instruction, sans prologue and epilogue.""" - # Write a static assertion that a family's cache size is correct - if family := self.family: - if self.name == family.name: - if cache_size := family.size: - out.emit( - f"static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");' - ) - - # Write input stack effect variable declarations and initializations - ieffects = list(reversed(self.input_effects)) - for i, ieffect in enumerate(ieffects): - isize = string_effect_size( - list_effect_size([ieff for ieff in ieffects[: i + 1]]) - ) - if ieffect.size: - src = StackEffect(f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **") - elif ieffect.cond: - src = StackEffect(f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", "") - else: - src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") - out.declare(ieffect, src) - - # Write output stack effect variable declarations - isize = string_effect_size(list_effect_size(self.input_effects)) - input_names = {ieffect.name for ieffect in self.input_effects} - for i, oeffect in enumerate(self.output_effects): - if oeffect.name not in input_names: - if oeffect.size: - osize = string_effect_size( - list_effect_size([oeff for oeff in self.output_effects[:i]]) - ) - offset = "stack_pointer" - if isize != osize: - if isize != "0": - offset += f" - ({isize})" - if osize != "0": - offset += f" + {osize}" - src = StackEffect(offset, "PyObject **") - out.declare(oeffect, src) - else: - out.declare(oeffect, None) - - # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") - - self.write_body(out, 0, self.active_caches, tier=tier) - - # Skip the rest if the block always exits - if self.always_exits: - return - - # Write net stack growth/shrinkage - out.stack_adjust( - [ieff for ieff in self.input_effects], - [oeff for oeff in self.output_effects], - ) - - # Write output stack effect assignments - oeffects = list(reversed(self.output_effects)) - for i, oeffect in enumerate(oeffects): - if oeffect.name in self.unmoved_names: - continue - osize = string_effect_size( - list_effect_size([oeff for oeff in oeffects[: i + 1]]) - ) - if oeffect.size: - dst = StackEffect(f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **") - else: - dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") - out.assign(dst, oeffect) - - # Write cache effect - if tier == TIER_ONE and self.cache_offset: - out.emit(f"next_instr += {self.cache_offset};") - - def write_body( - self, - out: Formatter, - dedent: int, - active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, - ) -> None: - """Write the instruction body.""" - # Write cache effect variable declarations and initializations - for active in active_caches: - ceffect = active.effect - bits = ceffect.size * BITS_PER_CODE_UNIT - if bits == 64: - # NOTE: We assume that 64-bit data in the cache - # is always an object pointer. - # If this becomes false, we need a way to specify - # syntactically what type the cache data is. - typ = "PyObject *" - func = "read_obj" - else: - typ = f"uint{bits}_t " - func = f"read_u{bits}" - if tier == TIER_ONE: - out.emit( - f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);" - ) - else: - out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") - - # Write the body, substituting a goto for ERROR_IF() and other stuff - assert dedent <= 0 - extra = " " * -dedent - names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) - offset = 0 - context = self.block.context - assert context is not None and context.owner is not None - filename = context.owner.filename - for line in self.block_text: - out.set_lineno(self.block_line + offset, filename) - offset += 1 - if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): - space, cond, label = m.groups() - space = extra + space - # ERROR_IF() must pop the inputs from the stack. - # The code block is responsible for DECREF()ing them. - # NOTE: If the label doesn't exist, just add it to ceval.c. - - # Don't pop common input/output effects at the bottom! - # These aren't DECREF'ed so they can stay. - ieffs = list(self.input_effects) - oeffs = list(self.output_effects) - while ieffs and oeffs and ieffs[0] == oeffs[0]: - ieffs.pop(0) - oeffs.pop(0) - ninputs, symbolic = list_effect_size(ieffs) - if ninputs: - label = f"pop_{ninputs}_{label}" - if symbolic: - out.write_raw( - f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" - ) - else: - out.write_raw(f"{space}if ({cond}) goto {label};\n") - elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): - out.reset_lineno() - space = extra + m.group(1) - for ieff in self.input_effects: - if ieff.name in names_to_skip: - continue - if ieff.size: - out.write_raw( - f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" - ) - out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") - out.write_raw(f"{space}}}\n") - else: - decref = "XDECREF" if ieff.cond else "DECREF" - out.write_raw(f"{space}Py_{decref}({ieff.name});\n") - else: - out.write_raw(extra + line) - out.reset_lineno() - - -InstructionOrCacheEffect = Instruction | parser.CacheEffect -StackEffectMapping = list[tuple[StackEffect, StackEffect]] - - -@dataclasses.dataclass -class Component: - instr: Instruction - input_mapping: StackEffectMapping - output_mapping: StackEffectMapping - active_caches: list[ActiveCacheEffect] - - def write_body(self, out: Formatter) -> None: - with out.block(""): - input_names = {ieffect.name for _, ieffect in self.input_mapping} - for var, ieffect in self.input_mapping: - out.declare(ieffect, var) - for _, oeffect in self.output_mapping: - if oeffect.name not in input_names: - out.declare(oeffect, None) - - self.instr.write_body(out, -4, self.active_caches) - - for var, oeffect in self.output_mapping: - out.assign(var, oeffect) - - -MacroParts = list[Component | parser.CacheEffect] - - -@dataclasses.dataclass -class MacroInstruction: - """A macro instruction.""" - - name: str - stack: list[StackEffect] - initial_sp: int - final_sp: int - instr_fmt: str - instr_flags: InstructionFlags - macro: parser.Macro - parts: MacroParts - cache_offset: int - predicted: bool = False - - -@dataclasses.dataclass -class PseudoInstruction: - """A pseudo instruction.""" - - name: str - targets: list[Instruction] - instr_fmt: str - instr_flags: InstructionFlags - - -@dataclasses.dataclass -class OverriddenInstructionPlaceHolder: - name: str - - -AnyInstruction = Instruction | MacroInstruction | PseudoInstruction -INSTR_FMT_PREFIX = "INSTR_FMT_" - - -class Analyzer: - """Parse input, analyze it, and write to output.""" - - input_filenames: list[str] - output_filename: str - metadata_filename: str - pymetadata_filename: str - executor_filename: str - errors: int = 0 - emit_line_directives: bool = False - - def __init__( - self, - input_filenames: list[str], - output_filename: str, - metadata_filename: str, - pymetadata_filename: str, - executor_filename: str, - ): - """Read the input file.""" - self.input_filenames = input_filenames - self.output_filename = output_filename - self.metadata_filename = metadata_filename - self.pymetadata_filename = pymetadata_filename - self.executor_filename = executor_filename - - def error(self, msg: str, node: parser.Node) -> None: - lineno = 0 - filename = "" - if context := node.context: - filename = context.owner.filename - # Use line number of first non-comment in the node - for token in context.owner.tokens[context.begin : context.end]: - lineno = token.line - if token.kind != "COMMENT": - break - print(f"{filename}:{lineno}: {msg}", file=sys.stderr) - self.errors += 1 - - everything: list[ - parser.InstDef | parser.Macro | parser.Pseudo | OverriddenInstructionPlaceHolder - ] - instrs: dict[str, Instruction] # Includes ops - macros: dict[str, parser.Macro] - macro_instrs: dict[str, MacroInstruction] - families: dict[str, parser.Family] - pseudos: dict[str, parser.Pseudo] - pseudo_instrs: dict[str, PseudoInstruction] - - def parse(self) -> None: - """Parse the source text. - - We only want the parser to see the stuff between the - begin and end markers. - """ - - self.everything = [] - self.instrs = {} - self.macros = {} - self.families = {} - self.pseudos = {} - - instrs_idx: dict[str, int] = dict() - - for filename in self.input_filenames: - self.parse_file(filename, instrs_idx) - - files = " + ".join(self.input_filenames) - print( - f"Read {len(self.instrs)} instructions/ops, " - f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " - f"and {len(self.families)} families from {files}", - file=sys.stderr, - ) - - def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: - with open(filename) as file: - src = file.read() - - - psr = parser.Parser(src, filename=prettify_filename(filename)) - - # Skip until begin marker - while tkn := psr.next(raw=True): - if tkn.text == BEGIN_MARKER: - break - else: - raise psr.make_syntax_error( - f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" - ) - start = psr.getpos() - - # Find end marker, then delete everything after it - while tkn := psr.next(raw=True): - if tkn.text == END_MARKER: - break - del psr.tokens[psr.getpos() - 1 :] - - # Parse from start - psr.setpos(start) - thing: parser.InstDef | parser.Macro | parser.Pseudo | parser.Family | None - thing_first_token = psr.peek() - while thing := psr.definition(): - if ws := [w for w in RESERVED_WORDS if variable_used(thing, w)]: - self.error(f"'{ws[0]}' is a reserved word. {RESERVED_WORDS[ws[0]]}", thing) - - match thing: - case parser.InstDef(name=name): - if name in self.instrs: - if not thing.override: - raise psr.make_syntax_error( - f"Duplicate definition of '{name}' @ {thing.context} " - f"previous definition @ {self.instrs[name].inst.context}", - thing_first_token, - ) - self.everything[instrs_idx[name]] = OverriddenInstructionPlaceHolder(name=name) - if name not in self.instrs and thing.override: - raise psr.make_syntax_error( - f"Definition of '{name}' @ {thing.context} is supposed to be " - "an override but no previous definition exists.", - thing_first_token, - ) - self.instrs[name] = Instruction(thing) - instrs_idx[name] = len(self.everything) - self.everything.append(thing) - case parser.Macro(name): - self.macros[name] = thing - self.everything.append(thing) - case parser.Family(name): - self.families[name] = thing - case parser.Pseudo(name): - self.pseudos[name] = thing - self.everything.append(thing) - case _: - typing.assert_never(thing) - if not psr.eof(): - raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") - - def analyze(self) -> None: - """Analyze the inputs. - - Raises SystemExit if there is an error. - """ - self.analyze_macros_and_pseudos() - self.find_predictions() - self.map_families() - self.check_families() - - def find_predictions(self) -> None: - """Find the instructions that need PREDICTED() labels.""" - for instr in self.instrs.values(): - targets: set[str] = set() - for line in instr.block_text: - if m := re.match(RE_PREDICTED, line): - targets.add(m.group(1)) - for target in targets: - if target_instr := self.instrs.get(target): - target_instr.predicted = True - elif target_macro := self.macro_instrs.get(target): - target_macro.predicted = True - else: - self.error( - f"Unknown instruction {target!r} predicted in {instr.name!r}", - instr.inst, # TODO: Use better location - ) - - def map_families(self) -> None: - """Link instruction names back to their family, if they have one.""" - for family in self.families.values(): - for member in [family.name] + family.members: - if member_instr := self.instrs.get(member): - if member_instr.family not in (family, None): - self.error( - f"Instruction {member} is a member of multiple families " - f"({member_instr.family.name}, {family.name}).", - family, - ) - else: - member_instr.family = family - elif not self.macro_instrs.get(member): - self.error( - f"Unknown instruction {member!r} referenced in family {family.name!r}", - family, - ) - - def check_families(self) -> None: - """Check each family: - - - Must have at least 2 members (including head) - - Head and all members must be known instructions - - Head and all members must have the same cache, input and output effects - """ - for family in self.families.values(): - if family.name not in self.macro_instrs and family.name not in self.instrs: - self.error( - f"Family {family.name!r} has unknown instruction {family.name!r}", - family, - ) - members = [ - member - for member in family.members - if member in self.instrs or member in self.macro_instrs - ] - if members != family.members: - unknown = set(family.members) - set(members) - self.error( - f"Family {family.name!r} has unknown members: {unknown}", family - ) - expected_effects = self.effect_counts(family.name) - for member in members: - member_effects = self.effect_counts(member) - if member_effects != expected_effects: - self.error( - f"Family {family.name!r} has inconsistent " - f"(cache, input, output) effects:\n" - f" {family.name} = {expected_effects}; " - f"{member} = {member_effects}", - family, - ) - - def effect_counts(self, name: str) -> tuple[int, int, int]: - if instr := self.instrs.get(name): - cache = instr.cache_offset - input = len(instr.input_effects) - output = len(instr.output_effects) - elif mac := self.macro_instrs.get(name): - cache = mac.cache_offset - input, output = 0, 0 - for part in mac.parts: - if isinstance(part, Component): - # A component may pop what the previous component pushed, - # so we offset the input/output counts by that. - delta_i = len(part.instr.input_effects) - delta_o = len(part.instr.output_effects) - offset = min(delta_i, output) - input += delta_i - offset - output += delta_o - offset - else: - assert False, f"Unknown instruction {name!r}" - return cache, input, output - - def analyze_macros_and_pseudos(self) -> None: - """Analyze each macro and pseudo instruction.""" - self.macro_instrs = {} - self.pseudo_instrs = {} - for name, macro in self.macros.items(): - self.macro_instrs[name] = self.analyze_macro(macro) - for name, pseudo in self.pseudos.items(): - self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) - - def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: - components = self.check_macro_components(macro) - stack, initial_sp = self.stack_analysis(components) - sp = initial_sp - parts: MacroParts = [] - flags = InstructionFlags.newEmpty() - offset = 0 - for component in components: - match component: - case parser.CacheEffect() as ceffect: - parts.append(ceffect) - offset += ceffect.size - case Instruction() as instr: - part, sp, offset = self.analyze_instruction(instr, stack, sp, offset) - parts.append(part) - flags.add(instr.instr_flags) - case _: - typing.assert_never(component) - final_sp = sp - format = "IB" - if offset: - format += "C" + "0"*(offset-1) - return MacroInstruction( - macro.name, stack, initial_sp, final_sp, format, flags, macro, parts, offset - ) - - def analyze_pseudo(self, pseudo: parser.Pseudo) -> PseudoInstruction: - targets = [self.instrs[target] for target in pseudo.targets] - assert targets - # Make sure the targets have the same fmt - fmts = list(set([t.instr_fmt for t in targets])) - assert(len(fmts) == 1) - assert(len(list(set([t.instr_flags.bitmap() for t in targets]))) == 1) - return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) - - def analyze_instruction( - self, instr: Instruction, stack: list[StackEffect], sp: int, offset: int - ) -> tuple[Component, int, int]: - input_mapping: StackEffectMapping = [] - for ieffect in reversed(instr.input_effects): - sp -= 1 - input_mapping.append((stack[sp], ieffect)) - output_mapping: StackEffectMapping = [] - for oeffect in instr.output_effects: - output_mapping.append((stack[sp], oeffect)) - sp += 1 - active_effects: list[ActiveCacheEffect] = [] - for ceffect in instr.cache_effects: - if ceffect.name != UNUSED: - active_effects.append(ActiveCacheEffect(ceffect, offset)) - offset += ceffect.size - return Component(instr, input_mapping, output_mapping, active_effects), sp, offset - - def check_macro_components( - self, macro: parser.Macro - ) -> list[InstructionOrCacheEffect]: - components: list[InstructionOrCacheEffect] = [] - for uop in macro.uops: - match uop: - case parser.OpName(name): - if name not in self.instrs: - self.error(f"Unknown instruction {name!r}", macro) - components.append(self.instrs[name]) - case parser.CacheEffect(): - components.append(uop) - case _: - typing.assert_never(uop) - return components - - def stack_analysis( - self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[StackEffect], int]: - """Analyze a macro. - - Ignore cache effects. - - Return the list of variables (as StackEffects) and the initial stack pointer. - """ - lowest = current = highest = 0 - conditions: dict[int, str] = {} # Indexed by 'current'. - last_instr: Instruction | None = None - for thing in components: - if isinstance(thing, Instruction): - last_instr = thing - for thing in components: - match thing: - case Instruction() as instr: - if any( - eff.size for eff in instr.input_effects + instr.output_effects - ): - # TODO: Eventually this will be needed, at least for macros. - self.error( - f"Instruction {instr.name!r} has variable-sized stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if any(eff.cond for eff in instr.input_effects): - self.error( - f"Instruction {instr.name!r} has conditional input stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if any(eff.cond for eff in instr.output_effects) and instr is not last_instr: - self.error( - f"Instruction {instr.name!r} has conditional output stack effect, " - "but is not the last instruction in a macro", - instr.inst, # TODO: Pass name+location of macro - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - for eff in instr.output_effects: - if eff.cond: - conditions[current] = eff.cond - current += 1 - highest = max(highest, current) - case parser.CacheEffect(): - pass - case _: - typing.assert_never(thing) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [ - StackEffect(f"_tmp_{i}", "", conditions.get(highest - i, "")) - for i in reversed(range(1, highest - lowest + 1)) - ] - return stack, -lowest - +class Generator(Analyzer): def get_stack_effect_info( - self, thing: parser.InstDef | parser.Macro | parser.Pseudo + self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo ) -> tuple[AnyInstruction | None, str | None, str | None]: def effect_str(effects: list[StackEffect]) -> str: n_effect, sym_effect = list_effect_size(effects) @@ -1053,8 +104,10 @@ def effect_str(effects: list[StackEffect]) -> str: return str(n_effect) instr: AnyInstruction | None + popped: str | None + pushed: str | None match thing: - case parser.InstDef(): + case parsing.InstDef(): if thing.kind != "op": instr = self.instrs[thing.name] popped = effect_str(instr.input_effects) @@ -1063,7 +116,7 @@ def effect_str(effects: list[StackEffect]) -> str: instr = None popped = "" pushed = "" - case parser.Macro(): + case parsing.Macro(): instr = self.macro_instrs[thing.name] parts = [comp for comp in instr.parts if isinstance(comp, Component)] # Note: stack_analysis() already verifies that macro components @@ -1084,7 +137,11 @@ def effect_str(effects: list[StackEffect]) -> str: if effect.cond in ("0", "1"): pushed_symbolic.append(effect.cond) else: - pushed_symbolic.append(maybe_parenthesize(f"{maybe_parenthesize(effect.cond)} ? 1 : 0")) + pushed_symbolic.append( + maybe_parenthesize( + f"{maybe_parenthesize(effect.cond)} ? 1 : 0" + ) + ) sp += 1 high = max(sp, high) if high != max(0, sp): @@ -1096,7 +153,7 @@ def effect_str(effects: list[StackEffect]) -> str: popped = str(-low) pushed_symbolic.append(str(sp - low - len(pushed_symbolic))) pushed = " + ".join(pushed_symbolic) - case parser.Pseudo(): + case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] popped = pushed = None # Calculate stack effect, and check that it's the the same @@ -1135,10 +192,14 @@ def write_function( ) -> None: self.out.emit("") self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit(f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);") + self.out.emit( + f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);" + ) self.out.emit("#else") self.out.emit("int") - self.out.emit(f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{") + self.out.emit( + f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{" + ) self.out.emit(" switch(opcode) {") for instr, effect in data: self.out.emit(f" case {instr.name}:") @@ -1159,7 +220,7 @@ def from_source_files(self) -> str: try: filename = os.path.relpath(filename, ROOT) except ValueError: - # May happen on Windows if root and temp on different volumes + # May happen on Windows if root and temp on different volumes pass filenames.append(filename) paths = f"\n{self.out.comment} ".join(filenames) @@ -1170,20 +231,21 @@ def write_provenance_header(self): self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") - def write_metadata(self) -> None: + def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" # Compute the set of all instruction formats. all_formats: set[str] = set() for thing in self.everything: + format: str | None match thing: case OverriddenInstructionPlaceHolder(): continue - case parser.InstDef(): + case parsing.InstDef(): format = self.instrs[thing.name].instr_fmt - case parser.Macro(): + case parsing.Macro(): format = self.macro_instrs[thing.name].instr_fmt - case parser.Pseudo(): + case parsing.Pseudo(): format = None for target in self.pseudos[thing.name].targets: target_instr = self.instrs.get(target) @@ -1192,13 +254,14 @@ def write_metadata(self) -> None: format = target_instr.instr_fmt else: assert format == target_instr.instr_fmt + assert format is not None case _: typing.assert_never(thing) all_formats.add(format) # Turn it into a list of enum definitions. format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] - with open(self.metadata_filename, "w") as f: + with open(metadata_filename, "w") as f: # Create formatter self.out = Formatter(f, 0) @@ -1220,7 +283,8 @@ def write_metadata(self) -> None: self.out.emit( "#define IS_VALID_OPCODE(OP) \\\n" " (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \\\n" - " (_PyOpcode_opcode_metadata[(OP)].valid_entry))") + " (_PyOpcode_opcode_metadata[(OP)].valid_entry))" + ) self.out.emit("") InstructionFlags.emit_macros(self.out) @@ -1234,17 +298,23 @@ def write_metadata(self) -> None: with self.out.block("struct opcode_macro_expansion", ";"): self.out.emit("int nuops;") - self.out.emit("struct { int16_t uop; int8_t size; int8_t offset; } uops[8];") + self.out.emit( + "struct { int16_t uop; int8_t size; int8_t offset; } uops[8];" + ) self.out.emit("") for key, value in OPARG_SIZES.items(): self.out.emit(f"#define {key} {value}") self.out.emit("") - self.out.emit("#define OPCODE_METADATA_FMT(OP) " - "(_PyOpcode_opcode_metadata[(OP)].instr_format)") + self.out.emit( + "#define OPCODE_METADATA_FMT(OP) " + "(_PyOpcode_opcode_metadata[(OP)].instr_format)" + ) self.out.emit("#define SAME_OPCODE_METADATA(OP1, OP2) \\") - self.out.emit(" (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2))") + self.out.emit( + " (OPCODE_METADATA_FMT(OP1) == OPCODE_METADATA_FMT(OP2))" + ) self.out.emit("") # Write metadata array declaration @@ -1253,27 +323,35 @@ def write_metadata(self) -> None: self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") self.out.emit("") self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit("extern const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];") - self.out.emit("extern const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];") - self.out.emit("extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];") + self.out.emit( + "extern const struct opcode_metadata " + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];" + ) + self.out.emit( + "extern const struct opcode_macro_expansion " + "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];" + ) + self.out.emit( + "extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];" + ) self.out.emit("#else // if NEED_OPCODE_METADATA") - self.out.emit("const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {") + self.out.emit( + "const struct opcode_metadata " + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {" + ) # Write metadata for each instruction for thing in self.everything: match thing: case OverriddenInstructionPlaceHolder(): continue - case parser.InstDef(): + case parsing.InstDef(): if thing.kind != "op": self.write_metadata_for_inst(self.instrs[thing.name]) - case parser.Macro(): + case parsing.Macro(): self.write_metadata_for_macro(self.macro_instrs[thing.name]) - case parser.Pseudo(): + case parsing.Pseudo(): self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) case _: typing.assert_never(thing) @@ -1291,32 +369,38 @@ def write_metadata(self) -> None: match thing: case OverriddenInstructionPlaceHolder(): pass - case parser.InstDef(name=name): + case parsing.InstDef(name=name): instr = self.instrs[name] # Since an 'op' is not a bytecode, it has no expansion; but 'inst' is if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used part = Component(instr, [], [], instr.active_caches) self.write_macro_expansions(instr.name, [part]) - elif instr.kind == "inst" and variable_used(instr.inst, "oparg1"): - assert variable_used(instr.inst, "oparg2"), "Half super-instr?" + elif instr.kind == "inst" and variable_used( + instr.inst, "oparg1" + ): + assert variable_used( + instr.inst, "oparg2" + ), "Half super-instr?" self.write_super_expansions(instr.name) - case parser.Macro(): + case parsing.Macro(): mac = self.macro_instrs[thing.name] self.write_macro_expansions(mac.name, mac.parts) - case parser.Pseudo(): + case parsing.Pseudo(): pass case _: typing.assert_never(thing) - with self.out.block("const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";"): - self.write_uop_items(lambda name, counter: f"[{name}] = \"{name}\",") + with self.out.block( + "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";" + ): + self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') self.out.emit("#endif // NEED_OPCODE_METADATA") - with open(self.pymetadata_filename, "w") as f: + with open(pymetadata_filename, "w") as f: # Create formatter - self.out = Formatter(f, 0, comment = "#") + self.out = Formatter(f, 0, comment="#") self.write_provenance_header() @@ -1324,10 +408,10 @@ def write_metadata(self) -> None: self.out.emit("_specializations = {") for name, family in self.families.items(): with self.out.indent(): - self.out.emit(f"\"{family.name}\": [") + self.out.emit(f'"{family.name}": [') with self.out.indent(): for m in family.members: - self.out.emit(f"\"{m}\",") + self.out.emit(f'"{m}",') self.out.emit(f"],") self.out.emit("}") @@ -1335,15 +419,17 @@ def write_metadata(self) -> None: self.out.emit("") self.out.emit("# An irregular case:") self.out.emit( - "_specializations[\"BINARY_OP\"].append(" - "\"BINARY_OP_INPLACE_ADD_UNICODE\")") + '_specializations["BINARY_OP"].append(' + '"BINARY_OP_INPLACE_ADD_UNICODE")' + ) # Make list of specialized instructions self.out.emit("") self.out.emit( "_specialized_instructions = [" - "opcode for family in _specializations.values() for opcode in family" - "]") + "opcode for family in _specializations.values() for opcode in family" + "]" + ) def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" @@ -1432,16 +518,18 @@ def write_super_expansions(self, name: str) -> None: ] self.write_expansions(name, expansions) - def write_expansions(self, name: str, expansions: list[tuple[str, int, int]]) -> None: - pieces = [f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions] + def write_expansions( + self, name: str, expansions: list[tuple[str, int, int]] + ) -> None: + pieces = [ + f"{{ {name}, {size}, {offset} }}" for name, size, offset in expansions + ] self.out.emit( f"[{name}] = " f"{{ .nuops = {len(pieces)}, .uops = {{ {', '.join(pieces)} }} }}," ) - def emit_metadata_entry( - self, name: str, fmt: str, flags: InstructionFlags - ) -> None: + def emit_metadata_entry(self, name: str, fmt: str, flags: InstructionFlags) -> None: flag_names = flags.names(value=True) if not flag_names: flag_names.append("0") @@ -1462,11 +550,13 @@ def write_metadata_for_pseudo(self, ps: PseudoInstruction) -> None: """Write metadata for a macro-instruction.""" self.emit_metadata_entry(ps.name, ps.instr_fmt, ps.instr_flags) - def write_instructions(self) -> None: + def write_instructions( + self, output_filename: str, emit_line_directives: bool + ) -> None: """Write instructions to output file.""" - with open(self.output_filename, "w") as f: + with open(output_filename, "w") as f: # Create formatter - self.out = Formatter(f, 8, self.emit_line_directives) + self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() @@ -1478,35 +568,37 @@ def write_instructions(self) -> None: match thing: case OverriddenInstructionPlaceHolder(): self.write_overridden_instr_place_holder(thing) - case parser.InstDef(): + case parsing.InstDef(): if thing.kind != "op": n_instrs += 1 self.write_instr(self.instrs[thing.name]) - case parser.Macro(): + case parsing.Macro(): n_macros += 1 self.write_macro(self.macro_instrs[thing.name]) - case parser.Pseudo(): + case parsing.Pseudo(): n_pseudos += 1 case _: typing.assert_never(thing) print( f"Wrote {n_instrs} instructions, {n_macros} macros, " - f"and {n_pseudos} pseudos to {self.output_filename}", + f"and {n_pseudos} pseudos to {output_filename}", file=sys.stderr, ) - def write_executor_instructions(self) -> None: + def write_executor_instructions( + self, executor_filename: str, emit_line_directives: bool + ) -> None: """Generate cases for the Tier 2 interpreter.""" - with open(self.executor_filename, "w") as f: - self.out = Formatter(f, 8, self.emit_line_directives) + with open(executor_filename, "w") as f: + self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() for thing in self.everything: match thing: case OverriddenInstructionPlaceHolder(): # TODO: Is this helpful? self.write_overridden_instr_place_holder(thing) - case parser.InstDef(): + case parsing.InstDef(): instr = self.instrs[thing.name] if instr.is_viable_uop(): self.out.emit("") @@ -1517,22 +609,24 @@ def write_executor_instructions(self) -> None: self.out.emit("break;") # elif instr.kind != "op": # print(f"NOTE: {thing.name} is not a viable uop") - case parser.Macro(): + case parsing.Macro(): pass - case parser.Pseudo(): + case parsing.Pseudo(): pass case _: typing.assert_never(thing) print( - f"Wrote some stuff to {self.executor_filename}", + f"Wrote some stuff to {executor_filename}", file=sys.stderr, ) - def write_overridden_instr_place_holder(self, - place_holder: OverriddenInstructionPlaceHolder) -> None: + def write_overridden_instr_place_holder( + self, place_holder: OverriddenInstructionPlaceHolder + ) -> None: self.out.emit("") self.out.emit( - f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition") + f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition" + ) def write_instr(self, instr: Instruction) -> None: name = instr.name @@ -1555,7 +649,7 @@ def write_macro(self, mac: MacroInstruction) -> None: cache_adjust = 0 for part in mac.parts: match part: - case parser.CacheEffect(size=size): + case parsing.CacheEffect(size=size): cache_adjust += size case Component() as comp: last_instr = comp.instr @@ -1603,7 +697,7 @@ def wrap_macro(self, mac: MacroInstruction): yield - self.out.stack_adjust(ieffects[:mac.initial_sp], mac.stack[:mac.final_sp]) + self.out.stack_adjust(ieffects[: mac.initial_sp], mac.stack[: mac.final_sp]) for i, var in enumerate(reversed(mac.stack[: mac.final_sp]), 1): dst = StackEffect(f"stack_pointer[-{i}]", "") @@ -1612,99 +706,6 @@ def wrap_macro(self, mac: MacroInstruction): self.out.emit(f"DISPATCH();") -def prettify_filename(filename: str) -> str: - # Make filename more user-friendly and less platform-specific, - # it is only used for error reporting at this point. - filename = filename.replace("\\", "/") - if filename.startswith("./"): - filename = filename[2:] - if filename.endswith(".new"): - filename = filename[:-4] - return filename - - -def extract_block_text(block: parser.Block) -> tuple[list[str], bool, int]: - # Get lines of text with proper dedent - blocklines = block.text.splitlines(True) - first_token: lx.Token = block.tokens[0] # IndexError means the context is broken - block_line = first_token.begin[0] - - # Remove blank lines from both ends - while blocklines and not blocklines[0].strip(): - blocklines.pop(0) - block_line += 1 - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Remove leading and trailing braces - assert blocklines and blocklines[0].strip() == "{" - assert blocklines and blocklines[-1].strip() == "}" - blocklines.pop() - blocklines.pop(0) - block_line += 1 - - # Remove trailing blank lines - while blocklines and not blocklines[-1].strip(): - blocklines.pop() - - # Separate CHECK_EVAL_BREAKER() macro from end - check_eval_breaker = \ - blocklines != [] and blocklines[-1].strip() == "CHECK_EVAL_BREAKER();" - if check_eval_breaker: - del blocklines[-1] - - return blocklines, check_eval_breaker, block_line - - -def always_exits(lines: list[str]) -> bool: - """Determine whether a block always ends in a return/goto/etc.""" - if not lines: - return False - line = lines[-1].rstrip() - # Indent must match exactly (TODO: Do something better) - if line[:12] != " " * 12: - return False - line = line[12:] - return line.startswith( - ( - "goto ", - "return ", - "DISPATCH", - "GO_TO_", - "Py_UNREACHABLE()", - "ERROR_IF(true, ", - ) - ) - - -def variable_used(node: parser.Node, name: str) -> bool: - """Determine whether a variable with a given name is used in a node.""" - return any( - token.kind == "IDENTIFIER" and token.text == name for token in node.tokens - ) - - -def variable_used_unspecialized(node: parser.Node, name: str) -> bool: - """Like variable_used(), but skips #if ENABLE_SPECIALIZATION blocks.""" - tokens: list[lx.Token] = [] - skipping = False - for i, token in enumerate(node.tokens): - if token.kind == "MACRO": - text = "".join(token.text.split()) - # TODO: Handle nested #if - if text == "#if": - if ( - i + 1 < len(node.tokens) - and node.tokens[i + 1].text == "ENABLE_SPECIALIZATION" - ): - skipping = True - elif text in ("#else", "#endif"): - skipping = False - if not skipping: - tokens.append(token) - return any(token.kind == "IDENTIFIER" and token.text == name for token in tokens) - - def main(): """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error @@ -1712,17 +713,17 @@ def main(): args.input.append(DEFAULT_INPUT) # Raises OSError if input unreadable - a = Analyzer(args.input, args.output, args.metadata, args.pymetadata, args.executor_cases) + a = Generator(args.input) - if args.emit_line_directives: - a.emit_line_directives = True a.parse() # Raises SyntaxError on failure a.analyze() # Prints messages and sets a.errors on failure if a.errors: sys.exit(f"Found {a.errors} errors") - a.write_instructions() # Raises OSError if output can't be written - a.write_metadata() - a.write_executor_instructions() + + # These raise OSError if output can't be written + a.write_instructions(args.output, args.emit_line_directives) + a.write_metadata(args.metadata, args.pymetadata) + a.write_executor_instructions(args.executor_cases, args.emit_line_directives) if __name__ == "__main__": diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py new file mode 100644 index 00000000000000..6f42699d900b46 --- /dev/null +++ b/Tools/cases_generator/instructions.py @@ -0,0 +1,424 @@ +import dataclasses +import re +import typing + +from flags import InstructionFlags, variable_used_unspecialized +from formatting import ( + Formatter, + UNUSED, + string_effect_size, + list_effect_size, + maybe_parenthesize, +) +import lexer as lx +import parsing +from parsing import StackEffect + +BITS_PER_CODE_UNIT = 16 + + +@dataclasses.dataclass +class ActiveCacheEffect: + """Wraps a CacheEffect that is actually used, in context.""" + + effect: parsing.CacheEffect + offset: int + + +FORBIDDEN_NAMES_IN_UOPS = ( + "resume_with_error", + "kwnames", + "next_instr", + "oparg1", # Proxy for super-instructions like LOAD_FAST_LOAD_FAST + "JUMPBY", + "DISPATCH", + "INSTRUMENTED_JUMP", + "throwflag", + "exception_unwind", + "import_from", + "import_name", + "_PyObject_CallNoArgs", # Proxy for BEFORE_WITH +) + + +# Interpreter tiers +TIER_ONE: typing.Final = 1 # Specializing adaptive interpreter (PEP 659) +TIER_TWO: typing.Final = 2 # Experimental tracing interpreter +Tiers: typing.TypeAlias = typing.Literal[1, 2] + + +@dataclasses.dataclass +class Instruction: + """An instruction with additional data and code.""" + + # Parts of the underlying instruction definition + inst: parsing.InstDef + kind: typing.Literal["inst", "op"] + name: str + block: parsing.Block + block_text: list[str] # Block.text, less curlies, less PREDICT() calls + block_line: int # First line of block in original code + + # Computed by constructor + always_exits: bool + cache_offset: int + cache_effects: list[parsing.CacheEffect] + input_effects: list[StackEffect] + output_effects: list[StackEffect] + unmoved_names: frozenset[str] + instr_fmt: str + instr_flags: InstructionFlags + active_caches: list[ActiveCacheEffect] + + # Set later + family: parsing.Family | None = None + predicted: bool = False + + def __init__(self, inst: parsing.InstDef): + self.inst = inst + self.kind = inst.kind + self.name = inst.name + self.block = inst.block + self.block_text, self.check_eval_breaker, self.block_line = extract_block_text( + self.block + ) + self.always_exits = always_exits(self.block_text) + self.cache_effects = [ + effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) + ] + self.cache_offset = sum(c.size for c in self.cache_effects) + self.input_effects = [ + effect for effect in inst.inputs if isinstance(effect, StackEffect) + ] + self.output_effects = inst.outputs # For consistency/completeness + unmoved_names: set[str] = set() + for ieffect, oeffect in zip(self.input_effects, self.output_effects): + if ieffect.name == oeffect.name: + unmoved_names.add(ieffect.name) + else: + break + self.unmoved_names = frozenset(unmoved_names) + + self.instr_flags = InstructionFlags.fromInstruction(inst) + + self.active_caches = [] + offset = 0 + for effect in self.cache_effects: + if effect.name != UNUSED: + self.active_caches.append(ActiveCacheEffect(effect, offset)) + offset += effect.size + + if self.instr_flags.HAS_ARG_FLAG: + fmt = "IB" + else: + fmt = "IX" + if offset: + fmt += "C" + "0" * (offset - 1) + self.instr_fmt = fmt + + def is_viable_uop(self) -> bool: + """Whether this instruction is viable as a uop.""" + dprint: typing.Callable[..., None] = lambda *args, **kwargs: None + # if self.name.startswith("CALL"): + # dprint = print + + if self.name == "EXIT_TRACE": + return True # This has 'return frame' but it's okay + if self.always_exits: + dprint(f"Skipping {self.name} because it always exits") + return False + if len(self.active_caches) > 1: + # print(f"Skipping {self.name} because it has >1 cache entries") + return False + res = True + for forbidden in FORBIDDEN_NAMES_IN_UOPS: + # NOTE: To disallow unspecialized uops, use + # if variable_used(self.inst, forbidden): + if variable_used_unspecialized(self.inst, forbidden): + dprint(f"Skipping {self.name} because it uses {forbidden}") + res = False + return res + + def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: + """Write one instruction, sans prologue and epilogue.""" + # Write a static assertion that a family's cache size is correct + if family := self.family: + if self.name == family.name: + if cache_size := family.size: + out.emit( + f"static_assert({cache_size} == " + f'{self.cache_offset}, "incorrect cache size");' + ) + + # Write input stack effect variable declarations and initializations + ieffects = list(reversed(self.input_effects)) + for i, ieffect in enumerate(ieffects): + isize = string_effect_size( + list_effect_size([ieff for ieff in ieffects[: i + 1]]) + ) + if ieffect.size: + src = StackEffect( + f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **" + ) + elif ieffect.cond: + src = StackEffect( + f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", + "", + ) + else: + src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") + out.declare(ieffect, src) + + # Write output stack effect variable declarations + isize = string_effect_size(list_effect_size(self.input_effects)) + input_names = {ieffect.name for ieffect in self.input_effects} + for i, oeffect in enumerate(self.output_effects): + if oeffect.name not in input_names: + if oeffect.size: + osize = string_effect_size( + list_effect_size([oeff for oeff in self.output_effects[:i]]) + ) + offset = "stack_pointer" + if isize != osize: + if isize != "0": + offset += f" - ({isize})" + if osize != "0": + offset += f" + {osize}" + src = StackEffect(offset, "PyObject **") + out.declare(oeffect, src) + else: + out.declare(oeffect, None) + + # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") + + self.write_body(out, 0, self.active_caches, tier=tier) + + # Skip the rest if the block always exits + if self.always_exits: + return + + # Write net stack growth/shrinkage + out.stack_adjust( + [ieff for ieff in self.input_effects], + [oeff for oeff in self.output_effects], + ) + + # Write output stack effect assignments + oeffects = list(reversed(self.output_effects)) + for i, oeffect in enumerate(oeffects): + if oeffect.name in self.unmoved_names: + continue + osize = string_effect_size( + list_effect_size([oeff for oeff in oeffects[: i + 1]]) + ) + if oeffect.size: + dst = StackEffect( + f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **" + ) + else: + dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") + out.assign(dst, oeffect) + + # Write cache effect + if tier == TIER_ONE and self.cache_offset: + out.emit(f"next_instr += {self.cache_offset};") + + def write_body( + self, + out: Formatter, + dedent: int, + active_caches: list[ActiveCacheEffect], + tier: Tiers = TIER_ONE, + ) -> None: + """Write the instruction body.""" + # Write cache effect variable declarations and initializations + for active in active_caches: + ceffect = active.effect + bits = ceffect.size * BITS_PER_CODE_UNIT + if bits == 64: + # NOTE: We assume that 64-bit data in the cache + # is always an object pointer. + # If this becomes false, we need a way to specify + # syntactically what type the cache data is. + typ = "PyObject *" + func = "read_obj" + else: + typ = f"uint{bits}_t " + func = f"read_u{bits}" + if tier == TIER_ONE: + out.emit( + f"{typ}{ceffect.name} = {func}(&next_instr[{active.offset}].cache);" + ) + else: + out.emit(f"{typ}{ceffect.name} = ({typ.strip()})operand;") + + # Write the body, substituting a goto for ERROR_IF() and other stuff + assert dedent <= 0 + extra = " " * -dedent + names_to_skip = self.unmoved_names | frozenset({UNUSED, "null"}) + offset = 0 + context = self.block.context + assert context is not None and context.owner is not None + filename = context.owner.filename + for line in self.block_text: + out.set_lineno(self.block_line + offset, filename) + offset += 1 + if m := re.match(r"(\s*)ERROR_IF\((.+), (\w+)\);\s*(?://.*)?$", line): + space, cond, label = m.groups() + space = extra + space + # ERROR_IF() must pop the inputs from the stack. + # The code block is responsible for DECREF()ing them. + # NOTE: If the label doesn't exist, just add it to ceval.c. + + # Don't pop common input/output effects at the bottom! + # These aren't DECREF'ed so they can stay. + ieffs = list(self.input_effects) + oeffs = list(self.output_effects) + while ieffs and oeffs and ieffs[0] == oeffs[0]: + ieffs.pop(0) + oeffs.pop(0) + ninputs, symbolic = list_effect_size(ieffs) + if ninputs: + label = f"pop_{ninputs}_{label}" + if symbolic: + out.write_raw( + f"{space}if ({cond}) {{ STACK_SHRINK({symbolic}); goto {label}; }}\n" + ) + else: + out.write_raw(f"{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): + out.reset_lineno() + space = extra + m.group(1) + for ieff in self.input_effects: + if ieff.name in names_to_skip: + continue + if ieff.size: + out.write_raw( + f"{space}for (int _i = {ieff.size}; --_i >= 0;) {{\n" + ) + out.write_raw(f"{space} Py_DECREF({ieff.name}[_i]);\n") + out.write_raw(f"{space}}}\n") + else: + decref = "XDECREF" if ieff.cond else "DECREF" + out.write_raw(f"{space}Py_{decref}({ieff.name});\n") + else: + out.write_raw(extra + line) + out.reset_lineno() + + +InstructionOrCacheEffect = Instruction | parsing.CacheEffect +StackEffectMapping = list[tuple[StackEffect, StackEffect]] + + +@dataclasses.dataclass +class Component: + instr: Instruction + input_mapping: StackEffectMapping + output_mapping: StackEffectMapping + active_caches: list[ActiveCacheEffect] + + def write_body(self, out: Formatter) -> None: + with out.block(""): + input_names = {ieffect.name for _, ieffect in self.input_mapping} + for var, ieffect in self.input_mapping: + out.declare(ieffect, var) + for _, oeffect in self.output_mapping: + if oeffect.name not in input_names: + out.declare(oeffect, None) + + self.instr.write_body(out, -4, self.active_caches) + + for var, oeffect in self.output_mapping: + out.assign(var, oeffect) + + +MacroParts = list[Component | parsing.CacheEffect] + + +@dataclasses.dataclass +class MacroInstruction: + """A macro instruction.""" + + name: str + stack: list[StackEffect] + initial_sp: int + final_sp: int + instr_fmt: str + instr_flags: InstructionFlags + macro: parsing.Macro + parts: MacroParts + cache_offset: int + predicted: bool = False + + +@dataclasses.dataclass +class PseudoInstruction: + """A pseudo instruction.""" + + name: str + targets: list[Instruction] + instr_fmt: str + instr_flags: InstructionFlags + + +@dataclasses.dataclass +class OverriddenInstructionPlaceHolder: + name: str + + +AnyInstruction = Instruction | MacroInstruction | PseudoInstruction + + +def extract_block_text(block: parsing.Block) -> tuple[list[str], bool, int]: + # Get lines of text with proper dedent + blocklines = block.text.splitlines(True) + first_token: lx.Token = block.tokens[0] # IndexError means the context is broken + block_line = first_token.begin[0] + + # Remove blank lines from both ends + while blocklines and not blocklines[0].strip(): + blocklines.pop(0) + block_line += 1 + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Remove leading and trailing braces + assert blocklines and blocklines[0].strip() == "{" + assert blocklines and blocklines[-1].strip() == "}" + blocklines.pop() + blocklines.pop(0) + block_line += 1 + + # Remove trailing blank lines + while blocklines and not blocklines[-1].strip(): + blocklines.pop() + + # Separate CHECK_EVAL_BREAKER() macro from end + check_eval_breaker = ( + blocklines != [] and blocklines[-1].strip() == "CHECK_EVAL_BREAKER();" + ) + if check_eval_breaker: + del blocklines[-1] + + return blocklines, check_eval_breaker, block_line + + +def always_exits(lines: list[str]) -> bool: + """Determine whether a block always ends in a return/goto/etc.""" + if not lines: + return False + line = lines[-1].rstrip() + # Indent must match exactly (TODO: Do something better) + if line[:12] != " " * 12: + return False + line = line[12:] + return line.startswith( + ( + "goto ", + "return ", + "DISPATCH", + "GO_TO_", + "Py_UNREACHABLE()", + "ERROR_IF(true, ", + ) + ) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parsing.py similarity index 96% rename from Tools/cases_generator/parser.py rename to Tools/cases_generator/parsing.py index ac77e7eae81ad3..290285dc03123c 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parsing.py @@ -1,7 +1,7 @@ """Parser for bytecodes.inst.""" from dataclasses import dataclass, field -from typing import NamedTuple, Callable, TypeVar, Literal +from typing import NamedTuple, Callable, TypeVar, Literal, cast import lexer as lx from plexer import PLexer @@ -19,7 +19,7 @@ def contextual_wrapper(self: P) -> N | None: res = func(self) if res is None: self.setpos(begin) - return + return None end = self.getpos() res.context = Context(begin, end, self) return res @@ -147,6 +147,7 @@ def definition(self) -> InstDef | Macro | Pseudo | Family | None: return family if pseudo := self.pseudo_def(): return pseudo + return None @contextual def inst_def(self) -> InstDef | None: @@ -166,7 +167,8 @@ def inst_header(self) -> InstHeader | None: # TODO: Make INST a keyword in the lexer. override = bool(self.expect(lx.OVERRIDE)) register = bool(self.expect(lx.REGISTER)) - if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("inst", "op"): + if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text in ("inst", "op"): + kind = cast(Literal["inst", "op"], tkn.text) if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text if self.expect(lx.COMMA): @@ -190,6 +192,7 @@ def inputs(self) -> list[InputEffect] | None: # input (',' input)* here = self.getpos() if inp := self.input(): + inp = cast(InputEffect, inp) near = self.getpos() if self.expect(lx.COMMA): if rest := self.inputs(): @@ -232,6 +235,7 @@ def cache_effect(self) -> CacheEffect | None: raise self.make_syntax_error(f"Expected integer, got {num!r}") else: return CacheEffect(tkn.text, size) + return None @contextual def stack_effect(self) -> StackEffect | None: @@ -258,6 +262,7 @@ def stack_effect(self) -> StackEffect | None: type_text = "PyObject **" size_text = size.text.strip() return StackEffect(tkn.text, type_text, cond_text, size_text) + return None @contextual def expression(self) -> Expression | None: @@ -288,6 +293,7 @@ def expression(self) -> Expression | None: def op(self) -> OpName | None: if tkn := self.expect(lx.IDENTIFIER): return OpName(tkn.text) + return None @contextual def macro_def(self) -> Macro | None: @@ -300,16 +306,20 @@ def macro_def(self) -> Macro | None: self.require(lx.SEMI) res = Macro(tkn.text, uops) return res + return None def uops(self) -> list[UOp] | None: if uop := self.uop(): + uop = cast(UOp, uop) uops = [uop] while self.expect(lx.PLUS): if uop := self.uop(): + uop = cast(UOp, uop) uops.append(uop) else: raise self.make_syntax_error("Expected op name or cache effect") return uops + return None @contextual def uop(self) -> UOp | None: @@ -327,6 +337,7 @@ def uop(self) -> UOp | None: raise self.make_syntax_error("Expected integer") else: return OpName(tkn.text) + return None @contextual def family_def(self) -> Family | None: @@ -385,6 +396,7 @@ def members(self) -> list[str] | None: def block(self) -> Block | None: if self.c_blob(): return Block() + return None def c_blob(self) -> list[lx.Token]: tokens: list[lx.Token] = []