From f7a7bc32c5f8220c8a7f2ea73dd27f3d8dfe9234 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 5 Dec 2022 16:50:03 -0700 Subject: [PATCH 1/7] Move p5s to _PyRuntimeState. --- Include/internal/pycore_dtoa.h | 11 +++++++---- Python/dtoa.c | 8 ++------ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index fdc6e74ecd25e3..67189cf0ade665 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -25,7 +25,7 @@ Bigint { #ifdef Py_USING_MEMORY_DEBUGGER struct _dtoa_runtime_state { - int _not_used; + int _not_used; }; #define _dtoa_runtime_state_INIT {0} @@ -41,9 +41,12 @@ struct _dtoa_runtime_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_runtime_state { - struct Bigint *freelist[Bigint_Kmax+1]; - double preallocated[Bigint_PREALLOC_SIZE]; - double *preallocated_next; + /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // XXX This should be freed during runtime fini. + struct Bigint *p5s; + struct Bigint *freelist[Bigint_Kmax+1]; + double preallocated[Bigint_PREALLOC_SIZE]; + double *preallocated_next; }; #define _dtoa_runtime_state_INIT(runtime) \ { \ diff --git a/Python/dtoa.c b/Python/dtoa.c index 1b47d83bf77a24..cff5f1b0658eae 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -673,10 +673,6 @@ mult(Bigint *a, Bigint *b) #ifndef Py_USING_MEMORY_DEBUGGER -/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ - -static Bigint *p5s; - /* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on failure; if the returned pointer is distinct from b then the original Bigint b will have been Bfree'd. Ignores the sign of b. */ @@ -696,7 +692,7 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - p5 = p5s; + p5 = _PyRuntime.dtoa.p5s; if (!p5) { /* first time */ p5 = i2b(625); @@ -704,7 +700,7 @@ pow5mult(Bigint *b, int k) Bfree(b); return NULL; } - p5s = p5; + _PyRuntime.dtoa.p5s = p5; p5->next = 0; } for(;;) { diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index af5b7eb8eaf024..c2ec83e195ceb4 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -306,7 +306,6 @@ Objects/sliceobject.c - _Py_EllipsisObject - ## effectively-const but initialized lazily ## idempotent -Python/dtoa.c - p5s - Objects/obmalloc.c new_arena debug_stats - ## others From 5c705504c17da2d78b9090174eac5ddc6fe9039a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 5 Dec 2022 17:16:24 -0700 Subject: [PATCH 2/7] Move ucnhash_capi to the interpreter. --- Include/internal/pycore_unicodeobject.h | 3 +++ Objects/unicodeobject.c | 8 +++++--- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index b315ca1ae5b64b..19faceebf1d8ee 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); @@ -52,6 +53,8 @@ struct _Py_unicode_ids { struct _Py_unicode_state { struct _Py_unicode_fs_codec fs_codec; + _PyUnicode_Name_CAPI *ucnhash_capi; + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() struct _Py_unicode_ids ids; }; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index deeca35714b766..b721ccd805edf1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5697,8 +5697,6 @@ PyUnicode_AsUTF16String(PyObject *unicode) /* --- Unicode Escape Codec ----------------------------------------------- */ -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject * _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, @@ -5711,6 +5709,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, const char *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; + _PyUnicode_Name_CAPI *ucnhash_capi; + PyInterpreterState *interp = _PyInterpreterState_Get(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -5858,6 +5858,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': + ucnhash_capi = interp->unicode.ucnhash_capi; if (ucnhash_capi == NULL) { /* load the unicode data module */ ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( @@ -5869,6 +5870,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, ); goto onError; } + interp->unicode.ucnhash_capi = ucnhash_capi; } message = "malformed \\N character escape"; @@ -15128,10 +15130,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) assert(get_interned_dict() == NULL); // bpo-47182: force a unicodedata CAPI capsule re-import on // subsequent initialization of main interpreter. - ucnhash_capi = NULL; } _PyUnicode_FiniEncodings(&state->fs_codec); + interp->unicode.ucnhash_capi = NULL; unicode_clear_identifiers(state); } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index c2ec83e195ceb4..5053c98140d7f3 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -310,7 +310,6 @@ Objects/obmalloc.c new_arena debug_stats - ## others Python/perf_trampoline.c - perf_map_file - -Objects/unicodeobject.c - ucnhash_capi - ##----------------------- ## state From e3bd40ea03ef65ad66f19345f8e2067b6b02ae30 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 5 Dec 2022 17:18:41 -0700 Subject: [PATCH 3/7] Drop levenshtein_distance():buffer from the known globals. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 5053c98140d7f3..64b2d5b65f703e 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -314,9 +314,6 @@ Python/perf_trampoline.c - perf_map_file - ##----------------------- ## state -## local buffer -Python/suggestions.c levenshtein_distance buffer - - ## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - From 19f5f29dab95f331fee5ce72b765a67e487d871d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 6 Dec 2022 10:57:57 -0700 Subject: [PATCH 4/7] Eliminate eintr_int. --- Modules/_io/bufferedio.c | 24 ++++++++++----------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6df55b5b8303c2..ba8969f0bcd100 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -746,26 +746,26 @@ _buffered_init(buffered *self) int _PyIO_trap_eintr(void) { - static PyObject *eintr_int = NULL; PyObject *typ, *val, *tb; PyOSErrorObject *env_err; - - if (eintr_int == NULL) { - eintr_int = PyLong_FromLong(EINTR); - assert(eintr_int != NULL); - } if (!PyErr_ExceptionMatches(PyExc_OSError)) return 0; PyErr_Fetch(&typ, &val, &tb); PyErr_NormalizeException(&typ, &val, &tb); env_err = (PyOSErrorObject *) val; assert(env_err != NULL); - if (env_err->myerrno != NULL && - PyObject_RichCompareBool(env_err->myerrno, eintr_int, Py_EQ) > 0) { - Py_DECREF(typ); - Py_DECREF(val); - Py_XDECREF(tb); - return 1; + if (env_err->myerrno != NULL) { + assert(EINTR > 0 && EINTR < INT_MAX); + assert(PyLong_CheckExact(env_err->myerrno)); + int overflow; + int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow); + PyErr_Clear(); + if (myerrno == EINTR) { + Py_DECREF(typ); + Py_DECREF(val); + Py_XDECREF(tb); + return 1; + } } /* This silences any error set by PyObject_RichCompareBool() */ PyErr_Restore(typ, val, tb); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 64b2d5b65f703e..1defa723b83358 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -393,7 +393,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - Modules/posixmodule.c - structseq_new - Modules/posixmodule.c - ticks_per_second - From c62b47b75c67d8a27e9e075cafe30eec554dfe4d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 Dec 2022 14:59:15 -0700 Subject: [PATCH 5/7] Move debug_stats to _PyRuntimeState. --- Include/internal/pycore_obmalloc.h | 1 + Include/internal/pycore_obmalloc_init.h | 1 + Objects/obmalloc.c | 3 ++- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 --- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 93349d89c6ab52..a5c7f4528f9126 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -658,6 +658,7 @@ struct _obmalloc_usage { struct _obmalloc_state { + int dump_debug_stats; struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; struct _obmalloc_usage usage; diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index c0fb057d06652b..c9f197e72de9f5 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -56,6 +56,7 @@ extern "C" { #define _obmalloc_state_INIT(obmalloc) \ { \ + .dump_debug_stats = -1, \ .pools = { \ .used = _obmalloc_pools_INIT(obmalloc.pools), \ }, \ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4c08bc214cd27a..276c5a276c06e6 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -908,11 +908,12 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; - static int debug_stats = -1; + int debug_stats = _PyRuntime.obmalloc.dump_debug_stats; if (debug_stats == -1) { const char *opt = Py_GETENV("PYTHONMALLOCSTATS"); debug_stats = (opt != NULL && *opt != '\0'); + _PyRuntime.obmalloc.dump_debug_stats = debug_stats; } if (debug_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 1defa723b83358..5c8164517f138a 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -305,9 +305,6 @@ Objects/sliceobject.c - _Py_EllipsisObject - ##----------------------- ## effectively-const but initialized lazily -## idempotent -Objects/obmalloc.c new_arena debug_stats - - ## others Python/perf_trampoline.c - perf_map_file - From cca8253ff52d3382be9fda6bfba7e15a693ce60c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 Dec 2022 15:02:15 -0700 Subject: [PATCH 6/7] Move orig_argv to _PyRuntimeState. --- Include/internal/pycore_runtime.h | 2 ++ Python/initconfig.c | 14 ++++++-------- Tools/c-analyzer/cpython/ignored.tsv | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c1829cb1bdadeb..c3dbbc6f55f294 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -129,6 +129,8 @@ typedef struct pyruntimestate { unsigned long main_thread; + PyWideStringList orig_argv; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; diff --git a/Python/initconfig.c b/Python/initconfig.c index 67f6777d3b1d9e..64ae987b3f34d9 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -595,17 +595,13 @@ _Py_ClearStandardStreamEncoding(void) /* --- Py_GetArgcArgv() ------------------------------------------- */ -/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ -static PyWideStringList orig_argv = {.length = 0, .items = NULL}; - - void _Py_ClearArgcArgv(void) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyWideStringList_Clear(&orig_argv); + _PyWideStringList_Clear(&_PyRuntime.orig_argv); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -620,7 +616,9 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - res = _PyWideStringList_Copy(&orig_argv, &argv_list); + // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(), + // so it it currently leaks for embedders. + res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; @@ -631,8 +629,8 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) void Py_GetArgcArgv(int *argc, wchar_t ***argv) { - *argc = (int)orig_argv.length; - *argv = orig_argv.items; + *argc = (int)_PyRuntime.orig_argv.length; + *argv = _PyRuntime.orig_argv.items; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 128336a997eba0..865f00f8ed1726 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -87,8 +87,6 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - -# XXX This only gets cleared by Py_Main(). -Python/initconfig.c - orig_argv - ##----------------------- ## public C-API From bcbefb7a5b917ebc3610311c24654c8c1c5942bf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 Dec 2022 15:21:08 -0700 Subject: [PATCH 7/7] Move memo_statistics to _PyRuntimeState. --- Include/internal/pycore_parser.h | 20 ++++++++++++++++++++ Include/internal/pycore_runtime.h | 3 +++ Parser/pegen.c | 4 ++-- Tools/c-analyzer/cpython/ignored.tsv | 9 --------- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index e2de24e2ca9734..2d2b56bd824cb4 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -8,12 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +#include "pycore_pyarena.h" // PyArena + + +#ifdef Py_DEBUG +#define _PYPEGEN_NSTATISTICS 2000 +#endif + +struct _parser_runtime_state { +#ifdef Py_DEBUG + long memo_statistics[_PYPEGEN_NSTATISTICS]; +#else + int _not_used; +#endif +}; + + + extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); + extern struct _mod* _PyParser_ASTFromFile( FILE *fp, PyObject *filename_ob, @@ -25,6 +44,7 @@ extern struct _mod* _PyParser_ASTFromFile( int *errcode, PyArena *arena); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c3dbbc6f55f294..0720e2ed4422df 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState +#include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state @@ -131,6 +132,8 @@ typedef struct pyruntimestate { PyWideStringList orig_argv; + struct _parser_runtime_state parser; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; diff --git a/Parser/pegen.c b/Parser/pegen.c index d34a86e9c883de..d84e06861edefc 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -246,8 +246,8 @@ _PyPegen_fill_token(Parser *p) // The array counts the number of tokens skipped by memoization, // indexed by type. -#define NSTATISTICS 2000 -static long memo_statistics[NSTATISTICS]; +#define NSTATISTICS _PYPEGEN_NSTATISTICS +#define memo_statistics _PyRuntime.parser.memo_statistics void _PyPegen_clear_memo_statistics() diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 865f00f8ed1726..257823574fa8b3 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -141,15 +141,6 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - -##----------------------- -## *not* tied to init/fini cycle - -# These do not ge reset with each init/fini cycle. -# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? - -# special-use diagnistic state -Parser/pegen.c - memo_statistics - - ##----------------------- ## one-off temporary state