From 7c18aeba59470c80023eca41b52b397286e0764b Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 11:09:43 -0700 Subject: [PATCH 01/15] Move the legacy config flag globals to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 19 ------------------ Tools/c-analyzer/cpython/ignored.tsv | 22 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index b60f16db9a28ac..f94e6d01d4c7dc 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -321,25 +321,6 @@ Objects/unicodeobject.c - bloom_linebreak - Python/bootstrap_hash.c - _Py_HashSecret_Initialized - Python/bootstrap_hash.c py_getrandom getrandom_works - Python/initconfig.c - _Py_global_config_int_max_str_digits - -Python/initconfig.c - Py_DebugFlag - -Python/initconfig.c - Py_UTF8Mode - -Python/initconfig.c - Py_DebugFlag - -Python/initconfig.c - Py_VerboseFlag - -Python/initconfig.c - Py_QuietFlag - -Python/initconfig.c - Py_InteractiveFlag - -Python/initconfig.c - Py_InspectFlag - -Python/initconfig.c - Py_OptimizeFlag - -Python/initconfig.c - Py_NoSiteFlag - -Python/initconfig.c - Py_BytesWarningFlag - -Python/initconfig.c - Py_FrozenFlag - -Python/initconfig.c - Py_IgnoreEnvironmentFlag - -Python/initconfig.c - Py_DontWriteBytecodeFlag - -Python/initconfig.c - Py_NoUserSiteDirectory - -Python/initconfig.c - Py_UnbufferedStdioFlag - -Python/initconfig.c - Py_HashRandomizationFlag - -Python/initconfig.c - Py_IsolatedFlag - -Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - -Python/initconfig.c - Py_LegacyWindowsStdioFlag - Python/initconfig.c - orig_argv - Python/pyhash.c - _Py_HashSecret - Python/pylifecycle.c - runtime_initialized - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 414e68df60dac9..1d049b9d2b84a1 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -10,6 +10,28 @@ Python/pylifecycle.c - _PyRuntime - # All uses of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. +#----------------------- +# legacy config flags + +Python/initconfig.c - Py_UTF8Mode - +Python/initconfig.c - Py_DebugFlag - +Python/initconfig.c - Py_VerboseFlag - +Python/initconfig.c - Py_QuietFlag - +Python/initconfig.c - Py_InteractiveFlag - +Python/initconfig.c - Py_InspectFlag - +Python/initconfig.c - Py_OptimizeFlag - +Python/initconfig.c - Py_NoSiteFlag - +Python/initconfig.c - Py_BytesWarningFlag - +Python/initconfig.c - Py_FrozenFlag - +Python/initconfig.c - Py_IgnoreEnvironmentFlag - +Python/initconfig.c - Py_DontWriteBytecodeFlag - +Python/initconfig.c - Py_NoUserSiteDirectory - +Python/initconfig.c - Py_UnbufferedStdioFlag - +Python/initconfig.c - Py_HashRandomizationFlag - +Python/initconfig.c - Py_IsolatedFlag - +Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - +Python/initconfig.c - Py_LegacyWindowsStdioFlag - + #----------------------- # others From dbe721928d6e6dca414a6677f98bc10723263029 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 11:14:24 -0700 Subject: [PATCH 02/15] Move a safe static buffer to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 --- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index f94e6d01d4c7dc..1937086bfd741c 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -313,9 +313,6 @@ Python/hamt.c - _empty_hamt - #----------------------- # initialized/set once -# pre-allocated buffer -Modules/getbuildinfo.c Py_GetBuildInfo buildinfo - - # during init Objects/unicodeobject.c - bloom_linebreak - Python/bootstrap_hash.c - _Py_HashSecret_Initialized - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1d049b9d2b84a1..d269bb3f34089a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -38,6 +38,9 @@ Python/initconfig.c - Py_LegacyWindowsStdioFlag - # XXX The analyzer should have ignored these. Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - +# a pre-allocated buffer with a very unlikely race +Modules/getbuildinfo.c Py_GetBuildInfo buildinfo - + ################################## # forward/extern references From f45f96039530144bc6904a3dc3fa127ac65c792a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 11:19:25 -0700 Subject: [PATCH 03/15] Move effectively-const globals initialized during init. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 30 ++------------------ Tools/c-analyzer/cpython/ignored.tsv | 31 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 1937086bfd741c..41b4e9f2669fd1 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -311,34 +311,8 @@ Python/hamt.c - _empty_hamt - # global non-objects to fix in core code #----------------------- -# initialized/set once - -# during init -Objects/unicodeobject.c - bloom_linebreak - -Python/bootstrap_hash.c - _Py_HashSecret_Initialized - -Python/bootstrap_hash.c py_getrandom getrandom_works - -Python/initconfig.c - _Py_global_config_int_max_str_digits - -Python/initconfig.c - orig_argv - -Python/pyhash.c - _Py_HashSecret - -Python/pylifecycle.c - runtime_initialized - -Python/sysmodule.c - _PySys_ImplCacheTag - -Python/sysmodule.c - _PySys_ImplName - -Python/sysmodule.c - _preinit_warnoptions - -Python/sysmodule.c - _preinit_xoptions - -Python/thread.c - initialized - -Python/thread_pthread.h - condattr_monotonic - -Python/thread_pthread.h init_condattr ca - - -# set by embedders during init -Python/initconfig.c - _Py_StandardStreamEncoding - -Python/initconfig.c - _Py_StandardStreamErrors - - -# lazy -Objects/floatobject.c - double_format - -Objects/floatobject.c - float_format - -Objects/longobject.c long_from_non_binary_base log_base_BASE - -Objects/longobject.c long_from_non_binary_base convwidth_base - -Objects/longobject.c long_from_non_binary_base convmultmax_base - +# effectively-const but initialized lazily + Python/perf_trampoline.c - perf_map_file - Objects/unicodeobject.c - ucnhash_capi - Parser/action_helpers.c _PyPegen_dummy_name cache - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index d269bb3f34089a..1dee11781c7f58 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -32,6 +32,35 @@ Python/initconfig.c - Py_IsolatedFlag - Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - Python/initconfig.c - Py_LegacyWindowsStdioFlag - +#----------------------- +# effectively const, initialized during init + +Objects/floatobject.c - double_format - +Objects/floatobject.c - float_format - +Objects/unicodeobject.c - bloom_linebreak - +Python/bootstrap_hash.c py_getrandom getrandom_works - +Python/bootstrap_hash.c - _Py_HashSecret_Initialized - +Python/initconfig.c - orig_argv - +Python/pyhash.c - _Py_HashSecret - +Python/pylifecycle.c - runtime_initialized - +Python/sysmodule.c - _preinit_warnoptions - +Python/sysmodule.c - _preinit_xoptions - +Python/thread.c - initialized - +Python/thread_pthread.h - condattr_monotonic - +Python/thread_pthread.h init_condattr ca - + +# set by embedders during init +Python/initconfig.c - _Py_StandardStreamEncoding - +Python/initconfig.c - _Py_StandardStreamErrors - + +#----------------------- +# effectively const but initialized lazily + +# idempotent +Objects/longobject.c long_from_non_binary_base log_base_BASE - +Objects/longobject.c long_from_non_binary_base convwidth_base - +Objects/longobject.c long_from_non_binary_base convmultmax_base - + #----------------------- # others @@ -537,4 +566,6 @@ Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - Python/specialize.c - compare_masks - Python/stdlib_module_names.h - _Py_stdlib_module_names - +Python/sysmodule.c - _PySys_ImplCacheTag - +Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - From acf1a9bda94f075836f511fb5d27fa46cb2636a9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 12:04:24 -0700 Subject: [PATCH 04/15] Move single-context globals to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 13 ------------- Tools/c-analyzer/cpython/ignored.tsv | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 41b4e9f2669fd1..72dff660aab231 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -335,19 +335,6 @@ Python/preconfig.c - Py_HasFileSystemDefaultEncoding - Python/preconfig.c - Py_FileSystemDefaultEncodeErrors - Python/preconfig.c - _Py_HasFileSystemDefaultEncodeErrors - -# REPL -Parser/myreadline.c - _PyOS_ReadlineLock - -Parser/myreadline.c - _PyOS_ReadlineTState - -Parser/myreadline.c - PyOS_InputHook - -Parser/myreadline.c - PyOS_ReadlineFunctionPointer - - -# handling C argv -Python/getopt.c - _PyOS_optarg - -Python/getopt.c - _PyOS_opterr - -Python/getopt.c - _PyOS_optind - -Python/getopt.c - opt_ptr - -Python/pathconfig.c - _Py_path_config - - #----------------------- # state diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1dee11781c7f58..901c58a27b2beb 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -61,6 +61,23 @@ Objects/longobject.c long_from_non_binary_base log_base_BASE - Objects/longobject.c long_from_non_binary_base convwidth_base - Objects/longobject.c long_from_non_binary_base convmultmax_base - +#----------------------- +# used only in the main thread + +# REPL + +Parser/myreadline.c - _PyOS_ReadlineLock - +Parser/myreadline.c - _PyOS_ReadlineTState - +Parser/myreadline.c - PyOS_InputHook - +Parser/myreadline.c - PyOS_ReadlineFunctionPointer - + +# handling C argv +Python/getopt.c - _PyOS_optarg - +Python/getopt.c - _PyOS_opterr - +Python/getopt.c - _PyOS_optind - +Python/getopt.c - opt_ptr - +Python/pathconfig.c - _Py_path_config - + #----------------------- # others From 72492eb03a2b4036a66a3846780f4295da4af19c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 12:19:09 -0700 Subject: [PATCH 05/15] Add a TODO comment about leaked memory. --- Parser/action_helpers.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index d1be679aff2e7b..27c093332f6725 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -12,6 +12,7 @@ _create_dummy_identifier(Parser *p) void * _PyPegen_dummy_name(Parser *p, ...) { + // XXX This leaks memory from the initial arena. static void *cache = NULL; if (cache != NULL) { From c90895884e145017892367a11e1fe5e4fc296350 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 12:45:06 -0700 Subject: [PATCH 06/15] Move more globals to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 9 ++++----- Tools/c-analyzer/cpython/ignored.tsv | 5 +++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 72dff660aab231..fa7c2f23c77f4d 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -313,13 +313,12 @@ Python/hamt.c - _empty_hamt - #----------------------- # effectively-const but initialized lazily +# idempotent +Python/dtoa.c - p5s - + +# others Python/perf_trampoline.c - perf_map_file - Objects/unicodeobject.c - ucnhash_capi - -Parser/action_helpers.c _PyPegen_dummy_name cache - -Python/dtoa.c - p5s - -Python/fileutils.c - _Py_open_cloexec_works - -Python/fileutils.c - force_ascii - -Python/fileutils.c set_inheritable ioctl_works - #----------------------- # unlikely to change after init (or main thread) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 901c58a27b2beb..bc1c0456637a28 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -40,6 +40,10 @@ Objects/floatobject.c - float_format - Objects/unicodeobject.c - bloom_linebreak - Python/bootstrap_hash.c py_getrandom getrandom_works - Python/bootstrap_hash.c - _Py_HashSecret_Initialized - +Python/fileutils.c - _Py_open_cloexec_works - +Python/fileutils.c - force_ascii - +Python/fileutils.c set_inheritable ioctl_works - +Python/import.c import_find_and_load header - Python/initconfig.c - orig_argv - Python/pyhash.c - _Py_HashSecret - Python/pylifecycle.c - runtime_initialized - @@ -60,6 +64,7 @@ Python/initconfig.c - _Py_StandardStreamErrors - Objects/longobject.c long_from_non_binary_base log_base_BASE - Objects/longobject.c long_from_non_binary_base convwidth_base - Objects/longobject.c long_from_non_binary_base convmultmax_base - +Parser/action_helpers.c _PyPegen_dummy_name cache - #----------------------- # used only in the main thread From 5357ea4b03da4491d1d656fc139c6adbaa4c74c2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 12:51:53 -0700 Subject: [PATCH 07/15] Re-organize the ignored list a tiny bit. --- Tools/c-analyzer/cpython/ignored.tsv | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index bc1c0456637a28..0a58c4c51ef230 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -4,8 +4,6 @@ filename funcname name reason ################################## # mutable but known to be safe -Python/import.c - inittab_copy - -Python/import.c - PyImport_Inittab - Python/pylifecycle.c - _PyRuntime - # All uses of _PyArg_Parser are handled in c-analyzr/cpython/_analyzer.py. @@ -32,6 +30,12 @@ Python/initconfig.c - Py_IsolatedFlag - Python/initconfig.c - Py_LegacyWindowsFSEncodingFlag - Python/initconfig.c - Py_LegacyWindowsStdioFlag - +#----------------------- +# effectively const, initialized before init + +Python/import.c - inittab_copy - +Python/import.c - PyImport_Inittab - + #----------------------- # effectively const, initialized during init From 6a3fb9524d4e10984e9fbc89cc47ab477fb90680 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 13:01:59 -0700 Subject: [PATCH 08/15] Move more globals to the ignored list. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 14 -------------- Tools/c-analyzer/cpython/ignored.tsv | 11 ++++++++++- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index fa7c2f23c77f4d..57029298db08a7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -320,20 +320,6 @@ Python/dtoa.c - p5s - Python/perf_trampoline.c - perf_map_file - Objects/unicodeobject.c - ucnhash_capi - -#----------------------- -# unlikely to change after init (or main thread) - -# through C-API -Python/frozen.c - PyImport_FrozenModules - -Python/frozen.c - _PyImport_FrozenAliases - -Python/frozen.c - _PyImport_FrozenBootstrap - -Python/frozen.c - _PyImport_FrozenStdlib - -Python/frozen.c - _PyImport_FrozenTest - -Python/preconfig.c - Py_FileSystemDefaultEncoding - -Python/preconfig.c - Py_HasFileSystemDefaultEncoding - -Python/preconfig.c - Py_FileSystemDefaultEncodeErrors - -Python/preconfig.c - _Py_HasFileSystemDefaultEncodeErrors - - #----------------------- # state diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 0a58c4c51ef230..951a91ecea5395 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -33,6 +33,7 @@ Python/initconfig.c - Py_LegacyWindowsStdioFlag - #----------------------- # effectively const, initialized before init +Python/frozen.c - PyImport_FrozenModules - Python/import.c - inittab_copy - Python/import.c - PyImport_Inittab - @@ -49,6 +50,10 @@ Python/fileutils.c - force_ascii - Python/fileutils.c set_inheritable ioctl_works - Python/import.c import_find_and_load header - Python/initconfig.c - orig_argv - +Python/preconfig.c - Py_FileSystemDefaultEncoding - +Python/preconfig.c - Py_HasFileSystemDefaultEncoding - +Python/preconfig.c - Py_FileSystemDefaultEncodeErrors - +Python/preconfig.c - _Py_HasFileSystemDefaultEncodeErrors - Python/pyhash.c - _Py_HashSecret - Python/pylifecycle.c - runtime_initialized - Python/sysmodule.c - _preinit_warnoptions - @@ -554,7 +559,6 @@ Objects/obmalloc.c - _PyMem_Debug - Objects/obmalloc.c - _PyMem_Raw - Objects/obmalloc.c - _PyObject - Objects/obmalloc.c - usedpools - -Python/perf_trampoline.c - _Py_perfmap_callbacks - Objects/typeobject.c - name_op - Objects/typeobject.c - slotdefs - Objects/unicodeobject.c - stripfuncnames - @@ -579,10 +583,15 @@ Python/frozen.c - aliases - Python/frozen.c - bootstrap_modules - Python/frozen.c - stdlib_modules - Python/frozen.c - test_modules - +Python/frozen.c - _PyImport_FrozenAliases - +Python/frozen.c - _PyImport_FrozenBootstrap - +Python/frozen.c - _PyImport_FrozenStdlib - +Python/frozen.c - _PyImport_FrozenTest - Python/getopt.c - longopts - Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - Python/opcode_targets.h - opcode_targets - +Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - From efad4c376959f07cbd5b939e10bc04f5a2f0c470 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 13:33:12 -0700 Subject: [PATCH 09/15] Only generate the version & build info once. --- Include/internal/pycore_pylifecycle.h | 1 + Modules/getbuildinfo.c | 12 +++++++++--- Python/getversion.c | 17 ++++++++++++++--- Python/pylifecycle.c | 2 ++ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - Tools/c-analyzer/cpython/ignored.tsv | 6 ++++-- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 85bf166d92fd47..359b809d5870a1 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -33,6 +33,7 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ +extern void _Py_InitVersion(void); extern PyStatus _PyImport_Init(void); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 7cb7397a22c8ab..a24750b76c09bc 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -31,12 +31,18 @@ #define GITBRANCH "" #endif +static int initialized = 0; +static char buildinfo[50 + sizeof(GITVERSION) + + ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? + sizeof(GITTAG) : sizeof(GITBRANCH))]; + const char * Py_GetBuildInfo(void) { - static char buildinfo[50 + sizeof(GITVERSION) + - ((sizeof(GITTAG) > sizeof(GITBRANCH)) ? - sizeof(GITTAG) : sizeof(GITBRANCH))]; + if (initialized) { + return buildinfo; + } + initialized = 1; const char *revision = _Py_gitversion(); const char *sep = *revision ? ":" : ""; const char *gitid = _Py_gitidentifier(); diff --git a/Python/getversion.c b/Python/getversion.c index 46910451fdf859..5db836ab4bfd6d 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -5,12 +5,23 @@ #include "patchlevel.h" -const char * -Py_GetVersion(void) +static int initialized = 0; +static char version[250]; + +void _Py_InitVersion(void) { - static char version[250]; + if (initialized) { + return; + } + initialized = 1; PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); +} + +const char * +Py_GetVersion(void) +{ + _Py_InitVersion(); return version; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 44f844249b13b9..7ca284ef1efb9d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -600,6 +600,8 @@ pycore_init_runtime(_PyRuntimeState *runtime, */ _PyRuntimeState_SetFinalizing(runtime, NULL); + _Py_InitVersion(); + status = _Py_HashRandomization_Init(config); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 57029298db08a7..76aa7282113406 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -332,7 +332,6 @@ Python/dtoa.c - freelist - Python/dtoa.c - private_mem - # local buffer -Python/getversion.c Py_GetVersion version - Python/suggestions.c levenshtein_distance buffer - # linked list diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 951a91ecea5395..39d3ab8f67d89c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -40,6 +40,8 @@ Python/import.c - PyImport_Inittab - #----------------------- # effectively const, initialized during init +Modules/getbuildinfo.c - buildinfo - +Modules/getbuildinfo.c - initialized - Objects/floatobject.c - double_format - Objects/floatobject.c - float_format - Objects/unicodeobject.c - bloom_linebreak - @@ -48,6 +50,8 @@ Python/bootstrap_hash.c - _Py_HashSecret_Initialized - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c - force_ascii - Python/fileutils.c set_inheritable ioctl_works - +Python/getversion.c - initialized - +Python/getversion.c - version - Python/import.c import_find_and_load header - Python/initconfig.c - orig_argv - Python/preconfig.c - Py_FileSystemDefaultEncoding - @@ -98,8 +102,6 @@ Python/pathconfig.c - _Py_path_config - # XXX The analyzer should have ignored these. Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - -# a pre-allocated buffer with a very unlikely race -Modules/getbuildinfo.c Py_GetBuildInfo buildinfo - ################################## From 3c97f388daa614385963a235bad2affa055fb1b9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 14:08:05 -0700 Subject: [PATCH 10/15] Tweak the ignored list. --- Tools/c-analyzer/cpython/ignored.tsv | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 39d3ab8f67d89c..0a6f3ec8152dd3 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -38,10 +38,16 @@ Python/import.c - inittab_copy - Python/import.c - PyImport_Inittab - #----------------------- -# effectively const, initialized during init +# effectively const, initialized before/during init Modules/getbuildinfo.c - buildinfo - Modules/getbuildinfo.c - initialized - +Python/getversion.c - initialized - +Python/getversion.c - version - + +#----------------------- +# effectively const, initialized during init + Objects/floatobject.c - double_format - Objects/floatobject.c - float_format - Objects/unicodeobject.c - bloom_linebreak - @@ -50,8 +56,6 @@ Python/bootstrap_hash.c - _Py_HashSecret_Initialized - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c - force_ascii - Python/fileutils.c set_inheritable ioctl_works - -Python/getversion.c - initialized - -Python/getversion.c - version - Python/import.c import_find_and_load header - Python/initconfig.c - orig_argv - Python/preconfig.c - Py_FileSystemDefaultEncoding - @@ -72,6 +76,7 @@ Python/initconfig.c - _Py_StandardStreamErrors - #----------------------- # effectively const but initialized lazily +# XXX Move them to _PyRuntimeState? # idempotent Objects/longobject.c long_from_non_binary_base log_base_BASE - @@ -83,7 +88,6 @@ Parser/action_helpers.c _PyPegen_dummy_name cache - # used only in the main thread # REPL - Parser/myreadline.c - _PyOS_ReadlineLock - Parser/myreadline.c - _PyOS_ReadlineTState - Parser/myreadline.c - PyOS_InputHook - @@ -96,17 +100,15 @@ Python/getopt.c - _PyOS_optind - Python/getopt.c - opt_ptr - Python/pathconfig.c - _Py_path_config - -#----------------------- -# others -# XXX The analyzer should have ignored these. +################################## +# The analyzer should have ignored these. +# XXX Fix the analyzer. + Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - - -################################## # forward/extern references -# XXX The analyzer should have ignored these. Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - From b9df4290946e14869adfad2b10b34ba524f97507 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 14:18:50 -0700 Subject: [PATCH 11/15] Move _Py_tracemalloc_config to _PyRuntimeState. --- Include/internal/pycore_pymem.h | 2 - Include/internal/pycore_runtime.h | 3 ++ Include/internal/pycore_runtime_init.h | 3 ++ Modules/_tracemalloc.c | 50 +++++++++++---------- Objects/object.c | 2 +- Objects/obmalloc.c | 6 --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 7 files changed, 33 insertions(+), 34 deletions(-) diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index b042a4cb268e54..5749af7465f6f0 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -113,8 +113,6 @@ struct _PyTraceMalloc_Config { .tracing = 0, \ .max_nframe = 1} -PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config; - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 6bcb35b35610f0..f13dde62d4928b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -125,6 +125,9 @@ typedef struct pyruntimestate { struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; struct _getargs_runtime_state getargs; + struct { + struct _PyTraceMalloc_Config config; + } tracemalloc; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ea98c3784ff300..ff91c2ce7ddd25 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -36,6 +36,9 @@ extern "C" { until _PyInterpreterState_Enable() is called. */ \ .next_id = -1, \ }, \ + .tracemalloc = { \ + .config = _PyTraceMalloc_Config_INIT, \ + }, \ .types = { \ .next_version_tag = 1, \ }, \ diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index fe73d63d411f5f..0d70f0cf34c8d6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -16,6 +16,8 @@ module _tracemalloc [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=708a98302fc46e5f]*/ +#define tracemalloc_config _PyRuntime.tracemalloc.config + _Py_DECLARE_STR(anon_unknown, ""); /* Trace memory blocks allocated by PyMem_RawMalloc() */ @@ -407,7 +409,7 @@ traceback_get_frames(traceback_t *traceback) if (pyframe == NULL) { break; } - if (traceback->nframe < _Py_tracemalloc_config.max_nframe) { + if (traceback->nframe < tracemalloc_config.max_nframe) { tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); assert(traceback->frames[traceback->nframe].filename != NULL); traceback->nframe++; @@ -505,7 +507,7 @@ tracemalloc_get_traces_table(unsigned int domain) static void tracemalloc_remove_trace(unsigned int domain, uintptr_t ptr) { - assert(_Py_tracemalloc_config.tracing); + assert(tracemalloc_config.tracing); _Py_hashtable_t *traces = tracemalloc_get_traces_table(domain); if (!traces) { @@ -529,7 +531,7 @@ static int tracemalloc_add_trace(unsigned int domain, uintptr_t ptr, size_t size) { - assert(_Py_tracemalloc_config.tracing); + assert(tracemalloc_config.tracing); traceback_t *traceback = traceback_new(); if (traceback == NULL) { @@ -863,13 +865,13 @@ tracemalloc_clear_traces(void) static int tracemalloc_init(void) { - if (_Py_tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { + if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); return -1; } - if (_Py_tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) + if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) return 0; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); @@ -919,7 +921,7 @@ tracemalloc_init(void) tracemalloc_empty_traceback.frames[0].lineno = 0; tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); - _Py_tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; + tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; return 0; } @@ -927,9 +929,9 @@ tracemalloc_init(void) static void tracemalloc_deinit(void) { - if (_Py_tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) + if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) return; - _Py_tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; + tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; tracemalloc_stop(); @@ -969,12 +971,12 @@ tracemalloc_start(int max_nframe) return -1; } - if (_Py_tracemalloc_config.tracing) { + if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; } - _Py_tracemalloc_config.max_nframe = max_nframe; + tracemalloc_config.max_nframe = max_nframe; /* allocate a buffer to store a new traceback */ size = TRACEBACK_SIZE(max_nframe); @@ -1010,7 +1012,7 @@ tracemalloc_start(int max_nframe) PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); /* everything is ready: start tracing Python memory allocations */ - _Py_tracemalloc_config.tracing = 1; + tracemalloc_config.tracing = 1; return 0; } @@ -1019,11 +1021,11 @@ tracemalloc_start(int max_nframe) static void tracemalloc_stop(void) { - if (!_Py_tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) return; /* stop tracing Python memory allocations */ - _Py_tracemalloc_config.tracing = 0; + tracemalloc_config.tracing = 0; /* unregister the hook on memory allocators */ #ifdef TRACE_RAW_MALLOC @@ -1051,7 +1053,7 @@ static PyObject * _tracemalloc_is_tracing_impl(PyObject *module) /*[clinic end generated code: output=2d763b42601cd3ef input=af104b0a00192f63]*/ { - return PyBool_FromLong(_Py_tracemalloc_config.tracing); + return PyBool_FromLong(tracemalloc_config.tracing); } @@ -1065,7 +1067,7 @@ static PyObject * _tracemalloc_clear_traces_impl(PyObject *module) /*[clinic end generated code: output=a86080ee41b84197 input=0dab5b6c785183a5]*/ { - if (!_Py_tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) Py_RETURN_NONE; set_reentrant(1); @@ -1345,7 +1347,7 @@ _tracemalloc__get_traces_impl(PyObject *module) if (get_traces.list == NULL) goto error; - if (!_Py_tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) return get_traces.list; /* the traceback hash table is used temporarily to intern traceback tuple @@ -1418,7 +1420,7 @@ static traceback_t* tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) { - if (!_Py_tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) return NULL; trace_t *trace; @@ -1498,7 +1500,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - if (!_Py_tracemalloc_config.tracing) { + if (!tracemalloc_config.tracing) { PUTS(fd, "Enable tracemalloc to get the memory block " "allocation traceback\n\n"); return; @@ -1572,7 +1574,7 @@ static PyObject * _tracemalloc_get_traceback_limit_impl(PyObject *module) /*[clinic end generated code: output=d556d9306ba95567 input=da3cd977fc68ae3b]*/ { - return PyLong_FromLong(_Py_tracemalloc_config.max_nframe); + return PyLong_FromLong(tracemalloc_config.max_nframe); } @@ -1630,7 +1632,7 @@ _tracemalloc_get_traced_memory_impl(PyObject *module) { Py_ssize_t size, peak_size; - if (!_Py_tracemalloc_config.tracing) + if (!tracemalloc_config.tracing) return Py_BuildValue("ii", 0, 0); TABLES_LOCK(); @@ -1654,7 +1656,7 @@ static PyObject * _tracemalloc_reset_peak_impl(PyObject *module) /*[clinic end generated code: output=140c2870f691dbb2 input=18afd0635066e9ce]*/ { - if (!_Py_tracemalloc_config.tracing) { + if (!tracemalloc_config.tracing) { Py_RETURN_NONE; } @@ -1735,7 +1737,7 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int res; PyGILState_STATE gil_state; - if (!_Py_tracemalloc_config.tracing) { + if (!tracemalloc_config.tracing) { /* tracemalloc is not tracing: do nothing */ return -2; } @@ -1754,7 +1756,7 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { - if (!_Py_tracemalloc_config.tracing) { + if (!tracemalloc_config.tracing) { /* tracemalloc is not tracing: do nothing */ return -2; } @@ -1777,7 +1779,7 @@ _PyTraceMalloc_NewReference(PyObject *op) { assert(PyGILState_Check()); - if (!_Py_tracemalloc_config.tracing) { + if (!tracemalloc_config.tracing) { /* tracemalloc is not tracing: do nothing */ return -1; } diff --git a/Objects/object.c b/Objects/object.c index 01eede94d58b5d..a499cb32b22f58 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2001,7 +2001,7 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) void _Py_NewReference(PyObject *op) { - if (_Py_tracemalloc_config.tracing) { + if (_PyRuntime.tracemalloc.config.tracing) { _PyTraceMalloc_NewReference(op); } #ifdef Py_REF_DEBUG diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 481cbde9fd3170..4c08bc214cd27a 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -201,12 +201,6 @@ _PyMem_ArenaFree(void *Py_UNUSED(ctx), void *ptr, #endif -/* bpo-35053: Declare tracemalloc configuration here rather than - Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic - library, whereas _Py_NewReference() requires it. */ -struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT; - - #define _PyMem_Raw (_PyRuntime.allocators.standard.raw) #define _PyMem (_PyRuntime.allocators.standard.mem) #define _PyObject (_PyRuntime.allocators.standard.obj) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 76aa7282113406..c8b2a78e856d76 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -324,7 +324,6 @@ Objects/unicodeobject.c - ucnhash_capi - # state # object allocator -Objects/obmalloc.c - _Py_tracemalloc_config - Objects/obmalloc.c new_arena debug_stats - # pre-allocated memory From a0a961ef67ed7c928becc8c0355c609677179c79 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 14:45:31 -0700 Subject: [PATCH 12/15] Adjust the whitelist. --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index c8b2a78e856d76..95ae31381ccab2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -315,6 +315,7 @@ Python/hamt.c - _empty_hamt - # idempotent Python/dtoa.c - p5s - +Objects/obmalloc.c new_arena debug_stats - # others Python/perf_trampoline.c - perf_map_file - @@ -323,9 +324,6 @@ Objects/unicodeobject.c - ucnhash_capi - #----------------------- # state -# object allocator -Objects/obmalloc.c new_arena debug_stats - - # pre-allocated memory Python/dtoa.c - freelist - Python/dtoa.c - private_mem - From c7e4333177e9a2fd1d08c6990a9da23124b655a9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 15:05:44 -0700 Subject: [PATCH 13/15] Move the Bigint freelist to _PyRuntimeState. --- Include/internal/pycore_dtoa.h | 18 +++++++++++++++ Include/internal/pycore_runtime.h | 6 +++++ Python/dtoa.c | 25 +++++++++------------ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index c77cf6e46cc3c3..1b83d604bf94bb 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -1,3 +1,5 @@ +#ifndef Py_INTERNAL_DTOA_H +#define Py_INTERNAL_DTOA_H #ifdef __cplusplus extern "C" { #endif @@ -11,6 +13,21 @@ extern "C" { #if _PY_SHORT_FLOAT_REPR == 1 +typedef uint32_t ULong; + +struct +Bigint { + struct Bigint *next; + int k, maxwds, sign, wds; + ULong x[1]; +}; + +#ifndef Py_USING_MEMORY_DEBUGGER +/* The size of the Bigint freelist */ +# define Bigint_Kmax 7 +#endif + + /* These functions are used by modules compiled as C extension like math: they must be exported. */ @@ -26,3 +43,4 @@ PyAPI_FUNC(double) _Py_dg_infinity(int sign); #ifdef __cplusplus } #endif +#endif /* !Py_INTERNAL_DTOA_H */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f13dde62d4928b..f8f37d58790e11 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_dtoa.h" // struct Bigint #include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state @@ -128,6 +129,11 @@ typedef struct pyruntimestate { struct { struct _PyTraceMalloc_Config config; } tracemalloc; + struct { +#ifndef Py_USING_MEMORY_DEBUGGER + struct Bigint *freelist[Bigint_Kmax+1]; +#endif + } dtoa; PyPreConfig preconfig; diff --git a/Python/dtoa.c b/Python/dtoa.c index 733e70bc791698..f953c353cdf250 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -119,6 +119,7 @@ #include "Python.h" #include "pycore_dtoa.h" // _PY_SHORT_FLOAT_REPR +#include "pycore_runtime.h" // _PyRuntime #include // exit() /* if _PY_SHORT_FLOAT_REPR == 0, then don't even try to compile @@ -156,7 +157,7 @@ #endif -typedef uint32_t ULong; +// ULong is defined in pycore_dtoa.h. typedef int32_t Long; typedef uint64_t ULLong; @@ -298,8 +299,6 @@ BCinfo { #define FFFFFFFF 0xffffffffUL -#define Kmax 7 - /* struct Bigint is used to represent arbitrary-precision integers. These integers are stored in sign-magnitude format, with the magnitude stored as an array of base 2**32 digits. Bigints are always normalized: if x is a @@ -322,13 +321,7 @@ BCinfo { significant (x[0]) to most significant (x[wds-1]). */ -struct -Bigint { - struct Bigint *next; - int k, maxwds, sign, wds; - ULong x[1]; -}; - +// struct Bigint is defined in pycore_dtoa.h. typedef struct Bigint Bigint; #ifndef Py_USING_MEMORY_DEBUGGER @@ -352,7 +345,7 @@ typedef struct Bigint Bigint; Bfree to PyMem_Free. Investigate whether this has any significant performance on impact. */ -static Bigint *freelist[Kmax+1]; +#define freelist _PyRuntime.dtoa.freelist /* Allocate space for a Bigint with up to 1<next; else { x = 1 << k; len = (sizeof(Bigint) + (x-1)*sizeof(ULong) + sizeof(double) - 1) /sizeof(double); - if (k <= Kmax && pmem_next - private_mem + len <= (Py_ssize_t)PRIVATE_mem) { + if (k <= Bigint_Kmax && + pmem_next - private_mem + len <= (Py_ssize_t)PRIVATE_mem + ) { rv = (Bigint*)pmem_next; pmem_next += len; } @@ -391,7 +386,7 @@ static void Bfree(Bigint *v) { if (v) { - if (v->k > Kmax) + if (v->k > Bigint_Kmax) FREE((void*)v); else { v->next = freelist[v->k]; @@ -400,6 +395,8 @@ Bfree(Bigint *v) } } +#undef freelist + #else /* Alternative versions of Balloc and Bfree that use PyMem_Malloc and diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 95ae31381ccab2..ec3431d4d2b9be 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -325,7 +325,6 @@ Objects/unicodeobject.c - ucnhash_capi - # state # pre-allocated memory -Python/dtoa.c - freelist - Python/dtoa.c - private_mem - # local buffer From 5c4fde41cfab87b43bf49ec366a4d2e2e5b3f306 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 14 Nov 2022 15:34:26 -0700 Subject: [PATCH 14/15] Move the Bigint preallocated memory to _PyRuntimeState. --- Include/internal/pycore_dtoa.h | 29 +++++++++++++++++++-- Include/internal/pycore_runtime.h | 8 ++---- Include/internal/pycore_runtime_init.h | 1 + Python/dtoa.c | 12 ++++----- Tools/c-analyzer/cpython/globals-to-fix.tsv | 4 --- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index 1b83d604bf94bb..fdc6e74ecd25e3 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -22,10 +22,35 @@ Bigint { ULong x[1]; }; -#ifndef Py_USING_MEMORY_DEBUGGER +#ifdef Py_USING_MEMORY_DEBUGGER + +struct _dtoa_runtime_state { + int _not_used; +}; +#define _dtoa_runtime_state_INIT {0} + +#else // !Py_USING_MEMORY_DEBUGGER + /* The size of the Bigint freelist */ -# define Bigint_Kmax 7 +#define Bigint_Kmax 7 + +#ifndef PRIVATE_MEM +#define PRIVATE_MEM 2304 #endif +#define Bigint_PREALLOC_SIZE \ + ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) + +struct _dtoa_runtime_state { + struct Bigint *freelist[Bigint_Kmax+1]; + double preallocated[Bigint_PREALLOC_SIZE]; + double *preallocated_next; +}; +#define _dtoa_runtime_state_INIT(runtime) \ + { \ + .preallocated_next = runtime.dtoa.preallocated, \ + } + +#endif // !Py_USING_MEMORY_DEBUGGER /* These functions are used by modules compiled as C extension like math: diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f8f37d58790e11..d50455b8036999 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -9,7 +9,7 @@ extern "C" { #endif #include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_dtoa.h" // struct Bigint +#include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state @@ -129,11 +129,7 @@ typedef struct pyruntimestate { struct { struct _PyTraceMalloc_Config config; } tracemalloc; - struct { -#ifndef Py_USING_MEMORY_DEBUGGER - struct Bigint *freelist[Bigint_Kmax+1]; -#endif - } dtoa; + struct _dtoa_runtime_state dtoa; PyPreConfig preconfig; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ff91c2ce7ddd25..fd87f1eea79af3 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -39,6 +39,7 @@ extern "C" { .tracemalloc = { \ .config = _PyTraceMalloc_Config_INIT, \ }, \ + .dtoa = _dtoa_runtime_state_INIT(runtime), \ .types = { \ .next_version_tag = 1, \ }, \ diff --git a/Python/dtoa.c b/Python/dtoa.c index f953c353cdf250..1b47d83bf77a24 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -172,12 +172,6 @@ typedef uint64_t ULLong; #define Bug(x) {fprintf(stderr, "%s\n", x); exit(1);} #endif -#ifndef PRIVATE_MEM -#define PRIVATE_MEM 2304 -#endif -#define PRIVATE_mem ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) -static double private_mem[PRIVATE_mem], *pmem_next = private_mem; - #ifdef __cplusplus extern "C" { #endif @@ -346,6 +340,8 @@ typedef struct Bigint Bigint; performance on impact. */ #define freelist _PyRuntime.dtoa.freelist +#define private_mem _PyRuntime.dtoa.preallocated +#define pmem_next _PyRuntime.dtoa.preallocated_next /* Allocate space for a Bigint with up to 1< Date: Mon, 14 Nov 2022 15:50:25 -0700 Subject: [PATCH 15/15] Move static_arg_parsers into _PyRuntimeState. --- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_runtime.h | 3 ++- Python/getargs.c | 11 ++++------- Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 --- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 976e16a3742bda..96e7008d4c0e02 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -26,6 +26,7 @@ extern "C" { #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state + struct _pending_calls { PyThread_type_lock lock; /* Request for running pending calls. */ diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d50455b8036999..8b2b9d7a85b231 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -19,7 +19,8 @@ extern "C" { #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { - PyThread_type_lock mutex; + PyThread_type_lock mutex; + struct _PyArg_Parser *static_parsers; }; /* ceval state */ diff --git a/Python/getargs.c b/Python/getargs.c index 703462242a0585..748209d7d713f8 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1846,9 +1846,6 @@ vgetargskeywords(PyObject *args, PyObject *kwargs, const char *format, } -/* List of static parsers. */ -static struct _PyArg_Parser *static_arg_parsers = NULL; - static int scan_keywords(const char * const *keywords, int *ptotal, int *pposonly) { @@ -2024,8 +2021,8 @@ _parser_init(struct _PyArg_Parser *parser) parser->initialized = owned ? 1 : -1; assert(parser->next == NULL); - parser->next = static_arg_parsers; - static_arg_parsers = parser; + parser->next = _PyRuntime.getargs.static_parsers; + _PyRuntime.getargs.static_parsers = parser; return 1; } @@ -2930,14 +2927,14 @@ _PyArg_NoKwnames(const char *funcname, PyObject *kwnames) void _PyArg_Fini(void) { - struct _PyArg_Parser *tmp, *s = static_arg_parsers; + struct _PyArg_Parser *tmp, *s = _PyRuntime.getargs.static_parsers; while (s) { tmp = s->next; s->next = NULL; parser_clear(s); s = tmp; } - static_arg_parsers = NULL; + _PyRuntime.getargs.static_parsers = NULL; } #ifdef __cplusplus diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 77fc09e88830dd..b4ac9120d98bb0 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -327,9 +327,6 @@ Objects/unicodeobject.c - ucnhash_capi - # local buffer Python/suggestions.c levenshtein_distance buffer - -# linked list -Python/getargs.c - static_arg_parsers - - # other Objects/dictobject.c - _pydict_global_version - Objects/dictobject.c - next_dict_keys_version -