From 1b41c7b722b77d9c02c670edd322b088860c00ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Jul 2026 01:29:53 +0200 Subject: [PATCH 1/3] gh-152132: Enhance Py_RunMain() and fix bugs (#153461) * Check for signals more often. Previously, a pending exception could be removed by PyErr_Clear(). * Only call _PyInterpreterState_SetNotRunningMain() if _PyInterpreterState_SetRunningMain() has been called. * Add pymain_error() and pyrun_error(): write the error message to sys.stderr instead of the C stream stderr. * Convert _PyPathConfig_UpdateGlobal() PyStatus error to an exception. * Simplify pymain_run_startup(): avoid strerror() which is decoded from the wrong encoding. Instead display the exception, Py_fopen() raises an OSError with the path object (and can raise other exceptions). * Add pymain_set_path0() function. * Add _PySys_FormatV() function. --- Include/internal/pycore_sysmodule.h | 6 + Lib/test/test_cmd_line.py | 2 +- Modules/main.c | 332 +++++++++++++++++----------- Python/pythonrun.c | 18 +- Python/sysmodule.c | 9 +- 5 files changed, 231 insertions(+), 136 deletions(-) diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 347b0a7a790c066..d86599cbc3a0452 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -21,6 +21,12 @@ extern int _PySys_SetIntMaxStrDigits(int maxdigits); extern int _PySysRemoteDebug_SendExec(int pid, int tid, const char *debugger_script_path); +extern void _PySys_FormatV( + PyObject *key, + FILE *fp, + const char *format, + va_list va); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index a8645af26b25d87..7640e50f19b7839 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -314,7 +314,7 @@ def test_undecodable_code(self): # decodable from ASCII) and run_command() failed on # PyUnicode_AsUTF8String(). This is the expected behaviour on # Linux. - pattern = b"Unable to decode the command from the command line:" + pattern = b"python: Unable to decode the command from the command line:" elif p.returncode == 0: # _Py_char2wchar() decoded b'\xff' as '\xff' even if the locale is # C and the locale encoding is ASCII. It occurs on FreeBSD, Solaris diff --git a/Modules/main.c b/Modules/main.c index a65247dc1e5c617..ef22331760906ce 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -11,6 +11,7 @@ #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_pythonrun.h" // _PyRun_AnyFile() +#include "pycore_sysmodule.h" // _PySys_FormatV() #include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_unicodeobject.h" // _PyUnicode_Dedent() @@ -80,6 +81,32 @@ pymain_init(const _PyArgv *args) /* --- pymain_run_python() ---------------------------------------- */ +static void +pymain_error_format(const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + + PySys_WriteStderr("python: "); + _PySys_FormatV(&_Py_ID(stderr), stderr, format, vargs); + PySys_WriteStderr("\n"); + + va_end(vargs); +} + +// See also pyrun_error() +static void +pymain_error(const char *msg) +{ + PySys_FormatStderr("python: %s\n", msg); +} + +static int +pymain_check_signals(void) +{ + return Py_MakePendingCalls(); +} + /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ static inline int config_run_code(const PyConfig *config) @@ -122,19 +149,29 @@ pymain_exit_err_print(void) } -/* Write an exitcode into *exitcode and return 1 if we have to exit Python. - Return 0 otherwise. */ +// If filename is a package (ex: directory or ZIP file) which contains +// __main__.py, main_importer_path is set to filename and will be prepended to +// sys.path. +// +// Otherwise, main_importer_path is left unchanged. +// +// Write an exitcode into *exitcode and return 1 if we have to exit Python. +// Return 0 otherwise. static int -pymain_get_importer(const wchar_t *filename, PyObject **importer_p, int *exitcode) +pymain_get_importer(const PyConfig *config, PyObject **importer_p, + int *exitcode) { - PyObject *sys_path0 = NULL, *importer; + const wchar_t *filename = config->run_filename; + if (filename == NULL) { + return 0; + } - sys_path0 = PyUnicode_FromWideChar(filename, -1); + PyObject *sys_path0 = PyUnicode_FromWideChar(filename, -1); if (sys_path0 == NULL) { goto error; } - importer = PyImport_GetImporter(sys_path0); + PyObject *importer = PyImport_GetImporter(sys_path0); if (importer == NULL) { goto error; } @@ -152,15 +189,16 @@ pymain_get_importer(const wchar_t *filename, PyObject **importer_p, int *exitcod error: Py_XDECREF(sys_path0); - PySys_WriteStderr("Failed checking if argv[0] is an import path entry\n"); + pymain_error("Failed checking if argv[0] is an import path entry:"); return pymain_err_print(exitcode); } static int -pymain_sys_path_add_path0(PyInterpreterState *interp, PyObject *path0) +pymain_sys_path_add_path0(PyObject *path0) { PyObject *sys_path; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *sysdict = interp->sysdict; if (sysdict != NULL) { sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); @@ -201,17 +239,21 @@ pymain_header(const PyConfig *config) } -static void +static int pymain_import_readline(const PyConfig *config) { + if (pymain_check_signals() < 0) { + return -1; + } + if (config->isolated) { - return; + return 0; } if (!config->inspect && config_run_code(config)) { - return; + return 0; } if (!isatty(fileno(stdin))) { - return; + return 0; } PyObject *mod = PyImport_ImportModule("readline"); @@ -221,6 +263,7 @@ pymain_import_readline(const PyConfig *config) else { Py_DECREF(mod); } + mod = PyImport_ImportModule("rlcompleter"); if (mod == NULL) { PyErr_Clear(); @@ -228,6 +271,7 @@ pymain_import_readline(const PyConfig *config) else { Py_DECREF(mod); } + return 0; } @@ -272,7 +316,7 @@ pymain_run_command(wchar_t *command) return 0; error: - PySys_WriteStderr("Unable to decode the command from the command line:\n"); + pymain_error("Unable to decode the command from the command line:"); return pymain_exit_err_print(); } @@ -289,13 +333,13 @@ pymain_run_pyrepl(int pythonstartup) PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); if (pyrepl == NULL) { - fprintf(stderr, "Could not import _pyrepl.main\n"); + pymain_error("Could not import _pyrepl.main"); goto error; } console = PyObject_GetAttrString(pyrepl, "interactive_console"); if (console == NULL) { - fprintf(stderr, "Could not access _pyrepl.main.interactive_console\n"); + pymain_error("Could not access _pyrepl.main.interactive_console"); goto error; } @@ -343,45 +387,56 @@ pymain_run_pyrepl(int pythonstartup) static int pymain_run_module(const wchar_t *modname, int set_argv0) { - PyObject *module, *runmodule, *runargs, *result; + int exitcode = 0; + PyObject *module = NULL; + PyObject *runmodule = NULL; + PyObject *runargs = NULL; + PyObject *result = NULL; + if (PySys_Audit("cpython.run_module", "u", modname) < 0) { - return pymain_exit_err_print(); + goto error; } + runmodule = PyImport_ImportModuleAttrString("runpy", "_run_module_as_main"); if (runmodule == NULL) { - fprintf(stderr, "Could not import runpy._run_module_as_main\n"); - return pymain_exit_err_print(); + pymain_error("Could not import runpy._run_module_as_main"); + goto error; } + module = PyUnicode_FromWideChar(modname, -1); if (module == NULL) { - fprintf(stderr, "Could not convert module name to unicode\n"); - Py_DECREF(runmodule); - return pymain_exit_err_print(); + pymain_error("Could not convert module name to unicode"); + goto error; } + runargs = _PyTuple_FromPair(module, set_argv0 ? Py_True : Py_False); if (runargs == NULL) { - fprintf(stderr, - "Could not create arguments for runpy._run_module_as_main\n"); - Py_DECREF(runmodule); - Py_DECREF(module); - return pymain_exit_err_print(); + pymain_error("Could not create arguments " + "for runpy._run_module_as_main"); + goto error; } + result = PyObject_Call(runmodule, runargs, NULL); - Py_DECREF(runmodule); - Py_DECREF(module); - Py_DECREF(runargs); if (result == NULL) { - return pymain_exit_err_print(); + goto error; } - Py_DECREF(result); - return 0; + +done: + Py_XDECREF(module); + Py_XDECREF(runmodule); + Py_XDECREF(runargs); + Py_XDECREF(result); + return exitcode; + +error: + exitcode = pymain_exit_err_print(); + goto done; } static int -pymain_run_file_obj(PyObject *program_name, PyObject *filename, - int skip_source_first_line) +pymain_run_file_obj(PyObject *filename, int skip_source_first_line) { if (PySys_Audit("cpython.run_file", "O", filename) < 0) { return pymain_exit_err_print(); @@ -389,11 +444,8 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, FILE *fp = Py_fopen(filename, "rb"); if (fp == NULL) { - // Ignore the OSError - PyErr_Clear(); - // TODO(picnixz): strerror() is locale dependent but not PySys_FormatStderr(). - PySys_FormatStderr("%S: can't open file %R: [Errno %d] %s\n", - program_name, filename, errno, strerror(errno)); + pymain_error_format("can't open file %R:", filename); + pymain_exit_err_print(); return 2; } @@ -410,14 +462,13 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, struct _Py_stat_struct sb; if (_Py_fstat_noraise(fileno(fp), &sb) == 0 && S_ISDIR(sb.st_mode)) { - PySys_FormatStderr("%S: %R is a directory, cannot continue\n", - program_name, filename); + pymain_error_format("%R is a directory, cannot continue", + filename); fclose(fp); return 1; } - // Call pending calls like signal handlers (SIGINT) - if (Py_MakePendingCalls() == -1) { + if (pymain_check_signals() < 0) { fclose(fp); return pymain_exit_err_print(); } @@ -439,16 +490,9 @@ pymain_run_file(const PyConfig *config) if (filename == NULL) { return pymain_exit_err_print(); } - PyObject *program_name = PyUnicode_FromWideChar(config->program_name, -1); - if (program_name == NULL) { - Py_DECREF(filename); - return pymain_exit_err_print(); - } - int exitcode = pymain_run_file_obj(program_name, filename, - config->skip_source_first_line); - Py_DECREF(filename); - Py_DECREF(program_name); + int exitcode = pymain_run_file_obj(filename, config->skip_source_first_line); + Py_XDECREF(filename); return exitcode; } @@ -457,15 +501,17 @@ static int pymain_run_startup(PyConfig *config, int *exitcode) { int ret = 0; + PyObject *startup = NULL; + if (!config->use_environment) { - return 0; + goto done; } - PyObject *startup = NULL; #ifdef MS_WINDOWS const wchar_t *env = _wgetenv(L"PYTHONSTARTUP"); if (env == NULL || env[0] == L'\0') { - return 0; + goto done; } + startup = PyUnicode_FromWideChar(env, -1); if (startup == NULL) { goto error; @@ -473,8 +519,9 @@ pymain_run_startup(PyConfig *config, int *exitcode) #else const char *env = _Py_GetEnv(config->use_environment, "PYTHONSTARTUP"); if (env == NULL) { - return 0; + goto done; } + startup = PyUnicode_DecodeFSDefault(env); if (startup == NULL) { goto error; @@ -486,12 +533,7 @@ pymain_run_startup(PyConfig *config, int *exitcode) FILE *fp = Py_fopen(startup, "r"); if (fp == NULL) { - int save_errno = errno; - PyErr_Clear(); - PySys_WriteStderr("Could not open PYTHONSTARTUP\n"); - - errno = save_errno; - PyErr_SetFromErrnoWithFilenameObjects(PyExc_OSError, startup, NULL); + pymain_error("Could not open PYTHONSTARTUP"); goto error; } @@ -537,53 +579,55 @@ pymain_run_interactive_hook(int *exitcode) goto error; } Py_DECREF(result); - return 0; error: - PySys_WriteStderr("Failed calling sys.__interactivehook__\n"); + pymain_error("Failed calling sys.__interactivehook__:"); return pymain_err_print(exitcode); } -static void -pymain_set_inspect(PyConfig *config, int inspect) +static int +pymain_set_inspect(PyConfig *config, int inspect, int *exitcode) { PyObject *value = PyLong_FromLong(inspect); if (value == NULL || PyConfig_Set("inspect", value) < 0) { - fprintf(stderr, "Could not set the inspect flag\n"); - PyErr_Print(); + Py_XDECREF(value); + pymain_error("Could not set the inspect flag"); + return pymain_err_print(exitcode); } else { assert(config->inspect == inspect); } Py_XDECREF(value); + return 0; } static int _pymain_run_repl(PyConfig *config, int startup) { - /* call pending calls like signal handlers (SIGINT) */ - if (Py_MakePendingCalls() == -1) { - return pymain_exit_err_print(); + if (pymain_check_signals() < 0) { + goto error; } if (PySys_Audit("cpython.run_stdin", NULL) < 0) { - return pymain_exit_err_print(); + goto error; } if (isatty(fileno(stdin)) - && !_Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + && !_Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) + { PyObject *pyrepl = PyImport_ImportModule("_pyrepl"); if (pyrepl != NULL) { int exitcode = pymain_run_pyrepl(startup); Py_DECREF(pyrepl); return exitcode; } + if (!PyErr_ExceptionMatches(PyExc_ModuleNotFoundError)) { - fprintf(stderr, "Could not import _pyrepl.main\n"); - return pymain_exit_err_print(); + pymain_error("Could not import _pyrepl.main"); + goto error; } PyErr_Clear(); } @@ -596,10 +640,13 @@ _pymain_run_repl(PyConfig *config, int startup) Py_DECREF(stdin_obj); } if (result == NULL) { - return pymain_exit_err_print(); + goto error; } Py_DECREF(result); return 0; + +error: + return pymain_exit_err_print(); } static int @@ -607,9 +654,11 @@ pymain_run_stdin(PyConfig *config) { if (stdin_is_interactive(config)) { // do exit on SystemExit - pymain_set_inspect(config, 0); - int exitcode; + if (pymain_set_inspect(config, 0, &exitcode)) { + return exitcode; + } + if (pymain_run_startup(config, &exitcode)) { return exitcode; } @@ -629,14 +678,18 @@ pymain_repl(PyConfig *config, int *exitcode) /* Check this environment variable at the end, to give programs the opportunity to set it from Python. */ if (!config->inspect && _Py_GetEnv(config->use_environment, "PYTHONINSPECT")) { - pymain_set_inspect(config, 1); + if (pymain_set_inspect(config, 1, exitcode)) { + return; + } } if (!(config->inspect && stdin_is_interactive(config) && config_run_code(config))) { return; } - pymain_set_inspect(config, 0); + if (pymain_set_inspect(config, 0, exitcode)) { + return; + } if (pymain_run_interactive_hook(exitcode)) { return; } @@ -645,16 +698,68 @@ pymain_repl(PyConfig *config, int *exitcode) } +static int +pymain_set_path0(PyObject *main_importer_path) +{ + PyConfig *config = (PyConfig *)_Py_GetConfig(); + + PyObject *path0 = NULL; + if (main_importer_path != NULL) { + path0 = Py_NewRef(main_importer_path); + } + else if (!config->safe_path) { + int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); + if (res < 0) { + return -1; + } + else if (res == 0) { + Py_CLEAR(path0); + } + } + + if (path0 == NULL) { + return 0; + } + + // XXX Apply config->sys_path_0 in init_interp_main(). We have + // to be sure to get readline/rlcompleter imported at the correct time. + wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); + if (wstr == NULL) { + goto error; + } + + PyStatus status = PyConfig_SetString(config, &config->sys_path_0, wstr); + PyMem_Free(wstr); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + goto error; + } + + int res = pymain_sys_path_add_path0(path0); + Py_DECREF(path0); + + return res; + +error: + Py_DECREF(path0); + return -1; +} + + static void pymain_run_python(int *exitcode) { + int set_running_main = 0; + PyObject *main_importer_path = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); /* pymain_repl() and pymain_run_stdin() modify the config */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); /* ensure path config is written into global variables */ - if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) { + PyStatus status = _PyPathConfig_UpdateGlobal(config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); goto error; } @@ -663,60 +768,29 @@ pymain_run_python(int *exitcode) // at that point. assert(config->sys_path_0 == NULL); - if (config->run_filename != NULL) { - /* If filename is a package (ex: directory or ZIP file) which contains - __main__.py, main_importer_path is set to filename and will be - prepended to sys.path. - - Otherwise, main_importer_path is left unchanged. */ - if (pymain_get_importer(config->run_filename, &main_importer_path, - exitcode)) { - return; - } + if (pymain_get_importer(config, &main_importer_path, exitcode)) { + return; } // import readline and rlcompleter before script dir is added to sys.path - pymain_import_readline(config); - - PyObject *path0 = NULL; - if (main_importer_path != NULL) { - path0 = Py_NewRef(main_importer_path); - } - else if (!config->safe_path) { - int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); - if (res < 0) { - goto error; - } - else if (res == 0) { - Py_CLEAR(path0); - } + if (pymain_import_readline(config) < 0) { + goto error; } - // XXX Apply config->sys_path_0 in init_interp_main(). We have - // to be sure to get readline/rlcompleter imported at the correct time. - if (path0 != NULL) { - wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); - if (wstr == NULL) { - Py_DECREF(path0); - goto error; - } - config->sys_path_0 = _PyMem_RawWcsdup(wstr); - PyMem_Free(wstr); - if (config->sys_path_0 == NULL) { - Py_DECREF(path0); - goto error; - } - int res = pymain_sys_path_add_path0(interp, path0); - Py_DECREF(path0); - if (res < 0) { - goto error; - } + + if (pymain_set_path0(main_importer_path) < 0) { + goto error; } pymain_header(config); _PyInterpreterState_SetRunningMain(interp); + set_running_main = 1; assert(!PyErr_Occurred()); + if (pymain_check_signals() < 0) { + goto error; + } + if (config->run_command) { *exitcode = pymain_run_command(config->run_command); } @@ -733,6 +807,10 @@ pymain_run_python(int *exitcode) *exitcode = pymain_run_stdin(config); } + if (pymain_check_signals() < 0) { + goto error; + } + pymain_repl(config, exitcode); goto done; @@ -740,7 +818,9 @@ pymain_run_python(int *exitcode) *exitcode = pymain_exit_err_print(); done: - _PyInterpreterState_SetNotRunningMain(interp); + if (set_running_main) { + _PyInterpreterState_SetNotRunningMain(interp); + } Py_XDECREF(main_importer_path); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index e0a752720dfcc2d..5abfdc04538c369 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -60,6 +60,14 @@ _PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags); +// See also pymain_error() +static void +pyrun_error(const char *msg) +{ + PySys_FormatStderr("python: %s\n", msg); +} + + PyObject* _PyRun_AnyFile(FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags) @@ -496,7 +504,7 @@ _PyRun_SimpleFile(FILE *fp, PyObject *filename, int closeit, } if (!has_file) { if (PyDict_SetItemString(dict, "__file__", filename) < 0) { - fprintf(stderr, "python: failed to set __main__.__file__\n"); + pyrun_error("failed to set __main__.__file__"); goto done; } set_file_name = 1; @@ -516,12 +524,12 @@ _PyRun_SimpleFile(FILE *fp, PyObject *filename, int closeit, pyc_fp = Py_fopen(filename, "rb"); if (pyc_fp == NULL) { - fprintf(stderr, "python: Can't reopen .pyc file\n"); + pyrun_error("Can't reopen .pyc file"); goto done; } if (set_main_loader(dict, filename, "SourcelessFileLoader") < 0) { - fprintf(stderr, "python: failed to set __main__.__loader__\n"); + pyrun_error("failed to set __main__.__loader__"); fclose(pyc_fp); goto done; } @@ -530,7 +538,7 @@ _PyRun_SimpleFile(FILE *fp, PyObject *filename, int closeit, /* When running from stdin, leave __main__.__loader__ alone */ if ((!PyUnicode_Check(filename) || !PyUnicode_EqualToUTF8(filename, "")) && set_main_loader(dict, filename, "SourceFileLoader") < 0) { - fprintf(stderr, "python: failed to set __main__.__loader__\n"); + pyrun_error("failed to set __main__.__loader__"); goto done; } res = _PyRun_File(fp, filename, Py_file_input, dict, dict, @@ -541,7 +549,7 @@ _PyRun_SimpleFile(FILE *fp, PyObject *filename, int closeit, done: if (set_file_name) { if (PyDict_PopString(dict, "__file__", NULL) < 0) { - fprintf(stderr, "python: failed to delete __main__.__file__\n"); + pyrun_error("failed to delete __main__.__file__"); Py_CLEAR(res); } } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d9f7b9c449cfb94..9442472b53abbe1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -4641,8 +4641,8 @@ PySys_WriteStderr(const char *format, ...) va_end(va); } -static void -sys_format(PyObject *key, FILE *fp, const char *format, va_list va) +void +_PySys_FormatV(PyObject *key, FILE *fp, const char *format, va_list va) { PyObject *file, *message; const char *utf8; @@ -4664,13 +4664,14 @@ sys_format(PyObject *key, FILE *fp, const char *format, va_list va) _PyErr_SetRaisedException(tstate, exc); } + void PySys_FormatStdout(const char *format, ...) { va_list va; va_start(va, format); - sys_format(&_Py_ID(stdout), stdout, format, va); + _PySys_FormatV(&_Py_ID(stdout), stdout, format, va); va_end(va); } @@ -4680,7 +4681,7 @@ PySys_FormatStderr(const char *format, ...) va_list va; va_start(va, format); - sys_format(&_Py_ID(stderr), stderr, format, va); + _PySys_FormatV(&_Py_ID(stderr), stderr, format, va); va_end(va); } From d83d26724b8c3be5f535e56ded88593cc6803b86 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Jul 2026 03:36:31 +0200 Subject: [PATCH 2/3] gh-152132: Test Py_CompileString() in test_capi.test_run (#153470) PyRun functions now raises ValueError if the start argument is invalid. * Add Modules/_testlimitedcapi/run.c. * Rename _Py_CompileStringObjectWithModule() to _Py_CompileStringObject(). --- Include/cpython/pythonrun.h | 5 + Include/internal/pycore_pythonrun.h | 2 +- Lib/test/test_capi/test_run.py | 74 ++++++++++++- Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/run.c | 126 ++++++++++++++++------- Modules/_testlimitedcapi.c | 3 + Modules/_testlimitedcapi/parts.h | 1 + Modules/_testlimitedcapi/run.c | 35 +++++++ PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + Python/bltinmodule.c | 6 +- Python/pythonrun.c | 33 +++++- 12 files changed, 243 insertions(+), 46 deletions(-) create mode 100644 Modules/_testlimitedcapi/run.c diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index edc40952254029f..b0bf527f4f0ec19 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags( PyCompilerFlags *flags); +PyAPI_FUNC(PyObject *) Py_CompileStringFlags( + const char *str, + const char *filename, + int start, + PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( const char *str, const char *filename, /* decoded from the filesystem encoding */ diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index b2126f3f71bb46e..d333eb2ccf7c413 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -28,7 +28,7 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); -extern PyObject * _Py_CompileStringObjectWithModule( +extern PyObject * _Py_CompileString( const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize, diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index d0418cdee628677..14f9833f63a4fb8 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -1,20 +1,27 @@ +import ast import os import sys import tempfile +import textwrap import unittest from collections import UserDict from test import support from test.support import import_helper from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + NULL = None -_testcapi = import_helper.import_module('_testcapi') Py_single_input = _testcapi.Py_single_input Py_file_input = _testcapi.Py_file_input Py_eval_input = _testcapi.Py_eval_input +INVALID_START = Py_single_input - 1 STDIN = '' STDERR_FD = 2 +PyCF_ONLY_AST = _testcapi.PyCF_ONLY_AST +PyCF_IGNORE_COOKIE = _testcapi.PyCF_IGNORE_COOKIE # Code raising a SyntaxError SYNTAX_ERROR = 'True = 1' @@ -68,6 +75,9 @@ def run(s, *args): run(b'raise ValueError("BUG")', {}) self.assertEqual(str(cm.exception), 'BUG') + with self.assertRaises(ValueError): + func(b'x = 1', INVALID_START, {}) + self.assertIsNone(run(b'a\n', dict(a=1))) self.assertIsNone(run(b'a\n', dict(a=1), {})) self.assertIsNone(run(b'a\n', {}, dict(a=1))) @@ -118,6 +128,9 @@ def run(*args): closeit = 1 self.assertIsNone(run(dict(a=1), {}, closeit)) + with self.assertRaises(ValueError): + func(filename, INVALID_START, {}) + self.assertRaises(NameError, run, {}) self.assertRaises(NameError, run, {}, {}) self.assertRaises(TypeError, run, dict(a=1), []) @@ -357,6 +370,65 @@ def test_run_simplestringflags(self): # Test PyRun_SimpleStringFlags() self.check_run_simplestring(_testcapi.run_simplestringflags) + def check_compilestring(self, compilestring, has_flags, encode_filename=True): + filename_str = TESTFN + if encode_filename: + filename = os.fsencode(filename_str) + else: + filename = filename_str + + def check_code(co, name, value): + ns = {} + exec(co, ns, ns) + self.assertEqual(ns[name], value) + + co = compilestring(b'x = 1', filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 1) + + if has_flags: + code = textwrap.dedent(""" + # encoding: latin1 + x = 'a\xe9' + """) + co = compilestring(code.encode(), filename, Py_file_input, PyCF_IGNORE_COOKIE) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xe9') + + co = compilestring(code.encode(), filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xc3\xa9') + + tree = compilestring(b'x = 1', filename, Py_file_input, PyCF_ONLY_AST) + self.assertIsInstance(tree, ast.AST) + + co = compilestring(b'raise ValueError("BUG")', filename, Py_file_input) + with self.assertRaises(ValueError): + exec(co, {}) + + with self.assertRaises(SyntaxError) as cm: + compilestring(SYNTAX_ERROR.encode(), filename, Py_file_input) + + with self.assertRaises(ValueError): + compilestring(b'x = 1', filename, INVALID_START) + + def test_compilestring(self): + # Test Py_CompileString() + self.check_compilestring(_testlimitedcapi.run_compilestring, False) + + def test_compilestringflags(self): + # Test Py_CompileStringFlags() + self.check_compilestring(_testcapi.run_compilestringflags, True) + + def test_compilestringexflags(self): + # Test Py_CompileStringExFlags() + self.check_compilestring(_testcapi.run_compilestringexflags, True) + + def test_compilestringobject(self): + # Test Py_CompileStringObject() + self.check_compilestring(_testcapi.run_compilestringobject, True, + encode_filename=False) + if __name__ == '__main__': unittest.main() diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8efea27824f0e88..524f1466b191f64 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index 7fc180e136559ba..6812af994a43302 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -24,34 +24,16 @@ #undef PyRun_InteractiveLoop -// Some PyRun functions crash if start is invalid, -// so validate the start argument. -static int -check_start(int start) -{ - if (start == Py_single_input || start == Py_file_input - || start == Py_eval_input || start == Py_func_type_input) - { - return 0; - } - PyErr_SetString(PyExc_ValueError, "invalid start argument"); - return -1; -} - - // Test PyRun_String() static PyObject* run_string(PyObject *mod, PyObject *args) { const char *str; - int start = 0; + int start; PyObject *globals = NULL; PyObject *locals = NULL; - if (!PyArg_ParseTuple(args, "y|iOO", &str, &start, &globals, &locals)) { - return NULL; - } - if (check_start(start) < 0) { + if (!PyArg_ParseTuple(args, "yi|OO", &str, &start, &globals, &locals)) { return NULL; } NULLABLE(globals); @@ -79,9 +61,6 @@ run_stringflags(PyObject *mod, PyObject *args) &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -481,9 +460,6 @@ run_file(PyObject *mod, PyObject *args) &globals, &locals)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); @@ -518,9 +494,6 @@ run_fileex(PyObject *mod, PyObject *args) &closeit)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); @@ -560,9 +533,6 @@ run_fileflags(PyObject *mod, PyObject *args) &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -606,9 +576,6 @@ run_fileexflags(PyObject *mod, PyObject *args) &closeit, &cf_flags, &cf_feature_version)) { return NULL; } - if (check_start(start) < 0) { - return NULL; - } NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -733,6 +700,86 @@ run_interactiveloopflags(PyObject *mod, PyObject *args) } +// Test Py_CompileStringFlags() +static PyObject* +run_compilestringflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(args, "yyi|ii", &str, &filename, &start, + &cf_flags, &cf_feature_version)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringFlags(str, filename, start, pflags); +} + + +// Test Py_CompileStringExFlags() +static PyObject* +run_compilestringexflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yyi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringExFlags(str, filename, start, pflags, optimize); +} + + +// Test Py_CompileStringObject() +static PyObject* +run_compilestringobject(PyObject *mod, PyObject *args) +{ + const char *str; + PyObject *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yOi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringObject(str, filename, start, pflags, optimize); +} + + static PyMethodDef test_methods[] = { {"run_string", run_string, METH_VARARGS}, {"run_stringflags", run_stringflags, METH_VARARGS}, @@ -754,6 +801,9 @@ static PyMethodDef test_methods[] = { {"run_simplestringflags", run_simplestringflags, METH_VARARGS}, {"run_interactiveloop", run_interactiveloop, METH_VARARGS}, {"run_interactiveloopflags", run_interactiveloopflags, METH_VARARGS}, + {"run_compilestringflags", run_compilestringflags, METH_VARARGS}, + {"run_compilestringexflags", run_compilestringexflags, METH_VARARGS}, + {"run_compilestringobject", run_compilestringobject, METH_VARARGS}, {NULL}, }; @@ -763,5 +813,11 @@ _PyTestCapi_Init_Run(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } + if (PyModule_AddIntMacro(mod, PyCF_ONLY_AST) < 0) { + return -1; + } + if (PyModule_AddIntMacro(mod, PyCF_IGNORE_COOKIE) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 9314fccc6c915a4..8ce704502af0104 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -101,5 +101,8 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Run(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index c51d285e19ab0d6..a11e0edb8311e31 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -46,5 +46,6 @@ int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); int _PyTestLimitedCAPI_Init_Version(PyObject *module); int _PyTestLimitedCAPI_Init_File(PyObject *module); int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); +int _PyTestLimitedCAPI_Init_Run(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/run.c b/Modules/_testlimitedcapi/run.c new file mode 100644 index 000000000000000..ca12e5faa5b3f96 --- /dev/null +++ b/Modules/_testlimitedcapi/run.c @@ -0,0 +1,35 @@ +#include "parts.h" +#include "util.h" + + +// Test functions, not macros +#undef Py_CompileString + + +// Test Py_CompileString() +static PyObject* +run_compilestring(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + + if (!PyArg_ParseTuple(args, "yyi", &str, &filename, &start)) { + return NULL; + } + + return Py_CompileString(str, filename, start); +} + + +static PyMethodDef test_methods[] = { + {"run_compilestring", run_compilestring, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Run(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 69558d204dbb8e7..7ddcee6d9735ce6 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -118,6 +118,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 2bcc3f6ff176bd9..66a0a47d8e5548b 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -34,6 +34,7 @@ + diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index fa64255be00e75d..cbe59c8883d5a57 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -960,9 +960,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = _Py_CompileStringObjectWithModule(str, filename, - start[compile_mode], &cf, - optimize, modname); + result = _Py_CompileString(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5abfdc04538c369..049eaea4994ac46 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1240,11 +1240,27 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } +static int +check_start(int start) +{ + if (start == Py_single_input || start == Py_file_input + || start == Py_eval_input || start == Py_func_type_input) + { + return 0; + } + PyErr_SetString(PyExc_ValueError, "invalid start argument"); + return -1; +} + static PyObject * _PyRun_String(const char *str, PyObject* name, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, int generate_new_source) { + if (check_start(start) < 0) { + return NULL; + } + PyObject *ret = NULL; mod_ty mod; PyArena *arena; @@ -1294,6 +1310,10 @@ static PyObject * _PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) { + if (check_start(start) < 0) { + return NULL; + } + PyArena *arena = _PyArena_New(); if (arena == NULL) { return NULL; @@ -1538,14 +1558,17 @@ PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) { - return _Py_CompileStringObjectWithModule(str, filename, start, - flags, optimize, NULL); + return _Py_CompileString(str, filename, start, flags, optimize, NULL); } PyObject * -_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, - PyCompilerFlags *flags, int optimize, PyObject *module) +_Py_CompileString(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { + if (check_start(start) < 0) { + return NULL; + } + PyCodeObject *co; mod_ty mod; PyArena *arena = _PyArena_New(); @@ -1754,7 +1777,7 @@ Py_CompileString(const char *str, const char *p, int s) } #undef Py_CompileStringFlags -PyAPI_FUNC(PyObject *) +PyObject* Py_CompileStringFlags(const char *str, const char *p, int s, PyCompilerFlags *flags) { From 6c389c446e487bcb9d09e85de30aa42052990f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Fri, 10 Jul 2026 10:38:58 +0800 Subject: [PATCH 3/3] gh-153062: Fix a crash iterating itertools.tee on the free-threaded build (gh-153063) itertools.tee branches share a linked list of teedataobject cells. On the free-threaded build, iterating one branch from multiple threads, or iterating sibling branches concurrently, raced on the shared cells and each branch's position, corrupting refcounts and crashing. Lock each teedataobject while reading, extending, or clearing it, and snapshot each branch's position under the tee object's own lock, revalidating before advancing, so the two locks are never nested. Concurrent iteration of one tee is undefined and may raise RuntimeError as documented, but no longer crashes. The free-threading snapshot and revalidation add per-element overhead on the default build, where the GIL already serializes access. Guard that path under Py_GIL_DISABLED so the default build keeps the original iteration and the free-threaded path is unchanged. Co-authored-by: Neil Schemenauer --- .../test_free_threading/test_itertools.py | 88 +++++++++++++++++-- ...-07-05-14-30-00.gh-issue-153062.teeFT1.rst | 2 + Modules/itertoolsmodule.c | 82 +++++++++++++++-- 3 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py index 670d4ca8835e0d2..7392bd739acd709 100644 --- a/Lib/test/test_free_threading/test_itertools.py +++ b/Lib/test/test_free_threading/test_itertools.py @@ -1,5 +1,14 @@ import unittest -from itertools import accumulate, batched, chain, combinations_with_replacement, cycle, permutations, zip_longest +from itertools import ( + accumulate, + batched, + chain, + combinations_with_replacement, + cycle, + permutations, + tee, + zip_longest, +) from test.support import threading_helper @@ -15,20 +24,23 @@ def work_iterator(it): class ItertoolsThreading(unittest.TestCase): - @threading_helper.reap_threads def test_accumulate(self): number_of_iterations = 10 for _ in range(number_of_iterations): it = accumulate(tuple(range(40))) - threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=10, args=[it] + ) @threading_helper.reap_threads def test_batched(self): number_of_iterations = 10 for _ in range(number_of_iterations): it = batched(tuple(range(1000)), 2) - threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=10, args=[it] + ) @threading_helper.reap_threads def test_cycle(self): @@ -46,28 +58,88 @@ def test_chain(self): number_of_iterations = 10 for _ in range(number_of_iterations): it = chain(*[(1,)] * 200) - threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=6, args=[it] + ) @threading_helper.reap_threads def test_combinations_with_replacement(self): number_of_iterations = 6 for _ in range(number_of_iterations): it = combinations_with_replacement(tuple(range(2)), 2) - threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=6, args=[it] + ) @threading_helper.reap_threads def test_permutations(self): number_of_iterations = 6 for _ in range(number_of_iterations): it = permutations(tuple(range(4)), 2) - threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=6, args=[it] + ) @threading_helper.reap_threads def test_zip_longest(self): number_of_iterations = 10 for _ in range(number_of_iterations): it = zip_longest(list(range(4)), list(range(8)), fillvalue=0) - threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + threading_helper.run_concurrently( + work_iterator, nthreads=10, args=[it] + ) + + +class TestTeeConcurrent(unittest.TestCase): + # itertools.tee branches share a linked list of internal data cells. + # Concurrent iteration must not corrupt that shared state or crash the + # free-threaded build. A crash shows up as the interpreter dying (not as a + # caught exception); tee is documented as not thread-safe, so a + # ``RuntimeError`` from the re-entrancy guard is an allowed outcome and is + # tolerated here. + + def test_same_branch(self): + # Many threads consume the same tee branch. + errors = [] + + def consume(it): + try: + for _ in it: + pass + except RuntimeError: + pass + except Exception as e: + errors.append(e) + + for _ in range(100): + a, _ = tee(iter(range(2000)), 2) + threading_helper.run_concurrently(consume, nthreads=8, args=(a,)) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + def test_sibling_branches(self): + # Each thread consumes a different sibling branch of the same tee. + errors = [] + + def make_worker(it): + def consume(): + try: + for _ in it: + pass + except RuntimeError: + pass + except Exception as e: + errors.append(e) + + return consume + + for _ in range(100): + branches = tee(iter(range(4000)), 8) + threading_helper.run_concurrently( + [make_worker(it) for it in branches] + ) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst b/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst new file mode 100644 index 000000000000000..ed1aaa15d37bd9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst @@ -0,0 +1,2 @@ +Fix a crash when concurrently iterating an :func:`itertools.tee` iterator on +the free-threaded build. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 0dd31dfbc5a3469..72bfab1abaf9cae 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -768,13 +768,17 @@ teedataobject_newinternal(itertools_state *state, PyObject *it) static PyObject * teedataobject_jumplink(itertools_state *state, teedataobject *tdo) { + PyObject *link; + Py_BEGIN_CRITICAL_SECTION(tdo); if (tdo->nextlink == NULL) tdo->nextlink = teedataobject_newinternal(state, tdo->it); - return Py_XNewRef(tdo->nextlink); + link = Py_XNewRef(tdo->nextlink); + Py_END_CRITICAL_SECTION(); + return link; } static PyObject * -teedataobject_getitem(teedataobject *tdo, int i) +teedataobject_getitem_lock_held(teedataobject *tdo, int i) { PyObject *value; @@ -800,6 +804,16 @@ teedataobject_getitem(teedataobject *tdo, int i) return Py_NewRef(value); } +static PyObject * +teedataobject_getitem(teedataobject *tdo, int i) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(tdo); + result = teedataobject_getitem_lock_held(tdo, i); + Py_END_CRITICAL_SECTION(); + return result; +} + static int teedataobject_traverse(PyObject *op, visitproc visit, void * arg) { @@ -819,8 +833,11 @@ teedataobject_safe_decref(PyObject *obj) { while (obj && _PyObject_IsUniquelyReferenced(obj)) { teedataobject *tmp = teedataobject_CAST(obj); - PyObject *nextlink = tmp->nextlink; + PyObject *nextlink; + Py_BEGIN_CRITICAL_SECTION(obj); + nextlink = tmp->nextlink; tmp->nextlink = NULL; + Py_END_CRITICAL_SECTION(); Py_SETREF(obj, nextlink); } Py_XDECREF(obj); @@ -833,11 +850,13 @@ teedataobject_clear(PyObject *op) PyObject *tmp; teedataobject *tdo = teedataobject_CAST(op); + Py_BEGIN_CRITICAL_SECTION(op); Py_CLEAR(tdo->it); for (i=0 ; inumread ; i++) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; + Py_END_CRITICAL_SECTION(); teedataobject_safe_decref(tmp); return 0; } @@ -930,20 +949,67 @@ static PyObject * tee_next(PyObject *op) { teeobject *to = teeobject_CAST(op); - PyObject *value, *link; + PyObject *value; +#ifndef Py_GIL_DISABLED + /* The GIL already serializes access, so keep the simple path without the + snapshot and revalidation that the free-threaded build needs. */ if (to->index >= LINKCELLS) { - link = teedataobject_jumplink(to->state, to->dataobj); - if (link == NULL) + PyObject *link = teedataobject_jumplink(to->state, to->dataobj); + if (link == NULL) { return NULL; + } Py_SETREF(to->dataobj, (teedataobject *)link); to->index = 0; } value = teedataobject_getitem(to->dataobj, to->index); - if (value == NULL) + if (value == NULL) { return NULL; + } to->index++; return value; +#else + for (;;) { + teedataobject *dataobj; + int index; + + /* Snapshot the branch position (strong ref to the shared data object) + under the tee lock; the data object is locked separately, not nested, + then the advance is revalidated. */ + Py_BEGIN_CRITICAL_SECTION(op); + dataobj = (teedataobject *)Py_NewRef((PyObject *)to->dataobj); + index = to->index; + Py_END_CRITICAL_SECTION(); + + if (index < LINKCELLS) { + value = teedataobject_getitem(dataobj, index); + if (value != NULL) { + Py_BEGIN_CRITICAL_SECTION(op); + if (to->dataobj == dataobj && to->index == index) { + to->index = index + 1; + } + Py_END_CRITICAL_SECTION(); + } + Py_DECREF(dataobj); + return value; + } + + PyObject *link = teedataobject_jumplink(to->state, dataobj); + if (link == NULL) { + Py_DECREF(dataobj); + return NULL; + } + Py_BEGIN_CRITICAL_SECTION(op); + if (to->dataobj == dataobj) { + Py_SETREF(to->dataobj, (teedataobject *)link); + to->index = 0; + link = NULL; + } + Py_END_CRITICAL_SECTION(); + Py_XDECREF(link); + Py_DECREF(dataobj); + } +#endif } static int @@ -962,8 +1028,10 @@ tee_copy_impl(teeobject *to) if (newto == NULL) { return NULL; } + Py_BEGIN_CRITICAL_SECTION(to); newto->dataobj = (teedataobject *)Py_NewRef(to->dataobj); newto->index = to->index; + Py_END_CRITICAL_SECTION(); newto->weakreflist = NULL; newto->state = to->state; PyObject_GC_Track(newto);