diff --git a/.editorconfig b/.editorconfig index 25bc5935258bd14..ab1f7ce8425769e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,toml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h,gram}] +[*.{py,c,cpp,h,toml,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index d654721239318ee..4d52f900b93179e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf @@ -68,6 +71,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/_static/tachyon-example-*.html generated Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated @@ -80,17 +84,22 @@ Include/internal/pycore_uop_ids.h generated Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated +Include/slots_generated.h generated Include/token.h generated Lib/_opcode_metadata.py generated -Lib/keyword.py generated Lib/idlelib/help.html generated +Lib/keyword.py generated +Lib/pydoc_data/topics.py generated +Lib/pydoc_data/module_docs.py generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated +Lib/test/test_zoneinfo/data/*.json generated Lib/token.py generated Misc/sbom.spdx.json generated -Objects/typeslots.inc generated +Modules/_testinternalcapi/test_cases.c.h generated +Modules/_testinternalcapi/test_targets.h generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated @@ -100,8 +109,11 @@ Python/executor_cases.c.h generated Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated +Python/record_functions.c.h generated +Python/slots_generated.c generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated *.min.js generated +package-lock.json generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1086b42620479df..0cdf17838b5798f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,16 +63,17 @@ .azure-pipelines/ @AA-Turner # GitHub & related scripts -.github/ @ezio-melotti @hugovk @AA-Turner -Tools/build/compute-changes.py @AA-Turner -Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro @JacobCoffee +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz +Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz +Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg # Pre-commit -.pre-commit-config.yaml @hugovk -.ruff.toml @hugovk @AlexWaygood @AA-Turner +.pre-commit-config.yaml @hugovk @JacobCoffee +.ruff.toml @hugovk @AlexWaygood @AA-Turner @JacobCoffee # Patchcheck -Tools/patchcheck/ @AA-Turner +Tools/patchcheck/ @AA-Turner @itamaro # ---------------------------------------------------------------------------- @@ -80,11 +81,11 @@ Tools/patchcheck/ @AA-Turner # ---------------------------------------------------------------------------- # Autotools -configure* @erlend-aasland @corona10 @AA-Turner @emmatyping -Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping -Modules/makesetup @erlend-aasland @AA-Turner @emmatyping -Modules/Setup* @erlend-aasland @AA-Turner @emmatyping -Tools/build/regen-configure.sh @AA-Turner +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping @itamaro +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping @itamaro +Tools/build/regen-configure.sh @AA-Turner @itamaro # generate-build-details Tools/build/generate-build-details.py @FFY00 @@ -99,17 +100,18 @@ Lib/test/test_build_details.py @FFY00 InternalDocs/ @AA-Turner # Tools, Configuration, etc -Doc/Makefile @AA-Turner @hugovk -Doc/_static/ @AA-Turner @hugovk -Doc/conf.py @AA-Turner @hugovk -Doc/make.bat @AA-Turner @hugovk -Doc/requirements.txt @AA-Turner @hugovk -Doc/tools/ @AA-Turner @hugovk +Doc/Makefile @AA-Turner @hugovk @StanFromIreland +Doc/_static/ @AA-Turner @hugovk @StanFromIreland +Doc/conf.py @AA-Turner @hugovk @StanFromIreland +Doc/make.bat @AA-Turner @hugovk @StanFromIreland +Doc/requirements.txt @AA-Turner @hugovk @StanFromIreland +Doc/tools/ @AA-Turner @hugovk @StanFromIreland # PR Previews .readthedocs.yml @AA-Turner # Sections +Doc/c-api/ @ZeroIntensity Doc/reference/ @willingc @AA-Turner Doc/whatsnew/ @AA-Turner @@ -126,8 +128,13 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner # C Analyser Tools/c-analyzer/ @ericsnowcurrently +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # Fuzzing -Modules/_xxtestfuzz/ @ammaraskar +Modules/_xxtestfuzz/ @python/fuzzers +Lib/test/test_xxtestfuzz.py @python/fuzzers +.github/workflows/reusable-cifuzz.yml @python/fuzzers # Limited C API & Stable ABI Doc/c-api/stable.rst @encukou @@ -140,13 +147,16 @@ Misc/externals.spdx.json @sethmlarson Misc/sbom.spdx.json @sethmlarson Tools/build/generate_sbom.py @sethmlarson +# ABI check +Misc/libabigail.abignore @encukou + # ---------------------------------------------------------------------------- # Platform Support # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -154,8 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team @@ -166,13 +175,14 @@ Lib/test/test__osx_support.py @python/macos-team Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) -Tools/wasm/wasi-env @brettcannon @emmatyping -Tools/wasm/wasi.py @brettcannon @emmatyping -Tools/wasm/wasi @brettcannon @emmatyping +Platforms/WASI @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski +Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski # Windows PC/ @python/windows-team @@ -251,45 +261,51 @@ Include/pyhash.h @gpshead @picnixz Python/pyhash.c @gpshead @picnixz # The import system (including importlib) -**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw -Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @kumaraditya303 +**/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 +Python/import.c @brettcannon @ericsnowcurrently @ncoghlan @warsaw @FFY00 @kumaraditya303 **/*freeze* @ericsnowcurrently **/*frozen* @ericsnowcurrently **/*modsupport* @ericsnowcurrently -**/*modulefinder* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently @FFY00 **/*moduleobject* @ericsnowcurrently **/*multiphase* @ericsnowcurrently -**/*pkgutil* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently @FFY00 **/*pythonrun* @ericsnowcurrently -**/*runpy* @ericsnowcurrently +**/*runpy* @ericsnowcurrently @FFY00 **/*singlephase* @ericsnowcurrently Doc/c-api/module.rst @ericsnowcurrently Lib/test/test_module/ @ericsnowcurrently -Python/dynload_*.c @ericsnowcurrently +Python/dynload_*.c @ericsnowcurrently @FFY00 # Initialisation -**/*initconfig* @ericsnowcurrently -**/*pathconfig* @ericsnowcurrently -**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently @FFY00 +**/*pathconfig* @ericsnowcurrently @FFY00 +**/*preconfig* @ericsnowcurrently @FFY00 Doc/library/sys_path_init.rst @FFY00 Doc/c-api/init_config.rst @FFY00 # Interpreter main program -Modules/main.c @ericsnowcurrently -Programs/_bootstrap_python.c @ericsnowcurrently -Programs/python.c @ericsnowcurrently +Modules/main.c @ericsnowcurrently @FFY00 +Programs/_bootstrap_python.c @ericsnowcurrently @FFY00 +Programs/python.c @ericsnowcurrently @FFY00 # JIT +.github/workflows/jit.yml @savannahostrowski Include/internal/pycore_jit.h @brandtbucher @savannahostrowski @diegorusso Python/jit.c @brandtbucher @savannahostrowski @diegorusso Tools/jit/ @brandtbucher @savannahostrowski @diegorusso InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner +# Lazy imports (PEP 810) +Objects/lazyimportobject.c @yhg1s @DinoV @pablogsal +Include/internal/pycore_lazyimportobject.h @yhg1s @DinoV @pablogsal +Lib/test/test_lazy_import @yhg1s @DinoV @pablogsal + # Micro-op / μop / Tier 2 Optimiser -Python/optimizer.c @markshannon -Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner -Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner -Python/optimizer_symbols.c @markshannon @tomasr8 +Python/optimizer.c @markshannon @Fidget-Spinner +Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski +Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner @savannahostrowski # Parser, Lexer, and Grammar Grammar/python.gram @pablogsal @lysnikolaou @@ -301,8 +317,8 @@ Tools/peg_generator/ @pablogsal @lysnikolaou # Runtime state/lifecycle **/*gil* @ericsnowcurrently -**/*pylifecycle* @ericsnowcurrently @ZeroIntensity -**/*pystate* @ericsnowcurrently @ZeroIntensity +**/*pylifecycle* @ericsnowcurrently @ZeroIntensity @FFY00 +**/*pystate* @ericsnowcurrently @ZeroIntensity @FFY00 Include/internal/pycore_*_init.h @ericsnowcurrently Include/internal/pycore_*_state.h @ericsnowcurrently Include/internal/pycore_atexit.h @ericsnowcurrently @@ -319,7 +335,7 @@ Tools/build/generate_global_objects.py @ericsnowcurrently # Remote Debugging Python/remote_debug.h @pablogsal Python/remote_debugging.c @pablogsal -Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 +Modules/_remote_debugging/ @pablogsal # Sub-Interpreters **/*crossinterp* @ericsnowcurrently @@ -410,18 +426,19 @@ Lib/dataclasses.py @ericvsmith Lib/test/test_dataclasses/ @ericvsmith # Dates and times -Doc/**/*time.rst @pganssle @abalkin -Doc/library/zoneinfo.rst @pganssle -Include/datetime.h @pganssle @abalkin -Include/internal/pycore_time.h @pganssle @abalkin -Lib/test/test_zoneinfo/ @pganssle -Lib/zoneinfo/ @pganssle -Lib/*time.py @pganssle @abalkin -Lib/test/datetimetester.py @pganssle @abalkin -Lib/test/test_*time.py @pganssle @abalkin -Modules/*zoneinfo* @pganssle -Modules/*time* @pganssle @abalkin -Python/pytime.c @pganssle @abalkin +Doc/**/*time.rst @pganssle @StanFromIreland +Doc/library/datetime-* @pganssle @StanFromIreland +Doc/library/zoneinfo.rst @pganssle @StanFromIreland +Include/datetime.h @pganssle @StanFromIreland +Include/internal/pycore_time.h @pganssle @StanFromIreland +Lib/test/test_zoneinfo/ @pganssle @StanFromIreland +Lib/zoneinfo/ @pganssle @StanFromIreland +Lib/*time.py @pganssle @StanFromIreland +Lib/test/datetimetester.py @pganssle @StanFromIreland +Lib/test/test_*time.py @pganssle @StanFromIreland +Modules/*zoneinfo* @pganssle @StanFromIreland +Modules/*time* @pganssle @StanFromIreland +Python/pytime.c @pganssle @StanFromIreland # Dbm Doc/library/dbm.rst @corona10 @erlend-aasland @serhiy-storchaka @@ -460,8 +477,9 @@ Lib/test/test_functools.py @rhettinger Modules/_functoolsmodule.c @rhettinger # Garbage collector -Modules/gcmodule.c @pablogsal -Doc/library/gc.rst @pablogsal +Modules/gcmodule.c @pablogsal +Doc/library/gc.rst @pablogsal +InternalDocs/garbage_collector.md @pablogsal # Gettext Doc/library/gettext.rst @tomasr8 @@ -488,13 +506,13 @@ Lib/idlelib/ @terryjreedy Lib/turtledemo/ @terryjreedy # importlib.metadata -Doc/library/importlib.metadata.rst @jaraco @warsaw -Lib/importlib/metadata/ @jaraco @warsaw -Lib/test/test_importlib/metadata/ @jaraco @warsaw +Doc/library/importlib.metadata.rst @jaraco @warsaw @FFY00 +Lib/importlib/metadata/ @jaraco @warsaw @FFY00 +Lib/test/test_importlib/metadata/ @jaraco @warsaw @FFY00 # importlib.resources -Doc/library/importlib.resources.abc.rst @jaraco @warsaw -Doc/library/importlib.resources.rst @jaraco @warsaw +Doc/library/importlib.resources.abc.rst @jaraco @warsaw @FFY00 +Doc/library/importlib.resources.rst @jaraco @warsaw @FFY00 Lib/importlib/resources/ @jaraco @warsaw @FFY00 Lib/test/test_importlib/resources/ @jaraco @warsaw @FFY00 @@ -534,6 +552,11 @@ Lib/pydoc.py @AA-Turner Lib/pydoc_data/ @AA-Turner Lib/test/test_pydoc/ @AA-Turner +# Profiling (Sampling) +Doc/library/profiling*.rst @pablogsal +Lib/profiling/ @pablogsal +Lib/test/test_profiling/ @pablogsal + # PyREPL Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv @@ -550,9 +573,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner @@ -563,10 +586,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -599,9 +622,6 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 224061f2c5a3508..94b67ce3dbe3414 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -28,13 +28,12 @@ Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in `Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: +comments they leave and their "Details" links, respectively. -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS``. +The final key part of our workflow is that all discussions that are not +directly related to the code in the pull request should happen on +`GitHub Issues `__, generally in the +pull request's parent issue. Setting Expectations diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index da70710b7ecfa39..177615621f6b8c7 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,13 +34,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce1601..de6e8756b03d800 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 470ad581367b103..81ae91e5b0af973 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,13 +27,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md deleted file mode 100644 index 174fd39171d47dd..000000000000000 --- a/.github/ISSUE_TEMPLATE/documentation.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -name: Documentation -about: Report a problem with the documentation -labels: "docs" ---- - -# Documentation - -(A clear and concise description of the issue.) diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 000000000000000..d720cf9c4de91e2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,16 @@ +name: Documentation +description: Report a problem with the documentation +labels: ["docs"] +body: + - type: markdown + attributes: + value: | + > [!NOTE] + > Trivial changes (for example typos) don’t require an issue before opening a PR. + - type: textarea + id: description + attributes: + label: "Documentation" + description: "A clear and concise description of the issue. Include a link to the page." + validations: + required: true diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 923720bce0bc3bf..5bf4b40947bea70 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,17 +1,24 @@ # Security Policy -## Supported Versions +Python [provides a security policy and threat model](https://devguide.python.org/security/policy/) +in the Python Developer's Guide documenting what bugs are vulnerabilities, +how to structure reports, and what versions of Python accept reports. -The Python team applies security fixes according to the table -in [the devguide]( -https://devguide.python.org/versions/#supported-versions -). +Python Security Response Team (PSRT) members +balance security work against many other responsibilities. Please be thoughtful +about the time and attention your report requires. Repeated failure to respect +the security policy will result in future reports being rejected, or the +reporter being banned from the ``python`` GitHub organization, regardless of +technical merit. ## Reporting a Vulnerability -Please read the guidelines on reporting security issues [on the -official website](https://www.python.org/dev/security/) for -instructions on how to report a security-related problem to -the Python team responsibly. +The [Python security policy](https://devguide.python.org/security/policy/) +documents [how to submit a vulnerability report](https://devguide.python.org/security/policy/#how-to-submit-a-vulnerability-report) +using GitHub Security Advisories. Please read the security policy +prior to filing a vulnerability report, especially the section on [what information to +include and exclude](https://devguide.python.org/security/policy/#what-to-include-and-how-to-structure-a-vulnerability-report) +in vulnerability reports. Following the security policy means the PSRT can +quickly and efficiently triage your report, not following the security policy +will only delay triaging your report. -To reach the response team, email `security at python dot org`. diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 267ff6b42a86556..61ee08db9919bb9 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,10 +1,12 @@ -self-hosted-runner: - # Pending https://github.com/rhysd/actionlint/issues/533 - # and https://github.com/rhysd/actionlint/issues/571 - labels: ["windows-11-arm", "macos-15-intel"] - config-variables: null +# Pending release of actionlint > 1.7.12 for ubuntu-26.04* support: +# https://github.com/rhysd/actionlint/pull/683 +self-hosted-runner: + labels: + - ubuntu-26.04 + - ubuntu-26.04-arm + paths: .github/workflows/**/*.yml: ignore: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d6903645..4b77646e22db4ba 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" @@ -12,10 +12,21 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + groups: + actions: + patterns: + - "*" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: - interval: "monthly" + interval: "quarterly" labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/setup-rust-windows/action.yml b/.github/setup-rust-windows/action.yml new file mode 100644 index 000000000000000..97e0288592b146f --- /dev/null +++ b/.github/setup-rust-windows/action.yml @@ -0,0 +1,38 @@ +name: Set up Rust and LLVM for Windows +description: >- + Install the pinned Rust toolchain with Windows cross-compile targets and + a version of LLVM whose libclang is loadable by bindgen. + +inputs: + arch: + description: CPU architecture (x64, Win32, arm64) + required: true + +runs: + using: composite + steps: + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: i686-pc-windows-msvc,x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + # LIBCLANG_PATH must be set explicitly so the vcxproj uses the LLVM we + # install here rather than the VS-bundled LLVM whose clang headers have + # AVX intrinsic bugs. The values are hardcoded literals (no attacker- + # controlled input), so the GITHUB_ENV writes are safe. + - name: Install LLVM for bindgen + if: inputs.arch != 'arm64' + shell: cmd + run: | # zizmor: ignore[github-env] + choco install llvm --allow-downgrade --no-progress --version 21.1.0 + if not exist "C:\Program Files\LLVM\bin\libclang.dll" exit /b 1 + echo LIBCLANG_PATH=C:\Program Files\LLVM\bin>> "%GITHUB_ENV%" + # Chocolatey's LLVM package only ships x64 binaries, which an ARM64-native + # cargo process cannot load. Install the official ARM64 build directly. + - name: Install LLVM for bindgen (ARM64) + if: inputs.arch == 'arm64' + shell: pwsh + run: | # zizmor: ignore[github-env] + $installer = Join-Path $env:RUNNER_TEMP 'LLVM-21.1.0-woa64.exe' + Invoke-WebRequest 'https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.0/LLVM-21.1.0-woa64.exe' -OutFile $installer + Start-Process -Wait -FilePath $installer -ArgumentList '/S','/D=C:\Program Files\LLVM' + if (!(Test-Path 'C:\Program Files\LLVM\bin\libclang.dll')) { exit 1 } + echo "LIBCLANG_PATH=C:\Program Files\LLVM\bin" >> $env:GITHUB_ENV diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3cbc23af578d106..55095f17bb4cce9 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,15 +12,17 @@ on: # Only ever run once - opened +permissions: + contents: read jobs: add-header: - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0f60c30ac8a60c..2a92a988d90e318 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 container: image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 @@ -64,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 1 persist-credentials: false @@ -96,23 +96,21 @@ jobs: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -143,6 +141,14 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + check-c-api-docs: + name: C API Docs + needs: build-context + if: >- + needs.build-context.outputs.run-tests == 'true' + || needs.build-context.outputs.run-docs == 'true' + uses: ./.github/workflows/reusable-check-c-api-docs.yml + build-windows: name: >- Windows @@ -159,13 +165,21 @@ jobs: free-threading: - false - true + interpreter: + - switch-case exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } + include: + # msvc::musttail is currently only supported on x64, + # and only supported on 3.15+. + - { arch: x64, free-threading: false, interpreter: tail-call } + - { arch: x64, free-threading: true, interpreter: tail-call } uses: ./.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + interpreter: ${{ matrix.interpreter }} build-windows-msi: # ${{ '' } is a hack to nest jobs under the same sidebar category. @@ -188,24 +202,23 @@ jobs: macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-macos == 'true' strategy: fail-fast: false matrix: - # macos-14 is M1, macos-15-intel is Intel. - # macos-15-intel only runs tests against the GIL-enabled CPython. + # macos-26 is Apple Silicon, macos-26-intel is Intel. + # macos-26-intel only runs tests against the GIL-enabled CPython. os: - - macos-14 - - macos-15-intel + - macos-26 + - macos-26-intel free-threading: - false - true exclude: - - os: macos-15-intel + - os: macos-26-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -215,7 +228,7 @@ jobs: ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -226,96 +239,59 @@ jobs: - false - true os: - - ubuntu-24.04 - - ubuntu-24.04-arm + - ubuntu-26.04 + - ubuntu-26.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true free-threading: true # BOLT currently crashes during instrumentation on aarch64 - - os: ubuntu-24.04-arm + - os: ubuntu-26.04-arm bolt: true + include: + # Enable CPU-intensive tests on ARM (default build only) + - os: ubuntu-26.04-arm + bolt: false + free-threading: false + test-opts: '-u cpu' uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} + test-opts: ${{ matrix.test-opts || '' }} - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - # Keep 1.1.1w in our list despite it being upstream EOL and otherwise - # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs - # supported by important vendors such as AWS-LC. - openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' + build-ubuntu-ssltests: + name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] + os: [ubuntu-26.04] + ssllib: + # See Tools/ssl/make_ssl_data.py for notes on adding a new version + ## OpenSSL + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + - { name: openssl, version: 1.1.1w } + - { name: openssl, version: 3.0.21 } + - { name: openssl, version: 3.4.6 } + - { name: openssl, version: 3.5.7 } + - { name: openssl, version: 3.6.3 } + - { name: openssl, version: 4.0.1 } + ## AWS-LC + - { name: aws-lc, version: 5.0.0 } env: - AWSLC_VER: ${{ matrix.awslc_ver}} + SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -324,79 +300,78 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 + - name: 'Restore SSL library build' + id: cache-ssl-lib + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' + path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} + - name: Install SSL Library + if: steps.cache-ssl-lib.outputs.cache-hit != 'true' run: | python3 Tools/ssl/multissltests.py \ --steps=library \ --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ + '--${{ matrix.ssllib.name }}' '${{ matrix.ssllib.version }}' \ --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure CPython run: | ./configure CFLAGS="-fdiagnostics-format=json" \ --config-cache \ --enable-slower-safety \ --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ + --with-openssl="$SSLLIB_DIR" \ --with-builtin-hashlib-hashes=blake2 \ --with-ssl-default-suites=openssl - name: Build CPython - run: make -j + run: make -j4 - name: Display build info run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: Verify python is linked to the right lib + run: | + ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' \ + | grep -iE '${{ matrix.ssllib.name }}.*${{ matrix.ssllib.version }}' - name: SSL tests run: ./python Lib/test/ssltests.py build-android: name: Android (${{ matrix.arch }}) needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-android == 'true' timeout-minutes: 60 strategy: fail-fast: false matrix: include: - arch: aarch64 - runs-on: macos-14 + runs-on: macos-26 - arch: x86_64 - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: ${{ matrix.arch }}-linux-android - name: Build and test - run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + run: JAVA_HOME="${JAVA_HOME_21_X64:-$JAVA_HOME_21_arm64}" python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android build-ios: name: iOS needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ios == 'true' timeout-minutes: 60 - runs-on: macos-15 + runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: aarch64-apple-ios-sim # GitHub recommends explicitly selecting the desired Xcode version: # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 # This became a necessity as a result of @@ -404,30 +379,34 @@ jobs: # https://github.com/actions/runner-images/issues/12751. - name: Select Xcode version run: | - sudo xcode-select --switch /Applications/Xcode_16.4.app + sudo xcode-select --switch /Applications/Xcode_15.4.app - name: Build and test - run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5' + run: python3 Platforms/Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml build-wasi: name: 'WASI' needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-wasi == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Register gcc problem matcher @@ -441,16 +420,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -491,7 +467,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -518,7 +494,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 if: always() with: name: hypothesis-example-db @@ -529,17 +505,17 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-26.04] env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -548,10 +524,6 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -559,18 +531,15 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -582,7 +551,7 @@ jobs: # ${{ '' } is a hack to nest jobs under the same sidebar category. name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -600,17 +569,16 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: name: Cross build Linux - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -634,49 +602,56 @@ jobs: run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - name: Install cross Python run: make -j8 install + - name: Display build info + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test.pythoninfo - name: Run test subset with host build run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: + sanitizer: + - address + oss-fuzz-project-name: + - cpython3 + - python3-libraries + include: + - sanitizer: undefined oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 + - sanitizer: memory oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -687,14 +662,14 @@ jobs: - check-docs - check-autoconf-regen - check-generated-files + - check-c-api-docs - build-windows - build-windows-msi - build-macos - build-ubuntu - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl - - build-android + - build-ubuntu-ssltests - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan @@ -708,50 +683,50 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- + build-android, + build-emscripten, build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, + build-ubuntu-ssltests, test-hypothesis, cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} + ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - build-android, - build-ios, - build-wasi, - test-hypothesis, - build-asan, - build-san, - cross-build-linux, ' || '' }} ${{ - !fromJSON(needs.build-context.outputs.run-windows-tests) - && ' - build-windows, - ' + !fromJSON(needs.build-context.outputs.run-tests) + && !fromJSON(needs.build-context.outputs.run-docs) + && 'check-c-api-docs,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' + }} + ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ubuntu) && ' - cifuzz, + build-ubuntu, + build-ubuntu-ssltests, + test-hypothesis, + build-asan, + build-san, + cross-build-linux, ' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/cargo.yml b/.github/workflows/cargo.yml new file mode 100644 index 000000000000000..c9fb50043d33a55 --- /dev/null +++ b/.github/workflows/cargo.yml @@ -0,0 +1,27 @@ +name: cargo + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + timeout-minutes: 15 + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + - run: ./configure + - run: cargo test + - run: cargo fmt --check + - run: cargo clippy diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index a09a30587b35ebb..000000000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb042425dd4..813589c2bc14c8b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -1,25 +1,18 @@ name: JIT on: pull_request: - paths: + paths: &paths - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' - 'Python/executor_cases.c.h' - 'Python/optimizer_cases.c.h' + - '**_testinternalcapi**' - '!Python/perf_jit_trampoline.c' - '!**/*.md' - '!**/*.ini' push: - paths: - - '**jit**' - - 'Python/bytecodes.c' - - 'Python/optimizer*.c' - - 'Python/executor_cases.c.h' - - 'Python/optimizer_cases.c.h' - - '!Python/perf_jit_trampoline.c' - - '!**/*.md' - - '!**/*.ini' + paths: *paths workflow_dispatch: permissions: @@ -31,16 +24,20 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-24.04 - timeout-minutes: 90 + runs-on: ubuntu-26.04 + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -48,71 +45,79 @@ jobs: - name: Test tier two interpreter run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit: + + windows: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - needs: interpreter + runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: target: -# To re-enable later when we support these. -# - i686-pc-windows-msvc/msvc -# - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc debug: - true - false - llvm: - - 21 include: -# To re-enable later when we support these. -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 -# - target: x86_64-pc-windows-msvc/msvc -# architecture: x64 -# runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-11-arm - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-15-intel - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-2025 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2025 + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - - name: Windows - if: runner.os == 'Windows' + - name: Build run: | ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + - name: Test + run: | ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: macOS - if: runner.os == 'macOS' + macos: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + debug: + - true + - false + include: + - target: x86_64-apple-darwin/clang + runner: macos-26-intel + - target: aarch64-apple-darwin/clang + runner: macos-26 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install LLVM run: | brew update - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to # make sure we don't break downstream distributors (like uv): @@ -120,92 +125,90 @@ jobs: export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Linux - if: runner.os == 'Linux' - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} - make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + linux: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - jit-with-disabled-gil: - name: Free-Threaded (Debug) - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 strategy: fail-fast: false matrix: - llvm: - - 21 + target: + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + debug: + - true + - false + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-26.04 + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-26.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - - name: Build with JIT enabled and GIL disabled + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit --with-pydebug --disable-gil + # On ubuntu-26.04 image, clang is clang-21 by default + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 - - name: Run tests + - name: Test run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - continue-on-error: true - no-opt-jit: - name: JIT without optimizations (Debug) - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - llvm: - - 21 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Build with JIT - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit --with-pydebug - make all --jobs 4 - - name: Run tests without optimizations - run: | - PYTHON_UOPS_OPTIMIZE=0 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + linux-extras: + name: ${{ matrix.name }} - tail-call-jit: - name: JIT with tail calling interpreter - needs: interpreter - runs-on: ubuntu-24.04 - timeout-minutes: 90 + runs-on: ubuntu-26.04 + timeout-minutes: 60 strategy: fail-fast: false matrix: - llvm: - - 21 + include: + - name: Free-Threaded (Debug) + configure_flags: --enable-experimental-jit --with-pydebug --disable-gil + continue_on_error: true + - name: JIT without optimizations (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with stress testing (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_JIT_STRESS=1" + - name: JIT with tail calling interpreter + configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug + use_clang: true + run_tests: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - - name: Build with JIT and tailcall + - name: Install dependencies run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-${{ matrix.llvm }} ./configure --enable-experimental-jit --with-tail-call-interp --with-pydebug + sudo ./.github/workflows/posix-deps-apt.sh + - name: Build + run: | + # On ubuntu-26.04 image, clang is clang-21 by default + if [ "${{ matrix.use_clang }}" = "true" ]; then + export CC=clang-${{ env.LLVM_VERSION }} + fi + ./configure ${{ matrix.configure_flags }} make all --jobs 4 + - name: Test + if: matrix.run_tests != false + run: | + ${{ matrix.test_env }} ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + continue-on-error: ${{ matrix.continue_on_error }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d74ce8fcc256dc8..8a79ea20d5f50bc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,10 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index fac0fa8aba30504..3cdce4f5952e3d0 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,6 +19,7 @@ on: - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" @@ -26,6 +27,7 @@ on: - "Tools/build/update_file.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" + - "Tools/check-c-api-docs/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" @@ -58,19 +60,20 @@ jobs: "Lib/tomllib", "Tools/build", "Tools/cases_generator", + "Tools/check-c-api-docs", "Tools/clinic", "Tools/jit", "Tools/peg_generator", ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + python-version: "3.15" + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5f19a..864d0f48904bcc3 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -6,19 +6,21 @@ on: - opened permissions: - issues: read + contents: read jobs: notify-new-bugs-announce: - runs-on: ubuntu-latest + runs-on: ubuntu-slim + permissions: + issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: @@ -44,7 +46,7 @@ jobs: // We need to truncate the body size, because the max size for // the whole payload is 16kb. We want to be safe and assume that // body can take up to ~8kb of space. - body : issue.data.body.substring(0, 8000) + body : (issue.data.body || "").substring(0, 8000) }; const data = { diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0b64367e6c45624..803204635abfe11 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,35 +1,27 @@ #!/bin/sh apt-get update -apt-get -yq install \ +apt-get -yq --no-install-recommends install \ build-essential \ pkg-config \ - ccache \ cmake \ + curl \ gdb \ lcov \ libb2-dev \ libbz2-dev \ libffi-dev \ - libgdbm-dev \ libgdbm-compat-dev \ + libgdbm-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ libzstd-dev \ - lzma \ - lzma-dev \ strace \ tk-dev \ uuid-dev \ xvfb \ zlib1g-dev - -# Workaround missing libmpdec-dev on ubuntu 24.04: -# https://launchpad.net/~ondrej/+archive/ubuntu/php -# https://deb.sury.org/ -sudo add-apt-repository ppa:ondrej/php -apt-get update -apt-get -yq install libmpdec-dev diff --git a/.github/workflows/regen-abidump.sh b/.github/workflows/regen-abidump.sh index 251bb3857ecfcb6..75a1a72e3702024 100644 --- a/.github/workflows/regen-abidump.sh +++ b/.github/workflows/regen-abidump.sh @@ -2,7 +2,7 @@ set -ex export DEBIAN_FRONTEND=noninteractive ./.github/workflows/posix-deps-apt.sh -apt-get install -yq abigail-tools python3 +apt-get install -yq --no-install-recommends abigail-tools python3 export CFLAGS="-g3 -O0" ./configure --enable-shared && make make regen-abidump diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 7e534c58c798d1c..8af254c10786e1b 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,18 +4,21 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + contents: read + jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -25,7 +28,7 @@ jobs: label-reviews: name: Unresolved review if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 @@ -33,7 +36,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -42,7 +45,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 @@ -53,7 +56,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml new file mode 100644 index 000000000000000..db030c80008b1ed --- /dev/null +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -0,0 +1,25 @@ +name: Reusable C API Docs Check + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-c-api-docs: + name: 'Check if all C APIs are documented' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Check for undocumented C APIs + run: python Tools/check-c-api-docs/main.py diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml new file mode 100644 index 000000000000000..41ba1288be1ecf4 --- /dev/null +++ b/.github/workflows/reusable-check-html-ids.yml @@ -0,0 +1,67 @@ +name: Reusable check HTML IDs + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-html-ids: + name: 'Check for removed HTML IDs' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: 'Check out PR head' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + - name: 'Find merge base' + id: merge-base + run: | + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + git fetch --depth=$((${{ github.event.pull_request.commits }} + 10)) --no-tags origin "$BASE" "$HEAD" + + if ! MERGE_BASE=$(git merge-base "$BASE" "$HEAD" 2>/dev/null); then + git fetch --deepen=1 --no-tags origin "$BASE" "$HEAD" + + OLDEST=$(git rev-list --reflog --max-parents=0 --reverse "${BASE}^" "${HEAD}^" | head -1) + TIMESTAMP=$(git show --format=%at --no-patch "$OLDEST") + + git fetch --shallow-since="$TIMESTAMP" --no-tags origin "$BASE" "$HEAD" + + MERGE_BASE=$(git merge-base "$BASE" "$HEAD") + fi + echo "sha=$MERGE_BASE" >> "$GITHUB_OUTPUT" + - name: 'Create worktree at merge base' + env: + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + run: git worktree add /tmp/merge-base "$MERGE_BASE" --detach + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: make -C /tmp/merge-base/Doc/ venv + - name: 'Build HTML documentation' + run: make -C /tmp/merge-base/Doc/ SPHINXOPTS="--quiet" html + - name: 'Collect HTML IDs' + run: python Doc/tools/check-html-ids.py collect /tmp/merge-base/Doc/build/html -o /tmp/html-ids-base.json.gz + - name: 'Download PR head HTML IDs' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: html-ids-head.json.gz + path: /tmp + - name: 'Check for removed HTML IDs' + run: | + # shellcheck disable=SC2046 + python Doc/tools/check-html-ids.py -v check \ + /tmp/html-ids-base.json.gz /tmp/html-ids-head.json.gz \ + $([ -f Doc/tools/removed-ids.txt ] && echo "--exclude-file Doc/tools/removed-ids.txt") diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 000000000000000..0d02232686339bf --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,49 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +permissions: + contents: read + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@38697555549f1db7851b81482ff19f1fa5c4fedc # v4.34.1 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1a3d8..6595aef0377e500 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,24 +17,45 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-android: + description: Whether to run the Android tests + value: ${{ jobs.compute-changes.outputs.run-android }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-ios: + description: Whether to run the iOS tests + value: ${{ jobs.compute-changes.outputs.run-ios }} # bool + run-macos: + description: Whether to run the macOS tests + value: ${{ jobs.compute-changes.outputs.run-macos }} # bool run-tests: description: Whether to run the regular tests value: ${{ jobs.compute-changes.outputs.run-tests }} # bool - run-windows-tests: - description: Whether to run the Windows tests - value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + run-ubuntu: + description: Whether to run the Ubuntu tests + value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool + run-wasi: + description: Whether to run the WASI tests + value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool run-windows-msi: description: Whether to run the MSI installer smoke tests value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool - run-ci-fuzz: - description: Whether to run the CIFuzz job - value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + +permissions: + contents: read jobs: compute-changes: @@ -42,22 +63,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} + run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} + run-ios: ${{ steps.changes.outputs.run-ios }} + run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} + run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} + run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false ref: >- @@ -100,8 +127,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 65154aae4c41d58..199e0fd8d181f0b 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -3,6 +3,17 @@ name: Reusable Docs on: workflow_call: workflow_dispatch: + # Pushes to CPython branches seed the pip caches under the exact keys every + # docs PR restores from. Without a branch-scoped copy, each PR saves a + # duplicate into its own refs/pull/N/merge scope that nothing else can read. + push: + branches: + - main + - '3.*' + paths: + - 'Doc/pylock.toml' + - 'Doc/requirements.txt' + - '.github/workflows/reusable-docs.yml' permissions: contents: read @@ -27,7 +38,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false ref: >- @@ -52,11 +63,11 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' + cache-dependency-path: 'Doc/pylock.toml' - name: 'Install build dependencies' run: make -C Doc/ venv @@ -75,24 +86,40 @@ jobs: --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit + - name: 'Collect HTML IDs' + if: github.event_name == 'pull_request' + run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz + - name: 'Upload HTML IDs' + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: html-ids-head + path: Doc/build/html-ids-head.json.gz + archive: false + + check-html-ids: + name: 'Check for removed HTML IDs' + needs: build-doc + if: github.event_name == 'pull_request' + uses: ./.github/workflows/reusable-check-html-ids.yml # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/cache@v4 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} restore-keys: | ubuntu-doc- - name: 'Install Dependencies' - run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install --no-install-recommends wamerican - name: 'Configure CPython' run: ./configure --with-pydebug - name: 'Build CPython' @@ -108,11 +135,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 000000000000000..0205203ef77a9f5 --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,79 @@ +name: Reusable Emscripten + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-26.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") + - name: "Install Node.js" + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + id: emsdk-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + - name: "Install Python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Display build info of the build Python" + run: python3 Platforms/emscripten pythoninfo-build + - name: "Make dependencies" + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} + - name: "Configure host/Emscripten Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host/Emscripten Python" + run: python3 Platforms/emscripten make-host + - name: "Display build info of the host/Emscripten Python" + run: python3 Platforms/emscripten pythoninfo-host + - name: "Test" + run: python3 Platforms/emscripten run --test diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d85c46b96f873db..5393e9e34040b95 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -15,6 +12,9 @@ on: required: true type: string +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -31,16 +31,15 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@9 make - # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@9 + brew bundle --file=Misc/Brewfile + brew install make - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -53,17 +52,17 @@ jobs: --enable-safety \ ${{ inputs.free-threading && '--disable-gil' || '' }} \ --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" + --with-openssl="$(brew --prefix openssl@3.5)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-26-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 7fe96d1b238b047..405fa36454a62bf 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,15 +6,15 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false type: boolean default: false +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -26,10 +26,10 @@ jobs: && ' (free-threading)' || '' }} - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -37,20 +37,11 @@ jobs: - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh - # Install clang - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh + # On ubuntu-26.04 image, clang is clang-21 by default if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 # Reduce ASLR to avoid TSan crashing sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 fi - name: Sanitizer option setup @@ -62,16 +53,13 @@ jobs: || '' }}.txt handle_segv=0" >> "$GITHUB_ENV" else - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} halt_on_error=1 suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" >> "$GITHUB_ENV" fi echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" env: SANITIZER: ${{ inputs.sanitizer }} SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure CPython run: >- ./configure @@ -79,7 +67,7 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' - || '--with-undefined-behavior-sanitizer' + || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} @@ -91,18 +79,18 @@ jobs: run: >- ./python -m test ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} - -j4 + -j4 -W --timeout=900 --slowest - name: Parallel tests if: >- inputs.sanitizer == 'TSan' && fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 -W --timeout=600 --slowest - name: Display logs if: always() run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7b93b5f51b00df3..4ecc541e6aa7d02 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false @@ -20,6 +17,14 @@ on: description: OS to run the job required: true type: string + test-opts: + description: Extra options to pass to the test runner via TESTOPTS + required: false + type: string + default: '' + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -30,11 +35,11 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.18 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Register gcc problem matcher @@ -44,9 +49,9 @@ jobs: - name: Install Clang and BOLT if: ${{ fromJSON(inputs.bolt-optimizations) }} run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 - sudo apt-get install bolt-19 - echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV + # On ubuntu-26.04 image, LLVM is LLVM-21 by default + sudo apt-get install --no-install-recommends bolt-21 + echo PATH="$(llvm-config-21 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -54,16 +59,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -114,4 +116,6 @@ jobs: run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Tests working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make ci + run: xvfb-run make ci EXTRATESTOPTS="${TEST_OPTS}" + env: + TEST_OPTS: ${{ inputs.test-opts }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8f412288f530bc5..6f02fa5053557d3 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,9 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -13,53 +12,60 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm timeout-minutes: 60 env: WASMTIME_VERSION: 38.0.3 - WASI_SDK_VERSION: 25 - WASI_SDK_PATH: /opt/wasi-sdk - CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@v1 + uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - - name: "Restore WASI SDK" - id: cache-wasi-sdk - uses: actions/cache@v4 - with: - path: ${{ env.WASI_SDK_PATH }} - key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} - - name: "Install WASI SDK" # Hard-coded to x64. - if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + - name: "Read WASI SDK version" + id: wasi-sdk-version run: | - mkdir "${WASI_SDK_PATH}" && \ - curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-arm64-linux.tar.gz" | \ - tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Add ccache to PATH" - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + import tomllib + from pathlib import Path + import os + config = tomllib.loads(Path("Platforms/WASI/config.toml").read_text()) + version = config["targets"]["wasi-sdk"] + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"version={version}\n") + shell: python + - name: "Install WASI SDK" + id: install-wasi-sdk + uses: bytecodealliance/setup-wasi-sdk-action@b2de090b44eb70013ee96b393727d473b35e1728 + with: + version: ${{ steps.wasi-sdk-version.outputs.version }} + add-to-path: false - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' + - uses: dtolnay/rust-toolchain@1.91.1 + with: + targets: wasm32-wasip1 - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: "Configure build Python" - run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug + run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Configure host" + run: python3 Platforms/WASI make-build-python + - name: "Display build info of the build Python" + run: python3 Platforms/WASI pythoninfo-build + - name: "Configure host/WASI Python" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi configure-host -- --config-cache - - name: "Make host" - run: python3 Tools/wasm/wasi make-host - - name: "Display build info" - run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + run: python3 Platforms/WASI configure-host -- --config-cache + env: + WASI_SDK_PATH: ${{ steps.install-wasi-sdk.outputs.wasi-sdk-path }} + - name: "Make host/WASI Python" + run: python3 Platforms/WASI make-host + - name: "Display build info of the host/WASI Python" + run: python3 Platforms/WASI pythoninfo-host - name: "Test" run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index c95e40a38095f95..b8c12e9f39961b0 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,15 +17,18 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - uses: ./.github/setup-rust-windows + with: + arch: ${{ inputs.arch }} - name: Build CPython installer run: ./Tools/msi/build.bat --doc -"${ARCH}" shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 0648b770753255f..953de57986f5837 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -12,30 +12,41 @@ on: required: false type: boolean default: false + interpreter: + description: Which interpreter to build (switch-case or tail-call) + required: true + type: string + +permissions: + contents: read env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: - name: Build and test (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} + name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - uses: ./.github/setup-rust-windows + with: + arch: ${{ inputs.arch }} - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython + # msvc::musttail is not supported for debug builds, so we have to + # switch to release. run: >- .\\PCbuild\\build.bat - -e -d -v + -e -v + ${{ inputs.interpreter == 'switch-case' && '-d' || '--tail-call-interp -c Release' }} -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash @@ -45,6 +56,7 @@ jobs: run: >- .\\PCbuild\\rt.bat -p "${ARCH}" - -d -q --fast-ci + -q --fast-ci + ${{ inputs.interpreter == 'switch-case' && '-d' || '' }} ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index febb2dd823a8fe2..0283e9f621dcf05 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,23 +4,27 @@ on: schedule: - cron: "0 */6 * * *" +permissions: + contents: read + jobs: stale: if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: + actions: write pull-requests: write timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v9 + uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' + stale-pr-message: 'This PR is stale because it has been open for 90 days with no activity.' stale-pr-label: 'stale' days-before-issue-stale: -1 - days-before-pr-stale: 30 + days-before-pr-stale: 90 days-before-close: -1 ascending: true operations-per-run: 120 diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index e99e317182eaa6c..10383150b755def 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,19 +1,14 @@ name: Tail calling interpreter on: pull_request: - paths: - - '.github/workflows/tail-call.yml' - - 'Python/bytecodes.c' - - 'Python/ceval.c' - - 'Python/ceval_macros.h' - - 'Python/generated_cases.c.h' + paths: &paths + - ".github/workflows/tail-call.yml" + - "Python/bytecodes.c" + - "Python/ceval.c" + - "Python/ceval_macros.h" + - "Python/generated_cases.c.h" push: - paths: - - '.github/workflows/tail-call.yml' - - 'Python/bytecodes.c' - - 'Python/ceval.c' - - 'Python/ceval_macros.h' - - 'Python/generated_cases.c.h' + paths: *paths workflow_dispatch: permissions: @@ -25,108 +20,72 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 21 jobs: - tail-call: + macos: name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: - target: -# Un-comment as we add support for more platforms for tail-calling interpreters. -# - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc - - free-threading - llvm: - - 20 include: -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-2022 - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-15-intel + runner: macos-26-intel - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm - - target: free-threading - architecture: x86_64 - runner: ubuntu-24.04 + runner: macos-26 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: - python-version: '3.11' - - - name: Native Windows (debug) - if: runner.os == 'Windows' && matrix.architecture != 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} - call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - # No tests (yet): - - name: Emulated Windows (release) - if: runner.os == 'Windows' && matrix.architecture == 'ARM64' - shell: cmd - run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} - - - name: Native macOS (release) - if: runner.os == 'macOS' + python-version: "3.11" + - name: Install dependencies run: | brew update - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" - export PATH="/usr/local/opt/llvm@${{ matrix.llvm }}/bin:$PATH" - export PATH="/opt/homebrew/opt/llvm@${{ matrix.llvm }}/bin:$PATH" - CC=clang-20 ./configure --with-tail-call-interp + export PATH="/usr/local/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux (debug) - if: runner.os == 'Linux' && matrix.target != 'free-threading' + linux: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-26.04 + configure_flags: --with-pydebug + - target: x86_64-unknown-linux-gnu/gcc-free-threading + runner: ubuntu-26.04 + configure_flags: --disable-gil + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-26.04-arm + configure_flags: --with-pydebug + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + # On ubuntu-26.04 image, clang is clang-21 by default + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp ${{ matrix.configure_flags }} make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - - name: Native Linux with free-threading (release) - if: matrix.target == 'free-threading' + - name: Test run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --disable-gil - make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 463e7bf3355cc39..7def8c9e78abb8d 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml new file mode 100644 index 000000000000000..b96e71b487dbe57 --- /dev/null +++ b/.github/workflows/verify-expat.yml @@ -0,0 +1,32 @@ +name: Verify bundled libexpat + +on: + workflow_dispatch: + push: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + pull_request: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-slim + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Download and verify bundled libexpat files + run: | + ./Modules/expat/refresh.sh + git diff --exit-code Modules/expat/ diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 9b42b47cc855456..7c776d5ea1f941a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,10 +1,6 @@ -# Configuration for the zizmor static analysis tool, run via pre-commit in CI -# https://woodruffw.github.io/zizmor/configuration/ +# Configuration for the zizmor static analysis tool, run via prek in CI +# https://docs.zizmor.sh/configuration/ rules: dangerous-triggers: ignore: - documentation-links.yml - unpinned-uses: - config: - policies: - "*": ref-pin diff --git a/.gitignore b/.gitignore index 2bf4925647ddcd3..d13750307535c7d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.cover *.iml *.o +*.o.tmp *.lto *.a *.so @@ -45,6 +46,7 @@ gmon.out .pytest_cache/ .ruff_cache/ .DS_Store +.pixi/ *.exe @@ -135,10 +137,11 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache -/cross-build/ +/cross-build*/ /jit_stencils*.h +/jit_unwind_info*.h +.jit-stamp /platform /profile-clean-stamp /profile-run-stamp @@ -178,3 +181,6 @@ CLAUDE.local.md #### main branch only stuff below this line, things to backport go above. #### # main branch only: ABI files are not checked/maintained. Doc/data/python*.abi + +# Rust build artifacts +/target/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0311f052798adc..615ba587c6a4005 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,15 +1,33 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.2 + rev: 3b3f7c3f57fe9925356faf5fe6230835138be230 # frozen: v0.15.17 hooks: + - id: ruff-check + name: Run Ruff (lint) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-check + name: Run Ruff (lint) on Lib/ + args: [--exit-non-zero-on-fix] + files: ^Lib/ + exclude: ^Lib/test/ - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ + - id: ruff-check + name: Run Ruff (lint) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ + - id: ruff-check + name: Run Ruff (lint) on Tools/ + args: [--exit-non-zero-on-fix] + files: ^Tools/ + exclude: ^Tools/(build|clinic|i18n|peg_generator|wasm)/ - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] @@ -30,34 +48,42 @@ repos: name: Run Ruff (lint) on Tools/wasm/ args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Platforms/Apple/ + args: [--exit-non-zero-on-fix, --config=Platforms/Apple/.ruff.toml] + files: ^Platforms/Apple/ - id: ruff-format name: Run Ruff (format) on Doc/ - args: [--check] + args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Platforms/WASI/ + args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] + files: ^Platforms/WASI/ - id: ruff-format name: Run Ruff (format) on Tools/build/check_warnings.py - args: [--check, --config=Tools/build/.ruff.toml] + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py - id: ruff-format name: Run Ruff (format) on Tools/wasm/ - args: [--check, --config=Tools/wasm/.ruff.toml] + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.9.0 + rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: remove-tabs types: [python] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -69,30 +95,33 @@ repos: exclude: Lib/test/tokenizedata/coding20731.py - id: end-of-file-fixer files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' - id: trailing-whitespace types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.34.0 + rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 hooks: - id: actionlint - - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.14.1 + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 hooks: - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: c883505f64b59c3c5c9375191e4ad9f98e727ccd # frozen: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/.readthedocs.yml b/.readthedocs.yml index 0a2c3f8345367f4..038417e4bb34385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,24 +11,50 @@ build: os: ubuntu-24.04 tools: python: "3" + apt_packages: + - jq - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html + jobs: + post_system_dependencies: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there are no changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref"); + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" diff --git a/Android/README.md b/Android/README.md deleted file mode 100644 index 9f71aeb934f386d..000000000000000 --- a/Android/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Python for Android - -If you obtained this README as part of a release package, then the only -applicable sections are "Prerequisites", "Testing", and "Using in your own app". - -If you obtained this README as part of the CPython source tree, then you can -also follow the other sections to compile Python for Android yourself. - -However, most app developers should not need to do any of these things manually. -Instead, use one of the tools listed -[here](https://docs.python.org/3/using/android.html), which will provide a much -easier experience. - - -## Prerequisites - -If you already have an Android SDK installed, export the `ANDROID_HOME` -environment variable to point at its location. Otherwise, here's how to install -it: - -* Download the "Command line tools" from . -* Create a directory `android-sdk/cmdline-tools`, and unzip the command line - tools package into it. -* Rename `android-sdk/cmdline-tools/cmdline-tools` to - `android-sdk/cmdline-tools/latest`. -* `export ANDROID_HOME=/path/to/android-sdk` - -The `android.py` script will automatically use the SDK's `sdkmanager` to install -any packages it needs. - -The script also requires the following commands to be on the `PATH`: - -* `curl` -* `java` (or set the `JAVA_HOME` environment variable) - - -## Building - -Python can be built for Android on any POSIX platform supported by the Android -development tools, which currently means Linux or macOS. - -First we'll make a "build" Python (for your development machine), then use it to -help produce a "host" Python for Android. So make sure you have all the usual -tools and libraries needed to build Python for your development machine. - -The easiest way to do a build is to use the `android.py` script. You can either -have it perform the entire build process from start to finish in one step, or -you can do it in discrete steps that mirror running `configure` and `make` for -each of the two builds of Python you end up producing. - -The discrete steps for building via `android.py` are: - -```sh -./android.py configure-build -./android.py make-build -./android.py configure-host HOST -./android.py make-host HOST -``` - -`HOST` identifies which architecture to build. To see the possible values, run -`./android.py configure-host --help`. - -To do all steps in a single command, run: - -```sh -./android.py build HOST -``` - -In the end you should have a build Python in `cross-build/build`, and a host -Python in `cross-build/HOST`. - -You can use `--` as a separator for any of the `configure`-related commands – -including `build` itself – to pass arguments to the underlying `configure` -call. For example, if you want a pydebug build that also caches the results from -`configure`, you can do: - -```sh -./android.py build HOST -- -C --with-pydebug -``` - - -## Packaging - -After building an architecture as described in the section above, you can -package it for release with this command: - -```sh -./android.py package HOST -``` - -`HOST` is defined in the section above. - -This will generate a tarball in `cross-build/HOST/dist`, whose structure is -similar to the `Android` directory of the CPython source tree. - - -## Testing - -The Python test suite can be run on Linux, macOS, or Windows. - -On Linux, the emulator needs access to the KVM virtualization interface. This may -require adding your user to a group, or changing your udev rules. On GitHub -Actions, the test script will do this automatically using the commands shown -[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - -The test suite can usually be run on a device with 2 GB of RAM, but this is -borderline, so you may need to increase it to 4 GB. As of Android -Studio Koala, 2 GB is the default for all emulators, although the user interface -may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, -and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these -manually to the same value, or use the Android Studio Device Manager, which will -update both files. - -You can run the test suite either: - -* Within the CPython repository, after doing a build as described above. On - Windows, you won't be able to do the build on the same machine, so you'll have - to copy the `cross-build/HOST/prefix` directory from somewhere else. - -* Or by taking a release package built using the `package` command, extracting - it wherever you want, and using its own copy of `android.py`. - -The test script supports the following modes: - -* In `--connected` mode, it runs on a device or emulator you have already - connected to the build machine. List the available devices with - `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the - script like this: - - ```sh - ./android.py test --connected emulator-5554 - ``` - -* In `--managed` mode, it uses a temporary headless emulator defined in the - `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, - but more reproducible. - - We currently define two devices: `minVersion` and `maxVersion`, corresponding - to our minimum and maximum supported Android versions. For example: - - ```sh - ./android.py test --managed maxVersion - ``` - -By default, the only messages the script will show are Python's own stdout and -stderr. Add the `-v` option to also show Gradle output, and non-Python logcat -messages. - -Any other arguments on the `android.py test` command line will be passed through -to `python -m test` – use `--` to separate them from android.py's own options. -See the [Python Developer's -Guide](https://devguide.python.org/testing/run-write-tests/) for common options -– most of them will work on Android, except for those that involve subprocesses, -such as `-j`. - -Every time you run `android.py test`, changes in pure-Python files in the -repository's `Lib` directory will be picked up immediately. Changes in C files, -and architecture-specific files such as sysconfigdata, will not take effect -until you re-run `android.py make-host` or `build`. - -The testbed app can also be used to test third-party packages. For more details, -run `android.py test --help`, paying attention to the options `--site-packages`, -`--cwd`, `-c` and `-m`. - - -## Using in your own app - -See https://docs.python.org/3/using/android.html. diff --git a/Android/android.py b/Android/android.py deleted file mode 100755 index 25bb4ca70b581f4..000000000000000 --- a/Android/android.py +++ /dev/null @@ -1,916 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import json -import os -import platform -import re -import shlex -import shutil -import signal -import subprocess -import sys -import sysconfig -from asyncio import wait_for -from contextlib import asynccontextmanager -from datetime import datetime, timezone -from glob import glob -from os.path import abspath, basename, relpath -from pathlib import Path -from subprocess import CalledProcessError -from tempfile import TemporaryDirectory - - -SCRIPT_NAME = Path(__file__).name -ANDROID_DIR = Path(__file__).resolve().parent -PYTHON_DIR = ANDROID_DIR.parent -in_source_tree = ( - ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() -) - -TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" - -HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] -APP_ID = "org.python.testbed" -DECODE_ARGS = ("UTF-8", "backslashreplace") - - -try: - android_home = Path(os.environ['ANDROID_HOME']) -except KeyError: - sys.exit("The ANDROID_HOME environment variable is required.") - -adb = Path( - f"{android_home}/platform-tools/adb" - + (".exe" if os.name == "nt" else "") -) - -gradlew = Path( - f"{TESTBED_DIR}/gradlew" - + (".bat" if os.name == "nt" else "") -) - -# Whether we've seen any output from Python yet. -python_started = False - -# Buffer for verbose output which will be displayed only if a test fails and -# there has been no output from Python. -hidden_output = [] - - -def log_verbose(context, line, stream=sys.stdout): - if context.verbose: - stream.write(line) - else: - hidden_output.append((stream, line)) - - -def delete_glob(pattern): - # Path.glob doesn't accept non-relative patterns. - for path in glob(str(pattern)): - path = Path(path) - print(f"Deleting {path} ...") - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - - -def subdir(*parts, create=False): - path = CROSS_BUILD_DIR.joinpath(*parts) - if not path.exists(): - if not create: - sys.exit( - f"{path} does not exist. Create it by running the appropriate " - f"`configure` subcommand of {SCRIPT_NAME}.") - else: - path.mkdir(parents=True) - return path - - -def run(command, *, host=None, env=None, log=True, **kwargs): - kwargs.setdefault("check", True) - if env is None: - env = os.environ.copy() - - if host: - host_env = android_env(host) - print_env(host_env) - env.update(host_env) - - if log: - print(">", join_command(command)) - return subprocess.run(command, env=env, **kwargs) - - -# Format a command so it can be copied into a shell. Like shlex.join, but also -# accepts arguments which are Paths, or a single string/Path outside of a list. -def join_command(args): - if isinstance(args, (str, Path)): - return str(args) - else: - return shlex.join(map(str, args)) - - -# Format the environment so it can be pasted into a shell. -def print_env(env): - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") - - -def android_env(host): - if host: - prefix = subdir(host) / "prefix" - else: - prefix = ANDROID_DIR / "prefix" - sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") - sysconfig_filename = next(sysconfig_files).name - host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] - - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={prefix}; " - f". {env_script}; " - f"export", - check=True, shell=True, capture_output=True, encoding='utf-8', - ).stdout - - env = {} - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if os.environ.get(key) != value: - env[key] = value - - if not env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) - return env - - -def build_python_path(): - """The path to the build Python binary.""" - build_dir = subdir("build") - binary = build_dir / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{build_dir}") - - return binary - - -def configure_build_python(context): - if context.clean: - clean("build") - os.chdir(subdir("build", create=True)) - - command = [relpath(PYTHON_DIR / "configure")] - if context.args: - command.extend(context.args) - run(command) - - -def make_build_python(context): - os.chdir(subdir("build")) - run(["make", "-j", str(os.cpu_count())]) - - -# To create new builds of these dependencies, usually all that's necessary is to -# push a tag to the cpython-android-source-deps repository, and GitHub Actions -# will do the rest. -# -# If you're a member of the Python core team, and you'd like to be able to push -# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. -def unpack_deps(host, prefix_dir): - os.chdir(prefix_dir) - deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0", - "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: - filename = f"{name_ver}-{host}.tar.gz" - download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename) - os.remove(filename) - - -def download(url, target_dir="."): - out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) - return out_path - - -def configure_host_python(context): - if context.clean: - clean(context.host) - - host_dir = subdir(context.host, create=True) - prefix_dir = host_dir / "prefix" - if not prefix_dir.exists(): - prefix_dir.mkdir() - unpack_deps(context.host, prefix_dir) - - os.chdir(host_dir) - command = [ - # Basic cross-compiling configuration - relpath(PYTHON_DIR / "configure"), - f"--host={context.host}", - f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", - f"--with-build-python={build_python_path()}", - "--without-ensurepip", - - # Android always uses a shared libpython. - "--enable-shared", - "--without-static-libpython", - - # Dependent libraries. The others are found using pkg-config: see - # android-env.sh. - f"--with-openssl={prefix_dir}", - ] - - if context.args: - command.extend(context.args) - run(command, host=context.host) - - -def make_host_python(context): - # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so - # delete any previous Python installation to prevent it being used during - # the build. - host_dir = subdir(context.host) - prefix_dir = host_dir / "prefix" - for pattern in ("include/python*", "lib/libpython*", "lib/python*"): - delete_glob(f"{prefix_dir}/{pattern}") - - # The Android environment variables were already captured in the Makefile by - # `configure`, and passing them again when running `make` may cause some - # flags to be duplicated. So we don't use the `host` argument here. - os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())]) - - # The `make install` output is very verbose and rarely useful, so - # suppress it by default. - run( - ["make", "install", f"prefix={prefix_dir}"], - capture_output=not context.verbose, - ) - - -def build_all(context): - steps = [configure_build_python, make_build_python, configure_host_python, - make_host_python] - for step in steps: - step(context) - - -def clean(host): - delete_glob(CROSS_BUILD_DIR / host) - - -def clean_all(context): - for host in HOSTS + ["build"]: - clean(host) - - -def setup_ci(): - # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ - if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": - run( - ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], - input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', - text=True, - ) - run(["sudo", "udevadm", "control", "--reload-rules"]) - run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) - - -def setup_sdk(): - sdkmanager = android_home / ( - "cmdline-tools/latest/bin/sdkmanager" - + (".bat" if os.name == "nt" else "") - ) - - # Gradle will fail if it needs to install an SDK package whose license - # hasn't been accepted, so pre-accept all licenses. - if not all((android_home / "licenses" / path).exists() for path in [ - "android-sdk-arm-dbt-license", "android-sdk-license" - ]): - run( - [sdkmanager, "--licenses"], - text=True, - capture_output=True, - input="y\n" * 100, - ) - - # Gradle may install this automatically, but we can't rely on that because - # we need to run adb within the logcat task. - if not adb.exists(): - run([sdkmanager, "platform-tools"]) - - -# To avoid distributing compiled artifacts without corresponding source code, -# the Gradle wrapper is not included in the CPython repository. Instead, we -# extract it from the Gradle GitHub repository. -def setup_testbed(): - paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): - return - - # The wrapper version isn't important, as any version of the wrapper can - # download any version of Gradle. The Gradle version actually used for the - # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. - version = "8.9.0" - - for path in paths: - out_path = TESTBED_DIR / path - out_path.parent.mkdir(exist_ok=True) - download( - f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}", - out_path.parent, - ) - os.chmod(out_path, 0o755) - - -# run_testbed will build the app automatically, but it's useful to have this as -# a separate command to allow running the app outside of this script. -def build_testbed(context): - setup_sdk() - setup_testbed() - run( - [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], - cwd=TESTBED_DIR, - ) - - -# Work around a bug involving sys.exit and TaskGroups -# (https://github.com/python/cpython/issues/101515). -def exit(*args): - raise MySystemExit(*args) - - -class MySystemExit(Exception): - pass - - -# The `test` subcommand runs all subprocesses through this context manager so -# that no matter what happens, they can always be cancelled from another task, -# and they will always be cleaned up on exit. -@asynccontextmanager -async def async_process(*args, **kwargs): - process = await asyncio.create_subprocess_exec(*args, **kwargs) - try: - yield process - finally: - if process.returncode is None: - # Allow a reasonably long time for Gradle to clean itself up, - # because we don't want stale emulators left behind. - timeout = 10 - process.terminate() - try: - await wait_for(process.wait(), timeout) - except TimeoutError: - print( - f"Command {args} did not terminate after {timeout} seconds " - f" - sending SIGKILL" - ) - process.kill() - - # Even after killing the process we must still wait for it, - # otherwise we'll get the warning "Exception ignored in __del__". - await wait_for(process.wait(), timeout=1) - - -async def async_check_output(*args, **kwargs): - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) as process: - stdout, stderr = await process.communicate() - if process.returncode == 0: - return stdout.decode(*DECODE_ARGS) - else: - raise CalledProcessError( - process.returncode, args, - stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) - ) - - -# Return a list of the serial numbers of connected devices. Emulators will have -# serials of the form "emulator-5678". -async def list_devices(): - serials = [] - header_found = False - - lines = (await async_check_output(adb, "devices")).splitlines() - for line in lines: - # Ignore blank lines, and all lines before the header. - line = line.strip() - if line == "List of devices attached": - header_found = True - elif header_found and line: - try: - serial, status = line.split() - except ValueError: - raise ValueError(f"failed to parse {line!r}") - if status == "device": - serials.append(serial) - - if not header_found: - raise ValueError(f"failed to parse {lines}") - return serials - - -async def find_device(context, initial_devices): - if context.managed: - print("Waiting for managed device - this may take several minutes") - while True: - new_devices = set(await list_devices()).difference(initial_devices) - if len(new_devices) == 0: - await asyncio.sleep(1) - elif len(new_devices) == 1: - serial = new_devices.pop() - print(f"Serial: {serial}") - return serial - else: - exit(f"Found more than one new device: {new_devices}") - else: - return context.connected - - -# An older version of this script in #121595 filtered the logs by UID instead. -# But logcat can't filter by UID until API level 31. If we ever switch back to -# filtering by UID, we'll also have to filter by time so we only show messages -# produced after the initial call to `stop_app`. -# -# We're more likely to miss the PID because it's shorter-lived, so there's a -# workaround in PythonSuite.kt to stop it being *too* short-lived. -async def find_pid(serial): - print("Waiting for app to start - this may take several minutes") - shown_error = False - while True: - try: - # `pidof` requires API level 24 or higher. The level 23 emulator - # includes it, but it doesn't work (it returns all processes). - pid = (await async_check_output( - adb, "-s", serial, "shell", "pidof", "-s", APP_ID - )).strip() - except CalledProcessError as e: - # If the app isn't running yet, pidof gives no output. So if there - # is output, there must have been some other error. However, this - # sometimes happens transiently, especially when running a managed - # emulator for the first time, so don't make it fatal. - if (e.stdout or e.stderr) and not shown_error: - print_called_process_error(e) - print("This may be transient, so continuing to wait") - shown_error = True - else: - # Some older devices (e.g. Nexus 4) return zero even when no process - # was found, so check whether we actually got any output. - if pid: - print(f"PID: {pid}") - return pid - - # Loop fairly rapidly to avoid missing a short-lived process. - await asyncio.sleep(0.2) - - -async def logcat_task(context, initial_devices): - # Gradle may need to do some large downloads of libraries and emulator - # images. This will happen during find_device in --managed mode, or find_pid - # in --connected mode. - startup_timeout = 600 - serial = await wait_for(find_device(context, initial_devices), startup_timeout) - pid = await wait_for(find_pid(serial), startup_timeout) - - # `--pid` requires API level 24 or higher. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - logcat_started = False - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): - logcat_started = True - level, message = match.groups() - else: - # If the regex doesn't match, this is either a logcat startup - # error, or the second or subsequent line of a multi-line - # message. Python won't produce multi-line messages, but other - # components might. - level, message = None, line - - # Exclude high-volume messages which are rarely useful. - if context.verbose < 2 and "from python test_syslog" in message: - continue - - # Put high-level messages on stderr so they're highlighted in the - # buildbot logs. This will include Python's own stderr. - stream = ( - sys.stderr - if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) - else sys.stdout - ) - - # To simplify automated processing of the output, e.g. a buildbot - # posting a failure notice on a GitHub PR, we strip the level and - # tag indicators from Python's stdout and stderr. - for prefix in ["python.stdout: ", "python.stderr: "]: - if message.startswith(prefix): - global python_started - python_started = True - stream.write(message.removeprefix(prefix)) - break - else: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - log_verbose(context, line, stream) - - # If the device disconnects while logcat is running, which always - # happens in --managed mode, some versions of adb return non-zero. - # Distinguish this from a logcat startup error by checking whether we've - # received any logcat messages yet. - status = await wait_for(process.wait(), timeout=1) - if status != 0 and not logcat_started: - raise CalledProcessError(status, args) - - -def stop_app(serial): - run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) - - -async def gradle_task(context): - env = os.environ.copy() - if context.managed: - task_prefix = context.managed - else: - task_prefix = "connected" - env["ANDROID_SERIAL"] = context.connected - - if context.ci_mode: - context.args[0:0] = [ - # See _add_ci_python_opts in libregrtest/main.py. - "-W", "error", "-bb", "-E", - - # Randomization is disabled because order-dependent failures are - # much less likely to pass on a rerun in single-process mode. - "-m", "test", - f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" - ] - - if not any(arg in context.args for arg in ["-c", "-m"]): - context.args[0:0] = ["-m", "test"] - - args = [ - gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - ] + [ - f"-P{name}={value}" - for name, value in [ - ("python.sitePackages", context.site_packages), - ("python.cwd", context.cwd), - ( - "android.testInstrumentationRunnerArguments.pythonArgs", - json.dumps(context.args), - ), - ] - if value - ] - if context.verbose >= 2: - args.append("--info") - log_verbose(context, f"> {join_command(args)}\n") - - try: - async with async_process( - *args, cwd=TESTBED_DIR, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - log_verbose(context, line) - - status = await wait_for(process.wait(), timeout=1) - if status == 0: - exit(0) - else: - raise CalledProcessError(status, args) - finally: - # Gradle does not stop the tests when interrupted. - if context.connected: - stop_app(context.connected) - - -async def run_testbed(context): - setup_ci() - setup_sdk() - setup_testbed() - - if context.managed: - # In this mode, Gradle will create a device with an unpredictable name. - # So we save a list of the running devices before starting Gradle, and - # find_device then waits for a new device to appear. - initial_devices = await list_devices() - else: - # In case the previous shutdown was unclean, make sure the app isn't - # running, otherwise we might show logs from a previous run. This is - # unnecessary in --managed mode, because Gradle creates a new emulator - # every time. - stop_app(context.connected) - initial_devices = None - - try: - async with asyncio.TaskGroup() as tg: - tg.create_task(logcat_task(context, initial_devices)) - tg.create_task(gradle_task(context)) - except* MySystemExit as e: - raise SystemExit(*e.exceptions[0].args) from None - except* CalledProcessError as e: - # If Python produced no output, then the user probably wants to see the - # verbose output to explain why the test failed. - if not python_started: - for stream, line in hidden_output: - stream.write(line) - - # Extract it from the ExceptionGroup so it can be handled by `main`. - raise e.exceptions[0] - - -def package_version(prefix_dir): - patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h" - patchlevel_paths = glob(patchlevel_glob) - if len(patchlevel_paths) != 1: - sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.") - - for line in open(patchlevel_paths[0]): - if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line): - version = match[1] - break - else: - sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.") - - # If not building against a tagged commit, add a timestamp to the version. - # Follow the PyPA version number rules, as this will make it easier to - # process with other tools. - if version.endswith("+"): - version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") - - return version - - -def package(context): - prefix_dir = subdir(context.host, "prefix") - version = package_version(prefix_dir) - - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - temp_dir = Path(temp_dir) - - # Include all tracked files from the Android directory. - for line in run( - ["git", "ls-files"], - cwd=ANDROID_DIR, capture_output=True, text=True, log=False, - ).stdout.splitlines(): - src = ANDROID_DIR / line - dst = temp_dir / line - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst, follow_symlinks=False) - - # Include anything from the prefix directory which could be useful - # either for embedding Python in an app, or building third-party - # packages against it. - for rel_dir, patterns in [ - ("include", ["openssl*", "python*", "sqlite*"]), - ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*", - "libssl*.so", "ossl-modules", "python*"]), - ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]), - ]: - for pattern in patterns: - for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"): - dst = temp_dir / relpath(src, prefix_dir.parent) - dst.parent.mkdir(parents=True, exist_ok=True) - if Path(src).is_dir(): - shutil.copytree( - src, dst, symlinks=True, - ignore=lambda *args: ["__pycache__"] - ) - else: - shutil.copy2(src, dst, follow_symlinks=False) - - # Strip debug information. - if not context.debug: - so_files = glob(f"{temp_dir}/**/*.so", recursive=True) - run([android_env(context.host)["STRIP"], *so_files], log=False) - - dist_dir = subdir(context.host, "dist", create=True) - package_path = shutil.make_archive( - f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir - ) - print(f"Wrote {package_path}") - return package_path - - -def ci(context): - for step in [ - configure_build_python, - make_build_python, - configure_host_python, - make_host_python, - package, - ]: - caption = ( - step.__name__.replace("_", " ") - .capitalize() - .replace("python", "Python") - ) - print(f"::group::{caption}") - result = step(context) - if step is package: - package_path = result - print("::endgroup::") - - if ( - "GITHUB_ACTIONS" in os.environ - and (platform.system(), platform.machine()) != ("Linux", "x86_64") - ): - print( - "Skipping tests: GitHub Actions does not support the Android " - "emulator on this platform." - ) - else: - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - print("::group::Tests") - - # Prove the package is self-contained by using it to run the tests. - shutil.unpack_archive(package_path, temp_dir) - launcher_args = [ - "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" - ] - run( - ["./android.py", "test", *launcher_args], - cwd=temp_dir - ) - print("::endgroup::") - - -def env(context): - print_env(android_env(getattr(context, "host", None))) - - -# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated -# by the buildbot worker, we'll make an attempt to clean up our subprocesses. -def install_signal_handler(): - def signal_handler(*args): - os.kill(os.getpid(), signal.SIGINT) - - signal.signal(signal.SIGTERM, signal_handler) - - -def parse_args(): - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand", required=True) - - def add_parser(*args, **kwargs): - parser = subcommands.add_parser(*args, **kwargs) - parser.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show verbose output. Use twice to be even more verbose.") - return parser - - # Subcommands - build = add_parser( - "build", help="Run configure-build, make-build, configure-host and " - "make-host") - configure_build = add_parser( - "configure-build", help="Run `configure` for the build Python") - add_parser( - "make-build", help="Run `make` for the build Python") - configure_host = add_parser( - "configure-host", help="Run `configure` for Android") - make_host = add_parser( - "make-host", help="Run `make` for Android") - - add_parser("clean", help="Delete all build directories") - add_parser("build-testbed", help="Build the testbed app") - test = add_parser("test", help="Run the testbed app") - package = add_parser("package", help="Make a release package") - ci = add_parser("ci", help="Run build, package and test") - env = add_parser("env", help="Print environment variables") - - # Common arguments - for subcommand in [build, configure_build, configure_host, ci]: - subcommand.add_argument( - "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build directories first") - - host_commands = [build, configure_host, make_host, package, ci] - if in_source_tree: - host_commands.append(env) - for subcommand in host_commands: - subcommand.add_argument( - "host", metavar="HOST", choices=HOSTS, - help="Host triplet: choices=[%(choices)s]") - - for subcommand in [build, configure_build, configure_host, ci]: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") - - # Test arguments - device_group = test.add_mutually_exclusive_group(required=True) - device_group.add_argument( - "--connected", metavar="SERIAL", help="Run on a connected device. " - "Connect it yourself, then get its serial from `adb devices`.") - device_group.add_argument( - "--managed", metavar="NAME", help="Run on a Gradle-managed device. " - "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") - - test.add_argument( - "--site-packages", metavar="DIR", type=abspath, - help="Directory to copy as the app's site-packages.") - test.add_argument( - "--cwd", metavar="DIR", type=abspath, - help="Directory to copy as the app's working directory.") - test.add_argument( - "args", nargs="*", help=f"Python command-line arguments. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " - f"If neither -c nor -m are included, `-m test` will be prepended, " - f"which will run Python's own test suite.") - - # Package arguments. - for subcommand in [package, ci]: - subcommand.add_argument( - "-g", action="store_true", default=False, dest="debug", - help="Include debug information in package") - - # CI arguments - for subcommand in [test, ci]: - group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) - group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") - group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") - - return parser.parse_args() - - -def main(): - install_signal_handler() - - # Under the buildbot, stdout is not a TTY, but we must still flush after - # every line to make sure our output appears in the correct order relative - # to the output of our subprocesses. - for stream in [sys.stdout, sys.stderr]: - stream.reconfigure(line_buffering=True) - - context = parse_args() - dispatch = { - "configure-build": configure_build_python, - "make-build": make_build_python, - "configure-host": configure_host_python, - "make-host": make_host_python, - "build": build_all, - "clean": clean_all, - "build-testbed": build_testbed, - "test": run_testbed, - "package": package, - "ci": ci, - "env": env, - } - - try: - result = dispatch[context.subcommand](context) - if asyncio.iscoroutine(result): - asyncio.run(result) - except CalledProcessError as e: - print_called_process_error(e) - sys.exit(1) - - -def print_called_process_error(e): - for stream_name in ["stdout", "stderr"]: - content = getattr(e, stream_name) - if isinstance(content, bytes): - content = content.decode(*DECODE_ARGS) - stream = getattr(sys, stream_name) - if content: - stream.write(content) - if not content.endswith("\n"): - stream.write("\n") - - # shlex uses single quotes, so we surround the command with double quotes. - print( - f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' - ) - - -if __name__ == "__main__": - main() diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts deleted file mode 100644 index 4de628a279ca3f7..000000000000000 --- a/Android/testbed/app/build.gradle.kts +++ /dev/null @@ -1,269 +0,0 @@ -import com.android.build.api.variant.* -import kotlin.math.max - -plugins { - id("com.android.application") - id("org.jetbrains.kotlin.android") -} - -val ANDROID_DIR = file("../..") -val PYTHON_DIR = ANDROID_DIR.parentFile!! -val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") -val inSourceTree = ( - ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() -) - -val KNOWN_ABIS = mapOf( - "aarch64-linux-android" to "arm64-v8a", - "x86_64-linux-android" to "x86_64", -) - -// Discover prefixes. -val prefixes = ArrayList() -if (inSourceTree) { - for ((triplet, _) in KNOWN_ABIS.entries) { - val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } - } -} else { - // Testbed is inside a release package. - val prefix = file("$ANDROID_DIR/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } -} -if (prefixes.isEmpty()) { - throw GradleException( - "No Android prefixes found: see README.md for testing instructions" - ) -} - -// Detect Python versions and ABIs. -lateinit var pythonVersion: String -var abis = HashMap() -for ((i, prefix) in prefixes.withIndex()) { - val libDir = file("$prefix/lib") - val version = run { - for (filename in libDir.list()!!) { - """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python version in $libDir") - } - if (i == 0) { - pythonVersion = version - } else if (pythonVersion != version) { - throw GradleException( - "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" - ) - } - - val libPythonDir = file("$libDir/python$pythonVersion") - val triplet = run { - for (filename in libPythonDir.list()!!) { - """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() - .matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python triplet in $libPythonDir") - } - abis[prefix] = KNOWN_ABIS[triplet]!! -} - - -android { - val androidEnvFile = file("../../android-env.sh").absoluteFile - - namespace = "org.python.testbed" - compileSdk = 34 - - defaultConfig { - applicationId = "org.python.testbed" - - minSdk = androidEnvFile.useLines { - for (line in it) { - """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1].toInt() - } - } - throw GradleException("Failed to find API level in $androidEnvFile") - } - targetSdk = 34 - - versionCode = 1 - versionName = "1.0" - - ndk.abiFilters.addAll(abis.values) - externalNativeBuild.cmake.arguments( - "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { - // AGP uses the ${} syntax for its own purposes, so use a Jinja style - // placeholder. - "$PYTHON_CROSS_DIR/{{triplet}}/prefix" - } else { - prefixes[0] - }, - "-DPYTHON_VERSION=$pythonVersion", - "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", - ) - - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } - - ndkVersion = androidEnvFile.useLines { - for (line in it) { - """ndk_version=(\S+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1] - } - } - throw GradleException("Failed to find NDK version in $androidEnvFile") - } - externalNativeBuild.cmake { - path("src/main/c/CMakeLists.txt") - } - - // Set this property to something non-empty, otherwise it'll use the default - // list, which ignores asset directories beginning with an underscore. - aaptOptions.ignoreAssetsPattern = ".git" - - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - kotlinOptions { - jvmTarget = "1.8" - } - - testOptions { - managedDevices { - localDevices { - create("minVersion") { - device = "Small Phone" - - // Managed devices have a minimum API level of 27. - apiLevel = max(27, defaultConfig.minSdk!!) - - // ATD devices are smaller and faster, but have a minimum - // API level of 30. - systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" - } - - create("maxVersion") { - device = "Small Phone" - apiLevel = defaultConfig.targetSdk!! - systemImageSource = "aosp-atd" - } - } - - // If the previous test run succeeded and nothing has changed, - // Gradle thinks there's no need to run it again. Override that. - afterEvaluate { - (localDevices.names + listOf("connected")).forEach { - tasks.named("${it}DebugAndroidTest") { - outputs.upToDateWhen { false } - } - } - } - } - } -} - -dependencies { - implementation("androidx.appcompat:appcompat:1.6.1") - implementation("com.google.android.material:material:1.11.0") - implementation("androidx.constraintlayout:constraintlayout:2.1.4") - androidTestImplementation("androidx.test.ext:junit:1.1.5") - androidTestImplementation("androidx.test:rules:1.5.0") -} - - -// Create some custom tasks to copy Python and its standard library from -// elsewhere in the repository. -androidComponents.onVariants { variant -> - val pyPlusVer = "python$pythonVersion" - generateTask(variant, variant.sources.assets!!) { - into("python") { - // Include files such as pyconfig.h are used by some of the tests. - into("include/$pyPlusVer") { - for (prefix in prefixes) { - from("$prefix/include/$pyPlusVer") - } - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } - - into("lib/$pyPlusVer") { - // To aid debugging, the source directory takes priority when - // running inside a CPython source tree. - if (inSourceTree) { - from("$PYTHON_DIR/Lib") - } - for (prefix in prefixes) { - from("$prefix/lib/$pyPlusVer") - } - - into("site-packages") { - from("$projectDir/src/main/python") - - val sitePackages = findProperty("python.sitePackages") as String? - if (!sitePackages.isNullOrEmpty()) { - if (!file(sitePackages).exists()) { - throw GradleException("$sitePackages does not exist") - } - from(sitePackages) - } - } - - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - exclude("**/__pycache__") - } - - into("cwd") { - val cwd = findProperty("python.cwd") as String? - if (!cwd.isNullOrEmpty()) { - if (!file(cwd).exists()) { - throw GradleException("$cwd does not exist") - } - from(cwd) - } - } - } - } - - generateTask(variant, variant.sources.jniLibs!!) { - for ((prefix, abi) in abis.entries) { - into(abi) { - from("$prefix/lib") - include("libpython*.*.so") - include("lib*_python.so") - } - } - } -} - - -fun generateTask( - variant: ApplicationVariant, directories: SourceDirectories, - configure: GenerateTask.() -> Unit -) { - val taskName = "generate" + - listOf(variant.name, "Python", directories.name) - .map { it.replaceFirstChar(Char::uppercase) } - .joinToString("") - - directories.addGeneratedSourceDirectory( - tasks.register(taskName) { - into(outputDir) - configure() - }, - GenerateTask::outputDir) -} - - -// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. -abstract class GenerateTask: Sync() { - @get:OutputDirectory - abstract val outputDir: DirectoryProperty -} diff --git a/Apple/__main__.py b/Apple/__main__.py deleted file mode 100644 index e76fc3517987073..000000000000000 --- a/Apple/__main__.py +++ /dev/null @@ -1,1051 +0,0 @@ -#!/usr/bin/env python3 -########################################################################## -# Apple XCframework build script -# -# This script simplifies the process of configuring, compiling and packaging an -# XCframework for an Apple platform. -# -# At present, it only supports iOS, but it has been constructed so that it -# could be used on any Apple platform. -# -# The simplest entry point is: -# -# $ python Apple ci iOS -# -# which will: -# * Clean any pre-existing build artefacts -# * Configure and make a Python that can be used for the build -# * Configure and make a Python for each supported iOS architecture and ABI -# * Combine the outputs of the builds from the previous step into a single -# XCframework, merging binaries into a "fat" binary if necessary -# * Clone a copy of the testbed, configured to use the XCframework -# * Construct a tarball containing the release artefacts -# * Run the test suite using the generated XCframework. -# -# This is the complete sequence that would be needed in CI to build and test -# a candidate release artefact. -# -# Each individual step can be invoked individually - there are commands to -# clean, configure-build, make-build, configure-host, make-host, package, and -# test. -# -# There is also a build command that can be used to combine the configure and -# make steps for the build Python, an individual host, all hosts, or all -# builds. -########################################################################## -from __future__ import annotations - -import argparse -import os -import platform -import re -import shlex -import shutil -import signal -import subprocess -import sys -import sysconfig -import time -from collections.abc import Sequence -from contextlib import contextmanager -from datetime import datetime, timezone -from os.path import basename, relpath -from pathlib import Path -from subprocess import CalledProcessError -from typing import Callable - -EnvironmentT = dict[str, str] -ArgsT = Sequence[str | Path] - -SCRIPT_NAME = Path(__file__).name -PYTHON_DIR = Path(__file__).resolve().parent.parent - -CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" - -HOSTS: dict[str, dict[str, dict[str, str]]] = { - # Structure of this data: - # * Platform identifier - # * an XCframework slice that must exist for that platform - # * a host triple: the multiarch spec for that host - "iOS": { - "ios-arm64": { - "arm64-apple-ios": "arm64-iphoneos", - }, - "ios-arm64_x86_64-simulator": { - "arm64-apple-ios-simulator": "arm64-iphonesimulator", - "x86_64-apple-ios-simulator": "x86_64-iphonesimulator", - }, - }, -} - - -def subdir(name: str, create: bool = False) -> Path: - """Ensure that a cross-build directory for the given name exists.""" - path = CROSS_BUILD_DIR / name - if not path.exists(): - if not create: - sys.exit( - f"{path} does not exist. Create it by running the appropriate " - f"`configure` subcommand of {SCRIPT_NAME}." - ) - else: - path.mkdir(parents=True) - return path - - -def run( - command: ArgsT, - *, - host: str | None = None, - env: EnvironmentT | None = None, - log: bool | None = True, - **kwargs, -) -> subprocess.CompletedProcess: - """Run a command in an Apple development environment. - - Optionally logs the executed command to the console. - """ - kwargs.setdefault("check", True) - if env is None: - env = os.environ.copy() - - if host: - host_env = apple_env(host) - print_env(host_env) - env.update(host_env) - - if log: - print(">", join_command(command)) - return subprocess.run(command, env=env, **kwargs) - - -def join_command(args: str | Path | ArgsT) -> str: - """Format a command so it can be copied into a shell. - - Similar to `shlex.join`, but also accepts arguments which are Paths, or a - single string/Path outside of a list. - """ - if isinstance(args, (str, Path)): - return str(args) - else: - return shlex.join(map(str, args)) - - -def print_env(env: EnvironmentT) -> None: - """Format the environment so it can be pasted into a shell.""" - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") - - -def apple_env(host: str) -> EnvironmentT: - """Construct an Apple development environment for the given host.""" - env = { - "PATH": ":".join( - [ - str(PYTHON_DIR / "Apple/iOS/Resources/bin"), - str(subdir(host) / "prefix"), - "/usr/bin", - "/bin", - "/usr/sbin", - "/sbin", - "/Library/Apple/usr/bin", - ] - ), - } - - return env - - -def delete_path(name: str) -> None: - """Delete the named cross-build directory, if it exists.""" - path = CROSS_BUILD_DIR / name - if path.exists(): - print(f"Deleting {path} ...") - shutil.rmtree(path) - - -def all_host_triples(platform: str) -> list[str]: - """Return all host triples for the given platform. - - The host triples are the platform definitions used as input to configure - (e.g., "arm64-apple-ios-simulator"). - """ - triples = [] - for slice_name, slice_parts in HOSTS[platform].items(): - triples.extend(list(slice_parts)) - return triples - - -def clean(context: argparse.Namespace, target: str = "all") -> None: - """The implementation of the "clean" command.""" - # If we're explicitly targeting the build, there's no platform or - # distribution artefacts. If we're cleaning tests, we keep all built - # artefacts. Otherwise, the built artefacts must be dirty, so we remove - # them. - if target not in {"build", "test"}: - paths = ["dist", context.platform] + list(HOSTS[context.platform]) - else: - paths = [] - - if target in {"all", "build"}: - paths.append("build") - - if target in {"all", "hosts"}: - paths.extend(all_host_triples(context.platform)) - elif target not in {"build", "test", "package"}: - paths.append(target) - - if target in {"all", "hosts", "test"}: - paths.extend( - [ - path.name - for path in CROSS_BUILD_DIR.glob( - f"{context.platform}-testbed.*" - ) - ] - ) - - for path in paths: - delete_path(path) - - -def build_python_path() -> Path: - """The path to the build Python binary.""" - build_dir = subdir("build") - binary = build_dir / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError( - f"Unable to find `python(.exe)` in {build_dir}" - ) - - return binary - - -@contextmanager -def group(text: str): - """A context manager that outputs a log marker around a section of a build. - - If running in a GitHub Actions environment, the GitHub syntax for - collapsible log sections is used. - """ - if "GITHUB_ACTIONS" in os.environ: - print(f"::group::{text}") - else: - print(f"===== {text} " + "=" * (70 - len(text))) - - yield - - if "GITHUB_ACTIONS" in os.environ: - print("::endgroup::") - else: - print() - - -@contextmanager -def cwd(subdir: Path): - """A context manager that sets the current working directory.""" - orig = os.getcwd() - os.chdir(subdir) - yield - os.chdir(orig) - - -def configure_build_python(context: argparse.Namespace) -> None: - """The implementation of the "configure-build" command.""" - if context.clean: - clean(context, "build") - - with ( - group("Configuring build Python"), - cwd(subdir("build", create=True)), - ): - command = [relpath(PYTHON_DIR / "configure")] - if context.args: - command.extend(context.args) - run(command) - - -def make_build_python(context: argparse.Namespace) -> None: - """The implementation of the "make-build" command.""" - with ( - group("Compiling build Python"), - cwd(subdir("build")), - ): - run(["make", "-j", str(os.cpu_count())]) - - -def apple_target(host: str) -> str: - """Return the Apple platform identifier for a given host triple.""" - for _, platform_slices in HOSTS.items(): - for slice_name, slice_parts in platform_slices.items(): - for host_triple, multiarch in slice_parts.items(): - if host == host_triple: - return ".".join(multiarch.split("-")[::-1]) - - raise KeyError(host) - - -def apple_multiarch(host: str) -> str: - """Return the multiarch descriptor for a given host triple.""" - for _, platform_slices in HOSTS.items(): - for slice_name, slice_parts in platform_slices.items(): - for host_triple, multiarch in slice_parts.items(): - if host == host_triple: - return multiarch - - raise KeyError(host) - - -def unpack_deps( - platform: str, - host: str, - prefix_dir: Path, - cache_dir: Path, -) -> None: - """Unpack binary dependencies into a provided directory. - - Downloads binaries if they aren't already present. Downloads will be stored - in provided cache directory. - - On iOS, as a safety mechanism, any dynamic libraries will be purged from - the unpacked dependencies. - """ - # To create new builds of these dependencies, usually all that's necessary - # is to push a tag to the cpython-apple-source-deps repository, and GitHub - # Actions will do the rest. - # - # If you're a member of the Python core team, and you'd like to be able to - # push these tags yourself, please contact Malcolm Smith or Russell - # Keith-Magee. - deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" - for name_ver in [ - "BZip2-1.0.8-2", - "libFFI-3.4.7-2", - "OpenSSL-3.0.18-1", - "XZ-5.6.4-2", - "mpdecimal-4.0.0-2", - "zstd-1.5.7-1", - ]: - filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz" - archive_path = download( - f"{deps_url}/{name_ver}/{filename}", - target_dir=cache_dir, - ) - shutil.unpack_archive(archive_path, prefix_dir) - - # Dynamic libraries will be preferentially linked over static; - # On iOS, ensure that no dylibs are available in the prefix folder. - if platform == "iOS": - for dylib in prefix_dir.glob("**/*.dylib"): - dylib.unlink() - - -def download(url: str, target_dir: Path) -> Path: - """Download the specified URL into the given directory. - - :return: The path to the downloaded archive. - """ - target_path = Path(target_dir).resolve() - target_path.mkdir(exist_ok=True, parents=True) - - out_path = target_path / basename(url) - if not Path(out_path).is_file(): - run( - [ - "curl", - "-Lf", - "--retry", - "5", - "--retry-all-errors", - "-o", - out_path, - url, - ] - ) - else: - print(f"Using cached version of {basename(url)}") - return out_path - - -def configure_host_python( - context: argparse.Namespace, - host: str | None = None, -) -> None: - """The implementation of the "configure-host" command.""" - if host is None: - host = context.host - - if context.clean: - clean(context, host) - - host_dir = subdir(host, create=True) - prefix_dir = host_dir / "prefix" - - with group(f"Downloading dependencies ({host})"): - if not prefix_dir.exists(): - prefix_dir.mkdir() - unpack_deps(context.platform, host, prefix_dir, context.cache_dir) - else: - print("Dependencies already installed") - - with ( - group(f"Configuring host Python ({host})"), - cwd(host_dir), - ): - command = [ - # Basic cross-compiling configuration - relpath(PYTHON_DIR / "configure"), - f"--host={host}", - f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", - f"--with-build-python={build_python_path()}", - "--with-system-libmpdec", - "--enable-framework", - # Dependent libraries. - f"--with-openssl={prefix_dir}", - f"LIBLZMA_CFLAGS=-I{prefix_dir}/include", - f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma", - f"LIBFFI_CFLAGS=-I{prefix_dir}/include", - f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi", - f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include", - f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec", - f"LIBZSTD_CFLAGS=-I{prefix_dir}/include", - f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd", - ] - - if context.args: - command.extend(context.args) - run(command, host=host) - - -def make_host_python( - context: argparse.Namespace, - host: str | None = None, -) -> None: - """The implementation of the "make-host" command.""" - if host is None: - host = context.host - - with ( - group(f"Compiling host Python ({host})"), - cwd(subdir(host)), - ): - run(["make", "-j", str(os.cpu_count())], host=host) - run(["make", "install"], host=host) - - -def framework_path(host_triple: str, multiarch: str) -> Path: - """The path to a built single-architecture framework product. - - :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) - :param multiarch: The multiarch identifier (e.g., arm64-simulator) - """ - return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" - - -def package_version(prefix_path: Path) -> str: - """Extract the Python version being built from patchlevel.h.""" - for path in prefix_path.glob("**/patchlevel.h"): - text = path.read_text(encoding="utf-8") - if match := re.search( - r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text - ): - version = match[1] - # If not building against a tagged commit, add a timestamp to the - # version. Follow the PyPA version number rules, as this will make - # it easier to process with other tools. The version will have a - # `+` suffix once any official release has been made; a freshly - # forked main branch will have a version of 3.X.0a0. - if version.endswith("a0"): - version += "+" - if version.endswith("+"): - version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") - - return version - - sys.exit("Unable to determine Python version being packaged.") - - -def lib_platform_files(dirname, names): - """A file filter that ignores platform-specific files in the lib directory. - """ - path = Path(dirname) - if ( - path.parts[-3] == "lib" - and path.parts[-2].startswith("python") - and path.parts[-1] == "lib-dynload" - ): - return names - elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - ignored_names = set( - name - for name in names - if ( - name.startswith("_sysconfigdata_") - or name.startswith("_sysconfig_vars_") - or name == "build-details.json" - ) - ) - else: - ignored_names = set() - - return ignored_names - - -def lib_non_platform_files(dirname, names): - """A file filter that ignores anything *except* platform-specific files - in the lib directory. - """ - path = Path(dirname) - if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} - else: - return set() - - -def create_xcframework(platform: str) -> str: - """Build an XCframework from the component parts for the platform. - - :return: The version number of the Python version that was packaged. - """ - package_path = CROSS_BUILD_DIR / platform - try: - package_path.mkdir() - except FileExistsError: - raise RuntimeError( - f"{platform} XCframework already exists; do you need to run with --clean?" - ) from None - - frameworks = [] - # Merge Frameworks for each component SDK. If there's only one architecture - # for the SDK, we can use the compiled Python.framework as-is. However, if - # there's more than architecture, we need to merge the individual built - # frameworks into a merged "fat" framework. - for slice_name, slice_parts in HOSTS[platform].items(): - # Some parts are the same across all slices, so we use can any of the - # host frameworks as the source for the merged version. Use the first - # one on the list, as it's as representative as any other. - first_host_triple, first_multiarch = next(iter(slice_parts.items())) - first_framework = ( - framework_path(first_host_triple, first_multiarch) - / "Python.framework" - ) - - if len(slice_parts) == 1: - # The first framework is the only framework, so copy it. - print(f"Copying framework for {slice_name}...") - frameworks.append(first_framework) - else: - print(f"Merging framework for {slice_name}...") - slice_path = CROSS_BUILD_DIR / slice_name - slice_framework = slice_path / "Python.framework" - slice_framework.mkdir(exist_ok=True, parents=True) - - # Copy the Info.plist - shutil.copy( - first_framework / "Info.plist", - slice_framework / "Info.plist", - ) - - # Copy the headers - shutil.copytree( - first_framework / "Headers", - slice_framework / "Headers", - ) - - # Create the "fat" library binary for the slice - run( - ["lipo", "-create", "-output", slice_framework / "Python"] - + [ - ( - framework_path(host_triple, multiarch) - / "Python.framework/Python" - ) - for host_triple, multiarch in slice_parts.items() - ] - ) - - # Add this merged slice to the list to be added to the XCframework - frameworks.append(slice_framework) - - print() - print("Build XCframework...") - cmd = [ - "xcodebuild", - "-create-xcframework", - "-output", - package_path / "Python.xcframework", - ] - for framework in frameworks: - cmd.extend(["-framework", framework]) - - run(cmd) - - # Extract the package version from the merged framework - version = package_version(package_path / "Python.xcframework") - version_tag = ".".join(version.split(".")[:2]) - - # On non-macOS platforms, each framework in XCframework only contains the - # headers, libPython, plus an Info.plist. Other resources like the standard - # library and binary shims aren't allowed to live in framework; they need - # to be copied in separately. - print() - print("Copy additional resources...") - has_common_stdlib = False - for slice_name, slice_parts in HOSTS[platform].items(): - # Some parts are the same across all slices, so we can any of the - # host frameworks as the source for the merged version. - first_host_triple, first_multiarch = next(iter(slice_parts.items())) - first_path = framework_path(first_host_triple, first_multiarch) - first_framework = first_path / "Python.framework" - - slice_path = package_path / f"Python.xcframework/{slice_name}" - slice_framework = slice_path / "Python.framework" - - # Copy the binary helpers - print(f" - {slice_name} binaries") - shutil.copytree(first_path / "bin", slice_path / "bin") - - # Copy the include path (this will be a symlink to the framework headers) - print(f" - {slice_name} include files") - shutil.copytree( - first_path / "include", - slice_path / "include", - symlinks=True, - ) - - # Copy in the cross-architecture pyconfig.h - shutil.copy( - PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", - slice_framework / "Headers/pyconfig.h", - ) - - print(f" - {slice_name} architecture-specific files") - for host_triple, multiarch in slice_parts.items(): - print(f" - {multiarch} standard library") - arch, _ = multiarch.split("-", 1) - - if not has_common_stdlib: - print(" - using this architecture as the common stdlib") - shutil.copytree( - framework_path(host_triple, multiarch) / "lib", - package_path / "Python.xcframework/lib", - ignore=lib_platform_files, - ) - has_common_stdlib = True - - shutil.copytree( - framework_path(host_triple, multiarch) / "lib", - slice_path / f"lib-{arch}", - ignore=lib_non_platform_files, - ) - - # Copy the host's pyconfig.h to an architecture-specific name. - arch = multiarch.split("-")[0] - host_path = ( - CROSS_BUILD_DIR - / host_triple - / "Apple/iOS/Frameworks" - / multiarch - ) - host_framework = host_path / "Python.framework" - shutil.copy( - host_framework / "Headers/pyconfig.h", - slice_framework / f"Headers/pyconfig-{arch}.h", - ) - - # Apple identifies certain libraries as "security risks"; if you - # statically link those libraries into a Framework, you become - # responsible for providing a privacy manifest for that framework. - xcprivacy_file = { - "OpenSSL": subdir(host_triple) / "prefix/share/OpenSSL.xcprivacy" - } - print(f" - {multiarch} xcprivacy files") - for module, lib in [ - ("_hashlib", "OpenSSL"), - ("_ssl", "OpenSSL"), - ]: - shutil.copy( - xcprivacy_file[lib], - slice_path - / f"lib-{arch}/python{version_tag}/lib-dynload/{module}.xcprivacy", - ) - - print(" - build tools") - shutil.copytree( - PYTHON_DIR / "Apple/testbed/Python.xcframework/build", - package_path / "Python.xcframework/build", - ) - - return version - - -def package(context: argparse.Namespace) -> None: - """The implementation of the "package" command.""" - if context.clean: - clean(context, "package") - - with group("Building package"): - # Create an XCframework - version = create_xcframework(context.platform) - - # Clone testbed - print() - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - CROSS_BUILD_DIR / context.platform / "Python.xcframework", - CROSS_BUILD_DIR / context.platform / "testbed", - ] - ) - - # Build the final archive - archive_name = ( - CROSS_BUILD_DIR - / "dist" - / f"python-{version}-{context.platform}-XCframework" - ) - - print() - print("Create package archive...") - shutil.make_archive( - str(CROSS_BUILD_DIR / archive_name), - format="gztar", - root_dir=CROSS_BUILD_DIR / context.platform, - base_dir=".", - ) - print() - print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.") - - -def build(context: argparse.Namespace, host: str | None = None) -> None: - """The implementation of the "build" command.""" - if host is None: - host = context.host - - if context.clean: - clean(context, host) - - if host in {"all", "build"}: - for step in [ - configure_build_python, - make_build_python, - ]: - step(context) - - if host == "build": - hosts = [] - elif host in {"all", "hosts"}: - hosts = all_host_triples(context.platform) - else: - hosts = [host] - - for step_host in hosts: - for step in [ - configure_host_python, - make_host_python, - ]: - step(context, host=step_host) - - if host in {"all", "hosts"}: - package(context) - - -def test(context: argparse.Namespace, host: str | None = None) -> None: - """The implementation of the "test" command.""" - if host is None: - host = context.host - - if context.clean: - clean(context, "test") - - with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"): - timestamp = str(time.time_ns())[:-6] - testbed_dir = ( - CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}" - ) - if host in {"all", "hosts"}: - framework_path = ( - CROSS_BUILD_DIR / context.platform / "Python.xcframework" - ) - else: - build_arch = platform.machine() - host_arch = host.split("-")[0] - - if not host.endswith("-simulator"): - print("Skipping test suite non-simulator build.") - return - elif build_arch != host_arch: - print( - f"Skipping test suite for an {host_arch} build " - f"on an {build_arch} machine." - ) - return - else: - framework_path = ( - CROSS_BUILD_DIR - / host - / f"Apple/{context.platform}" - / f"Frameworks/{apple_multiarch(host)}" - ) - - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - framework_path, - testbed_dir, - ] - ) - - run( - [ - sys.executable, - testbed_dir, - "run", - "--verbose", - ] - + ( - ["--simulator", str(context.simulator)] - if context.simulator - else [] - ) - + [ - "--", - "test", - f"--{context.ci_mode}-ci", - "--single-process", - "--no-randomize", - # Timeout handling requires subprocesses; explicitly setting - # the timeout to -1 disables the faulthandler. - "--timeout=-1", - # Adding Python options requires the use of a subprocess to - # start a new Python interpreter. - "--dont-add-python-opts", - ] - ) - - -def apple_sim_host(platform_name: str) -> str: - """Determine the native simulator target for this platform.""" - for _, slice_parts in HOSTS[platform_name].items(): - for host_triple in slice_parts: - parts = host_triple.split('-') - if parts[0] == platform.machine() and parts[-1] == "simulator": - return host_triple - - raise KeyError(platform_name) - - -def ci(context: argparse.Namespace) -> None: - """The implementation of the "ci" command. - - In "Fast" mode, this compiles the build python, and the simulator for the - build machine's architecture; and runs the test suite with `--fast-ci` - configuration. - - In "Slow" mode, it compiles the build python, plus all candidate - architectures (both device and simulator); then runs the test suite with - `--slow-ci` configuration. - """ - clean(context, "all") - if context.ci_mode == "slow": - # In slow mode, build and test the full XCframework - build(context, host="all") - test(context, host="all") - else: - # In fast mode, just build the simulator platform. - sim_host = apple_sim_host(context.platform) - build(context, host="build") - build(context, host=sim_host) - test(context, host=sim_host) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( - "A tool for managing the build, package and test process of " - "CPython on Apple platforms." - ), - ) - parser.suggest_on_error = True - subcommands = parser.add_subparsers(dest="subcommand", required=True) - - clean = subcommands.add_parser( - "clean", - help="Delete all build directories", - ) - - configure_build = subcommands.add_parser( - "configure-build", help="Run `configure` for the build Python" - ) - subcommands.add_parser( - "make-build", help="Run `make` for the build Python" - ) - configure_host = subcommands.add_parser( - "configure-host", - help="Run `configure` for a specific platform and target", - ) - make_host = subcommands.add_parser( - "make-host", - help="Run `make` for a specific platform and target", - ) - package = subcommands.add_parser( - "package", - help="Create a release package for the platform", - ) - build = subcommands.add_parser( - "build", - help="Build all platform targets and create the XCframework", - ) - test = subcommands.add_parser( - "test", - help="Run the testbed for a specific platform", - ) - ci = subcommands.add_parser( - "ci", - help="Run build, package, and test", - ) - - # platform argument - for cmd in [clean, configure_host, make_host, package, build, test, ci]: - cmd.add_argument( - "platform", - choices=HOSTS.keys(), - help="The target platform to build", - ) - - # host triple argument - for cmd in [configure_host, make_host]: - cmd.add_argument( - "host", - help="The host triple to build (e.g., arm64-apple-ios-simulator)", - ) - # optional host triple argument - for cmd in [clean, build, test]: - cmd.add_argument( - "host", - nargs="?", - default="all", - help=( - "The host triple to build (e.g., arm64-apple-ios-simulator), " - "or 'build' for just the build platform, or 'hosts' for all " - "host platforms, or 'all' for the build platform and all " - "hosts. Defaults to 'all'" - ), - ) - - # --clean option - for cmd in [configure_build, configure_host, build, package, test, ci]: - cmd.add_argument( - "--clean", - action="store_true", - default=False, - dest="clean", - help="Delete the relevant build directories first", - ) - - # --cache-dir option - for cmd in [configure_host, build, ci]: - cmd.add_argument( - "--cache-dir", - default="./cross-build/downloads", - help="The directory to store cached downloads.", - ) - - # --simulator option - for cmd in [test, ci]: - cmd.add_argument( - "--simulator", - help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " - "the most recently released 'entry level' iPhone device. Device " - "architecture and OS version can also be specified; e.g., " - "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " - "an ARM64 iPhone 16 Pro simulator running iOS 26.0." - ), - ) - group = cmd.add_mutually_exclusive_group() - group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") - group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") - - for subcommand in [configure_build, configure_host, build, ci]: - subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" - ) - - return parser.parse_args() - - -def print_called_process_error(e: subprocess.CalledProcessError) -> None: - for stream_name in ["stdout", "stderr"]: - content = getattr(e, stream_name) - stream = getattr(sys, stream_name) - if content: - stream.write(content) - if not content.endswith("\n"): - stream.write("\n") - - # shlex uses single quotes, so we surround the command with double quotes. - print( - f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' - ) - - -def main() -> None: - # Handle SIGTERM the same way as SIGINT. This ensures that if we're - # terminated by the buildbot worker, we'll make an attempt to clean up our - # subprocesses. - def signal_handler(*args): - os.kill(os.getpid(), signal.SIGINT) - - signal.signal(signal.SIGTERM, signal_handler) - - # Process command line arguments - context = parse_args() - dispatch: dict[str, Callable] = { - "clean": clean, - "configure-build": configure_build_python, - "make-build": make_build_python, - "configure-host": configure_host_python, - "make-host": make_host_python, - "package": package, - "build": build, - "test": test, - "ci": ci, - } - - try: - dispatch[context.subcommand](context) - except CalledProcessError as e: - print() - print_called_process_error(e) - sys.exit(1) - except RuntimeError as e: - print() - print(e) - sys.exit(2) - - -if __name__ == "__main__": - # Under the buildbot, stdout is not a TTY, but we must still flush after - # every line to make sure our output appears in the correct order relative - # to the output of our subprocesses. - for stream in [sys.stdout, sys.stderr]: - stream.reconfigure(line_buffering=True) - - main() diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md deleted file mode 100644 index 7ee257b5d648f46..000000000000000 --- a/Apple/iOS/README.md +++ /dev/null @@ -1,339 +0,0 @@ -# Python on iOS README - -**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** - -This document provides a quick overview of some iOS specific features in the -Python distribution. - -These instructions are only needed if you're planning to compile Python for iOS -yourself. Most users should *not* need to do this. If you're looking to -experiment with writing an iOS app in Python, tools such as [BeeWare's -Briefcase](https://briefcase.readthedocs.io) and [Kivy's -Buildozer](https://buildozer.readthedocs.io) will provide a much more -approachable user experience. - -## Compilers for building on iOS - -Building for iOS requires the use of Apple's Xcode tooling. It is strongly -recommended that you use the most recent stable release of Xcode. This will -require the use of the most (or second-most) recently released macOS version, -as Apple does not maintain Xcode for older macOS versions. The Xcode Command -Line Tools are not sufficient for iOS development; you need a *full* Xcode -install. - -If you want to run your code on the iOS simulator, you'll also need to install -an iOS Simulator Platform. You should be prompted to select an iOS Simulator -Platform when you first run Xcode. Alternatively, you can add an iOS Simulator -Platform by selecting an open the Platforms tab of the Xcode Settings panel. - -## Building Python on iOS - -### ABIs and Architectures - -iOS apps can be deployed on physical devices, and on the iOS simulator. Although -the API used on these devices is identical, the ABI is different - you need to -link against different libraries for an iOS device build (`iphoneos`) or an -iOS simulator build (`iphonesimulator`). - -Apple uses the `XCframework` format to allow specifying a single dependency -that supports multiple ABIs. An `XCframework` is a wrapper around multiple -ABI-specific frameworks that share a common API. - -iOS can also support different CPU architectures within each ABI. At present, -there is only a single supported architecture on physical devices - ARM64. -However, the *simulator* supports 2 architectures - ARM64 (for running on Apple -Silicon machines), and x86_64 (for running on older Intel-based machines). - -To support multiple CPU architectures on a single platform, Apple uses a "fat -binary" format - a single physical file that contains support for multiple -architectures. It is possible to compile and use a "thin" single architecture -version of a binary for testing purposes; however, the "thin" binary will not be -portable to machines using other architectures. - -### Building a multi-architecture iOS XCframework - -The `Apple` subfolder of the Python repository acts as a build script that -can be used to coordinate the compilation of a complete iOS XCframework. To use -it, run:: - - python Apple build iOS - -This will: - -* Configure and compile a version of Python to run on the build machine -* Download pre-compiled binary dependencies for each platform -* Configure and build a `Python.framework` for each required architecture and - iOS SDK -* Merge the multiple `Python.framework` folders into a single `Python.xcframework` -* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing - the `Python.xcframework`, plus a copy of the Testbed app pre-configured to - use the XCframework. - -The `Apple` build script has other entry points that will perform the -individual parts of the overall `build` target, plus targets to test the -build, clean the `cross-build` folder of iOS build products, and perform a -complete "build and test" CI run. The `--clean` flag can also be used on -individual commands to ensure that a stale build product are removed before -building. - -### Building a single-architecture framework - -If you're using the `Apple` build script, you won't need to build -individual frameworks. However, if you do need to manually configure an iOS -Python build for a single framework, the following options are available. - -#### iOS specific arguments to configure - -* `--enable-framework[=DIR]` - - This argument specifies the location where the Python.framework will be - installed. If `DIR` is not specified, the framework will be installed into - a subdirectory of the `iOS/Frameworks` folder. - - This argument *must* be provided when configuring iOS builds. iOS does not - support non-framework builds. - -* `--with-framework-name=NAME` - - Specify the name for the Python framework; defaults to `Python`. - - > [!NOTE] - > Unless you know what you're doing, changing the name of the Python - > framework on iOS is not advised. If you use this option, you won't be able - > to run the `Apple` build script without making significant manual - > alterations, and you won't be able to use any binary packages unless you - > compile them yourself using your own framework name. - -#### Building Python for iOS - -The Python build system will create a `Python.framework` that supports a -*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a -framework to contain non-library content, so the iOS build will produce a -`bin` and `lib` folder in the same output folder as `Python.framework`. -The `lib` folder will be needed at runtime to support the Python library. - -If you want to use Python in a real iOS project, you need to produce multiple -`Python.framework` builds, one for each ABI and architecture. iOS builds of -Python *must* be constructed as framework builds. To support this, you must -provide the `--enable-framework` flag when configuring the build. The build -also requires the use of cross-compilation. The minimal commands for building -Python for the ARM64 iOS simulator will look something like: -``` -export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" -./configure \ - --enable-framework \ - --host=arm64-apple-ios-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=/path/to/python.exe -make -make install -``` - -In this invocation: - -* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the - compilers and linkers needed by the build. Xcode requires the use of `xcrun` - to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the - result passed to `configure`, these results can embed user- and - version-specific paths into the sysconfig data, which limits the portability - of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler, - it requires that compiler variables like `CC` include spaces, which can - cause significant problems with many C configuration systems which assume that - `CC` will be a single executable. - - To work around this problem, the `Apple/iOS/Resources/bin` folder contains some - wrapper scripts that present as simple compilers and linkers, but wrap - underlying calls to `xcrun`. This allows configure to use a `CC` - definition without spaces, and without user- or version-specific paths, while - retaining the ability to adapt to the local Xcode install. These scripts are - included in the `bin` directory of an iOS install. - - These scripts will, by default, use the currently active Xcode installation. - If you want to use a different Xcode installation, you can use - `xcode-select` to set a new default Xcode globally, or you can use the - `DEVELOPER_DIR` environment variable to specify an Xcode install. The - scripts will use the default `iphoneos`/`iphonesimulator` SDK version for - the select Xcode install; if you want to use a different SDK, you can set the - `IOS_SDK_VERSION` environment variable. (e.g, setting - `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1` - and `iphonesimulator17.1` SDKs, regardless of the Xcode default.) - - The path has also been cleared of any user customizations. A common source of - bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS - build. Resetting the path to a known "bare bones" value is the easiest way to - avoid these problems. - -* `--host` is the architecture and ABI that you want to build, in GNU compiler - triple format. This will be one of: - - - `arm64-apple-ios` for ARM64 iOS devices. - - `arm64-apple-ios-simulator` for the iOS simulator running on Apple - Silicon devices. - - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel - devices. - -* `--build` is the GNU compiler triple for the machine that will be running - the compiler. This is one of: - - - `arm64-apple-darwin` for Apple Silicon devices. - - `x86_64-apple-darwin` for Intel devices. - -* `/path/to/python.exe` is the path to a Python binary on the machine that - will be running the compiler. This is needed because the Python compilation - process involves running some Python code. On a normal desktop build of - Python, you can compile a python interpreter and then use that interpreter to - run Python code. However, the binaries produced for iOS won't run on macOS, so - you need to provide an external Python interpreter. This interpreter must be - the same version as the Python that is being compiled. To be completely safe, - this should be the *exact* same commit hash. However, the longer a Python - release has been stable, the more likely it is that this constraint can be - relaxed - the same micro version will often be sufficient. - -* The `install` target for iOS builds is slightly different to other - platforms. On most platforms, `make install` will install the build into - the final runtime location. This won't be the case for iOS, as the final - runtime location will be on a physical device. - - However, you still need to run the `install` target for iOS builds, as it - performs some final framework assembly steps. The location specified with - `--enable-framework` will be the location where `make install` will - assemble the complete iOS framework. This completed framework can then - be copied and relocated as required. - -For a full CPython build, you also need to specify the paths to iOS builds of -the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL). -This can be done by defining library specific environment variables (such as -`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure -option. Versions of these libraries pre-compiled for iOS can be found in [this -repository](https://github.com/beeware/cpython-apple-source-deps/releases). -LibFFI is especially important, as many parts of the standard library -(including the `platform`, `sysconfig` and `webbrowser` modules) require -the use of the `ctypes` module at runtime. - -By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 13.0. To specify a different deployment -target, provide the version number as part of the `--host` argument - for -example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64 -simulator build with a deployment target of 15.4. - -## Testing Python on iOS - -### Testing a multi-architecture framework - -Once you have a built an XCframework, you can test that framework by running: - - $ python Apple test iOS - -This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or -iPhone 16e, or similar), and run the test suite on the most recent version of -iOS that is available. You can specify a simulator using the `--simulator` -command line argument, providing the name of the simulator (e.g., `--simulator -'iPhone 16 Pro'`). You can also use this argument to control the OS version used -for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the -tests on an iPhone 16 Pro running iOS 18.2. - -If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS` -environment variable will be exposed to the iOS process at runtime. - -### Testing a single-architecture framework - -The `Apple/testbed` folder that contains an Xcode project that is able to run -the Python test suite on Apple platforms. This project converts the Python test -suite into a single test case in Xcode's XCTest framework. The single XCTest -passes if the test suite passes. - -To run the test suite, configure a Python build for an iOS simulator (i.e., -`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` -), specifying a framework build (i.e. `--enable-framework`). Ensure that your -`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and -exclude any non-iOS tools, then run: -``` -make all -make install -make testios -``` - -This will: - -* Build an iOS framework for your chosen architecture; -* Finalize the single-platform framework; -* Make a clean copy of the testbed project; -* Install the Python iOS framework into the copy of the testbed project; and -* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE, - iPhone 16e, or a similar). - -On success, the test suite will exit and report successful completion of the -test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 -minutes to run; a couple of extra minutes is required to compile the testbed -project, and then boot and prepare the iOS simulator. - -### Debugging test failures - -Running `python Apple test iOS` generates a standalone version of the -`Apple/testbed` project, and runs the full test suite. It does this using -`Apple/testbed` itself - the folder is an executable module that can be used -to create and run a clone of the testbed project. The standalone version of the -testbed will be created in a directory named -`cross-build/iOS-testbed.`. - -You can generate your own standalone testbed instance by running: -``` -python cross-build/iOS/testbed clone my-testbed -``` - -In this invocation, `my-testbed` is the name of the folder for the new -testbed clone. - -If you've built your own XCframework, or you only want to test a single architecture, -you can construct a standalone testbed instance by running: -``` -python Apple/testbed clone --platform iOS --framework my-testbed -``` - -The framework path can be the path path to a `Python.xcframework`, or the -path to a folder that contains a single-platform `Python.framework`. - -You can then use the `my-testbed` folder to run the Python test suite, -passing in any command line arguments you may require. For example, if you're -trying to diagnose a failure in the `os` module, you might run: -``` -python my-testbed run -- test -W test_os -``` - -This is the equivalent of running `python -m test -W test_os` on a desktop -Python build. Any arguments after the `--` will be passed to testbed as if -they were arguments to `python -m` on a desktop machine. - -### Testing in Xcode - -You can also open the testbed project in Xcode by running: -``` -open my-testbed/iOSTestbed.xcodeproj -``` - -This will allow you to use the full Xcode suite of tools for debugging. - -The arguments used to run the test suite are defined as part of the test plan. -To modify the test plan, select the test plan node of the project tree (it -should be the first child of the root node), and select the "Configurations" -tab. Modify the "Arguments Passed On Launch" value to change the testing -arguments. - -The test plan also disables parallel testing, and specifies the use of the -`Testbed.lldbinit` file for providing configuration of the debugger. The -default debugger configuration disables automatic breakpoints on the -`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals. - -### Testing on an iOS device - -To test on an iOS device, the app needs to be signed with known developer -credentials. To obtain these credentials, you must have an iOS Developer -account, and your Xcode install will need to be logged into your account (see -the Accounts tab of the Preferences dialog). - -Once the project is open, and you're signed into your Apple Developer account, -select the root node of the project tree (labeled "iOSTestbed"), then the -"Signing & Capabilities" tab in the details page. Select a development team -(this will likely be your own name), and plug in a physical device to your -macOS machine with a USB cable. You should then be able to select your physical -device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py deleted file mode 100644 index 49974cb142853ca..000000000000000 --- a/Apple/testbed/__main__.py +++ /dev/null @@ -1,430 +0,0 @@ -import argparse -import json -import os -import re -import shlex -import shutil -import subprocess -import sys -from pathlib import Path - -TEST_SLICES = { - "iOS": "ios-arm64_x86_64-simulator", -} - -DECODE_ARGS = ("UTF-8", "backslashreplace") - -# The system log prefixes each line: -# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... -# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... - -LOG_PREFIX_REGEX = re.compile( - r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD - r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID -) - - -# Select a simulator device to use. -def select_simulator_device(platform): - # List the testing simulators, in JSON format - raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"]) - json_data = json.loads(raw_json) - - if platform == "iOS": - # Any iOS device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". - # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. - se_simulators = sorted( - (devicetype["minRuntimeVersion"], devicetype["name"]) - for devicetype in json_data["devicetypes"] - if devicetype["productFamily"] == "iPhone" - and ( - ( - "iPhone " in devicetype["name"] - and devicetype["name"].endswith("e") - ) - or "iPhone SE " in devicetype["name"] - ) - ) - simulator = se_simulators[-1][1] - else: - raise ValueError(f"Unknown platform {platform}") - - return simulator - - -def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): - # Build and run the test suite on the named simulator. - args = [ - "-project", - str(location / f"{platform}Testbed.xcodeproj"), - "-scheme", - f"{platform}Testbed", - "-destination", - f"platform={platform} Simulator,name={simulator}", - "-derivedDataPath", - str(location / "DerivedData"), - ] - verbosity_args = [] if verbose else ["-quiet"] - - print("Building test project...") - subprocess.run( - ["xcodebuild", "build-for-testing"] + args + verbosity_args, - check=True, - ) - - # Any environment variable prefixed with TEST_RUNNER_ is exposed into the - # test runner environment. There are some variables (like those identifying - # CI platforms) that can be useful to have access to. - test_env = os.environ.copy() - if "GITHUB_ACTIONS" in os.environ: - test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"] - - print("Running test project...") - # Test execution *can't* be run -quiet; verbose mode - # is how we see the output of the test output. - process = subprocess.Popen( - ["xcodebuild", "test-without-building"] + args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=test_env, - ) - while line := (process.stdout.readline()).decode(*DECODE_ARGS): - # Strip the timestamp/process prefix from each log line - line = LOG_PREFIX_REGEX.sub("", line) - sys.stdout.write(line) - sys.stdout.flush() - - status = process.wait(timeout=5) - exit(status) - - -def copy(src, tgt): - """An all-purpose copy. - - If src is a file, it is copied. If src is a symlink, it is copied *as a - symlink*. If src is a directory, the full tree is duplicated, with symlinks - being preserved. - """ - if src.is_file() or src.is_symlink(): - shutil.copyfile(src, tgt, follow_symlinks=False) - else: - shutil.copytree(src, tgt, symlinks=True) - - -def clone_testbed( - source: Path, - target: Path, - framework: Path, - platform: str, - apps: list[Path], -) -> None: - if target.exists(): - print(f"{target} already exists; aborting without creating project.") - sys.exit(10) - - if framework is None: - if not ( - source / "Python.xcframework" / TEST_SLICES[platform] / "bin" - ).is_dir(): - print( - f"The testbed being cloned ({source}) does not contain " - "a framework with slices. Re-run with --framework" - ) - sys.exit(11) - else: - if not framework.is_dir(): - print(f"{framework} does not exist.") - sys.exit(12) - elif not ( - framework.suffix == ".xcframework" - or (framework / "Python.framework").is_dir() - ): - print( - f"{framework} is not an XCframework, " - f"or a simulator slice of a framework build." - ) - sys.exit(13) - - print("Cloning testbed project:") - print(f" Cloning {source}...", end="") - # Only copy the files for the platform being cloned plus the files common - # to all platforms. The XCframework will be copied later, if needed. - target.mkdir(parents=True) - - for name in [ - "__main__.py", - "TestbedTests", - "Testbed.lldbinit", - f"{platform}Testbed", - f"{platform}Testbed.xcodeproj", - f"{platform}Testbed.xctestplan", - ]: - copy(source / name, target / name) - - print(" done") - - orig_xc_framework_path = source / "Python.xcframework" - xc_framework_path = target / "Python.xcframework" - test_framework_path = xc_framework_path / TEST_SLICES[platform] - if framework is not None: - if framework.suffix == ".xcframework": - print(" Installing XCFramework...", end="") - xc_framework_path.symlink_to( - framework.relative_to(xc_framework_path.parent, walk_up=True) - ) - print(" done") - else: - print(" Installing simulator framework...", end="") - # We're only installing a slice of a framework; we need - # to do a full tree copy to make sure we don't damage - # symlinked content. - shutil.copytree(orig_xc_framework_path, xc_framework_path) - if test_framework_path.is_dir(): - shutil.rmtree(test_framework_path) - else: - test_framework_path.unlink(missing_ok=True) - test_framework_path.symlink_to( - framework.relative_to(test_framework_path.parent, walk_up=True) - ) - print(" done") - else: - copy(orig_xc_framework_path, xc_framework_path) - - if ( - xc_framework_path.is_symlink() - and not xc_framework_path.readlink().is_absolute() - ): - # XCFramework is a relative symlink. Rewrite the symlink relative - # to the new location. - print(" Rewriting symlink to XCframework...", end="") - resolved_xc_framework_path = ( - source / xc_framework_path.readlink() - ).resolve() - xc_framework_path.unlink() - xc_framework_path.symlink_to( - resolved_xc_framework_path.relative_to( - xc_framework_path.parent, walk_up=True - ) - ) - print(" done") - elif ( - test_framework_path.is_symlink() - and not test_framework_path.readlink().is_absolute() - ): - print(" Rewriting symlink to simulator framework...", end="") - # Simulator framework is a relative symlink. Rewrite the symlink - # relative to the new location. - orig_test_framework_path = ( - source / "Python.XCframework" / test_framework_path.readlink() - ).resolve() - test_framework_path.unlink() - test_framework_path.symlink_to( - orig_test_framework_path.relative_to( - test_framework_path.parent, walk_up=True - ) - ) - print(" done") - else: - print(" Using pre-existing Python framework.") - - for app_src in apps: - print(f" Installing app {app_src.name!r}...", end="") - app_target = target / f"Testbed/app/{app_src.name}" - if app_target.is_dir(): - shutil.rmtree(app_target) - shutil.copytree(app_src, app_target) - print(" done") - - print(f"Successfully cloned testbed: {target.resolve()}") - - -def update_test_plan(testbed_path, platform, args): - # Modify the test plan to use the requested test arguments. - test_plan_path = testbed_path / f"{platform}Testbed.xctestplan" - with test_plan_path.open("r", encoding="utf-8") as f: - test_plan = json.load(f) - - test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": shlex.quote(arg)} for arg in args - ] - - with test_plan_path.open("w", encoding="utf-8") as f: - json.dump(test_plan, f, indent=2) - - -def run_testbed( - platform: str, - simulator: str | None, - args: list[str], - verbose: bool = False, -): - location = Path(__file__).parent - print("Updating test plan...", end="") - update_test_plan(location, platform, args) - print(" done.") - - if simulator is None: - simulator = select_simulator_device(platform) - print(f"Running test on {simulator}") - - xcode_test( - location, - platform=platform, - simulator=simulator, - verbose=verbose, - ) - - -def main(): - # Look for directories like `iOSTestbed` as an indicator of the platforms - # that the testbed folder supports. The original source testbed can support - # many platforms, but when cloned, only one platform is preserved. - available_platforms = [ - platform - for platform in ["iOS"] - if (Path(__file__).parent / f"{platform}Testbed").is_dir() - ] - - parser = argparse.ArgumentParser( - description=( - "Manages the process of testing an Apple Python project through Xcode." - ), - ) - - subcommands = parser.add_subparsers(dest="subcommand") - clone = subcommands.add_parser( - "clone", - description=( - "Clone the testbed project, copying in a Python framework and" - "any specified application code." - ), - help="Clone a testbed project to a new location.", - ) - clone.add_argument( - "--framework", - help=( - "The location of the XCFramework (or simulator-only slice of an " - "XCFramework) to use when running the testbed" - ), - ) - clone.add_argument( - "--platform", - dest="platform", - choices=available_platforms, - default=available_platforms[0], - help=f"The platform to target (default: {available_platforms[0]})", - ) - clone.add_argument( - "--app", - dest="apps", - action="append", - default=[], - help="The location of any code to include in the testbed project", - ) - clone.add_argument( - "location", - help="The path where the testbed will be cloned.", - ) - - run = subcommands.add_parser( - "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", - description=( - "Run a testbed project. The arguments provided after `--` will be " - "passed to the running iOS process as if they were arguments to " - "`python -m`." - ), - help="Run a testbed project", - ) - run.add_argument( - "--platform", - dest="platform", - choices=available_platforms, - default=available_platforms[0], - help=f"The platform to target (default: {available_platforms[0]})", - ) - run.add_argument( - "--simulator", - help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " - "the most recently released 'entry level' iPhone device. Device " - "architecture and OS version can also be specified; e.g., " - "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " - "an ARM64 iPhone 16 Pro simulator running iOS 26.0." - ), - ) - run.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose output", - ) - - try: - pos = sys.argv.index("--") - testbed_args = sys.argv[1:pos] - test_args = sys.argv[pos + 1 :] - except ValueError: - testbed_args = sys.argv[1:] - test_args = [] - - context = parser.parse_args(testbed_args) - - if context.subcommand == "clone": - clone_testbed( - source=Path(__file__).parent.resolve(), - target=Path(context.location).resolve(), - framework=Path(context.framework).resolve() - if context.framework - else None, - platform=context.platform, - apps=[Path(app) for app in context.apps], - ) - elif context.subcommand == "run": - if test_args: - if not ( - Path(__file__).parent - / "Python.xcframework" - / TEST_SLICES[context.platform] - / "bin" - ).is_dir(): - print( - f"Testbed does not contain a compiled Python framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." - ) - sys.exit(20) - - run_testbed( - platform=context.platform, - simulator=context.simulator, - verbose=context.verbose, - args=test_args, - ) - else: - print( - f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)" - ) - print() - parser.print_help(sys.stderr) - sys.exit(21) - else: - parser.print_help(sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - # Under the buildbot, stdout is not a TTY, but we must still flush after - # every line to make sure our output appears in the correct order relative - # to the output of our subprocesses. - for stream in [sys.stdout, sys.stderr]: - stream.reconfigure(line_buffering=True) - main() diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000000000..132b913b06d5e5f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,251 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "_base64" +version = "0.1.0" +dependencies = [ + "cpython-build-helper", + "cpython-sys", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cpython-build-helper" +version = "0.1.0" +dependencies = [ + "shlex", +] + +[[package]] +name = "cpython-rust-staticlib" +version = "0.1.0" +dependencies = [ + "_base64", +] + +[[package]] +name = "cpython-sys" +version = "0.1.0" +dependencies = [ + "bindgen", + "shlex", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.110" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000000000..768d289fd77653e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +resolver = "3" +members = [ + "Modules/_base64", "Modules/cpython-build-helper", "Modules/cpython-rust-staticlib", + "Modules/cpython-sys" +] diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 3e676e13c3f41ac..6b573fd58d089bc 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -32,6 +32,9 @@ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] +[lint.per-file-ignores] +"tools/check-html-ids.py" = ["I001"] # Unsorted imports + [format] preview = true quote-style = "preserve" diff --git a/Doc/Makefile b/Doc/Makefile index f16d9cacb1b6fb9..d77ece1e681bcfc 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,7 +13,7 @@ JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt +REQUIREMENTS = pylock.toml SPHINXERRORHANDLING = --fail-on-warning # Internal variables. @@ -58,7 +58,7 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + elif $(BLURB) --version && $(SPHINXBUILD) --version ; then \ if [ -d ../Misc/NEWS.d ]; then \ echo "Building NEWS from Misc/NEWS.d with blurb"; \ $(BLURB) merge -f build/NEWS; \ @@ -89,7 +89,8 @@ htmlhelp: build .PHONY: latex latex: BUILDER = latex -latex: build +latex: _ensure-sphinxcontrib-svg2pdfconverter + $(MAKE) build BUILDER=$(BUILDER) @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." @@ -140,7 +141,8 @@ doctest: pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \ + "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py" .PHONY: gettext gettext: BUILDER = gettext @@ -230,7 +232,7 @@ dist-text: @echo "Build finished and archived!" .PHONY: dist-pdf -dist-pdf: +dist-pdf: _ensure-sphinxcontrib-svg2pdfconverter # archive the A4 latex @echo "Building LaTeX (A4 paper)..." mkdir -p dist @@ -291,6 +293,10 @@ _ensure-pre-commit: _ensure-sphinx-autobuild: $(MAKE) _ensure-package PACKAGE=sphinx-autobuild +.PHONY: _ensure-sphinxcontrib-svg2pdfconverter +_ensure-sphinxcontrib-svg2pdfconverter: + $(MAKE) _ensure-package PACKAGE=sphinxcontrib-svg2pdfconverter + .PHONY: check check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files @@ -335,3 +341,9 @@ autobuild-stable-html: exit 1;; \ esac @$(MAKE) autobuild-dev-html + +# Collect HTML IDs to a JSON document +.PHONY: html-ids +html-ids: + $(PYTHON) tools/check-html-ids.py collect build/html \ + -o build/html/html-ids.json.gz diff --git a/Doc/_static/profiling-sampling-visualization.css b/Doc/_static/profiling-sampling-visualization.css new file mode 100644 index 000000000000000..6bfbec3b8a60449 --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.css @@ -0,0 +1,570 @@ +/** + * Sampling Profiler Visualization - Scoped CSS + */ + +.sampling-profiler-viz { + /* Match docs background colors */ + --bg-page: #ffffff; + --bg-panel: #ffffff; + --bg-subtle: #f8f8f8; + --bg-code: #f8f8f8; + + /* Match docs border style */ + --border-color: #e1e4e8; + --border-accent: #3776ab; + + /* Match docs text colors */ + --text-primary: #0d0d0d; + --text-secondary: #505050; + --text-muted: #6e6e6e; + --text-code: #333333; + + /* Accent colors */ + --color-python-blue: #306998; + --color-green: #388e3c; + --color-orange: #e65100; + --color-purple: #7b1fa2; + --color-red: #c62828; + --color-teal: #00897b; + --color-yellow: #d4a910; + --color-highlight: #fff9e6; + + --radius-lg: 8px; + --radius-md: 6px; + --radius-sm: 4px; + + /* Lighter shadows to match docs style */ + --shadow-card: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-sm: 0 1px 2px rgba(0, 0, 0, 0.04); + + --container-height: 520px; + --code-panel-width: 320px; + + /* Reset for isolation */ + font-family: var(--font-ui); + line-height: 1.5; + font-weight: 400; + color: var(--text-primary); + background-color: var(--bg-page); + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + + /* Layout */ + position: relative; + width: 100%; + max-width: 920px; + height: var(--container-height); + display: grid; + grid-template-columns: var(--code-panel-width) 1fr; + margin: 24px auto; + border-radius: var(--radius-lg); + overflow: hidden; + box-shadow: var(--shadow-card); + border: 1px solid var(--border-color); + background: var(--bg-panel); + /* Prevent any DOM changes inside from affecting page scroll */ + contain: strict; +} + +.sampling-profiler-viz * { + box-sizing: border-box; +} + +/* Code Panel - Left Column */ +.sampling-profiler-viz #code-panel { + background: var(--bg-panel); + border-right: 1px solid var(--border-color); + overflow-y: auto; + font-family: var(--font-mono); + font-size: 12px; + line-height: 1.6; + display: flex; + flex-direction: column; +} + +.sampling-profiler-viz #code-panel .code-panel-title { + padding: 12px 16px; + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + border-bottom: 1px solid var(--border-color); + background: var(--bg-code); + text-transform: uppercase; + letter-spacing: 1px; + flex-shrink: 0; +} + +.sampling-profiler-viz #code-panel .code-container { + margin: 0; + padding: 12px 0; + overflow-x: auto; + flex: 1; +} + +.sampling-profiler-viz #code-panel .line { + display: flex; + padding: 1px 0; + min-height: 20px; + transition: background-color 0.1s ease; +} + +.sampling-profiler-viz #code-panel .line-number { + color: var(--text-muted); + min-width: 40px; + text-align: right; + padding-right: 12px; + padding-left: 12px; + user-select: none; + flex-shrink: 0; + font-size: 11px; +} + +.sampling-profiler-viz #code-panel .line-content { + flex: 1; + color: var(--text-code); + padding-right: 12px; + white-space: pre; +} + +.sampling-profiler-viz #code-panel .line.highlighted { + background: var(--color-highlight); + border-left: 3px solid var(--color-yellow); +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-number { + color: var(--color-yellow); + padding-left: 9px; +} + +.sampling-profiler-viz #code-panel .line.highlighted .line-content { + font-weight: 600; +} + +/* Python Syntax Highlighting */ +.sampling-profiler-viz #code-panel .keyword { + color: var(--color-red); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .function { + color: var(--color-purple); + font-weight: 600; +} + +.sampling-profiler-viz #code-panel .number { + color: var(--color-python-blue); +} + +.sampling-profiler-viz #code-panel .string { + color: #032f62; +} + +.sampling-profiler-viz #code-panel .comment { + color: #6a737d; + font-style: italic; +} + +.sampling-profiler-viz #code-panel .builtin { + color: var(--color-python-blue); +} + +/* Visualization Column - Right Side */ +.sampling-profiler-viz .viz-column { + display: flex; + flex-direction: column; + background: var(--bg-subtle); + overflow: hidden; +} + +/* Stack Section */ +.sampling-profiler-viz .stack-section { + padding: 12px 16px; + flex: 1; + min-height: 150px; + overflow-y: auto; +} + +.sampling-profiler-viz .stack-section-title { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 1px; + margin-bottom: 10px; +} + +.sampling-profiler-viz .stack-visualization { + display: flex; + flex-direction: column; + gap: 4px; + min-height: 80px; +} + +/* Stack Frames - Vertical Layout */ +.sampling-profiler-viz .stack-frame { + position: relative; + width: 100%; + height: 32px; + cursor: pointer; + contain: layout style paint; + opacity: 0; + transform: translateY(-10px); +} + +.sampling-profiler-viz .stack-frame.visible { + opacity: 1; + transform: translateY(0); + transition: + opacity 0.3s ease, + transform 0.3s ease; +} + +.sampling-profiler-viz .stack-frame-bg { + position: absolute; + inset: 0; + border-radius: var(--radius-sm); + transition: opacity 0.15s; +} + +.sampling-profiler-viz .stack-frame-text { + position: absolute; + left: 10px; + top: 50%; + transform: translateY(-50%); + font: 500 11px var(--font-mono); + color: white; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame-flash { + position: absolute; + inset: 0; + background: white; + border-radius: var(--radius-sm); + opacity: 0; + pointer-events: none; +} + +.sampling-profiler-viz .stack-frame:hover .stack-frame-bg { + opacity: 0.85; +} + +/* Flying frames */ +.sampling-profiler-viz .stack-frame.flying { + pointer-events: none; + z-index: 1000; + position: fixed; + top: 0; + left: 0; + width: auto; + height: 32px; + opacity: 1; +} + +/* Flying stack clone */ +.stack-visualization.flying-clone { + transform-origin: center center; + will-change: transform, opacity; +} + +/* Sampling Panel */ +.sampling-profiler-viz .sampling-panel { + margin: 0 16px 12px 16px; + background: var(--bg-panel); + border: 1px solid var(--border-color); + border-radius: var(--radius-md); + box-shadow: var(--shadow-sm); + overflow: hidden; + display: flex; + flex-direction: column; + flex: 0 0 auto; + height: 200px; + /* Lock font size to prevent Sphinx responsive scaling */ + font-size: 12px; +} + +.sampling-profiler-viz .sampling-header { + padding: 8px 10px; + border-bottom: 1px solid var(--border-color); + flex-shrink: 0; +} + +.sampling-profiler-viz .sampling-title { + font: 600 10px var(--font-mono); + color: var(--text-primary); + margin: 0 0 3px 0; +} + +.sampling-profiler-viz .sampling-stats { + font: 400 9px var(--font-mono); + color: var(--text-secondary); + display: flex; + gap: 12px; +} + +.sampling-profiler-viz .sampling-stats .missed { + color: var(--color-red); +} + +.sampling-profiler-viz .sampling-bars { + flex: 1; + padding: 10px 12px; + overflow-y: auto; +} + +.sampling-profiler-viz .sampling-bar-row { + display: flex; + align-items: center; + height: 22px; + gap: 8px; +} + +.sampling-profiler-viz .bar-label { + font: 500 8px var(--font-mono); + color: var(--text-primary); + flex-shrink: 0; + width: 60px; + overflow: hidden; + text-overflow: ellipsis; +} + +.sampling-profiler-viz .bar-container { + flex: 1; + height: 12px; + background: var(--border-color); + border-radius: 3px; + position: relative; + overflow: hidden; +} + +.sampling-profiler-viz .bar-fill { + height: 100%; + border-radius: 3px; + transition: width 0.2s ease; + min-width: 2px; +} + +.sampling-profiler-viz .bar-percent { + font: 500 8px var(--font-mono); + color: var(--text-secondary); + width: 36px; + text-align: right; + flex-shrink: 0; +} + +/* Impact effect circle */ +.impact-circle { + position: fixed; + width: 30px; + height: 30px; + border-radius: 50%; + background: var(--color-teal); + transform: translate(-50%, -50%); + pointer-events: none; + z-index: 2000; +} + +/* Control Panel - Integrated */ +.sampling-profiler-viz #control-panel { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + background: var(--bg-panel); + border-top: 1px solid var(--border-color); + flex-shrink: 0; + flex-wrap: wrap; +} + +.sampling-profiler-viz .control-group { + display: flex; + gap: 6px; + align-items: center; +} + +.sampling-profiler-viz .control-group label { + font-size: 10px; + color: var(--text-muted); + font-weight: 500; + white-space: nowrap; +} + +.sampling-profiler-viz .control-btn { + background: var(--bg-panel); + color: var(--text-primary); + border: 1px solid var(--border-color); + padding: 6px 10px; + border-radius: var(--radius-sm); + cursor: pointer; + transition: all 0.15s ease; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + display: flex; + align-items: center; + gap: 4px; +} + +.sampling-profiler-viz .control-btn:hover { + background: var(--bg-subtle); + border-color: var(--text-muted); +} + +.sampling-profiler-viz .control-btn:active { + transform: scale(0.98); +} + +.sampling-profiler-viz .control-btn.active { + background: var(--color-python-blue); + color: white; + border-color: var(--color-python-blue); +} + +.sampling-profiler-viz .control-btn.active:hover { + background: #2f6493; +} + +/* Timeline Scrubber */ +.sampling-profiler-viz .timeline-scrubber { + flex: 1; + display: flex; + align-items: center; + gap: 8px; + min-width: 160px; +} + +.sampling-profiler-viz #timeline-scrubber { + flex: 1; + height: 5px; + border-radius: 3px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + min-width: 60px; +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb { + -webkit-appearance: none; + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #timeline-scrubber::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #timeline-scrubber::-moz-range-thumb { + width: 14px; + height: 14px; + border-radius: 50%; + background: var(--color-python-blue); + cursor: pointer; + border: none; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #time-display { + font: 500 10px var(--font-mono); + color: var(--text-secondary); + min-width: 90px; + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* Sample Interval Slider */ +.sampling-profiler-viz #sample-interval { + width: 80px; + height: 4px; + border-radius: 2px; + background: var(--border-color); + outline: none; + appearance: none; + -webkit-appearance: none; + cursor: pointer; + flex-shrink: 0; +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb { + -webkit-appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + transition: transform 0.15s; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); +} + +.sampling-profiler-viz #sample-interval::-webkit-slider-thumb:hover { + transform: scale(1.15); +} + +.sampling-profiler-viz #sample-interval::-moz-range-thumb { + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--color-teal); + cursor: pointer; + border: none; +} + +.sampling-profiler-viz #interval-display { + font: 500 9px var(--font-mono); + color: var(--text-secondary); + min-width: 40px; + font-variant-numeric: tabular-nums; +} + +/* Flash overlay */ +.sampling-profiler-viz .flash-overlay { + position: absolute; + inset: 0; + background: white; + pointer-events: none; + opacity: 0; +} + +/* Performance optimizations */ +.sampling-profiler-viz .stack-frame, +.sampling-profiler-viz .flying-frame, +.sampling-profiler-viz .sampling-bar-row { + will-change: transform, opacity; + contain: layout style paint; +} + +/* Reduced motion support */ +@media (prefers-reduced-motion: reduce) { + .sampling-profiler-viz .stack-frame, + .sampling-profiler-viz .flying-frame, + .sampling-profiler-viz .sampling-bar-row, + .impact-circle { + animation-duration: 0.01ms !important; + transition-duration: 0.01ms !important; + } +} + +/* Responsive adjustments for narrower viewports */ +@media (max-width: 800px) { + .sampling-profiler-viz { + grid-template-columns: 280px 1fr; + --container-height: 550px; + } + + .sampling-profiler-viz #code-panel { + font-size: 11px; + } + + .sampling-profiler-viz .control-btn { + padding: 5px 8px; + font-size: 10px; + } +} diff --git a/Doc/_static/profiling-sampling-visualization.js b/Doc/_static/profiling-sampling-visualization.js new file mode 100644 index 000000000000000..3729be6795c1c82 --- /dev/null +++ b/Doc/_static/profiling-sampling-visualization.js @@ -0,0 +1,1163 @@ +/** + * Sampling Profiler Visualization + */ +(function () { + "use strict"; + + // ============================================================================ + // Configuration + // ============================================================================ + + const TIMINGS = { + sampleIntervalMin: 100, + sampleIntervalMax: 500, + sampleIntervalDefault: 200, + sampleToFlame: 600, + defaultSpeed: 0.05, + }; + + const LAYOUT = { frameSpacing: 6 }; + + // Function name to color mapping + const FUNCTION_COLORS = { + main: "#306998", + fibonacci: "#D4A910", + add: "#E65100", + multiply: "#7B1FA2", + calculate: "#D4A910", + }; + const DEFAULT_FUNCTION_COLOR = "#306998"; + + // Easing functions - cubic-bezier approximations + const EASING_MAP = { + linear: "linear", + easeOutQuad: "cubic-bezier(0.25, 0.46, 0.45, 0.94)", + easeOutCubic: "cubic-bezier(0.215, 0.61, 0.355, 1)", + }; + + function getFunctionColor(funcName) { + return FUNCTION_COLORS[funcName] || DEFAULT_FUNCTION_COLOR; + } + + // ============================================================================ + // Animation Manager + // ============================================================================ + + class AnimationManager { + constructor() { + this.activeAnimations = new Set(); + } + + to(element, props, duration, easing = "easeOutQuad", onComplete = null) { + this.killAnimationsOf(element); + + const cssEasing = EASING_MAP[easing] || EASING_MAP.easeOutQuad; + + const transformProps = {}; + const otherProps = {}; + + for (const [key, value] of Object.entries(props)) { + if (key === "position") { + if (typeof value.x === "number") transformProps.x = value.x; + if (typeof value.y === "number") transformProps.y = value.y; + } else if (key === "x" || key === "y") { + transformProps[key] = value; + } else if (key === "scale") { + transformProps.scale = value; + } else if (key === "alpha" || key === "opacity") { + otherProps.opacity = value; + } else { + otherProps[key] = value; + } + } + + const computedStyle = getComputedStyle(element); + const matrix = new DOMMatrix(computedStyle.transform); + const currentScale = Math.sqrt( + matrix.m11 * matrix.m11 + matrix.m21 * matrix.m21, + ); + + transformProps.x ??= matrix.m41; + transformProps.y ??= matrix.m42; + transformProps.scale ??= currentScale; + + const initialTransform = this._buildTransformString( + matrix.m41, + matrix.m42, + currentScale, + ); + + const finalTransform = this._buildTransformString( + transformProps.x, + transformProps.y, + transformProps.scale, + ); + + const initialKeyframe = { transform: initialTransform }; + const finalKeyframe = { transform: finalTransform }; + + for (const [key, value] of Object.entries(otherProps)) { + const currentVal = + key === "opacity" + ? element.style.opacity || computedStyle.opacity + : element.style[key]; + initialKeyframe[key] = currentVal; + finalKeyframe[key] = value; + } + + const animation = element.animate([initialKeyframe, finalKeyframe], { + duration, + easing: cssEasing, + fill: "forwards", + }); + + this.activeAnimations.add(animation); + animation.onfinish = () => { + this.activeAnimations.delete(animation); + element.style.transform = finalTransform; + for (const [key, value] of Object.entries(finalKeyframe)) { + if (key !== "transform") { + element.style[key] = typeof value === "number" ? `${value}` : value; + } + } + if (onComplete) onComplete(); + }; + + return animation; + } + + killAnimationsOf(element) { + element.getAnimations().forEach((animation) => animation.cancel()); + this.activeAnimations.forEach((animation) => { + if (animation.effect && animation.effect.target === element) { + animation.cancel(); + this.activeAnimations.delete(animation); + } + }); + } + + _buildTransformString(x, y, scale = 1) { + return `translate(${x}px, ${y}px) scale(${scale})`; + } + } + + const anim = new AnimationManager(); + + // ============================================================================ + // Execution Trace Model + // ============================================================================ + + class ExecutionEvent { + constructor( + type, + functionName, + lineno, + timestamp, + args = null, + value = null, + ) { + this.type = type; + this.functionName = functionName; + this.lineno = lineno; + this.timestamp = timestamp; + this.args = args; + this.value = value; + } + } + + class ExecutionTrace { + constructor(source, events) { + this.source = source; + this.events = events.map( + (e) => + new ExecutionEvent(e.type, e.func, e.line, e.ts, e.args, e.value), + ); + this.duration = events.length > 0 ? events[events.length - 1].ts : 0; + } + + getEventsUntil(timestamp) { + return this.events.filter((e) => e.timestamp <= timestamp); + } + + getStackAt(timestamp) { + const stack = []; + const events = this.getEventsUntil(timestamp); + + for (const event of events) { + if (event.type === "call") { + stack.push({ + func: event.functionName, + line: event.lineno, + args: event.args, + }); + } else if (event.type === "return") { + stack.pop(); + } else if (event.type === "line") { + if (stack.length > 0) { + stack[stack.length - 1].line = event.lineno; + } + } + } + return stack; + } + + getNextEvent(timestamp) { + return this.events.find((e) => e.timestamp > timestamp); + } + + getSourceLines() { + return this.source.split("\n"); + } + } + + // ============================================================================ + // Demo Data + // ============================================================================ + + // This placeholder is replaced by the profiling_trace Sphinx extension + // during the documentation build with dynamically generated trace data. + const DEMO_SIMPLE = /* PROFILING_TRACE_DATA */ null; + + // ============================================================================ + // Code Panel Component + // ============================================================================ + + class CodePanel { + constructor(source) { + this.source = source; + this.currentLine = null; + + this.element = document.createElement("div"); + this.element.id = "code-panel"; + + const title = document.createElement("div"); + title.className = "code-panel-title"; + title.textContent = "source code"; + this.element.appendChild(title); + + this.codeContainer = document.createElement("pre"); + this.codeContainer.className = "code-container"; + this.element.appendChild(this.codeContainer); + + this._renderSource(); + } + + updateSource(source) { + this.source = source; + this.codeContainer.innerHTML = ""; + this._renderSource(); + this.currentLine = null; + } + + _renderSource() { + const lines = this.source.split("\n"); + + lines.forEach((line, index) => { + const lineNumber = index + 1; + const lineDiv = document.createElement("div"); + lineDiv.className = "line"; + lineDiv.dataset.line = lineNumber; + + const lineNumSpan = document.createElement("span"); + lineNumSpan.className = "line-number"; + lineNumSpan.textContent = lineNumber; + lineDiv.appendChild(lineNumSpan); + + const codeSpan = document.createElement("span"); + codeSpan.className = "line-content"; + codeSpan.innerHTML = this._highlightSyntax(line); + lineDiv.appendChild(codeSpan); + + this.codeContainer.appendChild(lineDiv); + }); + } + + _highlightSyntax(line) { + return line + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/(f?"[^"]*"|f?'[^']*')/g, '$1') + .replace(/(#.*$)/g, '$1') + .replace( + /\b(def|if|elif|else|return|for|in|range|print|__name__|__main__)\b/g, + '$1', + ) + .replace( + /def<\/span>\s+(\w+)/g, + 'def $1', + ) + .replace(/\b(\d+)\b/g, '$1'); + } + + highlightLine(lineNumber) { + if (this.currentLine === lineNumber) return; + + if (this.currentLine !== null) { + const prevLine = this.codeContainer.querySelector( + `[data-line="${this.currentLine}"]`, + ); + if (prevLine) prevLine.classList.remove("highlighted"); + } + + if (lineNumber === null || lineNumber === undefined) { + this.currentLine = null; + return; + } + + this.currentLine = lineNumber; + const newLine = this.codeContainer.querySelector( + `[data-line="${lineNumber}"]`, + ); + if (newLine) { + newLine.classList.add("highlighted"); + } + } + + reset() { + this.highlightLine(null); + this.codeContainer.scrollTop = 0; + } + + destroy() { + this.element.remove(); + } + } + + // ============================================================================ + // Stack Frame Component + // ============================================================================ + + class DOMStackFrame { + constructor(functionName, lineno, args = null) { + this.functionName = functionName; + this.lineno = lineno; + this.args = args; + this.isActive = false; + this.color = getFunctionColor(functionName); + + this.element = document.createElement("div"); + this.element.className = "stack-frame"; + this.element.dataset.function = functionName; + + this.bgElement = document.createElement("div"); + this.bgElement.className = "stack-frame-bg"; + this.bgElement.style.backgroundColor = this.color; + this.element.appendChild(this.bgElement); + + this.textElement = document.createElement("span"); + this.textElement.className = "stack-frame-text"; + this.textElement.textContent = functionName; + this.element.appendChild(this.textElement); + + this.flashElement = document.createElement("div"); + this.flashElement.className = "stack-frame-flash"; + this.element.appendChild(this.flashElement); + + this.element.addEventListener("pointerover", this._onHover.bind(this)); + this.element.addEventListener("pointerout", this._onHoverOut.bind(this)); + } + + destroy() { + this.element.parentNode?.removeChild(this.element); + } + + updateLine(lineno) { + this.lineno = lineno; + this.textElement.textContent = this.functionName; + } + + setActive(isActive) { + if (this.isActive === isActive) return; + this.isActive = isActive; + this.bgElement.style.opacity = isActive ? "1.0" : "0.9"; + } + + _onHover() { + this.bgElement.style.opacity = "0.8"; + } + + _onHoverOut() { + this.bgElement.style.opacity = this.isActive ? "1.0" : "0.9"; + } + + flash(duration = 150) { + this.flashElement.animate([{ opacity: 1 }, { opacity: 0 }], { + duration, + easing: "ease-out", + }); + } + + getPosition() { + const rect = this.element.getBoundingClientRect(); + return { x: rect.left, y: rect.top }; + } + } + + // ============================================================================ + // Stack Visualization Component + // ============================================================================ + + class DOMStackVisualization { + constructor() { + this.frames = []; + this.frameSpacing = LAYOUT.frameSpacing; + + this.element = document.createElement("div"); + this.element.className = "stack-visualization"; + } + + processEvent(event) { + if (event.type === "call") { + this.pushFrame(event.functionName, event.lineno, event.args); + } else if (event.type === "return") { + this.popFrame(); + } else if (event.type === "line") { + this.updateTopFrameLine(event.lineno); + } + } + + updateTopFrameLine(lineno) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].updateLine(lineno); + } + } + + pushFrame(functionName, lineno, args = null) { + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(false); + } + + const frame = new DOMStackFrame(functionName, lineno, args); + frame.setActive(true); + this.element.appendChild(frame.element); + this.frames.push(frame); + + requestAnimationFrame(() => { + frame.element.classList.add("visible"); + }); + } + + popFrame() { + if (this.frames.length === 0) return; + + const frame = this.frames.pop(); + frame.element.classList.remove("visible"); + setTimeout(() => frame.destroy(), 300); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + + clear() { + this.frames.forEach((frame) => frame.destroy()); + this.frames = []; + this.element.innerHTML = ""; + } + + flashAll() { + this.frames.forEach((frame) => frame.flash()); + } + + createStackClone(container) { + const clone = this.element.cloneNode(false); + clone.className = "stack-visualization flying-clone"; + + const elementRect = this.element.getBoundingClientRect(); + const containerRect = container.getBoundingClientRect(); + + // Position relative to container since contain: strict makes position:fixed relative to container + clone.style.position = "absolute"; + clone.style.left = elementRect.left - containerRect.left + "px"; + clone.style.top = elementRect.top - containerRect.top + "px"; + clone.style.width = elementRect.width + "px"; + clone.style.pointerEvents = "none"; + clone.style.zIndex = "1000"; + + this.frames.forEach((frame) => { + const frameClone = frame.element.cloneNode(true); + frameClone.classList.add("visible"); + frameClone.style.opacity = "1"; + frameClone.style.transform = "translateY(0)"; + frameClone.style.transition = "none"; + clone.appendChild(frameClone); + }); + + container.appendChild(clone); + return clone; + } + + updateToMatch(targetStack) { + while (this.frames.length > targetStack.length) { + this.popFrame(); + } + + targetStack.forEach(({ func, line, args }, index) => { + if (index < this.frames.length) { + const frame = this.frames[index]; + if (frame.functionName !== func) { + frame.updateLine(line); + } + frame.setActive(index === targetStack.length - 1); + } else { + this.pushFrame(func, line, args); + } + }); + + if (this.frames.length > 0) { + this.frames[this.frames.length - 1].setActive(true); + } + } + } + + // ============================================================================ + // Sampling Panel Component + // ============================================================================ + + class DOMSamplingPanel { + constructor() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.groundTruthFunctions = new Set(); + this.bars = {}; + + this.element = document.createElement("div"); + this.element.className = "sampling-panel"; + + const header = document.createElement("div"); + header.className = "sampling-header"; + + const title = document.createElement("h3"); + title.className = "sampling-title"; + title.textContent = "Sampling Profiler"; + header.appendChild(title); + + const stats = document.createElement("div"); + stats.className = "sampling-stats"; + + this.sampleCountEl = document.createElement("span"); + this.sampleCountEl.textContent = "Samples: 0"; + stats.appendChild(this.sampleCountEl); + + this.intervalEl = document.createElement("span"); + this.intervalEl.textContent = `Interval: ${this.sampleInterval}ms`; + stats.appendChild(this.intervalEl); + + this.missedFunctionsEl = document.createElement("span"); + this.missedFunctionsEl.className = "missed"; + stats.appendChild(this.missedFunctionsEl); + + header.appendChild(stats); + this.element.appendChild(header); + + this.barsContainer = document.createElement("div"); + this.barsContainer.className = "sampling-bars"; + this.element.appendChild(this.barsContainer); + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.intervalEl.textContent = `Interval: ${interval}ms`; + } + + setGroundTruth(allFunctions) { + this.groundTruthFunctions = new Set(allFunctions); + this._updateMissedCount(); + } + + addSample(stack) { + this.totalSamples++; + this.sampleCountEl.textContent = `Samples: ${this.totalSamples}`; + + stack.forEach((frame) => { + const funcName = frame.func; + this.functionCounts[funcName] = + (this.functionCounts[funcName] || 0) + 1; + }); + + this._updateBars(); + this._updateMissedCount(); + } + + reset() { + this.samples = []; + this.functionCounts = {}; + this.totalSamples = 0; + this.sampleCountEl.textContent = "Samples: 0"; + this.missedFunctionsEl.textContent = ""; + this.barsContainer.innerHTML = ""; + this.bars = {}; + } + + _updateMissedCount() { + if (this.groundTruthFunctions.size === 0) return; + + const capturedFunctions = new Set(Object.keys(this.functionCounts)); + const notYetSeen = [...this.groundTruthFunctions].filter( + (f) => !capturedFunctions.has(f), + ); + + if (notYetSeen.length > 0) { + this.missedFunctionsEl.textContent = `Not yet seen: ${notYetSeen.length}`; + this.missedFunctionsEl.classList.add("missed"); + this.missedFunctionsEl.style.color = ""; + } else if (this.totalSamples > 0) { + this.missedFunctionsEl.textContent = "All captured!"; + this.missedFunctionsEl.classList.remove("missed"); + this.missedFunctionsEl.style.color = "var(--color-green)"; + } else { + this.missedFunctionsEl.textContent = ""; + } + } + + _updateBars() { + const sorted = Object.entries(this.functionCounts).sort( + (a, b) => b[1] - a[1], + ); + + sorted.forEach(([funcName, count], index) => { + const percentage = + this.totalSamples > 0 ? count / this.totalSamples : 0; + + if (!this.bars[funcName]) { + const row = this._createBarRow(funcName); + this.barsContainer.appendChild(row); + this.bars[funcName] = row; + } + + const row = this.bars[funcName]; + const barFill = row.querySelector(".bar-fill"); + barFill.style.width = `${percentage * 100}%`; + + const percentEl = row.querySelector(".bar-percent"); + percentEl.textContent = `${(percentage * 100).toFixed(0)}%`; + + const currentIndex = Array.from(this.barsContainer.children).indexOf( + row, + ); + if (currentIndex !== index) { + this.barsContainer.insertBefore( + row, + this.barsContainer.children[index], + ); + } + }); + } + + _createBarRow(funcName) { + const row = document.createElement("div"); + row.className = "sampling-bar-row"; + row.dataset.function = funcName; + + const label = document.createElement("span"); + label.className = "bar-label"; + label.textContent = funcName; + row.appendChild(label); + + const barContainer = document.createElement("div"); + barContainer.className = "bar-container"; + + const barFill = document.createElement("div"); + barFill.className = "bar-fill"; + barFill.style.backgroundColor = getFunctionColor(funcName); + barContainer.appendChild(barFill); + + row.appendChild(barContainer); + + const percent = document.createElement("span"); + percent.className = "bar-percent"; + percent.textContent = "0%"; + row.appendChild(percent); + + return row; + } + + getTargetPosition() { + const rect = this.barsContainer.getBoundingClientRect(); + return { x: rect.left + rect.width / 2, y: rect.top + 50 }; + } + + showImpactEffect(position) { + const impact = document.createElement("div"); + impact.className = "impact-circle"; + impact.style.position = "fixed"; + impact.style.left = `${position.x}px`; + impact.style.top = `${position.y}px`; + + // Append to barsContainer parent to avoid triggering scroll + this.element.appendChild(impact); + + impact.animate( + [ + { transform: "translate(-50%, -50%) scale(1)", opacity: 0.6 }, + { transform: "translate(-50%, -50%) scale(4)", opacity: 0 }, + ], + { + duration: 300, + easing: "ease-out", + }, + ).onfinish = () => impact.remove(); + } + } + + // ============================================================================ + // Control Panel Component + // ============================================================================ + + class ControlPanel { + constructor( + container, + onPlay, + onPause, + onReset, + onSpeedChange, + onSeek, + onStep, + onSampleIntervalChange = null, + ) { + this.container = container; + this.onPlay = onPlay; + this.onPause = onPause; + this.onReset = onReset; + this.onSpeedChange = onSpeedChange; + this.onSeek = onSeek; + this.onStep = onStep; + this.onSampleIntervalChange = onSampleIntervalChange; + + this.isPlaying = false; + this.speed = TIMINGS.defaultSpeed; + + this._createControls(); + } + + _createControls() { + const panel = document.createElement("div"); + panel.id = "control-panel"; + + const sampleIntervalHtml = this.onSampleIntervalChange + ? ` +
+ + + ${TIMINGS.sampleIntervalDefault}ms +
+ ` + : ""; + + panel.innerHTML = ` +
+ + + +
+ + ${sampleIntervalHtml} + +
+ + 0ms +
+ `; + + this.container.appendChild(panel); + + this.playPauseBtn = panel.querySelector("#play-pause-btn"); + this.resetBtn = panel.querySelector("#reset-btn"); + this.stepBtn = panel.querySelector("#step-btn"); + this.scrubber = panel.querySelector("#timeline-scrubber"); + this.timeDisplay = panel.querySelector("#time-display"); + + this.playPauseBtn.addEventListener("click", () => + this._togglePlayPause(), + ); + this.resetBtn.addEventListener("click", () => this._handleReset()); + this.stepBtn.addEventListener("click", () => this._handleStep()); + this.scrubber.addEventListener("input", (e) => this._handleSeek(e)); + + if (this.onSampleIntervalChange) { + this.sampleIntervalSlider = panel.querySelector("#sample-interval"); + this.intervalDisplay = panel.querySelector("#interval-display"); + this.sampleIntervalSlider.addEventListener("input", (e) => + this._handleSampleIntervalChange(e), + ); + } + } + + _handleSampleIntervalChange(e) { + const interval = parseInt(e.target.value); + this.intervalDisplay.textContent = `${interval}ms`; + this.onSampleIntervalChange(interval); + } + + _togglePlayPause() { + this.isPlaying = !this.isPlaying; + + if (this.isPlaying) { + this.playPauseBtn.textContent = "⏸ Pause"; + this.playPauseBtn.classList.add("active"); + this.onPlay(); + } else { + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.onPause(); + } + } + + _handleReset() { + this.isPlaying = false; + this.playPauseBtn.textContent = "▶ Play"; + this.playPauseBtn.classList.remove("active"); + this.scrubber.value = 0; + this.timeDisplay.textContent = "0ms"; + this.onReset(); + } + + _handleStep() { + if (this.onStep) this.onStep(); + } + + _handleSeek(e) { + const percentage = parseFloat(e.target.value); + this.onSeek(percentage / 100); + } + + updateTimeDisplay(currentTime, totalTime) { + this.timeDisplay.textContent = `${Math.floor(currentTime)}ms / ${Math.floor(totalTime)}ms`; + const percentage = (currentTime / totalTime) * 100; + this.scrubber.value = percentage; + } + + setDuration(duration) { + this.duration = duration; + } + + pause() { + if (this.isPlaying) this._togglePlayPause(); + } + + destroy() { + const panel = this.container.querySelector("#control-panel"); + if (panel) panel.remove(); + } + } + + // ============================================================================ + // Visual Effects Manager + // ============================================================================ + + class VisualEffectsManager { + constructor(container) { + this.container = container; + this.flyingAnimationInProgress = false; + + this.flashOverlay = document.createElement("div"); + this.flashOverlay.className = "flash-overlay"; + this.container.appendChild(this.flashOverlay); + } + + triggerSamplingEffect(stackViz, samplingPanel, currentTime, trace) { + if (this.flyingAnimationInProgress) return; + + const stack = trace.getStackAt(currentTime); + + if (stack.length === 0) { + samplingPanel.addSample(stack); + return; + } + + this.flyingAnimationInProgress = true; + stackViz.flashAll(); + + const clone = stackViz.createStackClone(this.container); + const targetPosition = samplingPanel.getTargetPosition(); + + this._animateFlash(); + this._animateFlyingStack(clone, targetPosition, () => { + samplingPanel.showImpactEffect(targetPosition); + clone.remove(); + + const currentStack = trace.getStackAt(currentTime); + samplingPanel.addSample(currentStack); + this.flyingAnimationInProgress = false; + }); + } + + _animateFlash() { + anim.to(this.flashOverlay, { opacity: 0.1 }, 0).onfinish = () => { + anim.to(this.flashOverlay, { opacity: 0 }, 150, "easeOutQuad"); + }; + } + + _animateFlyingStack(clone, targetPosition, onComplete) { + const containerRect = this.container.getBoundingClientRect(); + const cloneRect = clone.getBoundingClientRect(); + + // Convert viewport coordinates to container-relative + const startX = cloneRect.left - containerRect.left + cloneRect.width / 2; + const startY = cloneRect.top - containerRect.top + cloneRect.height / 2; + const targetX = targetPosition.x - containerRect.left; + const targetY = targetPosition.y - containerRect.top; + + const deltaX = targetX - startX; + const deltaY = targetY - startY; + + anim.to( + clone, + { + x: deltaX, + y: deltaY, + scale: 0.3, + opacity: 0.6, + }, + TIMINGS.sampleToFlame, + "easeOutCubic", + onComplete, + ); + } + } + + // ============================================================================ + // Main Visualization Class + // ============================================================================ + + class SamplingVisualization { + constructor(container) { + this.container = container; + + this.trace = new ExecutionTrace(DEMO_SIMPLE.source, DEMO_SIMPLE.trace); + + this.currentTime = 0; + this.isPlaying = false; + this.playbackSpeed = TIMINGS.defaultSpeed; + this.eventIndex = 0; + + this.sampleInterval = TIMINGS.sampleIntervalDefault; + this.lastSampleTime = 0; + + this._createLayout(); + + this.effectsManager = new VisualEffectsManager(this.vizColumn); + + this.lastTime = performance.now(); + this._animate(); + } + + _createLayout() { + this.codePanel = new CodePanel(this.trace.source); + this.container.appendChild(this.codePanel.element); + + this.vizColumn = document.createElement("div"); + this.vizColumn.className = "viz-column"; + this.container.appendChild(this.vizColumn); + + const stackSection = document.createElement("div"); + stackSection.className = "stack-section"; + + const stackTitle = document.createElement("div"); + stackTitle.className = "stack-section-title"; + stackTitle.textContent = "Call Stack"; + stackSection.appendChild(stackTitle); + + this.stackViz = new DOMStackVisualization(); + stackSection.appendChild(this.stackViz.element); + this.vizColumn.appendChild(stackSection); + + this.samplingPanel = new DOMSamplingPanel(); + this.samplingPanel.setGroundTruth(this._getGroundTruthFunctions()); + this.vizColumn.appendChild(this.samplingPanel.element); + + this.controls = new ControlPanel( + this.vizColumn, + () => this.play(), + () => this.pause(), + () => this.reset(), + (speed) => this.setSpeed(speed), + (progress) => this.seek(progress), + () => this.step(), + (interval) => this.setSampleInterval(interval), + ); + this.controls.setDuration(this.trace.duration); + } + + _getGroundTruthFunctions() { + const functions = new Set(); + this.trace.events.forEach((event) => { + if (event.type === "call") { + functions.add(event.functionName); + } + }); + return [...functions]; + } + + play() { + this.isPlaying = true; + } + + pause() { + this.isPlaying = false; + } + + reset() { + this.currentTime = 0; + this.eventIndex = 0; + this.isPlaying = false; + this.lastSampleTime = 0; + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + this.controls.updateTimeDisplay(0, this.trace.duration); + } + + setSpeed(speed) { + this.playbackSpeed = speed; + } + + setSampleInterval(interval) { + this.sampleInterval = interval; + this.samplingPanel.setSampleInterval(interval); + } + + seek(progress) { + this.currentTime = progress * this.trace.duration; + this.eventIndex = 0; + this.lastSampleTime = 0; + this._rebuildState(); + } + + step() { + this.pause(); + + const nextEvent = this.trace.getNextEvent(this.currentTime); + + if (nextEvent) { + // Calculate delta to reach next event + epsilon + const targetTime = nextEvent.timestamp + 0.1; + const delta = targetTime - this.currentTime; + if (delta > 0) { + this._advanceTime(delta); + } + } + } + + _animate(currentTime = performance.now()) { + const deltaTime = currentTime - this.lastTime; + this.lastTime = currentTime; + + this.update(deltaTime); + requestAnimationFrame((t) => this._animate(t)); + } + + update(deltaTime) { + if (!this.isPlaying) { + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + return; + } + + const virtualDelta = deltaTime * this.playbackSpeed; + this._advanceTime(virtualDelta); + } + + _advanceTime(virtualDelta) { + this.currentTime += virtualDelta; + + if (this.currentTime >= this.trace.duration) { + this.currentTime = this.trace.duration; + this.isPlaying = false; + this.controls.pause(); + } + + while (this.eventIndex < this.trace.events.length) { + const event = this.trace.events[this.eventIndex]; + + if (event.timestamp > this.currentTime) break; + + this._processEvent(event); + this.eventIndex++; + } + + this.controls.updateTimeDisplay(this.currentTime, this.trace.duration); + + if (this.currentTime - this.lastSampleTime >= this.sampleInterval) { + this._takeSample(); + this.lastSampleTime = this.currentTime; + } + } + + _processEvent(event) { + this.stackViz.processEvent(event); + + if (event.type === "call") { + this.codePanel.highlightLine(event.lineno); + } else if (event.type === "return") { + const currentStack = this.trace.getStackAt(this.currentTime); + if (currentStack.length > 0) { + this.codePanel.highlightLine( + currentStack[currentStack.length - 1].line, + ); + } else { + this.codePanel.highlightLine(null); + } + } else if (event.type === "line") { + this.codePanel.highlightLine(event.lineno); + } + } + + _takeSample() { + this.effectsManager.triggerSamplingEffect( + this.stackViz, + this.samplingPanel, + this.currentTime, + this.trace, + ); + } + + _rebuildState() { + this.stackViz.clear(); + this.codePanel.reset(); + this.samplingPanel.reset(); + + for (let t = 0; t < this.currentTime; t += this.sampleInterval) { + const stack = this.trace.getStackAt(t); + this.samplingPanel.addSample(stack); + this.lastSampleTime = t; + } + + const stack = this.trace.getStackAt(this.currentTime); + this.stackViz.updateToMatch(stack); + + if (stack.length > 0) { + this.codePanel.highlightLine(stack[stack.length - 1].line); + } + + this.eventIndex = this.trace.getEventsUntil(this.currentTime).length; + } + } + + // ============================================================================ + // Initialize + // ============================================================================ + + function init() { + // If trace data hasn't been injected yet (local dev), don't initialize + if (!DEMO_SIMPLE) return; + + const appContainer = document.getElementById("sampling-profiler-viz"); + if (appContainer) { + new SamplingVisualization(appContainer); + } + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", init); + } else { + init(); + } +})(); diff --git a/Doc/_static/tachyon-example-flamegraph.html b/Doc/_static/tachyon-example-flamegraph.html new file mode 100644 index 000000000000000..701c959bda3ccc0 --- /dev/null +++ b/Doc/_static/tachyon-example-flamegraph.html @@ -0,0 +1,3260 @@ + + + + + + Tachyon Profiler - Flamegraph Report + + + + + + + +
+ +
+
+ + Tachyon + + Flamegraph Report +
+
+ + +
+
+ + + + + + + + +
+
+ + +
+ + + + +
+
+
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + + + + + +
+
+ + + + diff --git a/Doc/_static/tachyon-example-heatmap.html b/Doc/_static/tachyon-example-heatmap.html new file mode 100644 index 000000000000000..f725e9475009fe7 --- /dev/null +++ b/Doc/_static/tachyon-example-heatmap.html @@ -0,0 +1,3804 @@ + + + + + + /tmp/tachyon_selfcontained.py - Heatmap + + + +
+ +
+
+ + Tachyon + + /tmp/tachyon_selfcontained.py +
+
+ + + + + + + + + + + +
+
+ + +
+
+
+
1,054
+
Self Samples
+
+
+
4,236
+
Cumulative
+
+
+
24
+
Lines Hit
+
+
+
100.00%
+
% of Total
+
+
+
206
+
Max Self
+
+
+
1054
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + + Hot +
+ +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+ Heat +
+ Specialization +
+ + +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+
+
1
+
+
+
+
"""
+ +
+
+
2
+
+
+
+
Tachyon Demo - Self-contained profiling example.
+ +
+
+
3
+
+
+
+
Pure Python with no external imports for clean heatmap output.
+ +
+
+
4
+
+
+
+
"""
+ +
+
+
5
+
+
+
+
+ +
+
+
6
+
+
+
+
+ +
+
+
7
+
1
+
1
+ +
def fibonacci(n):
+ +
+ +
+
8
+
+
+
+
"""Recursive fibonacci - creates deep call stacks."""
+ +
+
+
9
+
3
+
3
+ +
if n <= 1:
+ +
+ +
+
10
+
17
+
17
+ +
return n
+
+
+ +
+
11
+
206
+
227
+ +
return fibonacci(n - 1) + fibonacci(n - 2)
+
+
+ +
+
12
+
+
+
+
+ +
+
+
13
+
+
+
+
+ +
+
+
14
+
+
+
+
def bubble_sort(arr):
+ +
+
+
15
+
+
+
+
"""Classic bubble sort - O(n^2) comparison sorting."""
+ +
+
+
16
+
+
+
+
n = len(arr)
+ +
+
+
17
+
+
+
+
for i in range(n):
+ +
+
+
18
+
2
+
2
+ +
for j in range(0, n - i - 1):
+ +
+ +
+
19
+
8
+
8
+ +
if arr[j] > arr[j + 1]:
+
+
+ +
+
20
+
2
+
2
+ +
arr[j], arr[j + 1] = arr[j + 1], arr[j]
+ +
+ +
+
21
+
+
+
+
return arr
+ +
+
+
22
+
+
+
+
+ +
+
+
23
+
+
+
+
+ +
+
+
24
+
+
+
+
def matrix_multiply(a, b):
+ +
+
+
25
+
+
+
+
"""Matrix multiplication using nested loops."""
+ +
+
+
26
+
+
+
+
rows_a, cols_a = len(a), len(a[0])
+ +
+
+
27
+
+
+
+
rows_b, cols_b = len(b), len(b[0])
+ +
+
+
28
+
+
+
+
result = [[0] * cols_b for _ in range(rows_a)]
+ +
+
+
29
+
+
+
+
+ +
+
+
30
+
+
+
+
for i in range(rows_a):
+ +
+
+
31
+
+
+
+
for j in range(cols_b):
+ +
+
+
32
+
8
+
8
+ +
for k in range(cols_a):
+ +
+ +
+
33
+
62
+
62
+ +
result[i][j] += a[i][k] * b[k][j]
+
+
+ +
+
34
+
+
+
+
return result
+ +
+
+
35
+
+
+
+
+ +
+
+
36
+
+
+
+
+ +
+
+
37
+
+
+
+
def prime_sieve(limit):
+ +
+
+
38
+
+
+
+
"""Sieve of Eratosthenes - find all primes up to limit."""
+ +
+
+
39
+
+
+
+
is_prime = [True] * (limit + 1)
+ +
+
+
40
+
+
+
+
is_prime[0] = is_prime[1] = False
+ +
+
+
41
+
+
+
+
+ +
+
+
42
+
+
+
+
for num in range(2, int(limit ** 0.5) + 1):
+ +
+
+
43
+
+
+
+
if is_prime[num]:
+ +
+
+
44
+
1
+
1
+ +
for multiple in range(num * num, limit + 1, num):
+
+
+ +
+
45
+
+
+
+
is_prime[multiple] = False
+ +
+
+
46
+
+
+
+
+ +
+
+
47
+
2
+
2
+ +
return [num for num, prime in enumerate(is_prime) if prime]
+ +
+ +
+
48
+
+
+
+
+ +
+
+
49
+
+
+
+
+ +
+
+
50
+
+
+
+
def string_processing(iterations):
+ +
+
+
51
+
+
+
+
"""String operations - concatenation and formatting."""
+ +
+
+
52
+
+
+
+
for _ in range(iterations):
+ +
+
+
53
+
+
+
+
result = ""
+ +
+
+
54
+
+
+
+
for i in range(50):
+ +
+
+
55
+
91
+
91
+ +
result += f"item_{i}_"
+
+
+ +
+
56
+
22
+
22
+ +
parts = result.split("_")
+ +
+ +
+
57
+
10
+
10
+ +
joined = "-".join(parts)
+ +
+ +
+
58
+
+
+
+
+ +
+
+
59
+
+
+
+
+ +
+
+
60
+
+
+
+
def list_operations(iterations):
+ +
+
+
61
+
+
+
+
"""List comprehensions and operations."""
+ +
+
+
62
+
+
+
+
for _ in range(iterations):
+ +
+
+
63
+
118
+
118
+ +
squares = [x ** 2 for x in range(500)]
+ +
+ +
+
64
+
119
+
119
+ +
evens = [x for x in squares if x % 2 == 0]
+
+
+ +
+
65
+
12
+
12
+ +
total = sum(evens)
+ +
+ +
+
66
+
+
+
+
+ +
+
+
67
+
+
+
+
+ +
+
+
68
+
+
+
+
def dict_operations(iterations):
+ +
+
+
69
+
+
+
+
"""Dictionary creation and lookups."""
+ +
+
+
70
+
+
+
+
for _ in range(iterations):
+ +
+
+
71
+
206
+
206
+ +
data = {f"key_{i}": i ** 2 for i in range(200)}
+
+
+ +
+
72
+
158
+
158
+ +
values = [data[f"key_{i}"] for i in range(200)]
+ +
+ +
+
73
+
5
+
5
+ +
total = sum(values)
+ +
+ +
+
74
+
+
+
+
+ +
+
+
75
+
+
+
+
+ +
+
+
76
+
+
+
+
def compute_heavy():
+ +
+
+
77
+
+
+
+
"""CPU-intensive computation section."""
+ +
+
+
78
+
1
+
301
+ +
fibonacci(30)
+
+
+ +
+
79
+
+
+
+
+ +
+
+
80
+
+
+
+
size = 60
+ +
+
+
81
+
+
+
+
a = [[i + j for j in range(size)] for i in range(size)]
+ +
+
+
82
+
+
+
+
b = [[i * j for j in range(size)] for i in range(size)]
+ +
+
+
83
+
+
+
+
matrix_multiply(a, b)
+ +
+
+
84
+
+
+
+
+ +
+
+
85
+
+
+
+
prime_sieve(10000)
+ +
+
+
86
+
+
+
+
+ +
+
+
87
+
+
+
+
+ +
+
+
88
+
+
+
+
def data_processing():
+ +
+
+
89
+
+
+
+
"""Data structure operations."""
+ +
+
+
90
+
+
753
+ +
string_processing(2000)
+
+
+ +
+
91
+
+
+
+
list_operations(1500)
+ +
+
+
92
+
+
+
+
dict_operations(1000)
+ +
+
+
93
+
+
+
+
+ +
+
+
94
+
+
+
+
data = list(range(800))
+ +
+
+
95
+
+
+
+
for _ in range(3):
+ +
+
+
96
+
+
+
+
data = data[::-1]
+ +
+
+
97
+
+
+
+
bubble_sort(data[:200])
+ +
+
+
98
+
+
+
+
+ +
+
+
99
+
+
+
+
+ +
+
+
100
+
+
+
+
def main():
+ +
+
+
101
+
+
+
+
"""Main entry point."""
+ +
+
+
102
+
+
1,054
+ +
compute_heavy()
+
+
+ +
+
103
+
+
+
+
data_processing()
+ +
+
+
104
+
+
+
+
+ +
+
+
105
+
+
+
+
+ +
+
+
106
+
+
+
+
if __name__ == "__main__":
+ +
+
+
107
+
+
1,054
+ +
main()
+
+
+ + +
+
+ + + + diff --git a/Doc/about.rst b/Doc/about.rst index 8f635d7f743a982..5c1b497ca6bcea8 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -32,8 +32,9 @@ Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. +library, and the Python documentation. See the `CPython +GitHub repository `__ +for a partial list of contributors. It is only with the input and contributions of the Python community that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 0683eebbaf677be..a6ea0a72e76f9db 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -9,9 +9,12 @@ stability. In order to maintain this reputation, the developers would like to know of any deficiencies you find in Python. It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to +Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. + +.. _reporting-documentation-bugs: + Documentation bugs ================== diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88cc168..09c9ed3ca54cf69 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 96050f59bd52508..cb98d4307ee5224 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -34,6 +34,23 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. This can be ``0xA`` for alpha, ``0xB`` for beta, ``0xC`` for release candidate or ``0xF`` for final. + + .. c:namespace:: NULL + .. c:macro:: PY_RELEASE_LEVEL_ALPHA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_BETA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_GAMMA + :no-typesetting: + .. c:macro:: PY_RELEASE_LEVEL_FINAL + :no-typesetting: + + For completeness, the values are available as macros: + :c:macro:`!PY_RELEASE_LEVEL_ALPHA` (``0xA``), + :c:macro:`!PY_RELEASE_LEVEL_BETA` (``0xB``), + :c:macro:`!PY_RELEASE_LEVEL_GAMMA` (``0xC``), and + :c:macro:`!PY_RELEASE_LEVEL_FINAL` (``0xF``). + .. c:macro:: PY_RELEASE_SERIAL The ``2`` in ``3.4.1a2``. Zero for final releases. @@ -46,6 +63,12 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Use this for numeric comparisons, for example, ``#if PY_VERSION_HEX >= ...``. +.. c:macro:: PY_VERSION + + The Python version as a string, for example, ``"3.4.1a2"``. + +These macros are defined in :source:`Include/patchlevel.h`. + Run-time version ---------------- diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index fd6be6a9b67a031..58456a36b96c151 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -516,6 +516,28 @@ API Functions } +.. c:function:: int PyArg_ParseArray(PyObject *const *args, Py_ssize_t nargs, const char *format, ...) + + Parse the parameters of a function that takes only array parameters into + local variables (that is, a function using the :c:macro:`METH_FASTCALL` + calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + +.. c:function:: int PyArg_ParseArrayAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, const char *format, const char * const *kwlist, ...) + + Parse the parameters of a function that takes both array and keyword + parameters into local variables (that is, a function using the + :c:macro:`METH_FASTCALL` ``|`` :c:macro:`METH_KEYWORDS` calling convention). + Returns true on success; on failure, it returns false and raises the + appropriate exception. + + .. versionadded:: 3.15 + + .. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) A simpler form of parameter retrieval which does not use a format string to diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 6bb72a2312be3bb..dc3e0f37c36c5b2 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -10,11 +10,6 @@ Buffer Protocol --------------- -.. sectionauthor:: Greg Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - - Certain objects available in Python wrap access to an underlying memory array or *buffer*. Such objects include the built-in :class:`bytes` and :class:`bytearray`, and some extension types like :class:`array.array`. @@ -263,7 +258,9 @@ readonly, format .. c:macro:: PyBUF_WRITEABLE - This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 .. c:macro:: PyBUF_FORMAT @@ -505,10 +502,11 @@ Buffer-related functions *indices* must point to an array of ``view->ndim`` indices. -.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char fort) +.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char order) Copy contiguous *len* bytes from *buf* to *view*. - *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style + ordering or either one). ``0`` is returned on success, ``-1`` on error. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index e2b22ec3c794ae4..2b36da997d42956 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -44,6 +44,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) @@ -58,6 +62,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -70,6 +78,9 @@ Direct API functions ``NULL`` pointer. The returned array always has an extra null byte appended. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) @@ -89,6 +100,9 @@ These macros trade safety for speed and they don't check pointers. Similar to :c:func:`PyByteArray_AsString`, but without error checking. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 82c2557368371fd..ff68ecafcda4d08 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -127,6 +127,10 @@ called with a non-bytes parameter. Return the bytes representation of object *o* that implements the buffer protocol. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytes object is being created. + .. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) @@ -180,11 +184,15 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be - created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + appended to *bytes*; the caller will own the new reference. + The reference to the old value of *bytes* will be ":term:`stolen `". + If the new object cannot be created, the old reference to *bytes* will still + be "stolen", the value of *\*bytes* will be set to ``NULL``, and + the appropriate exception will be set. + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -192,6 +200,10 @@ called with a non-bytes parameter. appended to *bytes*. This version releases the :term:`strong reference` to *newpart* (i.e. decrements its reference count). + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. + .. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) @@ -210,6 +222,9 @@ called with a non-bytes parameter. .. versionadded:: 3.14 + .. note:: + If *iterable* objects implement the buffer protocol, then the buffers + must not be mutated while the new bytes object is being created. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) @@ -224,9 +239,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) @@ -371,12 +385,20 @@ Getters Get the writer size. + The function does not invalidate pointers returned by + :c:func:`PyBytesWriter_GetData`. + + The function cannot fail. + .. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) Get the writer data: start of the internal buffer. - The pointer is valid until :c:func:`PyBytesWriter_Finish` or - :c:func:`PyBytesWriter_Discard` is called on *writer*. + The pointer remains valid until a :c:type:`PyBytesWriter` function other + than :c:func:`PyBytesWriter_GetData` or :c:func:`PyBytesWriter_GetSize` is + called on *writer*. + + The function cannot fail. Low-level API diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 7198d6bc056eb43..9838879a528934a 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -347,6 +347,8 @@ please see individual documentation for details. .. versionadded:: 3.9 +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + :no-typesetting: .. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -358,7 +360,12 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_PyObject_Vectorcall`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 45f5e83adc48c62..57b77f92a7d2e6a 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -7,8 +7,6 @@ Code Objects ------------ -.. sectionauthor:: Jeffrey Yasskin - Code objects are a low-level detail of the CPython implementation. Each one represents a chunk of executable code that hasn't yet been bound into a function. @@ -69,13 +67,14 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(...) + :no-typesetting: + .. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs (C function) - .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 @@ -213,7 +212,7 @@ bound into a function. .. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) - This is a :term:`soft deprecated` function that does nothing. + This is a function that does nothing. Prior to Python 3.10, this function would perform basic optimizations to a code object. @@ -221,6 +220,8 @@ bound into a function. .. versionchanged:: 3.10 This function now does nothing. + .. soft-deprecated:: 3.13 + .. _c_codeobject_flags: @@ -298,6 +299,9 @@ These functions are part of the unstable C API tier: this functionality is a CPython implementation detail, and the API may change without deprecation warnings. +.. c:function:: Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) + :no-typesetting: + .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) Return a new opaque index value used to adding data to code objects. @@ -310,8 +314,6 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex (C function) - .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` .. versionchanged:: 3.12 @@ -320,6 +322,9 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) Set *extra* to the extra data stored under the given index. @@ -328,8 +333,6 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_GetExtra`` .. versionchanged:: 3.12 @@ -338,13 +341,14 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_SetExtra`` .. versionchanged:: 3.12 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 629312bd771beb2..10f96c7cb75e882 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -130,7 +130,7 @@ rather than dereferencing them through pointers. Please note, that these functions are :term:`soft deprecated` since Python 3.15. Avoid using this API in a new code to do complex arithmetic: either use -the `Number Protocol `_ API or use native complex types, like +the :ref:`Number Protocol ` API or use native complex types, like :c:expr:`double complex`. diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index a5c5a53236c9a47..3f38411a52de6b0 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,8 +109,10 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index a18bbf4e0e37d76..f7c8ef8b22b9556 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -130,6 +130,8 @@ The following functions provide locale-independent string to number conversions. *flags* can be zero or more of the following values or-ed together: + .. c:namespace:: NULL + .. c:macro:: Py_DTSF_SIGN Always precede the returned string with a sign @@ -151,9 +153,21 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.11 - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. + If *ptype* is non-``NULL``, then the value it points to will be set to one + of the following constants depending on the type of *val*: + + .. list-table:: + :header-rows: 1 + :align: left + + * - *\*ptype* + - type of *val* + * - .. c:macro:: Py_DTST_FINITE + - finite number + * - .. c:macro:: Py_DTST_INFINITE + - infinite number + * - .. c:macro:: Py_DTST_NAN + - not a number The return value is a pointer to *buffer* with the converted string or ``NULL`` if the conversion failed. The caller is responsible for freeing the @@ -162,16 +176,33 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. + - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) + Case insensitive comparison of strings. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. Character classification and conversion diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index caa855a10d20ca9..06422fb63f34f1a 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -33,3 +33,9 @@ return. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index f311aad5f15499e..d7b4e116c49e355 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,51 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + + .. versionchanged:: 3.15 + + This macro is now thread safe. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + + .. versionchanged:: 3.15 + + This variable should not be accessed directly as direct access is not thread-safe. + Use :c:func:`PyDateTime_IMPORT` instead. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -325,3 +365,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 22c3b790cc3ec34..539c4610ce4f5bc 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -8,27 +8,115 @@ Descriptor Objects "Descriptors" are objects that describe some attribute of an object. They are found in the dictionary of type objects. -.. XXX document these! +.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) -.. c:var:: PyTypeObject PyProperty_Type + Create a new get-set descriptor for extension type *type* from the + :c:type:`PyGetSetDef` structure *getset*. - The type object for the built-in descriptor types. + Get-set descriptors expose attributes implemented by C getter and setter + functions rather than stored directly in the instance. This is the same kind + of descriptor created for entries in :c:member:`~PyTypeObject.tp_getset`, and + it appears in Python as a :class:`types.GetSetDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *member) + + Create a new member descriptor for extension type *type* from the + :c:type:`PyMemberDef` structure *member*. + + Member descriptors expose fields in the type's C struct as Python + attributes. This is the same kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_members`, and it appears in Python as a + :class:`types.MemberDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. -.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) +.. c:var:: PyTypeObject PyGetSetDescr_Type -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. They correspond to + :class:`types.GetSetDescriptorType` objects in Python. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + Create a new method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *meth*. + + Method descriptors expose C functions as methods on a type. This is the same + kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_methods`, and it appears in Python as a + :class:`types.MethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MethodDescriptorType` + objects in Python. + + +.. c:struct:: wrapperbase + + Describes a slot wrapper used by :c:func:`PyDescr_NewWrapper`. + + Each ``wrapperbase`` record stores the Python-visible name and metadata for a + special method implemented by a type slot, together with the wrapper + function used to adapt that slot to Python's calling convention. + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) + + Create a new wrapper descriptor for extension type *type* from the + :c:struct:`wrapperbase` structure *base* and the wrapped slot function + pointer + *wrapped*. -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) + Wrapper descriptors expose special methods implemented by type slots. This + is the same kind of descriptor that CPython creates for slot-based special + methods such as ``__repr__`` or ``__add__``, and it appears in Python as a + :class:`types.WrapperDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + Create a new class method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *method*. + + Class method descriptors expose C methods that receive the class rather than + an instance when accessed. This is the same kind of descriptor created for + ``METH_CLASS`` entries in :c:member:`~PyTypeObject.tp_methods`, and it + appears in Python as a :class:`types.ClassMethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:function:: int PyDescr_IsData(PyObject *descr) @@ -37,12 +125,41 @@ found in the dictionary of type objects. no error checking. -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:function:: PyObject* PyWrapper_New(PyObject *d, PyObject *self) + + Create a new bound wrapper object from the wrapper descriptor *d* and the + instance *self*. + + This is the bound form of a wrapper descriptor created by + :c:func:`PyDescr_NewWrapper`. CPython creates these objects when a slot + wrapper is accessed through an instance, and they appear in Python as + :class:`types.MethodWrapperType` objects. + + On success, return a :term:`strong reference` to the wrapper object. Return + ``NULL`` with an exception set on failure. + +.. c:macro:: PyDescr_COMMON + + This is a macro including the common fields for a + descriptor object. + + This was included in Python's C API by mistake; do not use it in extensions. + For creating custom descriptor objects, create a class implementing the + descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and + :c:member:`~PyTypeObject.tp_descr_set`). + + .. soft-deprecated:: 3.15 Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PyProperty_Type + + The type object for property objects. This is the same object as + :class:`property` in the Python layer. + + .. c:var:: PyTypeObject PySuper_Type The type object for super objects. This is the same object as @@ -55,6 +172,14 @@ Built-in descriptors :class:`classmethod` in the Python layer. +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined + in C extension types, and corresponds to + :class:`types.ClassMethodDescriptorType` objects in Python. + + .. c:function:: PyObject *PyClassMethod_New(PyObject *callable) Create a new :class:`classmethod` object wrapping *callable*. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b7f201811aad6cc..87d09ad2412e405 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -2,7 +2,7 @@ .. _dictobjects: -Dictionary Objects +Dictionary objects ------------------ .. index:: pair: object; dictionary @@ -42,11 +42,31 @@ Dictionary Objects enforces read-only behavior. This is normally used to create a view to prevent modification of the dictionary for non-dynamic class types. + The first argument can be a :class:`dict`, a :class:`frozendict`, or a + mapping. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + + +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. + Do nothing if the argument is not a :class:`dict` or a :class:`!dict` + subclass. + .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) @@ -54,6 +74,16 @@ Dictionary Objects *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) @@ -61,20 +91,29 @@ Dictionary Objects :c:expr:`const char*` UTF-8 encoded bytes string, rather than a :c:expr:`PyObject*`. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. - .. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) Insert *val* into the dictionary *p* with a key of *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. This function *does not* steal a - reference to *val*. + ``0`` on success or ``-1`` on failure. + This function *does not* ":term:`steal`" a reference to *val*. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) @@ -91,6 +130,11 @@ Dictionary Objects If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) @@ -107,10 +151,20 @@ Dictionary Objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. + + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + See also the :c:func:`PyObject_GetItem` function. @@ -120,16 +174,28 @@ Dictionary Objects has a key *key*. Return ``NULL`` if the key *key* is missing *without* setting an exception. + The first argument can be a :class:`dict` or a :class:`frozendict`. + .. note:: Exceptions that occur while this calls :meth:`~object.__hash__` and :meth:`~object.__eq__` methods are silently ignored. Prefer the :c:func:`PyDict_GetItemWithError` function instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.10 Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) @@ -138,6 +204,16 @@ Dictionary Objects occurred. Return ``NULL`` **without** an exception set if the key wasn't present. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) @@ -153,6 +229,16 @@ Dictionary Objects Prefer using the :c:func:`PyDict_GetItemWithError` function with your own :c:func:`PyUnicode_FromString` *key* instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemStringRef`, + which returns a :term:`strong reference`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) @@ -162,6 +248,9 @@ Dictionary Objects .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) @@ -173,6 +262,14 @@ Dictionary Objects .. versionadded:: 3.4 + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_SetDefaultRef`, + which returns a :term:`strong reference`. + + .. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) @@ -192,6 +289,11 @@ Dictionary Objects These may refer to the same object: in that case you hold two separate references to it. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 @@ -209,6 +311,11 @@ Dictionary Objects Similar to :meth:`dict.pop`, but without the default value and not raising :exc:`KeyError` if the key is missing. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 @@ -225,17 +332,32 @@ Dictionary Objects Return a :c:type:`PyListObject` containing all the items from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Keys(PyObject *p) Return a :c:type:`PyListObject` containing all the keys from the dictionary. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: PyObject* PyDict_Values(PyObject *p) Return a :c:type:`PyListObject` containing all the values from the dictionary *p*. + The first argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: Py_ssize_t PyDict_Size(PyObject *p) @@ -244,11 +366,19 @@ Dictionary Objects Return the number of items in the dictionary. This is equivalent to ``len(p)`` on a dictionary. + The argument can be a :class:`dict` or a :class:`frozendict`. + + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) Similar to :c:func:`PyDict_Size`, but without error checking. + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) @@ -263,6 +393,8 @@ Dictionary Objects value represents offsets within the internal dictionary structure, and since the structure is sparse, the offsets are not consecutive. + The first argument can be a :class:`dict` or a :class:`frozendict`. + For example:: PyObject *key, *value; @@ -296,7 +428,7 @@ Dictionary Objects } The function is not thread-safe in the :term:`free-threaded ` - build without external synchronization. You can use + build without external synchronization for a mutable :class:`dict`. You can use :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating over it:: @@ -306,6 +438,8 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + The function is thread-safe on a :class:`frozendict`. + .. note:: On the free-threaded build, this function can be used safely inside a @@ -316,6 +450,9 @@ Dictionary Objects :term:`strong reference ` (for example, using :c:func:`Py_NewRef`). + .. versionchanged:: 3.15 + Also accept :class:`frozendict`. + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) Iterate over mapping object *b* adding key-value pairs to dictionary *a*. @@ -325,6 +462,13 @@ Dictionary Objects only be added if there is not a matching key in *a*. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) @@ -334,6 +478,13 @@ Dictionary Objects argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) @@ -349,6 +500,13 @@ Dictionary Objects if override or key not in a: a[key] = value + .. note:: + + In the :term:`free-threaded ` build, only *a* is locked. + The iteration over *seq2* is not synchronized; *seq2* may be concurrently + modified by another thread. + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer @@ -356,6 +514,13 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_ClearWatcher(int watcher_id) @@ -364,6 +529,13 @@ Dictionary Objects :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) @@ -433,7 +605,7 @@ Dictionary Objects .. versionadded:: 3.12 -Dictionary View Objects +Dictionary view objects ^^^^^^^^^^^^^^^^^^^^^^^ .. c:function:: int PyDictViewSet_Check(PyObject *op) @@ -477,3 +649,143 @@ Dictionary View Objects Return true if *op* is an instance of a dictionary items view. This function always succeeds. + + +Frozen dictionary objects +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.15 + + +.. c:var:: PyTypeObject PyFrozenDict_Type + + This instance of :c:type:`PyTypeObject` represents the Python frozen + dictionary type. + This is the same object as :class:`frozendict` in the Python layer. + + +.. c:function:: int PyAnyDict_Check(PyObject *p) + + Return true if *p* is a :class:`dict` object, a :class:`frozendict` object, + or an instance of a subtype of the :class:`!dict` or :class:`!frozendict` + type. + This function always succeeds. + + +.. c:function:: int PyAnyDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`dict` object or a :class:`frozendict` object, + but not an instance of a subtype of the :class:`!dict` or + :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_Check(PyObject *p) + + Return true if *p* is a :class:`frozendict` object or an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: int PyFrozenDict_CheckExact(PyObject *p) + + Return true if *p* is a :class:`frozendict` object, but not an instance of a + subtype of the :class:`!frozendict` type. + This function always succeeds. + + +.. c:function:: PyObject* PyFrozenDict_New(PyObject *iterable) + + Return a new :class:`frozendict` from an iterable, or ``NULL`` on failure + with an exception set. + + Create an empty dictionary if *iterable* is ``NULL``. + + +Ordered dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d7fe9e2c9ec9b49..78c95791ab31041 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -412,7 +412,7 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. @@ -503,7 +503,8 @@ Querying the error indicator .. warning:: - This call steals a reference to *exc*, which must be a valid exception. + This call ":term:`steals `" a reference to *exc*, + which must be a valid exception. .. versionadded:: 3.12 @@ -641,7 +642,8 @@ Querying the error indicator Set the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. + freshly raised. This function ":term:`steals `" the references + of the arguments. To clear the exception state, pass ``NULL`` for all three arguments. This function is kept for backwards compatibility. Prefer using :c:func:`PyErr_SetHandledException`. @@ -658,8 +660,8 @@ Querying the error indicator .. versionchanged:: 3.11 The ``type`` and ``traceback`` arguments are no longer used and can be NULL. The interpreter now derives them from the exception - instance (the ``value`` argument). The function still steals - references of all three arguments. + instance (the ``value`` argument). The function still + ":term:`steals `" references of all three arguments. Signal Handling @@ -673,28 +675,51 @@ Signal Handling single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. + Handle external interruptions, such as signals or activating a debugger, + whose processing has been delayed until it is safe + to run Python code and/or raise exceptions. - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. + For example, pressing :kbd:`Ctrl-C` causes a terminal to send the + :py:data:`signal.SIGINT` signal. + This function executes the corresponding Python signal handler, which, + by default, raises the :exc:`KeyboardInterrupt` exception. - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). + :c:func:`!PyErr_CheckSignals` should be called by long-running C code + frequently enough so that the response appears immediate to humans. - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. + Handlers invoked by this function currently include: - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). + - Signal handlers, including Python functions registered using + the :mod:`signal` module. - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. + Signal handlers are only run in the main thread of the main interpreter. + + (This is where the function got the name: originally, signals + were the only way to interrupt the interpreter.) + + - Running the garbage collector, if necessary. + + - Executing a pending :ref:`remote debugger ` script. + + - Raise the exception set by :c:func:`PyThreadState_SetAsyncExc`. + + If any handler raises an exception, immediately return ``-1`` with that + exception set. + Any remaining interruptions are left to be processed on the next + :c:func:`PyErr_CheckSignals()` invocation, if appropriate. + + If all handlers finish successfully, or there are no handlers to run, + return ``0``. + + .. versionchanged:: 3.12 + This function may now invoke the garbage collector. + + .. versionchanged:: 3.14 + This function may now execute a remote debugger script, if remote + debugging is enabled. + + .. versionchanged:: 3.15 + The exception set by :c:func:`PyThreadState_SetAsyncExc` is now raised. .. c:function:: void PyErr_SetInterrupt() @@ -793,6 +818,19 @@ Exception Classes Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. +.. c:macro:: PyException_HEAD + + This is a macro including the base fields for an + exception object. + + This was included in Python's C API by mistake and is not designed for use + in extensions. For creating custom exception objects, use + :c:func:`PyErr_NewException` or otherwise create a class inheriting from + :c:data:`PyExc_BaseException`. + + .. soft-deprecated:: 3.15 + + Exception Objects ================= @@ -833,7 +871,7 @@ Exception Objects Set the context associated with the exception to *ctx*. Use ``NULL`` to clear it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. + This ":term:`steals `" a reference to *ctx*. .. c:function:: PyObject* PyException_GetCause(PyObject *ex) @@ -848,7 +886,8 @@ Exception Objects Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception - instance or ``None``. This steals a reference to *cause*. + instance or ``None``. + This ":term:`steals `" a reference to *cause*. The :attr:`~BaseException.__suppress_context__` attribute is implicitly set to ``True`` by this function. @@ -1002,7 +1041,7 @@ Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requ special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. +these are the C equivalent to :deco:`reprlib.recursive_repr`. .. c:function:: int Py_ReprEnter(PyObject *object) @@ -1108,6 +1147,8 @@ Exception types * :exc:`FloatingPointError` * * .. c:var:: PyObject *PyExc_GeneratorExit * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportCycleError + * :exc:`ImportCycleError` * * .. c:var:: PyObject *PyExc_ImportError * :exc:`ImportError` * * .. c:var:: PyObject *PyExc_IndentationError @@ -1310,3 +1351,67 @@ Tracebacks This function returns ``0`` on success, and returns ``-1`` with an exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: 3.15 + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate, Py_ssize_t max_threads) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + + This function is meant to debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first *max_threads* threads, + further output is truncated with the line ``...``. If *max_threads* is 0, the + function will use a default value of 100 for the argument. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 3d331e6ec12f76e..879b1ec50369ff4 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -8,7 +8,8 @@ Defining extension modules A C extension for CPython is a shared library (for example, a ``.so`` file on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process (for example, it is compiled with compatible compiler settings), and which -exports an :ref:`initialization function `. +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). To be importable by default (that is, by :py:class:`importlib.machinery.ExtensionFileLoader`), @@ -23,25 +24,151 @@ and must be named after the module name plus an extension listed in One suitable tool is Setuptools, whose documentation can be found at https://setuptools.pypa.io/en/latest/setuptools.html. -Normally, the initialization function returns a module definition initialized -using :c:func:`PyModuleDef_Init`. -This allows splitting the creation process into several phases: +.. _extension-export-hook: + +Extension export hook +..................... + +.. versionadded:: 3.15 + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PySlot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PySlot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PySlot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + PyABIInfo_VAR(abi_info); + + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_FUNC(Py_mod_init, spam_init_function), + ... + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +.. _pymodexport-api-caveats: + +The hook should be kept short. +If it does more than ``return`` a static array, several caveats apply: + +- If you need to use any Python C API, it is recommended to call + :c:func:`PyABIInfo_Check` first to raise an exception, + rather than crash, in common cases of ABI mismatch. +- Code in the export hook must never rely on the :term:`GIL`: + :term:`free-threaded builds ` of Python can only check + the :c:macro:`Py_mod_gil` slot (or the lack of it) after the hook returns, +- Similarly, the hook may be called in any subinterpreter, since the + :c:macro:`Py_mod_multiple_interpreters` slot (or lack of it) + is only checked after the hook returns. + +For example:: + + PyMODEXPORT_FUNC + PyModExport_modulename(void) + { + if (PyABIInfo_Check(&abi_info, "modulename") < 0) { + /* ABI mismatch. It's not safe to examine the raised exception. */ + return NULL; + } + + /* use Python API (as little as possible); don't rely on GIL */ + + return modulename_slots; + } + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. - Before any substantial code is executed, Python can determine which capabilities the module supports, and it can adjust the environment or refuse loading an incompatible extension. -- By default, Python itself creates the module object -- that is, it does - the equivalent of :py:meth:`object.__new__` for classes. - It also sets initial attributes like :attr:`~module.__package__` and - :attr:`~module.__loader__`. -- Afterwards, the module object is initialized using extension-specific - code -- the equivalent of :py:meth:`~object.__init__` on classes. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. This is called *multi-phase initialization* to distinguish it from the legacy -(but still supported) *single-phase initialization* scheme, -where the initialization function returns a fully constructed module. -See the :ref:`single-phase-initialization section below ` -for details. +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. .. versionchanged:: 3.5 @@ -53,7 +180,7 @@ Multiple module instances By default, extension modules are not singletons. For example, if the :py:attr:`sys.modules` entry is removed and the module -is re-imported, a new module object is created, and typically populated with +is re-imported, a new module object is created and, typically, populated with fresh method and type objects. The old module is subject to normal garbage collection. This mirrors the behavior of pure-Python modules. @@ -83,36 +210,34 @@ A module may also be limited to the main interpreter using the :c:data:`Py_mod_multiple_interpreters` slot. -.. _extension-export-hook: +.. _extension-pyinit: + +``PyInit`` function +................... -Initialization function -....................... +.. soft-deprecated:: 3.15 -The initialization function defined by an extension module has the -following signature: + This functionality will not get new features, + but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: .. c:function:: PyObject* PyInit_modulename(void) Its name should be :samp:`PyInit_{}`, with ```` replaced by the name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). -For modules with ASCII-only names, the function must instead be named -:samp:`PyInit_{}`, with ```` replaced by the name of the module. -When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -:samp:`PyInitU_{}`, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python: +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. -.. code-block:: python - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is recommended to define the initialization function using a helper macro: +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: .. c:macro:: PyMODINIT_FUNC @@ -123,43 +248,15 @@ It is recommended to define the initialization function using a helper macro: * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. -For example, a module called ``spam`` would be defined like this:: - - static struct PyModuleDef spam_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the `Multiple modules in one library `__ -section in :pep:`489` for details. -The initialization function is typically the only non-\ ``static`` -item defined in the module's C source. - - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. -Normally, the :ref:`initialization function ` -(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with -non-``NULL`` :c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` instance must be initialized using the following function: - .. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) Ensure a module definition is a properly initialized Python object that @@ -167,7 +264,8 @@ using the following function: Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - Calling this function is required for :ref:`multi-phase-initialization`. + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. It should not be used in other contexts. Note that Python assumes that ``PyModuleDef`` structures are statically @@ -178,18 +276,36 @@ using the following function: .. versionadded:: 3.5 +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + .. _single-phase-initialization: Legacy single-phase initialization -.................................. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 -.. attention:: Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. -In single-phase initialization, the -:ref:`initialization function ` (``PyInit_modulename``) + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initialization function ` (``PyInit_modulename``) should create, populate and return a module object. This is typically done using :c:func:`PyModule_Create` and functions like :c:func:`PyModule_AddObjectRef`. @@ -242,6 +358,8 @@ in the following ways: * Single-phase modules support module lookup functions like :c:func:`PyState_FindModule`. +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + .. [#testsinglephase] ``_testsinglephase`` is an internal module used in CPython's self-test suite; your installation may or may not include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 9d01254ddb2a11f..dcafefdc0458722 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -123,11 +123,34 @@ the :mod:`io` APIs instead. Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. + instead of the :func:`repr`. + + If *obj* is ``NULL``, write the string ``""``. + Return ``0`` on success or ``-1`` on failure; the + appropriate exception will be set. .. c:function:: int PyFile_WriteString(const char *s, PyObject *p) Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. + + +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are APIs that were included in Python's C API +by mistake. They are documented solely for completeness; use other +``PyFile*`` APIs instead. + +.. c:function:: PyObject *PyFile_NewStdPrinter(int fd) + + Use :c:func:`PyFile_FromFd` with defaults (``fd, NULL, "w", -1, NULL, NULL, NULL, 0``) instead. + +.. c:var:: PyTypeObject PyStdPrinter_Type + + Type of file-like objects used internally at Python startup when :py:mod:`io` is + not yet available. + Use Python :py:func:`open` or :c:func:`PyFile_FromFd` to create file objects instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 79de5daaa90d8f7..a12ad11abb107d0 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -80,25 +80,31 @@ Floating-Point Objects .. c:macro:: Py_INFINITY - This macro expands a to constant expression of type :c:expr:`double`, that + This macro expands to a constant expression of type :c:expr:`double`, that represents the positive infinity. It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is soft deprecated. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN - This macro expands a to constant expression of type :c:expr:`double`, that + This macro expands to a constant expression of type :c:expr:`double`, that represents a quiet not-a-number (qNaN) value. On most platforms, this is equivalent to the :c:macro:`!NAN` macro from the C11 standard ```` header. +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. soft-deprecated:: 3.14 + + .. c:macro:: Py_MATH_E The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. @@ -147,6 +153,34 @@ Floating-Point Objects return PyFloat_FromDouble(copysign(INFINITY, sign)); +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -154,24 +188,23 @@ The pack and unpack functions provide an efficient platform-independent way to store floating-point values as byte strings. The Pack routines produce a bytes string from a C :c:expr:`double`, and the Unpack routines produce a C :c:expr:`double` from such a bytes string. The suffix (2, 4 or 8) specifies the -number of bytes in the bytes string. +number of bytes in the bytes string: -On platforms that appear to use IEEE 754 formats these functions work by -copying bits. On other platforms, the 2-byte format is identical to the IEEE -754 binary16 half-precision format, the 4-byte format (32-bit) is identical to -the IEEE 754 binary32 single precision format, and the 8-byte format to the -IEEE 754 binary64 double precision format, although the packing of INFs and -NaNs (if such things exist on the platform) isn't handled correctly, and -attempting to unpack a bytes string containing an IEEE INF or NaN will raise an -exception. +* The 2-byte format is the IEEE 754 binary16 half-precision format. +* The 4-byte format is the IEEE 754 binary32 single-precision format. +* The 8-byte format is the IEEE 754 binary64 double-precision format. -Note that NaNs type may not be preserved on IEEE platforms (signaling NaN become -quiet NaN), for example on x86 systems in 32-bit mode. +The NaN type may not be preserved on some platforms while unpacking (signaling +NaNs become quiet NaNs), for example on x86 systems in 32-bit mode. +It's assumed that the :c:expr:`double` type has the IEEE 754 binary64 double +precision format. What happens if it's not true is partly accidental (alas). On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less -precision, or smaller dynamic range, not all values can be unpacked. What -happens in such cases is partly accidental (alas). +precision, or smaller dynamic range, not all values can be unpacked. The +packing of special numbers like INFs and NaNs (if such things exist on the +platform) may not be handled correctly, and attempting to unpack a bytes string +containing an IEEE INF or NaN may raise an exception. .. versionadded:: 3.11 @@ -180,19 +213,14 @@ Pack functions The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian -format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you -want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` -constant can be used to use the native endian: it is equal to ``1`` on big -endian processor, or ``0`` on little endian processor. +format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` and ``p+7``), zero if you +want big-endian format (exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` +constant to select the native endian: it is equal to ``0`` on big +endian processor, or ``1`` on little endian processor. Return value: ``0`` if all is OK, ``-1`` if error (and an exception is set, most likely :exc:`OverflowError`). -There are two problems on non-IEEE platforms: - -* What this does is undefined if *x* is a NaN or infinity. -* ``-0.0`` and ``+0.0`` produce the same bytes string. - .. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. @@ -205,6 +233,9 @@ There are two problems on non-IEEE platforms: Pack a C double as the IEEE 754 binary64 double precision format. + .. impl-detail:: + This function always succeeds in CPython. + Unpack functions ^^^^^^^^^^^^^^^^ @@ -212,16 +243,16 @@ Unpack functions The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if the bytes string is in little-endian format (exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian -(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to -use the native endian: it is equal to ``1`` on big endian processor, or ``0`` +(exponent first, at *p*). Use the :c:macro:`!PY_LITTLE_ENDIAN` constant to +select the native endian: it is equal to ``0`` on big endian processor, or ``1`` on little endian processor. Return value: The unpacked double. On error, this is ``-1.0`` and :c:func:`PyErr_Occurred` is true (and an exception is set, most likely :exc:`OverflowError`). -Note that on a non-IEEE platform this will refuse to unpack a bytes string that -represents a NaN or infinity. +.. impl-detail:: + These functions always succeed in CPython. .. c:function:: double PyFloat_Unpack2(const char *p, int le) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index fb17cf7f1da6b24..4159ff6e5965fbd 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -50,6 +50,7 @@ See also :ref:`Reflection `. Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. + This raises no exceptions. .. versionadded:: 3.9 @@ -146,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -168,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -177,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -218,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -248,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index f6fa52b36c5ab31..9c71bb3d59d5e99 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -220,50 +220,237 @@ The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter o detection; it's not expected that users will need to write their own visitor functions. -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` +if the object is immutable. +.. c:type:: int (*inquiry)(PyObject *self) + + Drop references that may have created reference cycles. Immutable objects + do not have to define this method since they can never directly create + reference cycles. Note that the object must still be valid after calling + this method (don't just call :c:func:`Py_DECREF` on a reference). The + collector will call this method if it detects that this object is involved + in a reference cycle. + + +.. _gc-traversal: + +Traversal +--------- + +The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: + .. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) - Traversal function for a container object. Implementations must call the + Traversal function for a garbage-collected object, used by the garbage + collector to detect reference cycles. + Implementations must call the *visit* function for each object directly contained by *self*, with the parameters to *visit* being the contained object and the *arg* value passed to the handler. The *visit* function must not be called with a ``NULL`` - object argument. If *visit* returns a non-zero value that value should be + object argument. If *visit* returns a non-zero value, that value should be returned immediately. -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation -must name its arguments exactly *visit* and *arg*: - - -.. c:macro:: Py_VISIT(o) - - If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* - and *arg*. If *visit* returns a non-zero value, then return it. - Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers - look like:: + A typical :c:member:`!tp_traverse` function calls the :c:func:`Py_VISIT` + convenience macro on each of the instance's members that are Python + objects that the instance owns. + For example, this is a (slightly outdated) traversal function for + the :py:class:`threading.local` class:: static int - my_traverse(Noddy *self, visitproc visit, void *arg) + local_traverse(PyObject *op, visitproc visit, void *arg) { - Py_VISIT(self->foo); - Py_VISIT(self->bar); + localobject *self = (localobject *) op; + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->args); + Py_VISIT(self->kw); + Py_VISIT(self->dict); return 0; } -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` -if the object is immutable. + .. note:: + :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to + :c:func:`!local_traverse` to have these specific names; don't name them just + anything. + Instances of :ref:`heap-allocated types ` hold a reference to + their type. Their traversal function must therefore visit the type:: -.. c:type:: int (*inquiry)(PyObject *self) + Py_VISIT(Py_TYPE(self)); - Drop references that may have created reference cycles. Immutable objects - do not have to define this method since they can never directly create - reference cycles. Note that the object must still be valid after calling - this method (don't just call :c:func:`Py_DECREF` on a reference). The - collector will call this method if it detects that this object is involved - in a reference cycle. + Alternately, the type may delegate this responsibility by + calling ``tp_traverse`` of a heap-allocated superclass (or another + heap-allocated type, if applicable). + If they do not, the type object may not be garbage-collected. + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + int err = PyObject_VisitManagedDict((PyObject*)self, visit, arg); + if (err) { + return err; + } + + Only the members that the instance *owns* (by having + :term:`strong references ` to them) must be + visited. For instance, if an object supports weak references via the + :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting + the linked list (what *tp_weaklist* points to) must **not** be + visited as the instance does not directly own the weak references to itself. + + The traversal function has a limitation: + + .. warning:: + + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects, directly or indirectly. + + This means that *most* Python C API functions may not be used, since + they can raise a new exception, return a new reference to a result object, + have internal logic that uses side effects. + Also, unless documented otherwise, functions that happen to not have side + effects may start having them in future versions, without warning. + + For a list of safe functions, see a + :ref:`separate section ` below. + + .. note:: + + The :c:func:`Py_VISIT` call may be skipped for those members that provably + cannot participate in reference cycles. + In the ``local_traverse`` example above, there is also a ``self->key`` + member, but it can only be ``NULL`` or a Python string and therefore + cannot be part of a reference cycle. + + On the other hand, even if you know a member can never be part of a cycle, + as a debugging aid you may want to visit it anyway just so the :mod:`gc` + module's :func:`~gc.get_referents` function will include it. + + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + + .. impl-detail:: + + Garbage collection is a "stop-the-world" operation: + even in :term:`free threading` builds, only one thread state is + :term:`attached ` when :c:member:`!tp_traverse` + handlers run. + + .. versionchanged:: 3.9 + + Heap-allocated types are expected to visit ``Py_TYPE(self)`` in + ``tp_traverse``. In earlier versions of Python, due to + `bug 40217 `_, doing this + may lead to crashes in subclasses. + +To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, +a :c:func:`Py_VISIT` macro is provided. +In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` +implementation must name its arguments exactly *visit* and *arg*: + +.. c:macro:: Py_VISIT(o) + + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* + callback, with arguments *o* and *arg*. + If *visit* returns a non-zero value, then return it. + + This corresponds roughly to:: + + #define Py_VISIT(o) \ + if (op) { \ + int visit_result = visit(o, arg); \ + if (visit_result != 0) { \ + return visit_result; \ + } \ + } + + +Traversal-safe functions +^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and macros are safe to use in a +:c:member:`~PyTypeObject.tp_traverse` handler: + +* the *visit* function passed to ``tp_traverse`` +* :c:func:`Py_VISIT` +* :c:func:`Py_SIZE` +* :c:func:`Py_TYPE`: if called from a :c:member:`!tp_traverse` handler, + :c:func:`!Py_TYPE`'s result will be valid for the duration of the handler call +* :c:func:`PyObject_VisitManagedDict` +* :c:func:`PyObject_TypeCheck`, :c:func:`PyType_IsSubtype`, + :c:func:`PyType_HasFeature` +* :samp:`Py{}_Check` and :samp:`Py{}_CheckExact` -- for example, + :c:func:`PyTuple_Check` +* :ref:`duringgc-functions` + +.. _duringgc-functions: + +"DuringGC" functions +^^^^^^^^^^^^^^^^^^^^ + +The following functions should *only* be used in a +:c:member:`~PyTypeObject.tp_traverse` handler; calling them in other +contexts may have unintended consequences. + +These functions act like their counterparts without the ``_DuringGC`` suffix, +but they are guaranteed to not have side effects, they do not set an exception +on failure, and they return/set :term:`borrowed references ` +as detailed in the individual documentation. + +Note that these functions may fail (return ``NULL`` or ``-1``), +but as they do not set an exception, no error information is available. +In some cases, failure is not distinguishable from a successful ``NULL`` result. + +.. c:function:: void *PyObject_GetTypeData_DuringGC(PyObject *o, PyTypeObject *cls) + void *PyObject_GetItemData_DuringGC(PyObject *o) + void *PyType_GetModuleState_DuringGC(PyTypeObject *type) + void *PyModule_GetState_DuringGC(PyObject *module) + int PyModule_GetToken_DuringGC(PyObject *module, void** result) + + See :ref:`duringgc-functions` for common information. + + .. versionadded:: 3.15 + + .. seealso:: + + :c:func:`PyObject_GetTypeData`, + :c:func:`PyObject_GetItemData`, + :c:func:`PyType_GetModuleState`, + :c:func:`PyModule_GetState`, + :c:func:`PyModule_GetToken`, + :c:func:`PyType_GetBaseByToken` + +.. c:function:: int PyType_GetBaseByToken_DuringGC(PyTypeObject *type, void *tp_token, PyTypeObject **result) + + See :ref:`duringgc-functions` for common information. + + Sets *\*result* to a :term:`borrowed reference` rather than a strong one. + The reference is valid for the duration + of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: 3.15 + + .. seealso:: :c:func:`PyType_GetBaseByToken` + +.. c:function:: PyObject* PyType_GetModule_DuringGC(PyTypeObject *type) + PyObject* PyType_GetModuleByToken_DuringGC(PyTypeObject *type, const void *mod_token) + + See :ref:`duringgc-functions` for common information. + + These functions return a :term:`borrowed reference`, which is + valid for the duration of the :c:member:`!tp_traverse` handler call. + + .. versionadded:: 3.15 + + .. seealso:: + + :c:func:`PyType_GetModule`, + :c:func:`PyType_GetModuleByToken` Controlling the Garbage Collector State diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da75f2..88c3cc31186826a 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -35,12 +35,83 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be - ``NULL``. + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. + *frame* is ":term:`stolen `" by this function (even on error) and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PyAsyncGenASend_CheckExact(op) + + This is an API that was included in Python's C API + by mistake. + + It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b11ce5..b48cf951137e511 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -129,8 +129,7 @@ Importing Modules of :class:`~importlib.machinery.SourceFileLoader` otherwise. The module's :attr:`~module.__file__` attribute will be set to the code - object's :attr:`~codeobject.co_filename`. If applicable, - :attr:`~module.__cached__` will also be set. + object's :attr:`~codeobject.co_filename`. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -142,10 +141,13 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + The setting of ``__cached__`` and :attr:`~module.__loader__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) @@ -157,16 +159,19 @@ Importing Modules .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. + Like :c:func:`PyImport_ExecCodeModuleEx`, but the path to any compiled file + via *cpathname* is used appropriately when non-``NULL``. Of the three + functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`~module.__cached__` is deprecated. See + Setting ``__cached__`` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` no longer set. + .. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) @@ -299,6 +304,11 @@ Importing Modules Initialization function for a module built into the interpreter. + Note that the inittab uses "``PyInit``" + :ref:`initialization functions `; + there is currently no way to include "``PyModExport_``" + :ref:`export hooks `. + .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) @@ -314,6 +324,13 @@ Importing Modules initialization. +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) Import the module *mod_name* and get its attribute *attr_name*. @@ -333,3 +350,73 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyImport_LazyImportsMode PyImport_GetLazyImportsMode() + + Gets the current lazy imports mode. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_GetLazyImportsFilter() + + Return a :term:`strong reference` to the current lazy imports filter, + or ``NULL`` if none exists. This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + This function always returns ``0``. + + .. versionadded:: 3.15 + +.. c:function:: int PyImport_SetLazyImportsFilter(PyObject *filter) + + Sets the current lazy imports filter. The *filter* should be a callable that + will receive ``(importing_module_name, imported_module_name, [fromlist])`` + when an import can potentially be lazy. The ``imported_module_name`` value + is the resolved module name, so ``lazy from .spam import eggs`` passes + ``package.spam``. The callable must return ``True`` if the import should be + lazy and ``False`` otherwise. + + Return ``0`` on success and ``-1`` with an exception set otherwise. + + .. versionadded:: 3.15 + +.. c:type:: PyImport_LazyImportsMode + + Enumeration of possible lazy import modes. + + .. c:enumerator:: PyImport_LAZY_NORMAL + + Respect the ``lazy`` keyword in source code. This is the default mode. + + .. c:enumerator:: PyImport_LAZY_ALL + + Make all imports lazy by default. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index e9df2a304d975b4..051f6fd765e8506 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -1,7 +1,7 @@ .. _c-api-index: ################################## - Python/C API Reference Manual + Python/C API reference manual ################################## This manual documents the API used by C and C++ programmers who want to write @@ -18,10 +18,16 @@ document the API functions in detail. refcounting.rst exceptions.rst extension-modules.rst + slots.rst utilities.rst abstract.rst concrete.rst - init.rst + interp-lifecycle.rst + threads.rst + synchronization.rst + tls.rst + subinterpreters.rst + profiling.rst init_config.rst memory.rst objimpl.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 18ee16118070eb2..e56c67f95348c78 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,2522 +1,13 @@ -.. highlight:: c +:orphan: +Initialization, finalization, and threads +========================================= -.. _initialization: +This page has been split up into the following: -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See :ref:`Python Initialization Configuration ` for details -on how to configure the interpreter prior to initialization. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Functions that initialize the interpreter: - - * :c:func:`Py_Initialize` - * :c:func:`Py_InitializeEx` - * :c:func:`Py_InitializeFromConfig` - * :c:func:`Py_BytesMain` - * :c:func:`Py_Main` - * the runtime pre-initialization functions covered in :ref:`init-config` - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * the configuration functions covered in :ref:`init-config` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - * :c:func:`Py_IsInitialized` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - * the status reporting and utility functions covered in :ref:`init-config` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -* Synchronization: - - * :c:func:`PyMutex_Lock` - * :c:func:`PyMutex_Unlock` - -.. note:: - - Despite their apparent similarity to some of the functions listed above, - the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and - :c:func:`Py_RunMain`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: Py_FinalizeEx (C function) - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes the table of loaded modules (``sys.modules``), and creates - the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. - It also initializes the module search path (``sys.path``). It does not set - ``sys.argv``; use the :ref:`Python Initialization Configuration ` - API for that. This is a no-op when called for a second time (without calling - :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal - error if the initialization fails. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, - which will also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, - which may be useful when CPython is embedded as part of a larger application. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration, as described in - :ref:`init-from-config`. - - See the :ref:`init-config` section for details on pre-initializing the - interpreter, populating the runtime configuration structure, and querying - the returned status structure. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_IsFinalizing() - - Return true (non-zero) if the main Python interpreter is - :term:`shutting down `. Return false (zero) otherwise. - - .. versionadded:: 3.13 - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). - - Since this is the reverse of :c:func:`Py_Initialize`, it should be called - in the same thread with the same interpreter active. That means - the main thread and the main interpreter. - This should never be called while :c:func:`Py_RunMain` is running. - - Normally the return value is ``0``. - If there were errors during finalization (flushing buffered data), - ``-1`` is returned. - - Note that Python will do a best effort at freeing all memory allocated by the Python - interpreter. Therefore, any C-Extension should make sure to correctly clean up all - of the preveiously allocated PyObjects before using them in subsequent calls to - :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect - behavior. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regardless of their reference count. - Some memory allocated by extension modules may not be freed. Some extensions may not - work properly if their initialization routine is called more than once; this can - happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. :c:func:`Py_FinalizeEx` must not be called recursively from - within itself. Therefore, it must not be called by any code that may be run - as part of the interpreter shutdown process, such as :py:mod:`atexit` - handlers, object finalizers, or any code that may be run while flushing the - stdout and stderr files. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, - allowing the calling application to delegate the text decoding step to - the CPython runtime. - - .. versionadded:: 3.8 - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter, encapsulating a full - initialization/finalization cycle, as well as additional - behaviour to implement reading configurations settings from the environment - and command line, and then executing ``__main__`` in accordance with - :ref:`using-on-cmdline`. - - This is made available for programs which wish to support the full CPython - command line interface, rather than just embedding a Python runtime in a - larger application. - - The *argc* and *argv* parameters are similar to those which are passed to a - C program's :c:func:`main` function, except that the *argv* entries are first - converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also - important to note that the argument list entries may be modified to point to - strings other than those passed in (however, the contents of the strings - pointed to by the argument list are not modified). - - The return value is ``2`` if the argument list does not represent a valid - Python command line, and otherwise the same as :c:func:`Py_RunMain`. - - In terms of the CPython runtime configuration APIs documented in the - :ref:`runtime configuration ` section (and without accounting - for error handling), ``Py_Main`` is approximately equivalent to:: - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetArgv(&config, argc, argv); - Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - - Py_RunMain(); - - In normal usage, an embedding application will call this function - *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or - :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied - as described elsewhere in this documentation. If this function is instead - called *after* a preceding runtime initialization API call, then exactly - which environmental and command line configuration settings will be updated - is version dependent (as it depends on which settings correctly support - being modified after they have already been set once when the runtime was - first initialized). - - -.. c:function:: int Py_RunMain(void) - - Executes the main module in a fully configured CPython runtime. - - Executes the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. If none of these values are set, runs the interactive Python - prompt (REPL) using the ``__main__`` module's global namespace. - - If :c:member:`PyConfig.inspect` is not set (the default), the return value - will be ``0`` if the interpreter exits normally (that is, without raising - an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` - for any other unhandled exception. - - If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option - is used), rather than returning when the interpreter exits, execution will - instead resume in an interactive Python prompt (REPL) using the ``__main__`` - module's global namespace. If the interpreter exited with an exception, it - is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: ``0``, ``1``, or - the status of a :exc:`SystemExit`, as specified above. - - This function always finalizes the Python interpreter before it returns. - - See :ref:`Python Configuration ` for an example of a - customized Python that always runs in isolated mode using - :c:func:`Py_RunMain`. - -.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) - - Register an :mod:`atexit` callback for the target interpreter *interp*. - This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and - data pointer for the callback. - - There must be an :term:`attached thread state` for *interp*. - - .. versionadded:: 3.13 - -Process-wide parameters -======================= - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See :cve:`2008-5983`. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded ` build of :term:`CPython`, -the Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the :term:`GIL`, except on -:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, -:term:`attaching ` a thread state will block until the :term:`GIL` -can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required -to have an attached thread state to call most of the C API. - -In general, there will always be an :term:`attached thread state` when using Python's C API. -Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the -thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns -``NULL``. - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching -the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach -a thread state to their own thread, thus getting the :term:`GIL` and can start executing. -The pointer to the prior :term:`attached thread state` is stored as a local variable. -Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was -previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. -This function will block until another releases its :term:`thread state `, -thus allowing the old :term:`thread state ` to get re-attached and the -C API can be called again. - -For :term:`free-threaded ` builds, the :term:`GIL` is normally -out of the question, but detaching the :term:`thread state ` is still required -for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` -to be released to attach their thread state, allowing true multi-core parallelism. - -.. note:: - Calling system I/O functions is the most common use case for detaching - the :term:`thread state `, but it can also be useful before calling - long-running computations which don't need access to Python objects, such - as compression or cryptographic functions operating over memory buffers. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the :term:`GIL`, because they don't have an -:term:`attached thread state`. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating an :term:`attached thread state` before you can start using the Python/C -API. When you are done, you should detach the :term:`thread state `, and -finally free it. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` -and similar functions default to :term:`attaching ` a -:term:`thread state` for the main interpreter, meaning that the thread can't safely -interact with the calling subinterpreter. - -Supporting subinterpreters in non-Python threads ------------------------------------------------- - -If you would like to support subinterpreters with non-Python created threads, you -must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` -API. - -In particular, you must store the interpreter state from the calling -function and pass it to :c:func:`PyThreadState_New`, which will ensure that -the :term:`thread state` is targeting the correct interpreter:: - - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = ThreadData->interp; - PyThreadState *tstate = PyThreadState_New(interp); - PyThreadState_Swap(tstate); - - /* GIL of the subinterpreter is now held. - Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - -.. _cautions-regarding-runtime-finalization: - -Cautions regarding runtime finalization ---------------------------------------- - -In the late stage of :term:`interpreter shutdown`, after attempting to wait for -non-daemon threads to exit (though this can be interrupted by -:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - .. versionchanged:: 3.12 - - :pep:`684` introduced the possibility - of a :ref:`per-interpreter GIL `. - See :c:func:`Py_NewInterpreterFromConfig`. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso:: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function may return non-``NULL`` even when the :term:`thread state` - is detached. - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso:: :c:func:`PyThreadState_Get` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL` - and sets an :term:`attached thread state`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. An :term:`attached thread state` is not needed, - but may optionally exist if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the interpreter. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. There **should not** be an - :term:`attached thread state` for the target interpreter. The interpreter - state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`!PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`!PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) - - Set the stack protection start address and stack protection size - of a Python thread state. - - On success, return ``0``. - On failure, set an exception and return ``-1``. - - CPython implements :ref:`recursion control ` for C code by raising - :py:exc:`RecursionError` when it notices that the machine execution stack is close - to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. - For this, it needs to know the location of the current thread's stack, which it - normally gets from the operating system. - When the stack is changed, for example using context switching techniques like the - Boost library's ``boost::context``, you must call - :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. - - Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before - or after changing the stack. - Do not call any other Python C API between the call and the stack - change. - - See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - - .. versionadded:: next - - -.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) - - Reset the stack protection start address and stack protection size - of a Python thread state to the operating system defaults. - - See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - - .. versionadded:: next - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no :term:`attached thread state`. - It cannot return NULL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must have an :term:`attached thread state`. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - The returned dictionary is borrowed from the interpreter and is valid until - interpreter shutdown. - - .. versionadded:: 3.8 - - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with an :term:`attached thread state`. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:type:: PyInterpreterConfig - - Structure containing most parameters to configure a sub-interpreter. - Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and - never modified by the runtime. - - .. versionadded:: 3.12 - - Structure fields: - - .. c:member:: int use_main_obmalloc - - If this is ``0`` then the sub-interpreter will use its own - "object" allocator state. - Otherwise it will use (share) the main interpreter's. - - If this is ``0`` then - :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` - must be ``1`` (non-zero). - If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` - must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. - - .. c:member:: int allow_fork - - If this is ``0`` then the runtime will not support forking the - process in any thread where the sub-interpreter is currently active. - Otherwise fork is unrestricted. - - Note that the :mod:`subprocess` module still works - when fork is disallowed. - - .. c:member:: int allow_exec - - If this is ``0`` then the runtime will not support replacing the - current process via exec (e.g. :func:`os.execv`) in any thread - where the sub-interpreter is currently active. - Otherwise exec is unrestricted. - - Note that the :mod:`subprocess` module still works - when exec is disallowed. - - .. c:member:: int allow_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create threads. - Otherwise threads are allowed. - - .. c:member:: int allow_daemon_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create daemon threads. - Otherwise daemon threads are allowed (as long as - :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). - - .. c:member:: int check_multi_interp_extensions - - If this is ``0`` then all extension modules may be imported, - including legacy (single-phase init) modules, - in any thread where the sub-interpreter is currently active. - Otherwise only multi-phase init extension modules - (see :pep:`489`) may be imported. - (Also see :c:macro:`Py_mod_multiple_interpreters`.) - - This must be ``1`` (non-zero) if - :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. - - .. c:member:: int gil - - This determines the operation of the GIL for the sub-interpreter. - It may be one of the following: - - .. c:namespace:: NULL - - .. c:macro:: PyInterpreterConfig_DEFAULT_GIL - - Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). - - .. c:macro:: PyInterpreterConfig_SHARED_GIL - - Use (share) the main interpreter's GIL. - - .. c:macro:: PyInterpreterConfig_OWN_GIL - - Use the sub-interpreter's own GIL. - - If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then - :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. - - -.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The given *config* controls the options with which the interpreter - is initialized. - - Upon success, *tstate_p* will be set to the first :term:`thread state` - created in the new sub-interpreter. This thread state is - :term:`attached `. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, - *tstate_p* is set to ``NULL``; - no exception is set since the exception state is stored in the - :term:`attached thread state`, which might not exist. - - Like all other Python/C API functions, an :term:`attached thread state` - must be present before calling this function, but it might be detached upon - returning. On success, the returned thread state will be :term:`attached `. - If the sub-interpreter is created with its own :term:`GIL` then the - :term:`attached thread state` of the calling interpreter will be detached. - When the function returns, the new interpreter's :term:`thread state` - will be :term:`attached ` to the current thread and - the previous interpreter's :term:`attached thread state` will remain detached. - - .. versionadded:: 3.12 - - Sub-interpreters are most effective when isolated from each other, - with certain functionality restricted:: - - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - Note that the config is used only briefly and does not get modified. - During initialization the config's values are converted into various - :c:type:`PyInterpreterState` values. A read-only copy of the config - may be stored internally on the :c:type:`PyInterpreterState`. - - .. index:: - single: Py_FinalizeEx (C function) - single: Py_Initialize (C function) - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close (in module os) - - -.. c:function:: PyThreadState* Py_NewInterpreter(void) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is essentially just a wrapper - around :c:func:`Py_NewInterpreterFromConfig` with a config that - preserves the existing behavior. The result is an unisolated - sub-interpreter that shares the main interpreter's GIL, allows - fork/exec, allows daemon threads, and allows single-phase init - modules. - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx (C function) - - Destroy the (sub-)interpreter represented by the given :term:`thread state`. - The given thread state must be :term:`attached `. - When the call returns, there will be no :term:`attached thread state`. - All thread states associated with this interpreter are destroyed. - - :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -.. _per-interpreter-gil: - -A Per-Interpreter GIL ---------------------- - -Using :c:func:`Py_NewInterpreterFromConfig` you can create -a sub-interpreter that is completely isolated from other interpreters, -including having its own GIL. The most important benefit of this -isolation is that such an interpreter can execute Python code without -being blocked by other interpreters or blocking any others. Thus a -single Python process can truly take advantage of multiple CPU cores -when running Python code. The isolation also encourages a different -approach to concurrency than that of just using threads. -(See :pep:`554` and :pep:`684`.) - -Using an isolated interpreter requires vigilance in preserving that -isolation. That especially means not sharing any objects or mutable -state without guarantees about thread-safety. Even objects that are -otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared -because of the refcount. One simple but less-efficient approach around -this is to use a global lock around all use of some state (or object). -Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them :term:`immortal`. -In fact, this has been done for the builtin singletons, small integers, -and a number of other builtin objects. - -If you preserve isolation then you will have access to proper multi-core -computing without the complications that come with free-threading. -Failure to preserve isolation will expose you to the full consequences -of free-threading, including races and hard-to-debug crashes. - -Aside from that, one of the main challenges of using multiple isolated -interpreters is how to communicate between them safely (not break -isolation) and efficiently. The runtime and stdlib do not provide -any standard approach to this yet. A future stdlib module would help -mitigate the effort of preserving isolation and expose effective tools -for communicating (and sharing) data between interpreters. - -.. versionadded:: 3.12 - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - - .. versionchanged:: 3.12 - This function now always schedules *func* to be run in the main - interpreter. - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to - *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`~frame.f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while - setting the profile functions in all threads. - -.. versionadded:: 3.12 - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while - setting the trace functions in all threads. - -.. versionadded:: 3.12 - -Reference tracing -================= - -.. versionadded:: 3.13 - -.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. - The first parameter is a Python object that has been just created (when **event** - is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** - is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer - that was provided when :c:func:`PyRefTracer_SetTracer` was called. - - If a new tracing function is registered replacing the current a call to the - trace function will be made with the object set to **NULL** and **event** set to - :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new - function is registered. - -.. versionadded:: 3.13 - -.. c:var:: int PyRefTracer_CREATE - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been created. - -.. c:var:: int PyRefTracer_DESTROY - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been destroyed. - -.. c:var:: int PyRefTracer_TRACKER_REMOVED - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when the - current tracer is about to be replaced by a new one. - - .. versionadded:: 3.14 - -.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new - Python has been created or when an object is going to be destroyed. If - **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. - - Not that tracer functions **must not** create Python objects inside or - otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. A :term:`thread state` will be active - every time the tracer function is called. - - There must be an :term:`attached thread state` when calling this function. - - If another tracer function was already registered, the old function will be - called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before - the new function is registered. - -.. versionadded:: 3.13 - -.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - - Get the registered reference tracer function and the value of the opaque data - pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. - If no tracer was registered this function will return NULL and will set the - **data** pointer to NULL. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they supply their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - -Synchronization Primitives -========================== - -The C-API provides a basic mutual exclusion lock. - -.. c:type:: PyMutex - - A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to - zero to represent the unlocked state. For example:: - - PyMutex mutex = {0}; - - Instances of :c:type:`!PyMutex` should not be copied or moved. Both the - contents and address of a :c:type:`!PyMutex` are meaningful, and it must - remain at a fixed, writable location in memory. - - .. note:: - - A :c:type:`!PyMutex` currently occupies one byte, but the size should be - considered unstable. The size may change in future Python releases - without a deprecation period. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Lock(PyMutex *m) - - Lock mutex *m*. If another thread has already locked it, the calling - thread will block until the mutex is unlocked. While blocked, the thread - will temporarily detach the :term:`thread state ` if one exists. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Unlock(PyMutex *m) - - Unlock mutex *m*. The mutex must be locked --- otherwise, the function will - issue a fatal error. - - .. versionadded:: 3.13 - -.. c:function:: int PyMutex_IsLocked(PyMutex *m) - - Returns non-zero if the mutex *m* is currently locked, zero otherwise. - - .. note:: - - This function is intended for use in assertions and debugging only and - should not be used to make concurrency control decisions, as the lock - state may change immediately after the check. - - .. versionadded:: 3.14 - -.. _python-critical-section-api: - -Python Critical Section API ---------------------------- - -The critical section API provides a deadlock avoidance layer on top of -per-object locks for :term:`free-threaded ` CPython. They are -intended to replace reliance on the :term:`global interpreter lock`, and are -no-ops in versions of Python with the global interpreter lock. - -Critical sections are intended to be used for custom types implemented -in C-API extensions. They should generally not be used with built-in types like -:class:`list` and :class:`dict` because their public C-APIs -already use critical sections internally, with the notable -exception of :c:func:`PyDict_Next`, which requires critical section -to be acquired externally. - -Critical sections avoid deadlocks by implicitly suspending active critical -sections, hence, they do not provide exclusive access such as provided by -traditional locks like :c:type:`PyMutex`. When a critical section is started, -the per-object lock for the object is acquired. If the code executed inside the -critical section calls C-API functions then it can suspend the critical section thereby -releasing the per-object lock, so other threads can acquire the per-object lock -for the same object. - -Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also -available. Use these variants to start a critical section in a situation where -there is no :c:type:`PyObject` -- for example, when working with a C type that -does not extend or wrap :c:type:`PyObject` but still needs to call into the C -API in a manner that might lead to deadlocks. - -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - -.. note:: - - Operations that need to lock two objects at once must use - :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical - sections to lock more than one object at once, because the inner critical - section may suspend the outer critical sections. This API does not provide - a way to lock more than two objects at once. - -Example usage:: - - static PyObject * - set_field(MyObject *self, PyObject *value) - { - Py_BEGIN_CRITICAL_SECTION(self); - Py_SETREF(self->field, Py_XNewRef(value)); - Py_END_CRITICAL_SECTION(); - Py_RETURN_NONE; - } - -In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which -can call arbitrary code through an object's deallocation function. The critical -section API avoids potential deadlocks due to reentrancy and lock ordering -by allowing the runtime to temporarily suspend the critical section if the -code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) - - Acquires the per-object lock for the object *op* and begins a - critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) - - Locks the mutex *m* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_BeginMutex(&_py_cs, m) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for - the argument of the macro - it must be a :c:type:`PyMutex` pointer. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION() - - Ends the critical section and releases the per-object lock. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection_End(&_py_cs); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) - - Acquires the per-objects locks for the objects *a* and *b* and begins a - critical section. The locks are acquired in a consistent order (lowest - address first) to avoid lock ordering deadlocks. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) - - Locks the mutexes *m1* and *m2* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for - the arguments of the macro - they must be :c:type:`PyMutex` pointers. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION2() - - Ends the critical section and releases the per-object locks. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection2_End(&_py_cs2); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 +- :ref:`initialization` +- :ref:`threads` +- :ref:`synchronization` +- :ref:`thread-local-storage` +- :ref:`sub-interpreter-support` +- :ref:`profiling` diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c345029e4acd490..d6b9837987a3999 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -544,9 +544,9 @@ Configuration Options Visibility: -* Public: Can by get by :c:func:`PyConfig_Get` and set by +* Public: Can be retrieved by :c:func:`PyConfig_Get` and set by :c:func:`PyConfig_Set`. -* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by +* Read-only: Can be retrieved by :c:func:`PyConfig_Get`, but cannot be set by :c:func:`PyConfig_Set`. @@ -623,6 +623,10 @@ Some options are read from the :mod:`sys` attributes. For example, the option .. versionadded:: 3.14 + .. versionchanged:: next + The function now replaces :data:`sys.flags` (create a new object), + instead of modifying :data:`sys.flags` in-place. + .. _pyconfig_api: @@ -1153,7 +1157,7 @@ PyConfig Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + (:c:type:`PyPreConfig`) is based on the :c:type:`PyConfig`. If configuration fields which are in common with :c:type:`PyPreConfig` are tuned, they must be set before calling a :c:type:`PyConfig` method: @@ -1807,10 +1811,10 @@ PyConfig .. c:member:: wchar_t* run_presite - ``package.module`` path to module that should be imported before - ``site.py`` is run. + ``module`` or ``module:func`` entry point that should be executed before + the :mod:`site` module is imported. - Set by the :option:`-X presite=package.module <-X>` command-line + Set by the :option:`-X presite=module:func <-X>` command-line option and the :envvar:`PYTHON_PRESITE` environment variable. The command-line option takes precedence. @@ -2299,13 +2303,91 @@ Py_GetArgcArgv() See also :c:member:`PyConfig.orig_argv` member. -Delaying main module execution -============================== -In some embedding use cases, it may be desirable to separate interpreter initialization -from the execution of the main module. +Multi-Phase Initialization Private Provisional API +================================================== + +This section is a private provisional API introducing multi-phase +initialization, the core feature of :pep:`432`: + +* "Core" initialization phase, "bare minimum Python": + + * Builtin types; + * Builtin exceptions; + * Builtin and frozen modules; + * The :mod:`sys` module is only partially initialized + (ex: :data:`sys.path` doesn't exist yet). + +* "Main" initialization phase, Python is fully initialized: + + * Install and configure :mod:`importlib`; + * Apply the :ref:`Path Configuration `; + * Install signal handlers; + * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` + and :data:`sys.path`); + * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; + * Import the :mod:`site` module; + * etc. + +Private provisional API: + +.. c:member:: int PyConfig._init_main + + If set to ``0``, :c:func:`Py_InitializeFromConfig` stops at the "Core" + initialization phase. + +.. c:function:: PyStatus _Py_InitializeMain(void) + + Move to the "Main" initialization phase, finish the Python initialization. + +No module is imported during the "Core" phase and the ``importlib`` module is +not configured: the :ref:`Path Configuration ` is only +applied during the "Main" phase. It may allow to customize Python in Python to +override or tune the :ref:`Path Configuration `, maybe +install a custom :data:`sys.meta_path` importer or an import hook, etc. + +It may become possible to calculate the :ref:`Path Configuration +` in Python, after the Core phase and before the Main phase, +which is one of the :pep:`432` motivation. + +The "Core" phase is not properly defined: what should be and what should +not be available at this phase is not specified yet. The API is marked +as private and provisional: the API can be modified or even be removed +anytime until a proper public API is designed. + +Example running Python code between "Core" and "Main" initialization +phases:: + + void init_python(void) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config._init_main = 0; + + /* ... customize 'config' configuration ... */ + + status = Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + /* Use sys.stderr because sys.stdout is only created + by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + /* ... put more configuration code here ... */ -This separation can be achieved by setting ``PyConfig.run_command`` to the empty -string during initialization (to prevent the interpreter from dropping into the -interactive prompt), and then subsequently executing the desired main module -code using ``__main__.__dict__`` as the global namespace. + status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + } diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst new file mode 100644 index 000000000000000..46f5b1dd33963c0 --- /dev/null +++ b/Doc/c-api/interp-lifecycle.rst @@ -0,0 +1,974 @@ +.. highlight:: c + +.. _initialization: + +Interpreter initialization and finalization +=========================================== + +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. + +.. _pre-init-safe: + +Before Python initialization +---------------------------- + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`!PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * the configuration functions covered in :ref:`init-config` + +* Informative functions: + + * :c:func:`Py_IsInitialized` + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + +.. note:: + + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. + + +.. _global-conf-vars: + +Global configuration variables +------------------------------ + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + + +.. c:var:: int Py_BytesWarningFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python + Initialization Configuration `. + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_DebugFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python + Initialization Configuration `. + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_DontWriteBytecodeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python + Initialization Configuration `. + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_FrozenFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.pathconfig_warnings` should be used instead, see + :ref:`Python Initialization Configuration `. + + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_HashRandomizationFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should + be used instead, see :ref:`Python Initialization Configuration + `. + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_IgnoreEnvironmentFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.use_environment` should be used instead, see + :ref:`Python Initialization Configuration `. + + Ignore all :envvar:`!PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_InspectFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.inspect` should be used instead, see + :ref:`Python Initialization Configuration `. + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_InteractiveFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.interactive` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-i` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_IsolatedFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.isolated` should be used instead, see + :ref:`Python Initialization Configuration `. + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_LegacyWindowsFSEncodingFlag + + This API is kept for backward compatibility: setting + :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_LegacyWindowsStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_NoSiteFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.site_import` should be used instead, see + :ref:`Python Initialization Configuration `. + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_NoUserSiteDirectory + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.user_site_directory` should be used instead, see + :ref:`Python Initialization Configuration `. + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_OptimizeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.optimization_level` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_QuietFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python + Initialization Configuration `. + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_UnbufferedStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python + Initialization Configuration `. + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + + .. deprecated-removed:: 3.12 3.16 + + +.. c:var:: int Py_VerboseFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python + Initialization Configuration `. + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + + .. deprecated-removed:: 3.12 3.16 + + +Initializing and finalizing the interpreter +------------------------------------------- + +.. c:function:: void Py_Initialize() + + .. index:: + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + pair: module; builtins + pair: module; __main__ + pair: module; sys + triple: module; search; path + single: Py_FinalizeEx (C function) + + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + .. note:: + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + .. versionchanged:: 3.15 + This function no longer returns true until initialization has fully + completed, including import of the :mod:`site` module. Previously it + could return true while :c:func:`Py_Initialize` was still running. + + +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + +.. c:function:: int Py_FinalizeEx() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the previously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. + + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. + + +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 + + +.. _cautions-regarding-runtime-finalization: + +Cautions regarding interpreter finalization +------------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as finalizing, meaning that :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the finalization +thread (the thread that initiated finalization; this is typically the main thread) +is allowed to :term:`attach ` a thread state. + +Other threads that attempt to attach during finalization, either explicitly +(such as via :c:func:`PyThreadState_Ensure` or :c:macro:`Py_END_ALLOW_THREADS`) +or implicitly (such as in-between bytecode instructions), will enter a +**permanently blocked state**. Generally, this is harmless, but this can +result in deadlocks. For example, a thread may be permanently blocked while +holding a lock, meaning that the finalization thread can never acquire that +lock. + +Prior to CPython 3.13, the thread would exit instead of hanging, +which led to other issues (see the warning note at +:c:func:`PyThread_exit_thread`). + +Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make +it possible to avoid these issues by temporarily preventing finalization: + +.. _interpreter-guards: + +.. seealso:: + + :pep:`788` explains the design, motivation and rationale + for these APIs. + +.. c:type:: PyInterpreterGuard + + An opaque interpreter guard structure. + + By holding an interpreter guard, the caller can ensure that the interpreter + will not finalize until the guard is closed (through + :c:func:`PyInterpreterGuard_Close`). + + When a guard is held, a thread attempting to finalize the interpreter will + block until the guard is closed before starting finalization. + After finalization has started, threads are forever unable to acquire + guards for that interpreter. This means that if you forget to close an + interpreter guard, the process will **permanently hang** during + finalization! + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) + + Create a finalization guard for the current interpreter. This will prevent + finalization until the guard is closed. + + For example: + + .. code-block:: c + + // Temporarily prevent finalization. + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + // Finalization has already started or we're out of memory. + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + // Do some critical processing here. For example, we can safely acquire + // locks that might be acquired by the finalization thread. + Py_END_ALLOW_THREADS; + + // Now that we're done with our critical processing, the interpreter is + // allowed to finalize again. + PyInterpreterGuard_Close(guard); + + On success, this function returns a guard for the current interpreter; + on failure, it returns ``NULL`` with an exception set. + + This function will fail only if the current interpreter has already started + finalizing, or if the process is out of memory. + + The guard pointer returned by this function must be eventually closed + with :c:func:`PyInterpreterGuard_Close`; failing to do so will result in + the Python process infinitely hanging. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromView(PyInterpreterView *view) + + Create a finalization guard for an interpreter through a view. + + On success, this function returns a guard to the interpreter + represented by *view*. The view is still valid after calling this + function. The guard must eventually be closed with + :c:func:`PyInterpreterGuard_Close`. + + If the interpreter no longer exists, is already finalizing, or out of memory, + then this function returns ``NULL`` without setting an exception. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterGuard_Close(PyInterpreterGuard *guard) + + Close an interpreter guard, allowing the interpreter to start + finalization if no other guards remain. If an interpreter guard + is never closed, the interpreter will infinitely wait when trying + to enter finalization! + + After an interpreter guard is closed, it may not be used in + :c:func:`PyThreadState_Ensure`. Doing so will result in undefined + behavior. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. _interpreter-views: + +Interpreter views +----------------- + +In some cases, it may be necessary to access an interpreter that may have been +deleted. This can be done using interpreter views. + +.. c:type:: PyInterpreterView + + An opaque view of an interpreter. + + This is a thread-safe way to access an interpreter that may have be + finalizing or already destroyed. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromCurrent(void) + + Create a view to the current interpreter. + + This function is generally meant to be used alongside + :c:func:`PyInterpreterGuard_FromView` or :c:func:`PyThreadState_EnsureFromView`. + + On success, this function returns a view to the current interpreter; on + failure, it returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterView_Close(PyInterpreterView *view) + + Close an interpreter view. + + If an interpreter view is never closed, the view's memory will never be + freed, but there are no other consequences. (In contrast, forgetting to + close a guard will infinitely hang the main thread during finalization.) + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromMain(void) + + Create a view for the main interpreter (the first and default + interpreter in a Python process; see + :c:func:`PyInterpreterState_Main`). + + On success, this function returns a view to the main + interpreter; on failure, it returns ``NULL`` without an exception set. + Failure indicates that the process is out of memory. + + Use this function when an interpreter pointer or view cannot be supplied + by the caller, such as when a native threading library does not provide a + ``void *arg`` parameter that could carry a :c:type:`PyInterpreterGuard` or + :c:type:`PyInterpreterView`. In code that supports subinterpreters, prefer + :c:func:`PyInterpreterView_FromCurrent` so the guard tracks the calling + interpreter rather than the main one. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +Process-wide parameters +----------------------- + +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + See also the :c:var:`Py_Version` constant. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.16 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.16 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c76cc2f70ecccff..4c0c9af45e8360d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -107,6 +107,46 @@ header files properly declare the entry points to be ``extern "C"``. As a result there is no need to do anything special to use the API from C++. +.. _capi-system-includes: + +System includes +--------------- + + :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + The implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.13 or newer: + + * ```` + * ```` (on POSIX) + + The following are included for backwards compatibility, unless using + :ref:`Limited API ` 3.11 or newer: + + * ```` + * ```` + * ```` + +.. note:: + + Since Python may define some pre-processor definitions which affect the standard + headers on some systems, you *must* include :file:`Python.h` before any standard + headers are included. + + Useful macros ============= @@ -116,97 +156,152 @@ defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: Py_CAN_START_THREADS -.. c:macro:: Py_ABS(x) + If this macro is defined, then the current system is able to start threads. - Return the absolute value of ``x``. + Currently, all systems supported by CPython (per :pep:`11`), with the + exception of some WebAssembly platforms, support starting threads. - If the result cannot be represented (for example, if ``x`` has - :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is - undefined. + .. versionadded:: 3.13 - .. versionadded:: 3.3 +.. c:macro:: Py_GETENV(s) -.. c:macro:: Py_ALWAYS_INLINE + Like :samp:`getenv({s})`, but returns ``NULL`` if :option:`-E` was passed + on the command line (see :c:member:`PyConfig.use_environment`). - Ask the compiler to always inline a static inline function. The compiler can - ignore it and decide to not inline the function. - It can be used to inline performance critical static inline functions when - building Python in debug mode with function inlining disabled. For example, - MSC disables function inlining when building in debug mode. - - Marking blindly a static inline function with Py_ALWAYS_INLINE can result in - worse performances (due to increased code size for example). The compiler is - usually smarter than the developer for the cost/benefit analysis. +Docstring macros +---------------- - If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` - macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. +.. c:macro:: PyDoc_STRVAR(name, str) - It must be specified before the function return type. Usage:: + Creates a variable with name *name* that can be used in docstrings. + If Python is built without docstrings (:option:`--without-doc-strings`), + the value will be an empty string. - static inline Py_ALWAYS_INLINE int random(void) { return 4; } + Example:: - .. versionadded:: 3.11 + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); -.. c:macro:: Py_CHARMASK(c) + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. + Expands to :samp:`PyDoc_VAR({name}) = PyDoc_STR({str})`. -.. c:macro:: Py_DEPRECATED(version) +.. c:macro:: PyDoc_STR(str) - Use this for deprecated declarations. The macro must be placed before the - symbol name. + Expands to the given input string, or an empty string + if docstrings are disabled (:option:`--without-doc-strings`). Example:: - Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; - .. versionchanged:: 3.8 - MSVC support was added. +.. c:macro:: PyDoc_VAR(name) -.. c:macro:: Py_GETENV(s) + Declares a static character array variable with the given *name*. + Expands to :samp:`static const char {name}[]` - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (see :c:member:`PyConfig.use_environment`). + For example:: -.. c:macro:: Py_MAX(x, y) + PyDoc_VAR(python_doc) = PyDoc_STR( + "A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); - Return the maximum value between ``x`` and ``y``. - .. versionadded:: 3.3 +General utility macros +---------------------- -.. c:macro:: Py_MEMBER_SIZE(type, member) +The following macros are for common tasks not specific to Python. - Return the size of a structure (``type``) ``member`` in bytes. +.. c:macro:: Py_UNUSED(arg) - .. versionadded:: 3.6 + Use this for unused arguments in a function definition to silence compiler + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. -.. c:macro:: Py_MIN(x, y) + .. versionadded:: 3.4 + +.. c:macro:: Py_GCC_ATTRIBUTE(name) + + Use a GCC attribute *name*, hiding it from compilers that don't support GCC + attributes (such as MSVC). + + This expands to :samp:`__attribute__(({name)})` on a GCC compiler, + and expands to nothing on compilers that don't support GCC attributes. - Return the minimum value between ``x`` and ``y``. + +Numeric utilities +^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_ABS(x) + + Return the absolute value of ``x``. + + The argument may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + + Corresponds roughly to :samp:`(({x}) < 0 ? -({x}) : ({x}))` .. versionadded:: 3.3 -.. c:macro:: Py_NO_INLINE +.. c:macro:: Py_MAX(x, y) + Py_MIN(x, y) - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). + Return the larger or smaller of the arguments, respectively. - Usage:: + Any arguments may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. - Py_NO_INLINE static int random(void) { return 4; } + :c:macro:`!Py_MAX` corresponds roughly to + :samp:`((({x}) > ({y})) ? ({x}) : ({y}))`. - .. versionadded:: 3.11 + .. versionadded:: 3.3 -.. c:macro:: Py_STRINGIFY(x) +.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions) - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. + Similar to :samp:`{integer} >> {positions}`, but forces sign extension, + as the C standard does not define whether a right-shift of a signed + integer will perform sign extension or a zero-fill. - .. versionadded:: 3.4 + *integer* should be any signed integer type. + *positions* is the number of positions to shift to the right. + + Both *integer* and *positions* can be evaluated more than once; + consequently, avoid directly passing a function call or some other + operation with side-effects to this macro. Instead, store the result as a + variable and then pass it. + + *type* is unused and only kept for backwards compatibility. Historically, + *type* was used to cast *integer*. + + .. versionchanged:: 3.1 + + This macro is now valid for all signed integer types, not just those for + which ``unsigned type`` is legal. As a result, *type* is no longer + used. + +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + + +Assertion utilities +^^^^^^^^^^^^^^^^^^^ .. c:macro:: Py_UNREACHABLE() @@ -219,8 +314,11 @@ complete listing. avoids a warning about unreachable code. For example, the macro is implemented with ``__builtin_unreachable()`` on GCC in release mode. - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + In debug mode, and on unsupported compilers, the macro expands to a call to + :c:func:`Py_FatalError`. + + A use for ``Py_UNREACHABLE()`` is following a call to a function that + never returns but that is not declared ``_Noreturn``. If a code path is very unlikely code but can be reached under exceptional case, this macro must not be used. For example, under low memory condition @@ -230,18 +328,29 @@ complete listing. .. versionadded:: 3.7 -.. c:macro:: Py_UNUSED(arg) +.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller) - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + Cast *value* to type *smaller* from type *larger*, validating that no + information was lost. - .. versionadded:: 3.4 + On release builds of Python, this is roughly equivalent to + :samp:`(({smaller}) {value})` + (in C++, :samp:`static_cast<{smaller}>({value})` will be used instead). + + On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts + that no information was lost with the cast from *larger* to *smaller*. + + *value*, *larger*, and *smaller* may all be evaluated more than once in the + expression; consequently, do not pass an expression with side-effects + directly to this macro. .. c:macro:: Py_BUILD_ASSERT(cond) Asserts a compile-time condition *cond*, as a statement. The build will fail if the condition is false or cannot be evaluated at compile time. + Corresponds roughly to :samp:`static_assert({cond})` on C23 and above. + For example:: Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); @@ -260,48 +369,256 @@ complete listing. .. versionadded:: 3.3 -.. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name *name* that can be used in docstrings. - If Python is built without docstrings, the value will be empty. +Type size utilities +^^^^^^^^^^^^^^^^^^^ - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. +.. c:macro:: Py_ARRAY_LENGTH(array) - Example:: + Compute the length of a statically allocated C array at compile time. - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } + This is roughly equivalent to:: -.. c:macro:: PyDoc_STR(str) + sizeof(array) / sizeof((array)[0]) + +.. c:macro:: Py_MEMBER_SIZE(type, member) + + Return the size of a structure (*type*) *member* in bytes. + + Corresponds roughly to :samp:`sizeof((({type} *)NULL)->{member})`. + + .. versionadded:: 3.6 + + +Macro definition utilities +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_FORCE_EXPANSION(X) - Creates a docstring for the given input string or an empty string - if docstrings are disabled. + This is equivalent to :samp:`{X}`, which is useful for token-pasting in + macros, as macro expansions in *X* are forcefully evaluated by the + preprocessor. - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. +.. c:macro:: Py_STRINGIFY(x) + + Convert ``x`` to a C string. For example, ``Py_STRINGIFY(123)`` returns + ``"123"``. + + .. versionadded:: 3.4 + + +Declaration utilities +--------------------- + +The following macros can be used in declarations. +They are most useful for defining the C API itself, and have limited use +for extension authors. +Most of them expand to compiler-specific spellings of common extensions +to the C language. + +.. c:macro:: Py_ALWAYS_INLINE + + Ask the compiler to always inline a static inline function. The compiler can + ignore it and decide to not inline the function. + + Corresponds to ``always_inline`` attribute in GCC and ``__forceinline`` + in MSVC. + + It can be used to inline performance critical static inline functions when + building Python in debug mode with function inlining disabled. For example, + MSC disables function inlining when building in debug mode. + + Marking blindly a static inline function with Py_ALWAYS_INLINE can result in + worse performances (due to increased code size for example). The compiler is + usually smarter than the developer for the cost/benefit analysis. + + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` + macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. + + It must be specified before the function return type. Usage:: + + static inline Py_ALWAYS_INLINE int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_NO_INLINE + + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Corresponds to the ``noinline`` attribute/specification on GCC and MSVC. + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_DEPRECATED(version) + + Use this to declare APIs that were deprecated in a specific CPython version. + The macro must be placed before the symbol name. Example:: - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); -.. c:macro:: PyDoc_VAR(name) + .. versionchanged:: 3.8 + MSVC support was added. - Declares a static character array variable with the given name *name*. +.. c:macro:: Py_LOCAL(type) - For example:: + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to :samp:`static {type}`. + +.. c:macro:: Py_LOCAL_INLINE(type) + + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. + +.. c:macro:: Py_LOCAL_SYMBOL + + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. + + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. + +.. c:macro:: Py_EXPORTED_SYMBOL + + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: Py_IMPORTED_SYMBOL + + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. + + +.. c:macro:: PyAPI_FUNC(type) + + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + + +.. c:macro:: PyAPI_DATA(type) + + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. + + +Outdated macros +--------------- + +The following :term:`soft deprecated` macros have been used to features that +have been standardized in C11 (or previous standards). + +.. c:macro:: Py_ALIGNED(num) + + On some GCC-like compilers, specify alignment to *num* bytes. + This does nothing on other compilers. + + Use the standard ``alignas`` specifier rather than this macro. + + .. soft-deprecated:: 3.15 - PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " - "to the tropics and subtropics of the Eastern Hemisphere."); +.. c:macro:: PY_FORMAT_SIZE_T + + The :c:func:`printf` formatting modifier for :c:type:`size_t`. + Use ``"z"`` directly instead. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_LL(number) + Py_ULL(number) + + Use *number* as a ``long long`` or ``unsigned long long`` integer literal, + respectively. + + Expands to *number* followed by ``LL`` or ``LLU``, respectively, but will + expand to some compiler-specific suffixes on some older compilers. + + Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. + + .. soft-deprecated:: 3.15 + +.. c:macro:: PY_LONG_LONG + PY_INT32_T + PY_UINT32_T + PY_INT64_T + PY_UINT64_T + + Aliases for the types :c:type:`!long long`, :c:type:`!int32_t`, + :c:type:`!uint32_t`. :c:type:`!int64_t` and :c:type:`!uint64_t`, + respectively. + Historically, these types needed compiler-specific extensions. + + .. soft-deprecated:: 3.15 + +.. c:macro:: PY_LLONG_MIN + PY_LLONG_MAX + PY_ULLONG_MAX + PY_SIZE_MAX + + Aliases for the values :c:macro:`!LLONG_MIN`, :c:macro:`!LLONG_MAX`, + :c:macro:`!ULLONG_MAX`, and :c:macro:`!SIZE_MAX`, respectively. + Use these standard names instead. + + The required header, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is an alias to :c:func:`!memcpy`. + + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. + +.. c:macro:: Py_UNICODE_SIZE + + Size of the :c:type:`!wchar_t` type. + Use ``sizeof(wchar_t)`` or ``WCHAR_WIDTH/8`` instead. + + The required header for the latter, ````, + :ref:`is included ` in ``Python.h``. + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_UNICODE_WIDE + + Defined if ``wchar_t`` can hold a Unicode character (UCS-4). + Use ``sizeof(wchar_t) >= 4`` instead + + .. soft-deprecated:: 3.15 + +.. c:macro:: Py_VA_COPY + + This is an alias to the C99-standard ``va_copy`` function. + + Historically, this would use a compiler-specific method to copy a ``va_list``. + + .. versionchanged:: 3.6 + This is now an alias to ``va_copy``. + + .. soft-deprecated:: 3.15 .. _api-objects: @@ -420,9 +737,12 @@ the caller is said to *borrow* the reference. Nothing needs to be done for a Conversely, when a calling function passes in a reference to an object, there are two possibilities: the function *steals* a reference to the object, or it -does not. *Stealing a reference* means that when you pass a reference to a -function, that function assumes that it now owns that reference, and you are not -responsible for it any longer. +does not. + +*Stealing a reference* means that when you pass a reference to a +function, that function assumes that it now owns that reference. +Since the new owner can use :c:func:`!Py_DECREF` at its discretion, +you (the caller) must not use that reference after the call. .. index:: single: PyList_SetItem (C function) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 7eaf72ec55fd771..bfbfe3c92799809 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -108,6 +108,7 @@ Other Iterator Objects .. c:var:: PyTypeObject PyDictRevIterValue_Type .. c:var:: PyTypeObject PyDictIterItem_Type .. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type Type objects for iterators of various built-in objects. diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf deleted file mode 100644 index ed5b5039c83e2c4..000000000000000 Binary files a/Doc/c-api/lifecycle.dot.pdf and /dev/null differ diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst index 5a170862a26f440..5e3dca29f2ad901 100644 --- a/Doc/c-api/lifecycle.rst +++ b/Doc/c-api/lifecycle.rst @@ -19,7 +19,7 @@ object's life. An arrow from *A* to *B* indicates that event *B* can occur after event *A* has occurred, with the arrow's label indicating the condition that must be true for *B* to occur after *A*. -.. only:: html and not epub +.. only:: builder_html .. raw:: html @@ -50,20 +50,13 @@ that must be true for *B* to occur after *A*. })(); -.. only:: epub or not (html or latex) +.. only:: not builder_html .. image:: lifecycle.dot.svg :align: center :class: invert-in-dark-mode :alt: Diagram showing events in an object's life. Explained in detail below. -.. only:: latex - - .. image:: lifecycle.dot.pdf - :align: center - :class: invert-in-dark-mode - :alt: Diagram showing events in an object's life. Explained in detail below. - .. container:: :name: life-events-graph-description @@ -256,6 +249,8 @@ To allocate and free memory, see :ref:`allocating-objects`. collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may change in the future. + .. versionadded:: 3.4 + .. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) @@ -266,6 +261,8 @@ To allocate and free memory, see :ref:`allocating-objects`. should happen. Otherwise, this function returns 0 and destruction can continue normally. + .. versionadded:: 3.4 + .. seealso:: :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 758415a76e5cb41..21769c4c5c869c8 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -74,11 +74,25 @@ List Objects Like :c:func:`PyList_GetItemRef`, but returns a :term:`borrowed reference` instead of a :term:`strong reference`. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) Similar to :c:func:`PyList_GetItem`, but without error checking. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) @@ -88,8 +102,10 @@ List Objects .. note:: - This function "steals" a reference to *item* and discards a reference to - an item already in the list at the affected position. + This function ":term:`steals `" a reference to *item*, + even on error. + On success, it discards a reference to an item already in the list + at the affected position (unless it was ``NULL``). .. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) @@ -103,11 +119,19 @@ List Objects .. note:: - This macro "steals" a reference to *item*, and, unlike + This macro ":term:`steals `" a reference to *item*, and, unlike :c:func:`PyList_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in *list* at position *i* will be leaked. + .. note:: + + In the :term:`free-threaded build`, this macro has no internal + synchronization. It is normally only used to fill in new lists where no + other thread has a reference to the list. If the list may be shared, + use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object + lock`. + .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) @@ -138,6 +162,12 @@ List Objects Return ``0`` on success, ``-1`` on failure. Indexing from the end of the list is not supported. + .. note:: + + In the :term:`free-threaded build`, when *itemlist* is a :class:`list`, + both *list* and *itemlist* are locked for the duration of the operation. + For other iterables (or ``NULL``), only *list* is locked. + .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) @@ -150,6 +180,14 @@ List Objects .. versionadded:: 3.13 + .. note:: + + In the :term:`free-threaded build`, when *iterable* is a :class:`list`, + :class:`set`, :class:`dict`, or dict view, both *list* and *iterable* + (or its underlying dict) are locked for the duration of the operation. + For other iterables, only *list* is locked; *iterable* may be + concurrently modified by another thread. + .. c:function:: int PyList_Clear(PyObject *list) @@ -168,6 +206,14 @@ List Objects Sort the items of *list* in place. Return ``0`` on success, ``-1`` on failure. This is equivalent to ``list.sort()``. + .. note:: + + In the :term:`free-threaded build`, element comparison via + :meth:`~object.__lt__` can execute arbitrary Python code, during which + the :term:`per-object lock` may be temporarily released. For built-in + types (:class:`str`, :class:`int`, :class:`float`), the lock is not + released during comparison. + .. c:function:: int PyList_Reverse(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index ed34efe716d8b95..874e422d4701dd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -71,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -81,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -453,8 +451,8 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, returns the number of bytes required to store the value. If this is equal to or less than *n_bytes*, the entire value was copied. - All *n_bytes* of the buffer are written: large buffers are padded with - zeroes. + All *n_bytes* of the buffer are written: remaining bytes filled by + copies of the sign bit. If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, @@ -687,7 +685,7 @@ Export API .. versionadded:: 3.14 -.. c:struct:: PyLongLayout +.. c:type:: PyLongLayout Layout of an array of "digits" ("limbs" in the GMP terminology), used to represent absolute value for arbitrary precision integers. @@ -727,7 +725,7 @@ Export API Get the native layout of Python :class:`int` objects. - See the :c:struct:`PyLongLayout` structure. + See the :c:type:`PyLongLayout` structure. The function must not be called before Python initialization nor after Python finalization. The returned layout is valid until Python is @@ -735,7 +733,7 @@ Export API in a process, and so it can be cached. -.. c:struct:: PyLongExport +.. c:type:: PyLongExport Export of a Python :class:`int` object. @@ -769,7 +767,7 @@ Export API Export a Python :class:`int` object. - *export_long* must point to a :c:struct:`PyLongExport` structure allocated + *export_long* must point to a :c:type:`PyLongExport` structure allocated by the caller. It must not be ``NULL``. On success, fill in *\*export_long* and return ``0``. @@ -799,7 +797,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. versionadded:: 3.14 -.. c:struct:: PyLongWriter +.. c:type:: PyLongWriter A Python :class:`int` writer instance. @@ -827,7 +825,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] - (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + (where the :c:type:`~PyLongLayout.bits_per_digit` is the number of bits per digit). Any unused most significant digits must be set to ``0``. @@ -855,3 +853,31 @@ The :c:type:`PyLongWriter` API can be used to import an integer. If *writer* is ``NULL``, no operation is performed. The writer instance and the *digits* array are invalid after the call. + + +Deprecated API +^^^^^^^^^^^^^^ + +These macros are :term:`soft deprecated`. They describe parameters +of the internal representation of :c:type:`PyLongObject` instances. + +Use :c:func:`PyLong_GetNativeLayout` instead, along with :c:func:`PyLong_Export` +to read integer data or :c:type:`PyLongWriter` to write it. +These currently use the same layout, but are designed to continue working correctly +even if CPython's internal integer representation changes. + + +.. c:macro:: PyLong_SHIFT + + This is equivalent to :c:member:`~PyLongLayout.bits_per_digit` in + the output of :c:func:`PyLong_GetNativeLayout`. + + +.. c:macro:: PyLong_BASE + + This is currently equivalent to :c:expr:`1 << PyLong_SHIFT`. + + +.. c:macro:: PyLong_MASK + + This is currently equivalent to :c:expr:`(1 << PyLong_SHIFT) - 1` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 2395898010214d9..73310670ac371c9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -7,10 +7,6 @@ Memory Management ***************** -.. sectionauthor:: Vladimir Marangozov - - - .. _memoryoverview: Overview @@ -81,7 +77,7 @@ memory footprint as a whole. Consequently, under certain circumstances, the Python memory manager may or may not trigger appropriate actions, like garbage collection, memory compaction or other preventive procedures. Note that by using the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. +for the I/O buffer completely escapes the Python memory manager. .. seealso:: @@ -161,7 +157,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -208,8 +204,11 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. +In the GIL-enabled build (default build) the +:ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `, whereas in the +:term:`free-threaded build`, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -219,6 +218,11 @@ The :ref:`default memory allocator ` uses the The default allocator is now pymalloc instead of system :c:func:`malloc`. +.. versionchanged:: 3.13 + + In the :term:`free-threaded ` build, the default allocator + is now :ref:`mimalloc `. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the @@ -231,7 +235,7 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -293,17 +297,39 @@ The following type-oriented macros are provided for convenience. Note that Same as :c:func:`PyMem_Free`. -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` +Deprecated aliases +------------------ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Corresponding function or macro + * * .. c:macro:: PyMem_MALLOC(size) + * :c:func:`PyMem_Malloc` + * * .. c:macro:: PyMem_NEW(type, size) + * :c:macro:`PyMem_New` + * * .. c:macro:: PyMem_REALLOC(ptr, size) + * :c:func:`PyMem_Realloc` + * * .. c:macro:: PyMem_RESIZE(ptr, type, size) + * :c:macro:`PyMem_Resize` + * * .. c:macro:: PyMem_FREE(ptr) + * :c:func:`PyMem_Free` + * * .. c:macro:: PyMem_DEL(ptr) + * :c:func:`PyMem_Free` + +.. versionchanged:: 3.4 + + The macros are now aliases of the corresponding functions and macros. + Previously, their behavior was the same, but their use did not necessarily + preserve binary compatibility across Python versions. + +.. deprecated:: 2.0 .. _objectinterface: @@ -322,7 +348,9 @@ memory from the Python heap. the :ref:`Customize Memory Allocators ` section. The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. +:ref:`pymalloc memory allocator `. In the +:term:`free-threaded ` build, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -340,7 +368,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -402,14 +430,16 @@ Default Memory Allocators Default memory allocators: -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== +=================================== ======================= ==================== ====================== ====================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=================================== ======================= ==================== ====================== ====================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` +Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug +=================================== ======================= ==================== ====================== ====================== Legend: @@ -417,8 +447,7 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. -* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc - allocator will be used if mimalloc support isn't available. +* ``mimalloc``: :ref:`mimalloc memory allocator `. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -655,7 +684,11 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit -platforms. It falls back to :c:func:`PyMem_RawMalloc` and +platforms. When Python is configured with :option:`--with-pymalloc-hugepages`, +the arena size on 64-bit platforms is increased to 2 MiB to match the huge page +size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on +Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages. +It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator ` of the @@ -711,9 +744,27 @@ The mimalloc allocator .. versionadded:: 3.13 -Python supports the mimalloc allocator when the underlying platform support is available. -mimalloc "is a general purpose allocator with excellent performance characteristics. -Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." +Python supports the `mimalloc `__ +allocator when the underlying platform support is available. +mimalloc is a general purpose allocator with excellent performance +characteristics, initially developed by Daan Leijen for the runtime systems +of the Koka and Lean languages. + +Unlike :ref:`pymalloc `, which is optimized for small objects (512 +bytes or fewer), mimalloc handles allocations of any size. + +In the :term:`free-threaded ` build, mimalloc is the default +and **required** allocator for the :c:macro:`PYMEM_DOMAIN_MEM` and +:c:macro:`PYMEM_DOMAIN_OBJ` domains. It cannot be disabled in free-threaded +builds. The free-threaded build uses per-thread mimalloc heaps, which allows +allocation and deallocation to proceed without locking in most cases. + +In the default (non-free-threaded) build, mimalloc is available but not the +default allocator. It can be selected at runtime using +:envvar:`PYTHONMALLOC`\ ``=mimalloc`` (or ``mimalloc_debug`` to include +:ref:`debug hooks `). It can be disabled at build time +using the :option:`--without-mimalloc` configure option, but this option +cannot be combined with :option:`--disable-gil`. tracemalloc C API ================= diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 1994a3c7d01ca75..31df4451f15e663 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,11 +3,10 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) @@ -97,13 +96,6 @@ Module Objects Note that Python code may rename a module by setting its :py:attr:`~module.__name__` attribute. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - - .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was @@ -141,180 +133,192 @@ Module Objects unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _pymoduledef: +.. _c_module_slots: +.. _pymoduledef_slot: -Module definitions ------------------- +Module definition +----------------- -The functions in the previous section work on any module object, including -modules imported from Python code. +Modules created using the C API are typically defined using an +array of :c:type:`PySlot` structs, which provides a "description" of how a +module should be created. +See :ref:`capi-slots` for more information on slots in general. -Modules defined using the C API typically use a *module definition*, -:c:type:`PyModuleDef` -- a statically allocated, constant “description" of -how a module should be created. +.. versionchanged:: 3.15 -The definition is usually used to define an extension's “main” module object -(see :ref:`extension-modules` for details). -It is also used to -:ref:`create extension modules dynamically `. + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. -Unlike :c:func:`PyModule_New`, the definition allows management of -*module state* -- a piece of memory that is allocated and cleared together -with the module object. -Unlike the module's Python attributes, Python code cannot replace or delete -data stored in module state. +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. -.. c:type:: PyModuleDef +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - The module definition struct, which holds all information needed to create - a module object. - This structure must be statically allocated (or be otherwise guaranteed - to be valid while any modules created from it exist). - Usually, there is only one variable of this type for each extension module. - .. c:member:: PyModuleDef_Base m_base +Metadata slots +.............. - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. +.. c:macro:: Py_mod_name - .. c:member:: const char *m_name + :c:member:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - Name for the new module. + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - .. c:member:: const char *m_doc + .. versionadded:: 3.15 - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - .. c:member:: Py_ssize_t m_size +.. c:macro:: Py_mod_doc - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. + .. versionadded:: 3.15 - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Negative ``m_size`` is only allowed when using - :ref:`legacy single-phase initialization ` - or when :ref:`creating modules dynamically `. + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - See :PEP:`3121` for more details. - .. c:member:: PyMethodDef* m_methods +Feature slots +............. - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. +.. c:macro:: Py_mod_abi - .. c:member:: PyModuleDef_Slot* m_slots + :c:member:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using legacy single-phase initialization, *m_slots* must be ``NULL``. + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: - .. versionchanged:: 3.5 + .. code-block:: c - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: + PyABIInfo_VAR(abi_info); - .. c:member:: inquiry m_reload + static PySlot mymodule_slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + ... + }; - .. c:member:: traverseproc m_traverse + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. + This slot is required, except for modules created from + :c:struct:`PyModuleDef`. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. versionadded:: 3.15 - .. versionchanged:: 3.9 - No longer called before the module state is allocated. +.. c:macro:: Py_mod_multiple_interpreters - .. c:member:: inquiry m_clear + :c:member:`Slot ID ` whose value is one of: - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. + .. c:namespace:: NULL - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. + The module does not support being imported in subinterpreters. - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED - .. c:member:: freefunc m_free + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) - A function to call during deallocation of the module object, or ``NULL`` - if not needed. + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + This slot determines whether or not importing this module + in a subinterpreter will fail. + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. -Module slots -............ + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_multiple_interpreters`: -.. c:type:: PyModuleDef_Slot + .. code-block:: c - .. c:member:: int slot + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) - A slot ID, chosen from the available values explained below. + .. versionadded:: 3.12 - .. c:member:: void* value +.. c:macro:: Py_mod_gil - Value of the slot, whose meaning depends on the slot ID. + :c:member:`Slot ID ` whose value is one of: - .. versionadded:: 3.5 + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED -The available slot types are: + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_gil`: + + .. code-block:: c + + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED) + + .. versionadded:: 3.13 + + +Creation and initialization slots +................................. .. c:macro:: Py_mod_create - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: + :c:member:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) :no-index-entry: :no-contents-entry: - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error + The function will be called with: + + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may result in an infinite loop. - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - If ``Py_mod_create`` is not specified, the import machinery will create a normal module object using :c:func:`PyModule_New`. The name is taken from *spec*, not the definition, to allow extension modules to dynamically adjust @@ -322,118 +326,498 @@ The available slot types are: names through symlinks, all while sharing a single module definition. There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. + + .. versionadded:: 3.5 + + .. versionchanged:: 3.15 + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. .. c:macro:: Py_mod_exec - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. + :c:member:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. The signature of the function is: .. c:function:: int exec_module(PyObject* module) :no-index-entry: :no-contents-entry: - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. + See the :ref:`capi-module-support-functions` section for some useful + functions to call. -.. c:macro:: Py_mod_multiple_interpreters + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. - Specifies one of the following values: + .. versionadded:: 3.5 - .. c:namespace:: NULL + .. versionchanged:: 3.15 - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. - The module does not support being imported in subinterpreters. +.. c:macro:: Py_mod_methods - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + :c:member:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. - The module supports being imported in subinterpreters, - but only when they share the main interpreter's GIL. - (See :ref:`isolating-extensions-howto`.) + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. - .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). - The module supports being imported in subinterpreters, - even when they have their own GIL. - (See :ref:`isolating-extensions-howto`.) + .. versionadded:: 3.15 - This slot determines whether or not importing this module - in a subinterpreter will fail. + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. +.. _ext-module-state: - If ``Py_mod_multiple_interpreters`` is not specified, the import - machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. +Module state +------------ - .. versionadded:: 3.12 +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. -.. c:macro:: Py_mod_gil +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- - Specifies one of the following values: +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. - .. c:namespace:: NULL +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. - .. c:macro:: Py_MOD_GIL_USED +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. - The module depends on the presence of the global interpreter lock (GIL), - and may access global state without synchronization. +To retrieve the state from a given module, use the following functions: - .. c:macro:: Py_MOD_GIL_NOT_USED +.. c:function:: void* PyModule_GetState(PyObject *module) - The module is safe to run without an active GIL. + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. - This slot is ignored by Python builds not configured with - :option:`--disable-gil`. Otherwise, it determines whether or not importing - this module will cause the GIL to be automatically enabled. See - :ref:`whatsnew313-free-threaded-cpython` for more detail. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. - If ``Py_mod_gil`` is not specified, the import machinery defaults to - ``Py_MOD_GIL_USED``. +.. c:function:: int PyModule_GetStateSize(PyObject *module, Py_ssize_t *result) - .. versionadded:: 3.13 + Set *\*result* to the size of *module*'s state, as specified + using :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. -.. c:macro:: Py_mod_abi + On error, set *\*result* to -1, and return -1 with an exception set. - A pointer to a :c:struct:`PyABIInfo` structure that describes the ABI that - the extension is using. + .. versionadded:: 3.15 - When the module is loaded, the :c:struct:`!PyABIInfo` in this slot is checked - using :c:func:`PyABIInfo_Check`. - A suitable :c:struct:`!PyABIInfo` variable can be defined using the - :c:macro:`PyABIInfo_VAR` macro, as in: - .. code-block:: c +.. _ext-module-state-slots: - PyABIInfo_VAR(abi_info); +Slots for defining module state +............................... - static PyModuleDef_Slot mymodule_slots[] = { - {Py_mod_abi, &abi_info}, - ... - }; +The following :c:member:`slot IDs ` are available for +defining the module state. + +.. c:macro:: Py_mod_state_size + + :c:member:`Slot ID ` for the size of the module state, + in bytes. + + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + + See :PEP:`3121` for more details. + + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. .. versionadded:: 3.15 + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. -.. _moduledef-dynamic: +.. c:macro:: Py_mod_state_traverse + + :c:member:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_traverse` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:member:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. + +.. c:macro:: Py_mod_state_free + + :c:member:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: 3.15 + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overridden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:member:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` always use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module token for *module* and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: 3.15 + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: Creating extension modules dynamically -------------------------------------- -The following functions may be used to create a module outside of an -extension's :ref:`initialization function `. -They are also used in -:ref:`single-phase initialization `. +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PySlot` + structures, terminated by an entry with slot ID of 0 + (typically written as :c:macro:`PySlot_END`). + The array must include a :c:data:`Py_mod_abi` entry. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + .. versionadded:: 3.15 + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: 3.15 + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + The struct, including all members, is part of the + :ref:`Stable ABI ` for non-free-threaded builds (``abi3``). + In the Stable ABI for free-threaded builds (``abi3t``), + this struct is opaque, and unusable in practice; see :ref:`pymoduledef_slot` + for a replacement. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + The struct is part of the :ref:`Stable ABI ` for + non-free-threaded builds (``abi3``). + In the Stable ABI for Free-Threaded Builds + (``abi3t``), this struct is opaque, and unusable in practice. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + Note that the entries use the older :c:type:`PyModuleDef_Slot` structure, + rather than :c:type:`PySlot`. + + If the array contains slots corresponding to :c:type:`PyModuleDef` + members, the values must match. + For example, if you use :c:macro:`Py_mod_name` in :c:member:`!m_slots`, + :c:member:`PyModuleDef.m_name` must be set to the same pointer + (not just an equal string). + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Slot + + Older structure defining additional slots of a module. + + Note that a :c:type:`!PyModuleDef_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_mod_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. + + Each :c:type:`!PyModuleDef_Slot` structure ``modslot`` is interpreted + as the following :c:type:`PySlot` structure:: + + (PySlot){ + .sl_id=modslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=modslot.value + } + + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_mod_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_mod_slots` slot (if any). + + .. c:member:: int slot + + Corresponds to :c:member:`PySlot.sl_id`. + + .. c:member:: void* value + + Corresponds to :c:member:`PySlot.sl_ptr`. + + .. versionadded:: 3.5 + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are no longer called before the module state is allocated. + + +.. c:var:: PyTypeObject PyModuleDef_Type + + The type of ``PyModuleDef`` objects. + + +.. c:macro:: Py_mod_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyModuleDef_Slot` structures. + + .. versionadded:: 3.15 + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: .. c:function:: PyObject* PyModule_Create(PyModuleDef *def) @@ -469,6 +853,10 @@ They are also used in .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) Create a new module object, given the definition in *def* and the @@ -489,33 +877,47 @@ They are also used in .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. .. versionadded:: 3.5 + .. soft-deprecated:: next + + To run a module's own execution slots, prefer :c:func:`PyModule_Exec`, + which works on modules that were not created from a + :c:type:`PyModuleDef` structure. + .. c:macro:: PYTHON_API_VERSION + PYTHON_API_STRING - The C API version. Defined for backwards compatibility. + The C API version, as an integer (``1013``) and string (``"1013"``), respectively. + Defined for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. .. c:macro:: PYTHON_ABI_VERSION + PYTHON_ABI_STRING - Defined as ``3`` for backwards compatibility. + Defined as ``3`` and ``"3"``, respectively, for backwards compatibility. Currently, this constant is not updated in new Python versions, and is not useful for versioning. This may change in the future. +.. _capi-module-support-functions: + Support functions ----------------- -The following functions are provided to help initialize a module -state. -They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), the initialization function for legacy :ref:`single-phase initialization `, or code that creates modules dynamically. @@ -572,8 +974,8 @@ or code that creates modules dynamically. .. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference - to *value*. + Similar to :c:func:`PyModule_AddObjectRef`, but ":term:`steals `" + a reference to *value* (even on error). It can be called with a result of function that returns a new reference without bothering to check its result or even saving it to a variable. @@ -588,8 +990,8 @@ or code that creates modules dynamically. .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to - *value* on success (if it returns ``0``). + Similar to :c:func:`PyModule_AddObjectRef`, but :term:`steals ` + a reference to *value* on success (if it returns ``0``). The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions are recommended, since it is @@ -617,9 +1019,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -693,6 +1093,9 @@ or code that creates modules dynamically. ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Return ``0`` on success. + Return ``-1`` with an exception set on error. + .. versionadded:: 3.5 .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4faf15c..4bfcb86abf58ed1 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 96353266ac7300d..eedeb180c6b7606 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyObject_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: 3.15 + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. @@ -334,6 +363,8 @@ Object Protocol representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -348,6 +379,8 @@ Object Protocol a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. Called by the :func:`ascii` built-in function. + If argument is ``NULL``, return the string ``''``. + .. index:: string; PyObject_Str (C function) @@ -358,6 +391,8 @@ Object Protocol Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -373,6 +408,8 @@ Object Protocol a TypeError is raised when *o* is an integer instead of a zero-initialized bytes object. + If argument is ``NULL``, return the :class:`bytes` object ``b''``. + .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) @@ -682,10 +719,10 @@ Object Protocol :c:func:`PyUnstable_EnableTryIncRef` must have been called earlier on *obj* or this function may spuriously return ``0`` in the - :term:`free threading` build. + :term:`free-threaded build`. This function is logically equivalent to the following C code, except that - it behaves atomically in the :term:`free threading` build:: + it behaves atomically in the :term:`free-threaded build`:: if (Py_REFCNT(op) > 0) { Py_INCREF(op); @@ -762,13 +799,30 @@ Object Protocol On GIL-enabled builds, this function is equivalent to :c:expr:`Py_REFCNT(op) == 1`. - On a :term:`free threaded ` build, this checks if *op*'s + On a :term:`free-threaded build`, this checks if *op*'s :term:`reference count` is equal to one and additionally checks if *op* is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** - thread-safe on free threaded builds; prefer this function. + thread-safe on free-threaded builds; prefer this function. The caller must hold an :term:`attached thread state`, despite the fact that this function doesn't call into the Python interpreter. This function cannot fail. .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_SetImmortal(PyObject *op) + + Marks the object *op* :term:`immortal`. The argument should be uniquely referenced by + the calling thread. This is intended to be used for reducing reference counting contention + in the :term:`free-threaded build` for objects which are shared across threads. + + This is a one-way process: objects can only be made immortal; they cannot be + made mortal once again. Immortal objects do not participate in reference counting + and will never be garbage collected. If the object is GC-tracked, it is untracked. + + This function is intended to be used soon after *op* is created, by the code that + creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` slot. + Returns 1 if the object was made immortal and returns 0 if it was not. + This function cannot fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 76a1e9f528dc70c..a962c4ee09ad77d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -31,7 +31,7 @@ Note that holding an :term:`attached thread state` is not required for these API or ``-2`` on failure to create a lock. Check ``errno`` for more information about the cause of a failure. -.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, size_t code_size, const char *entry_name) Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is thread safe. Here is what an example entry looks like:: @@ -49,3 +49,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 000000000000000..9e2d92341b0f93e --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst new file mode 100644 index 000000000000000..0200f2eac6d9086 --- /dev/null +++ b/Doc/c-api/profiling.rst @@ -0,0 +1,239 @@ +.. highlight:: c + +.. _profiling: + +Profiling and tracing +===================== + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receive + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is about to return. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`~frame.f_trace_opcodes` to *1* on the frame. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or ``NULL``. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. + + See also the :func:`sys.setprofile` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while + setting the profile functions in all threads. + +.. versionadded:: 3.12 + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. + + See also the :func:`sys.settrace` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while + setting the trace functions in all threads. + +.. versionadded:: 3.12 + + +Reference tracing +================= + +.. versionadded:: 3.13 + + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + + If a new tracing function is registered replacing the current one, a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + +.. versionadded:: 3.13 + + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + + +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python object has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Note that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + +.. versionadded:: 3.13 + + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 57a0728d4e9af42..4d56a92bf2af797 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -25,7 +25,7 @@ of Python objects. .. note:: - On :term:`free threaded ` builds of Python, returning 1 + On :term:`free-threaded builds ` of Python, returning 1 isn't sufficient to determine if it's safe to treat *o* as having no access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` for that instead. diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 000000000000000..b1b7329a5d42c59 --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,47 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name, const char *repr) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. If *repr* is ``NULL``, ``repr()`` returns :attr:`~sentinel.__name__`. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: 3.15 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a00..90490cf6749b59c 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -67,7 +67,7 @@ Sequence Protocol Assign object *v* to the *i*\ th element of *o*. Raise an exception and return ``-1`` on failure; return ``0`` on success. This is the equivalent of the Python statement ``o[i] = v``. This function *does - not* steal a reference to *v*. + not* ":term:`steal`" a reference to *v*. If *v* is ``NULL``, the element is deleted, but this feature is deprecated in favour of using :c:func:`PySequence_DelItem`. @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027bd6d..db537aff2e6ce5a 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -5,9 +5,6 @@ Set Objects ----------- -.. sectionauthor:: Raymond D. Hettinger - - .. index:: pair: object; set pair: object; frozenset @@ -92,6 +89,11 @@ the constructor functions work with any iterable Python object. actually iterable. The constructor is also useful for copying a set (``c=set(s)``). + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) @@ -100,6 +102,11 @@ the constructor functions work with any iterable Python object. set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + The following functions and macros are available for instances of :class:`set` or :class:`frozenset` or instances of their subtypes. @@ -127,6 +134,10 @@ or :class:`frozenset` or instances of their subtypes. the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PySet_Add(PyObject *set, PyObject *key) @@ -138,6 +149,12 @@ or :class:`frozenset` or instances of their subtypes. :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + The following functions are available for instances of :class:`set` or its subtypes but not for instances of :class:`frozenset` or its subtypes. @@ -147,11 +164,16 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: PyObject* PySet_Pop(PyObject *set) @@ -166,3 +188,28 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Empty an existing set of all elements. Return ``0`` on success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + + .. note:: + + In the :term:`free-threaded build`, the set is emptied before its entries + are cleared, so other threads will observe an empty set rather than + intermediate states. + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PySet_MINSIZE + + A constant representing the size of an internal + preallocated table inside :c:type:`PySetObject` instances. + + This is documented solely for completeness, as there are no guarantees + that a given version of CPython uses preallocated tables with a fixed + size. + In code that does not deal with unstable set internals, + :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. + + If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/slots.rst b/Doc/c-api/slots.rst new file mode 100644 index 000000000000000..84a125cb60bae7e --- /dev/null +++ b/Doc/c-api/slots.rst @@ -0,0 +1,240 @@ +.. highlight:: c + +.. _capi-slots: + +Definition slots +================ + +To define :ref:`module objects ` and +:ref:`classes ` using the C API, you may use +an array of *slots* -- essentially, key-value pairs that describe features +of the object to create. +This decouples the data from the structures used at runtime, allowing CPython +-- and other Python C API implementations -- to update the structures without +breaking backwards compatibility. + +This section documents slots in general. +For object-specific behavior and slot values, see documentation for functions +that apply slots: + +- :c:func:`PyType_FromSlots` for types; +- :c:func:`PyModule_FromSlotsAndSpec` and :ref:`extension-export-hook` + for modules. + +When slots are passed to a function that applies them, the function will +not modify the slot array, nor any data it points to (recursively). +After the function is done, the caller is allowed to modify or deallocate +the array and any data it points to (recursively), except data +explicitly marked with :c:macro:`PySlot_STATIC`. + +Except when documented otherwise, multiple slots with the same ID +(:c:member:`~PySlot.sl_id`) may not occur in a single slots array. + +.. versionadded:: 3.15 + + Slot arrays generalize an earlier way of defining objects: + using :c:type:`PyType_Spec` with :c:type:`PyType_Slot` for types, and + :c:type:`PyModuleDef` with :c:type:`PyModuleDef_Slot` for modules. + The earlier API is :term:`soft deprecated`; there are no plans to remove it. + +Entries of the slots array use the following structure: + +.. c:type:: PySlot + + An entry in a slots array. Defined as: + + .. code-block:: c + + typedef struct { + uint16_t sl_id; + uint16_t sl_flags; + uint32_t _reserved; // must be 0 + union { + void *sl_ptr; + void (*sl_func)(void); + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; + } PySlot; + + .. c:member:: uint16_t sl_id + + A slot ID, chosen from: + + - ``Py_slot_*`` values documented in :ref:`pyslot-common-ids` below; + - ``Py_mod_*`` values for modules, as documented in :ref:`c_module_slots`; + - Values for types, as documented in :ref:`pyslot_type_slot_ids`. + + A :c:member:`!sl_id` of zero (:c:macro:`Py_slot_end`) marks the end of a + slots array. + + .. c:member:: void *sl_ptr + void (*sl_func)(void) + Py_ssize_t sl_size + int64_t sl_int64 + uint64_t sl_uint64 + + The data for the slot. + These members are part of an anonymous union; + the member to use depends on which data type is required by the slot ID: + data pointer, function pointer, size, signed or unsigned + integer, respectively. + + Except when documented otherwise for a specific slot ID, pointers + (that is :c:member:`!sl_ptr` and :c:member:`!sl_func`) may not be NULL. + + .. c:member:: uint16_t sl_flags + + Zero or more of the following flags, OR-ed together: + + .. c:namespace:: NULL + + .. c:macro:: PySlot_STATIC + + All data the slot points to is statically allocated and constant. + Thus, the interpreter does not need to copy the information. + + This flag is implied for function pointers. + + The flag applies even to data the slot points to “indirectly”, + except for slots nested via :c:macro:`Py_slot_subslots` which may + have their own :c:macro:`!PySlot_STATIC` flags. + For example, if applied to a :c:macro:`Py_tp_members` slot that + points to an array of :c:type:`PyMemberDef` structures, + then the entire array, as well as the name and doc strings + in its elements, must be static and constant. + + .. c:macro:: PySlot_INTPTR + + The data is stored in ``sl_ptr``; CPython will cast it to + the appropriate type. + + This flag can simplify porting from the older :c:type:`PyType_Slot` + and :c:type:`PyModuleDef_Slot` structures. + + .. c:macro:: PySlot_OPTIONAL + + If the slot ID is unknown, the interpreter should ignore the + slot, rather than fail. + + For example, if Python 3.16 adds a new feature with a new slot ID,attr + the corresponding slot may be marked :c:macro:`!PySlot_OPTIONAL` + so that Python 3.15 ignores it. + + Note that the "optionality" only applies to unknown slot IDs. + This flag does not make Python skip invalid values of known slots. + + .. versionadded:: 3.15 + + +Convenience macros +------------------ + +.. c:macro:: PySlot_DATA(name, value) + PySlot_FUNC(name, value) + PySlot_SIZE(name, value) + PySlot_INT64(name, value) + PySlot_UINT64(name, value) + PySlot_STATIC_DATA(name, value) + + Convenience macros to define :c:type:`!PySlot` structures with + :c:member:`~PySlot.sl_id` and a particular union member set. + + :c:macro:`!PySlot_STATIC_DATA` sets the :c:macro:`PySlot_STATIC` flag; + others set no flags. + + Note that these macros use *designated initializers*, a C language feature + that C++ added in the 2020 version of the standard. + If your code needs to be compatible with C++11 or older, + use :c:macro:`PySlot_PTR` instead. + + Defined as:: + + #define PySlot_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_ptr=(void*)(VALUE)} + + #define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=NAME, .sl_func=(VALUE)} + + #define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=NAME, .sl_size=(VALUE)} + + #define PySlot_INT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_int64=(VALUE)} + + #define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_uint64=(VALUE)} + + #define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_END + + Convenience macro to mark the end of a :c:type:`!PySlot` array. + + Defined as:: + + #define PySlot_END {0} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_PTR(name, value) + PySlot_PTR_STATIC(name, value) + + Convenience macros for use in C++11-compatible code. + This version of C++ does not allow setting arbitrary union members in + literals; instead, these macros set the :c:macro:`PySlot_INTPTR` flag and cast + the value to ``(void*)``. + + Defined as:: + + #define PySlot_PTR(NAME, VALUE) \ + {NAME, PySlot_INTPTR, {0}, {(void*)(VALUE)}} + + #define PySlot_PTR_STATIC(NAME, VALUE) \ + {NAME, PySlot_INTPTR|Py_SLOT_STATIC, {0}, {(void*)(VALUE)}} + + .. versionadded:: 3.15 + +.. _pyslot-common-ids: + +Common slot IDs +--------------- + +The following slot IDs may be used in both type and module definitions. + +.. c:macro:: Py_slot_end + + Marks the end of a slots array. + Defined as zero. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_subslots + + Nested slots array. + + The value (:c:member:`~PySlot.sl_ptr`) should point to an array of + :c:type:`PySlot` structures. + The slots in the array (up to but not including the zero-ID + terminator) will be treated as if they were inserted if the current + slot array, at the point :c:macro:`!Py_slot_subslots` appears. + + Slot nesting depth is limited to 5 levels. + This restriction may be lifted in the future. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_invalid + + Reserved; will always be treated as an unknown slot ID. + Defined as ``UINT16_MAX`` (``0xFFFF``). + + When used with the :c:macro:`PySlot_OPTIONAL` flag, defines a slot with + no effect. + Without the flag, processing a slot with this ID will fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index f5e6b7ad157e999..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -51,135 +51,212 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-abi: .. _stable-application-binary-interface: -Stable Application Binary Interface -=================================== +Stable Application Binary Interfaces +==================================== -For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python. +Python's :dfn:`Stable ABI` allows extensions to be compatible with multiple +versions of Python, without recompilation. -.. _limited-c-api: +.. note:: -Limited C API -------------- + For simplicity, this document talks about *extensions*, but Stable ABI + works the same way for all uses of the API – for example, embedding Python. -Python 3.2 introduced the *Limited API*, a subset of Python's C API. -Extensions that only use the Limited API can be -compiled once and be loaded on multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. +There are two Stable ABIs: -.. c:macro:: Py_LIMITED_API +- ``abi3``, introduced in Python 3.2, is compatible with + **non**-:term:`free-threaded ` builds of CPython. - Define this macro before including ``Python.h`` to opt in to only use - the Limited API, and to select the Limited API version. +- ``abi3t``, introduced in Python 3.15, is compatible with + :term:`free-threaded ` builds of CPython. + It has stricter API limitations than ``abi3``. - Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` - corresponding to the lowest Python version your extension supports. - The extension will be ABI-compatible with all Python 3 releases - from the specified one onward, and can use Limited API introduced up to that - version. + .. versionadded:: 3.15 - Rather than using the ``PY_VERSION_HEX`` macro directly, hardcode a minimum - minor version (e.g. ``0x030A0000`` for Python 3.10) for stability when - compiling with future Python versions. + ``abi3t`` was added in :pep:`803` - You can also define ``Py_LIMITED_API`` to ``3``. This works the same as - ``0x03020000`` (Python 3.2, the version that introduced Limited API). +It is possible for an extension to be compiled for *both* ``abi3`` and +``abi3t`` at the same time; the result will be compatible with +both free-threaded and non-free-threaded builds of Python. +Currently, this has no downsides compared to compiling for ``abi3t`` only. +Each Stable ABI is versioned using the first two numbers of the Python version. +For example, Stable ABI 3.14 corresponds to Python 3.14. +An extension compiled for Stable ABI 3.x is ABI-compatible with Python 3.x +and above. -.. _stable-abi: +Extensions that target a stable ABI must only use a limited subset of +the C API. This subset is known as the :dfn:`Limited API`; its contents +are :ref:`listed below `. -Stable ABI ----------- +On Windows, extensions that use a Stable ABI should be linked against +``python3.dll`` rather than a version-specific library such as +``python39.dll``. +This library only exposes the relevant symbols. + +On some platforms, Python will look for and load shared library files named +with the ``abi3`` or ``abi3t`` tag (for example, ``mymodule.abi3.so``). +:term:`Free-threaded ` interpreters only recognize the +``abi3t`` tag, while non-free-threaded ones will prefer ``abi3`` but fall back +to ``abi3t``. +Thus, extensions compatible with both ABIs should use the ``abi3t`` tag. + +Python does not necessarily check that extensions it loads +have compatible ABI. +Extension authors are encouraged to add a check using the :c:macro:`Py_mod_abi` +slot or the :c:func:`PyABIInfo_Check` function, but the user +(or their packaging tool) is ultimately responsible for ensuring that, +for example, extensions built for Stable ABI 3.10 are not installed for lower +versions of Python. -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain ABI-compatible across Python 3.x versions. +All functions in Stable ABI are present as functions in Python's shared +library, not solely as macros. +This makes them usable in languages that don't use the C +preprocessor, including Python's :py:mod:`ctypes`. + + +.. _abi3-compiling: + +Compiling for Stable ABI +------------------------ .. note:: - The Stable ABI prevents ABI issues, like linker errors due to missing - symbols or data corruption due to changes in structure layouts or function - signatures. - However, other changes in Python can change the *behavior* of extensions. - See Python's Backwards Compatibility Policy (:pep:`387`) for details. + Build tools (such as, for example, meson-python, scikit-build-core, + or Setuptools) often have a mechanism for setting macros and synchronizing + them with extension filenames and other metadata. + Prefer using such a mechanism, if it exists, over defining the + macros manually. -The Stable ABI contains symbols exposed in the :ref:`Limited API -`, but also other ones – for example, functions necessary to -support older versions of the Limited API. + The rest of this section is mainly relevant for tool authors, and for + people who compile extensions manually. -On Windows, extensions that use the Stable ABI should be linked against -``python3.dll`` rather than a version-specific library such as -``python39.dll``. + .. seealso:: `list of recommended tools`_ in the Python Packaging User Guide -On some platforms, Python will look for and load shared library files named -with the ``abi3`` tag (e.g. ``mymodule.abi3.so``). -It does not check if such extensions conform to a Stable ABI. -The user (or their packaging tools) need to ensure that, for example, -extensions built with the 3.10+ Limited API are not installed for lower -versions of Python. + .. _list of recommended tools: https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules + +To compile for a Stable ABI, define one or both of the following macros +to the lowest Python version your extension should support, in +:c:macro:`Py_PACK_VERSION` format. +Typically, you should choose a specific value rather than the version of +the Python headers you are compiling against. + +The macros must be defined before including ``Python.h``. +Since :c:macro:`Py_PACK_VERSION` is not available at this point, you +will need to use the numeric value directly. +For reference, the values for a few recent Python versions are: + +.. version-hex-cheatsheet:: -All functions in the Stable ABI are present as functions in Python's shared -library, not solely as macros. This makes them usable from languages that don't -use the C preprocessor. +When one of the macros is defined, ``Python.h`` will only expose API that is +compatible with the given Stable ABI -- that is, the +:ref:`Limited API ` plus some definitions that need to be +visible to the compiler but should not be used directly. +When both are defined, ``Python.h`` will only expose API compatible with +both Stable ABIs. +.. c:macro:: Py_LIMITED_API + + Target ``abi3``, that is, + non-:term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + +.. c:macro:: Py_TARGET_ABI3T + + Target ``abi3t``, that is, + :term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + + .. versionadded:: 3.15 + +Both macros specify a target ABI; the different naming style is due to +backwards compatibility. + +.. admonition:: Historical note + + You can also define ``Py_LIMITED_API`` as ``3``. This works the same as + ``0x03020000`` (Python 3.2, the version that introduced Stable ABI). + +When both are defined, ``Python.h`` may, or may not, redefine +:c:macro:`!Py_LIMITED_API` to match :c:macro:`!Py_TARGET_ABI3T`. + +On a :term:`free-threaded build` -- that is, when +:c:macro:`Py_GIL_DISABLED` is defined -- :c:macro:`!Py_TARGET_ABI3T` +defaults to the value of :c:macro:`!Py_LIMITED_API`. +This means that there are two ways to build for both ``abi3`` and ``abi3t``: -Limited API Scope and Performance ---------------------------------- +- define both :c:macro:`!Py_LIMITED_API` and :c:macro:`!Py_TARGET_ABI3T`, or +- define only :c:macro:`!Py_LIMITED_API` and: -The goal for the Limited API is to allow everything that is possible with the + - on Windows, define :c:macro:`!Py_GIL_DISABLED`; + - on other systems, use the headers of free-threaded build of Python. + + +.. _limited-api-scope-and-performance: + +Stable ABI Scope and Performance +-------------------------------- + +The goal for Stable ABI is to allow everything that is possible with the full C API, but possibly with a performance penalty. +Generally, compatibility with Stable ABI will require some changes to an +extension's source code. -For example, while :c:func:`PyList_GetItem` is available, its “unsafe” macro +For example, while :c:func:`PyList_GetItem` is available, its "unsafe" macro variant :c:func:`PyList_GET_ITEM` is not. The macro can be faster because it can rely on version-specific implementation details of the list object. -Without ``Py_LIMITED_API`` defined, some C API functions are inlined or -replaced by macros. -Defining ``Py_LIMITED_API`` disables this inlining, allowing stability as +For another example, when *not* compiling for Stable ABI, some C API +functions are inlined or replaced by macros. +Compiling for Stable ABI disables this inlining, allowing stability as Python's data structures are improved, but possibly reducing performance. -By leaving out the ``Py_LIMITED_API`` definition, it is possible to compile -a Limited API extension with a version-specific ABI. This can improve -performance for that Python version, but will limit compatibility. -Compiling with ``Py_LIMITED_API`` will then yield an extension that can be -distributed where a version-specific one is not available – for example, -for prereleases of an upcoming Python version. +By leaving out the :c:macro:`!Py_LIMITED_API` or :c:macro:`!Py_TARGET_ABI3T` +definition, it is possible to compile Stable-ABI-compatible source +for a version-specific ABI. +A potentially faster version-specific extension can then be distributed +alongside a version compiled for Stable ABI -- a slower but more compatible +fallback. + +.. _limited-api-caveats: -Limited API Caveats -------------------- +Stable ABI Caveats +------------------ -Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI -`. ``Py_LIMITED_API`` only covers definitions, but an API also -includes other issues, such as expected semantics. +Note that compiling for Stable ABI is *not* a complete guarantee that code will +be compatible with the expected Python versions. +Stable ABI prevents *ABI* issues, like linker errors due to missing +symbols or data corruption due to changes in structure layouts or function +signatures. +However, other changes in Python can change the *behavior* of extensions. -One issue that ``Py_LIMITED_API`` does not guard against is calling a function -with arguments that are invalid in a lower Python version. +One issue that the :c:macro:`Py_TARGET_ABI3T` and :c:macro:`Py_LIMITED_API` +macros do not guard against is calling a function with arguments that are +invalid in a lower Python version. For example, consider a function that starts accepting ``NULL`` for an argument. In Python 3.9, ``NULL`` now selects a default behavior, but in Python 3.8, the argument will be used directly, causing a ``NULL`` dereference and crash. A similar argument works for fields of structs. -Another issue is that some struct fields are currently not hidden when -``Py_LIMITED_API`` is defined, even though they're part of the Limited API. - For these reasons, we recommend testing an extension with *all* minor Python -versions it supports, and preferably to build with the *lowest* such version. +versions it supports. We also recommend reviewing documentation of all used API to check if it is explicitly part of the Limited API. Even with ``Py_LIMITED_API`` defined, a few private declarations are exposed for technical reasons (or even unintentionally, as bugs). -Also note that the Limited API is not necessarily stable: compiling with -``Py_LIMITED_API`` with Python 3.8 means that the extension will -run with Python 3.12, but it will not necessarily *compile* with Python 3.12. -In particular, parts of the Limited API may be deprecated and removed, -provided that the Stable ABI stays stable. +Also note that while compiling with ``Py_LIMITED_API`` 3.8 means that the +extension should *load* on Python 3.12, and *compile* with Python 3.12, +the same source will not necessarily compile with ``Py_LIMITED_API`` +set to 3.12. +In general, parts of the Limited API may be deprecated and removed, +provided that Stable ABI stays stable. .. _stable-abi-platform: @@ -189,12 +266,12 @@ Platform Considerations ABI stability depends not only on Python, but also on the compiler used, lower-level libraries and compiler options. For the purposes of -the :ref:`Stable ABI `, these details define a “platform”. They +the :ref:`Stable ABIs `, these details define a “platform”. They usually depend on the OS type and processor architecture It is the responsibility of each particular distributor of Python to ensure that all Python versions on a particular platform are built -in a way that does not break the Stable ABI. +in a way that does not break the Stable ABIs, or the version-specific ABIs. This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. @@ -302,7 +379,7 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_STABLE - Specifies that the stable ABI is used. + Specifies that Stable ABI is used. .. c:macro:: PyABIInfo_INTERNAL @@ -313,15 +390,22 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_FREETHREADED - Specifies ABI compatible with free-threading builds of CPython. + Specifies ABI compatible with :term:`free-threaded builds + ` of CPython. (That is, ones compiled with :option:`--disable-gil`; with ``t`` in :py:data:`sys.abiflags`) .. c:macro:: PyABIInfo_GIL - Specifies ABI compatible with non-free-threading builds of CPython + Specifies ABI compatible with non-free-threaded builds of CPython (ones compiled *without* :option:`--disable-gil`). + .. c:macro:: PyABIInfo_FREETHREADING_AGNOSTIC + + Specifies ABI compatible with both free-threaded and + non-free-threaded builds of CPython, that is, both + ``abi3`` and ``abi3t``. + .. c:member:: uint32_t build_version The version of the Python headers used to build the code, in the format @@ -335,10 +419,11 @@ The full API is described below for advanced use cases. The ABI version. - For the Stable ABI, this field should be the value of - :c:macro:`Py_LIMITED_API` - (except if :c:macro:`Py_LIMITED_API` is ``3``; use - :c:expr:`Py_PACK_VERSION(3, 2)` in that case). + For Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` or :c:macro:`Py_TARGET_ABI3T`. + If both are defined, use the smaller value. + (If :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` instead of ``3``.) Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. @@ -355,12 +440,13 @@ The full API is described below for advanced use cases. .. versionadded:: 3.15 +.. _limited-c-api: .. _limited-api-list: Contents of Limited API ======================= - -Currently, the :ref:`Limited API ` includes the following items: +This is the definitive list of :ref:`Limited API ` for +Python |version|: .. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f619a..1ddde5d136b3bb1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -33,6 +33,13 @@ under :ref:`reference counting `. The members must not be accessed directly; instead use macros such as :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_refcnt` and :c:member:`!ob_type` fields are available, + but using them directly is discouraged. + .. c:member:: Py_ssize_t ob_refcnt The object's reference count, as returned by :c:macro:`Py_REFCNT`. @@ -48,6 +55,19 @@ under :ref:`reference counting `. Do not use this field directly; use :c:macro:`Py_TYPE` and :c:func:`Py_SET_TYPE` instead. + .. c:member:: PyMutex ob_mutex + + A :ref:`per-object lock `, present only in the :term:`free-threaded ` + build (when :c:macro:`Py_GIL_DISABLED` is defined). + + This field is **reserved for use by the critical section API** + (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / :c:macro:`Py_END_CRITICAL_SECTION`). + Do **not** lock it directly with ``PyMutex_Lock``; doing so can cause + deadlocks. If you need your own lock, add a separate :c:type:`PyMutex` + field to your object struct. + + .. versionadded:: 3.13 + .. c:type:: PyVarObject @@ -59,6 +79,19 @@ under :ref:`reference counting `. instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_base` and :c:member:`!ob_size` fields are available, + but using them directly is discouraged. + + .. c:member:: PyObject ob_base + + Common object header. + Typically, this field is not accessed directly; instead + :c:type:`!PyVarObject` can be cast to :c:type:`PyObject`. + .. c:member:: Py_ssize_t ob_size A size field, whose contents should be considered an object's internal @@ -280,6 +313,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. @@ -408,7 +443,7 @@ There are these calling conventions: These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions +binding when used with methods of classes. These may not be used for functions defined for modules. At most one of these flags may be set for any given method. @@ -419,8 +454,8 @@ method. The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. + similar to what is created when using the :deco:`classmethod` built-in + decorator. .. c:macro:: METH_STATIC @@ -429,7 +464,7 @@ method. The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. + what is created when using the :deco:`staticmethod` built-in decorator. One other constant controls whether a method is loaded in place of another definition with the same method name. @@ -447,6 +482,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +526,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +554,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -605,14 +733,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst new file mode 100644 index 000000000000000..83c3fc3d801e9bd --- /dev/null +++ b/Doc/c-api/subinterpreters.rst @@ -0,0 +1,491 @@ +.. highlight:: c + +.. _sub-interpreter-support: + +Multiple interpreters in a Python process +========================================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: + + +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + :term:`attached thread state`, which might not exist. + + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. + + .. index:: + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) + + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using legacy + :ref:`single-phase initialization `, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. + + .. index:: single: close (in module os) + + +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_FinalizeEx (C function) + + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +.. _per-interpreter-gil: + +A per-interpreter GIL +--------------------- + +.. versionadded:: 3.12 + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554` and :pep:`684`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. + +Also note that combining this functionality with ``PyGILState_*`` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +High-level APIs +--------------- + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there is no :term:`attached thread state`. + It cannot return NULL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then ``-1`` is returned and an error is set. + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.7 + + +.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) + + Return a dictionary in which interpreter-specific data may be stored. + If this function returns ``NULL`` then no exception has been raised and + the caller should assume no interpreter-specific dict is available. + + This is not a replacement for :c:func:`PyModule_GetState()`, which + extensions should use to store interpreter-specific state information. + + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + + .. versionadded:: 3.8 + + +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + + .. versionchanged:: 3.11 + The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. + + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + +.. c:function:: void _PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp, int allow_specialization) + + Enables or disables specialization why a custom frame evaluator is in place. + + If *allow_specialization* is non-zero, the adaptive specializer will + continue to specialize bytecodes even though a custom eval frame function + is set. When *allow_specialization* is zero, setting a custom eval frame + disables specialization. The standard interpreter loop will continue to deopt + while a frame evaluation API is in place - the frame evaluation function needs + to handle the specialized opcodes to take advantage of this. + + .. versionadded:: 3.15 + +.. c:function:: int _PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp) + + Return non-zero if adaptive specialization is enabled for the interpreter. + Specialization is enabled when no custom eval frame function is set, or + when one is set with *allow_specialization* enabled. + + .. versionadded:: 3.15 + + +Low-level APIs +-------------- + +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this + function. + + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. + + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. + + +.. _advanced-debugging: + +Advanced debugger support +------------------------- + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst new file mode 100644 index 000000000000000..6f18c047a24a92f --- /dev/null +++ b/Doc/c-api/synchronization.rst @@ -0,0 +1,351 @@ +.. highlight:: c + +.. _synchronization: + +Synchronization primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state ` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + +.. _python-critical-section-api: + +Python critical section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. _critical-section-macros: + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-object locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +Low-level critical section API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and structs are exposed for cases where C macros +are not available. + +.. c:function:: void PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) + void PyCriticalSection_End(PyCriticalSection *c) + void PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) + void PyCriticalSection2_End(PyCriticalSection2 *c); + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.13 + +.. c:type:: PyCriticalSection + PyCriticalSection2 + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + Note that the contents of the structures are private and their meaning may + change in future Python versions. + + .. versionadded:: 3.13 + +.. c:function:: void PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + void PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + + .. (These need to be in a separate section without a Stable ABI annotation.) + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.14 + + +Legacy locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + +.. versionchanged:: 3.15 + These APIs are now a simple wrapper around ``PyMutex``. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionchanged:: 3.15 + This function now always uses :c:type:`PyMutex`. In prior versions, this + would use a lock provided by the operating system. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 336e3ef96400f4a..ee73c1c8adaa7b3 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst new file mode 100644 index 000000000000000..f48af654e7422bc --- /dev/null +++ b/Doc/c-api/threads.rst @@ -0,0 +1,1049 @@ +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. + +.. note:: + These functions require an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. + + +For example:: + + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; + + // Create a new thread state for the interpreter. + PyThreadStateToken *token = PyThreadState_Ensure(guard); + if (token == NULL) { + PyInterpreterGuard_Close(guard); + return; + } + + // We have a valid thread state -- perform Python actions here. + result = CallSomeFunction(); + // Evaluate result or handle exceptions. + + // Release the thread state. No calls to the C API are allowed beyond this + // point. + PyThreadState_Release(token); + PyInterpreterGuard_Close(guard); + + +Keep in mind that calling ``PyThreadState_Ensure`` might not always create a new +thread state, and calling ``PyThreadState_Release`` might not always detach it. +These functions may reuse an existing attached thread state, or may re-attach +a thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-reuse-thread-state: + +Reusing a thread state across repeated calls +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Creating and destroying a :c:type:`PyThreadState` is not free, and is more +expensive on a :term:`free-threaded build`. A foreign thread that calls into +the interpreter many times -- for example, a worker thread in a native thread +pool -- should avoid creating a fresh thread state on every entry and +destroying it on every exit. Instead, set up one thread state when the thread +starts (or lazily on its first call into Python), attach and detach it around +each call, and tear it down once when the thread exits. + +Manage the thread state explicitly with :c:func:`PyThreadState_New`, attaching +and detaching it with :c:func:`PyEval_RestoreThread` and +:c:func:`PyEval_SaveThread`. This happens in three distinct phases, at +different points in the thread's life. + +When the thread starts, create one thread state for it. ``interp`` is the +target interpreter, captured by the code that created this thread while it held +an attached thread state (for example via :c:func:`PyInterpreterState_Get`):: + + PyThreadState *tstate = PyThreadState_New(interp); + +Then, on each call into Python -- which may happen many times over the thread's +life -- attach the thread state, make the Python C API calls that require it, +and detach again so the thread does not hold the GIL while off doing non-Python +work:: + + PyEval_RestoreThread(tstate); + result = CallSomeFunction(); /* your Python C API calls go here */ + PyEval_SaveThread(); + +When the thread is finished calling into Python, destroy the thread state once:: + + PyEval_RestoreThread(tstate); + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); + +The general-purpose entry points for calling in from a foreign thread -- +:c:func:`PyThreadState_Ensure` and the older :c:func:`PyGILState_Ensure` -- do +*not* guarantee a persistent thread state: their thread-state lifetime is +deliberately implementation-defined, so a matched acquire/release pair may +create and destroy a thread state each time. Use :c:func:`PyThreadState_New`, +as shown here, whenever you specifically want to reuse one thread state across +calls. + +The code that created the foreign thread must arrange for the shutdown sequence +to run before the thread exits, and before :c:func:`Py_FinalizeEx` is called. +If interpreter finalization begins first, the shutdown +:c:func:`PyEval_RestoreThread` call will hang the thread rather than return (see +:ref:`cautions-regarding-runtime-finalization`). If the thread exits without +running the shutdown sequence, the thread state is leaked for the remainder of +the process. + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadStateToken *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one (meaning that "nested" + calls to this function are permitted). + + The function's effect (if any) will be reversed by the matching call to + :c:func:`PyThreadState_Release`. + + On error, this function returns ``NULL`` *without* an exception set. + Do not call :c:func:`!PyThreadState_Release` in this case. + + On success, this function returns a pointer value that must be passed + to the matching call to :c:func:`!PyThreadState_Release`. + + The conditions in which this function creates a new :term:`thread state` are + considered unstable and implementation-dependent. If you need to control the + exact lifetime of a thread state, consider using :c:func:`PyThreadState_New`. + However, do not avoid this function solely on the basis that the lifetime + of the thread state may be inconsistent across versions; changes to this + function will be done with caution and in a backwards-compatible manner. + In particular, the saving of thread-local variables and similar state will + be retained across Python versions. + + .. impl-detail:: + + The exact behavior of whether this function creates a new thread state is + described below, but be aware that this may change in the future. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + .. versionadded:: 3.15 + + +.. c:function:: PyThreadStateToken *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + The behavior and return value are the same as for :c:func:`PyThreadState_Ensure`; + additionally, if the function succeeds, the interpreter referenced by *view* will + be implicitly guarded. The guard will be released upon the corresponding + :c:func:`PyThreadState_Release` call. + + .. versionadded:: 3.15 + + +.. c:function:: void PyThreadState_Release(PyThreadStateToken *token) + + Undo a :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` call. + + This must be called exactly once for each successful *Ensure* call, with + *token* set to that call's return value. + + The state that was attached before the corresponding *Ensure* call + (if any) will be attached when :c:func:`PyThreadState_Release` returns. + + The exact behavior of whether this function deletes a thread state is + considered unstable and implementation-dependent. + + .. impl-detail:: + + Currently, this function will decrement an internal counter on the + attached thread state. If this counter ever reaches below zero, this + function emits a fatal error (via :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + .. versionadded:: 3.15 + +.. c:type:: PyThreadStateToken + + An opaque token retrieved from a :c:func:`PyThreadState_Ensure` call + and passed to a corresponding :c:func:`PyThreadState_Release` call. + + +.. _legacy-api: +.. _gilstate: + +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. + + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to store a state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL`` and no exception has been raised, then the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Schedule an exception to be raised asynchronously in a thread. + If the thread has a previously scheduled exception, it is overwritten. + + The *id* argument is the thread id of the target thread, as returned by + :c:func:`PyThread_get_thread_ident`. + *exc* is the class of the exception to be raised, or ``NULL`` to clear + the pending exception (if any). + + Return the number of affected thread states. + This is normally ``1`` if *id* is found, even when no change was + made (the given *exc* was already pending, or *exc* is ``NULL`` but + no exception is pending). + If the thread id isn't found, return ``0``. This raises no exceptions. + + To prevent naive misuse, you must write your own C extension to call this. + This function must be called with an :term:`attached thread state`. + This function does not :term:`steal` any references to *exc*. + This function does not necessarily interrupt system calls such as + :py:func:`~time.sleep`. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` and :py:attr:`threading.Thread.ident` + expose this identifier to Python. + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/tls.rst b/Doc/c-api/tls.rst new file mode 100644 index 000000000000000..93ac5557141e258 --- /dev/null +++ b/Doc/c-api/tls.rst @@ -0,0 +1,155 @@ +.. highlight:: c + +.. _thread-local-storage: + +Thread-local storage support +============================ + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread-local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:expr:`void*` value per +thread. + +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread-specific storage API +--------------------------- + +The thread-specific storage (TSS) API was introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:expr:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the initializer for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic allocation +------------------ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is ``NULL``. + + .. note:: + A freed key becomes a dangling pointer. You should reset the key to + ``NULL``. + + +Methods +------- + +The parameter *key* of these functions must not be ``NULL``. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:expr:`void*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:expr:`void*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:expr:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Legacy APIs +----------- + +.. deprecated:: 3.7 + This API is superseded by the + :ref:`thread-specific storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index d0add48d7e84571..e8be4762dc33a12 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -99,25 +99,27 @@ Tuple Objects Insert a reference to object *o* at position *pos* of the tuple pointed to by *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` - and set an :exc:`IndexError` exception. + and set an :exc:`IndexError` exception. This function should only be used to fill in brand new tuples; + using it on an existing tuple is thread-unsafe. .. note:: - This function "steals" a reference to *o* and discards a reference to - an item already in the tuple at the affected position. + This function ":term:`steals `" a reference to *o* and discards + a reference to an item already in the tuple at the affected position + (unless it was NULL). .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be - used to fill in brand new tuples. + used to fill in brand new tuples, using it on an existing tuple is thread-unsafe. Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. .. note:: - This function "steals" a reference to *o*, and, unlike + This function ":term:`steals `" a reference to *o*, and, unlike :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in the tuple at position *pos* will be leaked. @@ -148,8 +150,11 @@ Tuple Objects Struct Sequence Objects ----------------------- -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. +A struct sequence object is a :term:`named tuple`, that is, a sequence +whose items can also be accessed through attributes. +It is similar to :func:`collections.namedtuple`, but provides a slightly +different interface. + To create a struct sequence, you first have to create a specific struct sequence type. @@ -233,6 +238,8 @@ type. .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the struct sequence pointed to by *p*. + The returned reference is borrowed from the struct sequence *p* + (that is: it is only valid as long as you hold a reference to *p*). Bounds checking is performed as an assertion if Python is built in :ref:`debug mode ` or :option:`with assertions <--with-assertions>`. @@ -257,7 +264,7 @@ type. .. note:: - This function "steals" a reference to *o*. + This function ":term:`steals `" a reference to *o*. .. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b608f815160f76c..6b68c0d085589a1 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -3,7 +3,7 @@ .. _typeobjects: Type Objects ------------- +============ .. index:: pair: object; type @@ -110,11 +110,16 @@ Type Objects :c:func:`!_PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) + The callback is also invoked when a watched heap type is deallocated. + An extension should never call ``PyType_Watch`` with a *watcher_id* that was not returned to it by a previous call to :c:func:`PyType_AddWatcher`. .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback is now also invoked when a watched heap type is deallocated. + .. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) @@ -138,8 +143,17 @@ Type Objects called on *type* or any type in its MRO; violating this rule could cause infinite recursion. + The callback may be called during type deallocation. In this case, the type + object is temporarily resurrected (its reference count is at least 1) and all + its attributes are still valid. However, the callback should not store new + strong references to the type, as this would resurrect the object and prevent + its deallocation. + .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback may now be called during deallocation of a watched heap type. + .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) @@ -274,6 +288,10 @@ Type Objects Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. @@ -283,8 +301,8 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 @@ -304,10 +322,10 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -317,6 +335,23 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + The returned reference is :term:`borrowed ` from *type*, and will be valid as long as you hold a reference to *type*. Do not release it with :c:func:`Py_DECREF` or similar. @@ -324,10 +359,10 @@ Type Objects .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -338,7 +373,7 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 @@ -363,36 +398,19 @@ Type Objects * :py:mod:`weakref` -Creating Heap-Allocated Types -............................. +.. _creating-heap-types: -The following functions and structs are used to create -:ref:`heap types `. - -.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) - - Create and return a :ref:`heap type ` from the *spec* - (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). - - The metaclass *metaclass* is used to construct the resulting type object. - When *metaclass* is ``NULL``, the metaclass is derived from *bases* - (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). +Creating Heap-Allocated Types +----------------------------- - Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported, except if ``tp_new`` is ``NULL``. +The following function is used to create :ref:`heap types `: - The *bases* argument can be used to specify base classes; it can either - be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. - If that also is ``NULL``, the new type derives from :class:`object`. +.. c:function:: PyObject *PyType_FromSlots(const PySlot *slots) - The *module* argument can be used to record the module in which the new - class is defined. It must be a module object or ``NULL``. - If not ``NULL``, the module is associated with the new type and can later be - retrieved with :c:func:`PyType_GetModule`. - The associated module is not inherited by subclasses; it must be specified - for each class individually. + Create and return a :ref:`heap type ` from a :c:type:`!PySlot` + array. + See :ref:`capi-slots` for general information on slots, + and :ref:`pyslot_type_slot_ids` for slots specific to type creation. This function calls :c:func:`PyType_Ready` on the new type. @@ -409,8 +427,407 @@ The following functions and structs are used to create * :py:meth:`~object.__init_subclass__` is not called on any bases. * :py:meth:`~object.__set_name__` is not called on new descriptors. + Slots are typically defined as a global static constant arrays. + However, sometimes slot values are not statically known at compile time. + For example, slots like :c:data:`Py_tp_bases`, :c:data:`Py_tp_metaclass` + and :c:data:`Py_tp_module` require live Python objects. + In this case, it is recommended to put such slots on the stack, + and use :c:macro:`Py_slot_subslots` to refer to an array of static slots. + For example:: + + static const PySlot my_slots[] = { + PySlot_STATIC_DATA(Py_tp_name, "MyClass"), + PySlot_FUNC(Py_tp_repr, my_repr_func), + ... + PySlot_END + }; + + PyObject *make_my_class(PyObject *module) { + PySlot all_slots[] = { + PySlot_STATIC_DATA(Py_slot_subslots, my_slots), + PySlot_DATA(Py_tp_module, module), + PySlot_END + }; + return PyType_FromSlots(all_slots); + } + +Heap types created without the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag may be +modified, for example by setting attributes on them, as with classes defined +in Python code. +Sometimes, such modifications are necessary to fully initialize a type, +but you may wish to prevent users from changing the type after +the initialization is done: + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 + + +.. _pyslot_type_slot_ids: + +Type slot IDs +............. + +Most type slot IDs are named like the field names of the structures +:c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, +:c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and +:c:type:`PyAsyncMethods` with an added ``Py_`` prefix. +For example, use: + +* :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` +* :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` +* :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` + +The following slots need additional considerations when specified as slots: + +* :c:data:`Py_tp_name` +* :c:data:`Py_tp_basicsize` and :c:data:`Py_tp_extra_basicsize` +* :c:data:`Py_tp_itemsize` +* :c:data:`Py_tp_flags` + +Additional slots do not directly correspond to a :c:type:`!PyTypeObject` +struct field: + +* :c:data:`Py_tp_token` +* :c:data:`Py_tp_metaclass` +* :c:data:`Py_tp_module` + +The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + +* :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) +* :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) +* :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in :ref:`PyMemberDef `) + +If it is not possible to switch to a ``MANAGED`` flag (for example, +for vectorcall or to support Python older than 3.12), specify the +offset in :c:data:`Py_tp_members`. +See :ref:`PyMemberDef documentation ` +for details. + +The following internal fields cannot be set at all when creating a heap +type: + +* :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. + +The :c:data:`Py_tp_base` slot is equivalent to :c:data:`Py_tp_bases`; +both may be set either to a type or a tuple of types. +If both are specified, the value of :c:data:`Py_tp_bases` +is used. + +Slot values may not be ``NULL``, except for the following: + +* :c:data:`Py_tp_doc` +* :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + +.. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. + +.. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation + for details. + +.. versionchanged:: 3.15 + The :c:data:`Py_tp_bases` slot may be set to a single type object, + making it equivalent to the :c:data:`Py_tp_base` slot. + Previously, a tuple of types was required. + +The following slots correspond to fields in the underlying type structure, +but need extra remarks for use as slots: + +.. c:macro:: Py_tp_name + + :c:member:`Slot ID ` for the name of the type, + used to set :c:member:`PyTypeObject.tp_name`. + + This slot (or :c:member:`PyType_Spec.name`) is required to create a type. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.name` instead. + + .. impl-detail:: + + CPython processes slots in order. + It is recommended to put ``Py_tp_name`` at the beginning of the slots + array, so that if processing of a later slots fails, error messages + can include the name. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_basicsize + + :c:member:`Slot ID ` for the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. + + The value must be positive. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:member:`PyTypeObject.tp_basicsize` instead if needed, but be aware + that a type's size is often considered an implementation detail. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_extra_basicsize + + :c:member:`Slot ID ` for type data size in bytes, that is, + how much space instances of the class need *in addition* + to space needed for superclasses. + + The value is used, together with the size of superclasses, to set + :c:member:`PyTypeObject.tp_basicsize`. + Python will insert padding as needed to meet + :c:member:`!tp_basicsize`'s alignment requirements. + + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + The value must be positive. + To specify that instances need no additional size (that is, size should be + inherited), omit the :c:macro:`!Py_tp_extra_basicsize` slot rather than + set it to zero. + + Specifying both :c:macro:`Py_tp_basicsize` and + :c:macro:`!Py_tp_extra_basicsize` is an error. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_itemsize + + :c:member:`Slot ID ` for the size of one element of a + variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See :c:member:`!tp_itemsize` documentation for caveats. + + The value must be positive. + + If this slot is missing, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize` is zero). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.itemsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_flags + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_flags`. + + The ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetFlags` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_bases + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_bases`. + + The slot can be set to a tuple of type objects which the newly created + type should inherit from, like the "positional arguments" of + a Python :ref:`class definition `. + + Alternately, the slot can be set to a single type object to specify + a single base. + The effect is the same as specifying a one-element tuple. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_bases` required a tuple of types. + +.. c:macro:: Py_tp_base + + Equivalent to :c:macro:`Py_tp_bases` (with ``s`` at the end). + If both are specified, :c:macro:`!Py_tp_bases` takes priority and + this slot is ignored. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_base` required a single type, not a tuple. + + .. soft-deprecated:: 3.15 + + When not targetting older Python versions, pefer :c:macro:`!Py_tp_bases`. + +The following slots do not correspond to public fields in the +underlying structures: + +.. c:macro:: Py_tp_metaclass + + :c:member:`Slot ID ` for the metaclass used to construct + the resulting type object. + When omitted the metaclass is derived from bases + (:c:macro:`Py_tp_bases` or the *bases* argument of + :c:func:`PyType_FromMetaclass`). + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported, except if ``tp_new`` is ``NULL``. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a metaclass with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`Py_TYPE` on the type object instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_module + + :c:member:`Slot ID ` for recording the module in which + the new class is defined. + + The value must be a module object. + The module is associated with the new type and can later be + retrieved with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a module with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetModule` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_token + + :c:member:`Slot ID ` for recording a static memory layout ID + for a class. + + If the class is defined using a :c:type:`PyType_Spec`, and that spec is + statically allocated, the token can be set to the spec using the special + value :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + May only be used for classes defined using :c:type:`!PyType_Spec`. + + Expands to ``NULL``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_tp_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyType_Slot` structures. + + .. versionadded:: 3.15 + + +Soft-deprecated API +------------------- + +The following functions are :term:`soft deprecated`. +They will continue to work, but new features will be added as slots for +:c:func:`PyType_FromSlots`, not as arguments to new ``PyType_From*`` functions. + +.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) + + Create and return a :ref:`heap type ` from the *spec* + (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). + + A non-``NULL`` *metaclass* argument corresponds to the + :c:macro:`Py_tp_metaclass` slot. + + A non-``NULL`` *bases* argument corresponds to the :c:data:`Py_tp_bases` + slot, and takes precedence over :c:data:`Py_tp_bases` and + :c:data:`Py_tp_bases` slots. + + A non-``NULL`` *module* argument corresponds to the + :c:macro:`Py_tp_module` slot. + + This function calls :c:func:`PyType_Ready` on the new type. + + Note that this function does *not* fully match the behavior of + calling :py:class:`type() ` or using the :keyword:`class` statement. + See the note in :c:func:`PyType_FromSlots` documentation for details. + .. versionadded:: 3.12 + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) @@ -438,6 +855,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -460,6 +881,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -481,20 +906,9 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next -.. c:function:: int PyType_Freeze(PyTypeObject *type) - - Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. - - All base classes of *type* must be immutable. - - On success, return ``0``. - On error, set an exception and return ``-1``. - - The type must not be used before it's made immutable. For example, type - instances must not be created before the type is made immutable. - - .. versionadded:: 3.14 + Prefer :c:func:`PyType_FromSlots` in new code. .. raw:: html @@ -507,27 +921,23 @@ The following functions and structs are used to create .. c:type:: PyType_Spec - Structure defining a type's behavior. + Structure defining a type's behavior, used for soft-deprecated functions + like :c:func:`PyType_FromMetaclass`. + + This structure contains several members that can instead be specified + as :ref:`slots ` for :c:func:`PyType_FromSlots`, + and an array of slot entries with a simpler structure. .. c:member:: const char* name - Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + Corresponds to :c:macro:`Py_tp_name`. .. c:member:: int basicsize - If positive, specifies the size of the instance in bytes. - It is used to set :c:member:`PyTypeObject.tp_basicsize`. + If positive, corresponds to :c:macro:`Py_tp_basicsize`. - If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` - should be inherited. - - If negative, the absolute value specifies how much space instances of the - class need *in addition* to the superclass. - Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific - memory reserved this way. - For negative :c:member:`!basicsize`, Python will insert padding when - needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment - requirements. + If negative, corresponds to :c:macro:`Py_tp_extra_basicsize` set to + the absolute value. .. versionchanged:: 3.12 @@ -535,160 +945,53 @@ The following functions and structs are used to create .. c:member:: int itemsize - Size of one element of a variable-size type, in bytes. - Used to set :c:member:`PyTypeObject.tp_itemsize`. - See ``tp_itemsize`` documentation for caveats. - - If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. - Extending arbitrary variable-sized classes is dangerous, - since some types use a fixed offset for variable-sized memory, - which can then overlap fixed-sized memory used by a subclass. - To help prevent mistakes, inheriting ``itemsize`` is only possible - in the following situations: - - - The base is not variable-sized (its - :c:member:`~PyTypeObject.tp_itemsize`). - - The requested :c:member:`PyType_Spec.basicsize` is positive, - suggesting that the memory layout of the base class is known. - - The requested :c:member:`PyType_Spec.basicsize` is zero, - suggesting that the subclass does not access the instance's memory - directly. - - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + Corresponds to :c:macro:`Py_tp_itemsize`. .. c:member:: unsigned int flags - Type flags, used to set :c:member:`PyTypeObject.tp_flags`. - - If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, - :c:func:`PyType_FromSpecWithBases` sets it automatically. + Corresponds to :c:macro:`Py_tp_flags`. .. c:member:: PyType_Slot *slots - Array of :c:type:`PyType_Slot` structures. - Terminated by the special slot value ``{0, NULL}``. + Array of :c:type:`PyType_Slot` (not :c:type:`PySlot`) structures. + Terminated by the special slot value ``{0, NULL}``. Each slot ID should be specified at most once. -.. raw:: html - - - - - -.. c:type:: PyType_Slot - - Structure defining optional functionality of a type, containing a slot ID - and a value pointer. - - .. c:member:: int slot - - A slot ID. - - Slot IDs are named like the field names of the structures - :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, - :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and - :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. - For example, use: - - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - - An additional slot is supported that does not correspond to a - :c:type:`!PyTypeObject` struct field: - - * :c:data:`Py_tp_token` - - The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. - - The following internal fields cannot be set at all when creating a heap - type: - - * :c:member:`~PyTypeObject.tp_dict`, - :c:member:`~PyTypeObject.tp_mro`, - :c:member:`~PyTypeObject.tp_cache`, - :c:member:`~PyTypeObject.tp_subclasses`, and - :c:member:`~PyTypeObject.tp_weaklist`. + .. c:namespace:: NULL - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be - problematic on some platforms. - To avoid issues, use the *bases* argument of - :c:func:`PyType_FromSpecWithBases` instead. + .. raw:: html - .. versionchanged:: 3.9 - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + + + - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. c:type:: PyType_Slot - .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation - for details. + Structure defining optional functionality of a type, used for + soft-deprecated functions like :c:func:`PyType_FromMetaclass`. - .. c:member:: void *pfunc + Note that a :c:type:`!PyType_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_tp_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. - The desired value of the slot. In most cases, this is a pointer - to a function. + Each :c:type:`!PyType_Slot` structure ``tpslot`` is interpreted + as the following :c:type:`PySlot` structure:: - *pfunc* values may not be ``NULL``, except for the following slots: + (PySlot){ + .sl_id=tpslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=tpslot.func + } - * ``Py_tp_doc`` - * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` - rather than ``NULL``) + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_tp_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_tp_slots` slot (if any). + .. c:member:: int slot -.. c:macro:: Py_tp_token - - A :c:member:`~PyType_Slot.slot` that records a static memory layout ID - for a class. - - If the :c:type:`PyType_Spec` of the class is statically - allocated, the token can be set to the spec using the special value - :c:data:`Py_TP_USE_SPEC`: - - .. code-block:: c - - static PyType_Slot foo_slots[] = { - {Py_tp_token, Py_TP_USE_SPEC}, - - It can also be set to an arbitrary pointer, but you must ensure that: - - * The pointer outlives the class, so it's not reused for something else - while the class exists. - * It "belongs" to the extension module where the class lives, so it will not - clash with other extensions. + Corresponds to :c:member:`PySlot.sl_id`. - Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has - a given token -- that is, check whether the memory layout is compatible. - - To get the token for a given class (without considering superclasses), - use :c:func:`PyType_GetSlot` with ``Py_tp_token``. - - .. versionadded:: 3.14 - - .. c:namespace:: NULL - - .. c:macro:: Py_TP_USE_SPEC - - Used as a value with :c:data:`Py_tp_token` to set the token to the - class's :c:type:`PyType_Spec`. - Expands to ``NULL``. + .. c:member:: void *pfunc - .. versionadded:: 3.14 + Corresponds to :c:member:`PySlot.sl_ptr`. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 34d19acdf17868b..16dcb880712d244 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -555,6 +555,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_name + See :c:macro:`Py_tp_name` for the corresponding + :c:member:`Slot ID `. + Pointer to a NUL-terminated string containing the name of the type. For types that are accessible as module globals, the string should be the full module name, followed by a dot, followed by the type name; for built-in types, it @@ -594,6 +597,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) These fields allow calculating the size in bytes of instances of the type. + See :c:macro:`Py_tp_basicsize`, :c:macro:`Py_tp_extra_basicsize` and + :c:macro:`Py_tp_itemsize` for the corresponding + :c:member:`Slot IDs `. + There are two kinds of types: types with fixed-length instances have a zero :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all @@ -1133,6 +1140,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: unsigned long PyTypeObject.tp_flags + See :c:macro:`Py_tp_flags` for the corresponding + :c:member:`Slot ID `. + This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via @@ -1373,6 +1383,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) type structure. + .. c:macro:: _Py_TPFLAGS_HAVE_VECTORCALL + :no-typesetting: + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements @@ -1384,7 +1397,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit is inherited if :c:member:`~PyTypeObject.tp_call` is also inherited. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_Py_TPFLAGS_HAVE_VECTORCALL`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1491,106 +1509,75 @@ and :c:data:`PyType_Type` effectively act as defaults.) It will be removed in a future version of CPython -.. c:member:: const char* PyTypeObject.tp_doc + .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - .. corresponding-type-slot:: Py_tp_doc + This macro does nothing. + Historically, this would indicate that the + :c:member:`~PyTypeObject.tp_version_tag` field was available and + initialized. - An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`~type.__doc__` attribute on the - type and instances of the type. + .. soft-deprecated:: 3.13 - **Inheritance:** - This field is *not* inherited by subtypes. + .. c:macro:: Py_TPFLAGS_INLINE_VALUES + This bit indicates that instances of this type will have an "inline values" + array (containing the object's attributes) placed directly after the end + of the object. -.. c:member:: traverseproc PyTypeObject.tp_traverse + This requires that :c:macro:`Py_TPFLAGS_HAVE_GC` is set. - .. corresponding-type-slot:: Py_tp_traverse + **Inheritance:** - An optional pointer to a traversal function for the garbage collector. This is - only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + This flag is not inherited. - int tp_traverse(PyObject *self, visitproc visit, void *arg); + .. versionadded:: 3.13 - More information about Python's garbage collection scheme can be found - in section :ref:`supporting-cycle-detection`. - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function - simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the - :mod:`!_thread` extension module:: + .. c:macro:: Py_TPFLAGS_IS_ABSTRACT - static int - local_traverse(PyObject *op, visitproc visit, void *arg) - { - localobject *self = (localobject *) op; - Py_VISIT(self->args); - Py_VISIT(self->kw); - Py_VISIT(self->dict); - return 0; - } + This bit indicates that this is an abstract type and therefore cannot + be instantiated. - Note that :c:func:`Py_VISIT` is called only on those members that can participate - in reference cycles. Although there is also a ``self->key`` member, it can only - be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. + **Inheritance:** - On the other hand, even if you know a member can never be part of a cycle, as a - debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. + This flag is not inherited. - Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: + .. seealso:: + :mod:`abc` - Py_VISIT(Py_TYPE(self)); - It is only needed since Python 3.9. To support Python 3.8 and older, this - line must be conditional:: + .. c:macro:: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION - #if PY_VERSION_HEX >= 0x03090000 - Py_VISIT(Py_TYPE(self)); - #endif + Internal. Do not set or unset this flag. + Historically, this was a reserved flag for use in Stackless Python. - If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call - :c:func:`PyObject_VisitManagedDict` like this:: + .. warning:: + This flag is present in header files, but is not be used. + This may be removed in a future version of CPython. - PyObject_VisitManagedDict((PyObject*)self, visit, arg); - .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the - members that the instance *owns* (by having :term:`strong references - ` to them) must be - visited. For instance, if an object supports weak references via the - :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting - the linked list (what *tp_weaklist* points to) must **not** be - visited as the instance does not directly own the weak references to itself - (the weakreference list is there to support the weak reference machinery, - but the instance has no strong reference to the elements inside it, as they - are allowed to be removed even if the instance is still alive). - - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`!local_traverse` to have these specific names; don't name them just - anything. - - Instances of :ref:`heap-allocated types ` hold a reference to - their type. Their traversal function must therefore either visit - :c:func:`Py_TYPE(self) `, or delegate this responsibility by - calling ``tp_traverse`` of another heap-allocated type (such as a - heap-allocated superclass). - If they do not, the type object may not be garbage-collected. +.. c:member:: const char* PyTypeObject.tp_doc - .. note:: + .. corresponding-type-slot:: Py_tp_doc - The :c:member:`~PyTypeObject.tp_traverse` function can be called from any - thread. + An optional pointer to a NUL-terminated C string giving the docstring for this + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. - .. versionchanged:: 3.9 + **Inheritance:** + + This field is *not* inherited by subtypes. - Heap-allocated types are expected to visit ``Py_TYPE(self)`` in - ``tp_traverse``. In earlier versions of Python, due to - `bug 40217 `_, doing this - may lead to crashes in subclasses. + +.. c:member:: traverseproc PyTypeObject.tp_traverse + + .. corresponding-type-slot:: Py_tp_traverse + + An optional pointer to a traversal function for the garbage collector. This is + only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. + + See :ref:`gc-traversal` for documentation. **Inheritance:** @@ -1949,12 +1936,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base - .. corresponding-type-slot:: Py_tp_base - An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. + For the corresponding slot ID, see :c:macro:`Py_tp_base`. + .. note:: .. from Modules/xxmodule.c @@ -2266,17 +2253,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases - .. corresponding-type-slot:: Py_tp_bases - Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` - :c:type:`slot ` can be used instead of the *bases* argument - of :c:func:`PyType_FromSpecWithBases`. - The argument form is preferred. + For the corresponding slot ID, see :c:macro:`Py_tp_bases`. .. warning:: @@ -2619,9 +2601,6 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling Number Object Structures ------------------------ -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyNumberMethods This structure holds pointers to the functions which an object uses to @@ -2839,9 +2818,6 @@ Number Object Structures Mapping Object Structures ------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PyMappingMethods This structure holds pointers to the functions which an object uses to @@ -2882,9 +2858,6 @@ Mapping Object Structures Sequence Object Structures -------------------------- -.. sectionauthor:: Amaury Forgeot d'Arc - - .. c:type:: PySequenceMethods This structure holds pointers to the functions which an object uses to @@ -2978,10 +2951,6 @@ Sequence Object Structures Buffer Object Structures ------------------------ -.. sectionauthor:: Greg J. Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - .. c:type:: PyBufferProcs This structure holds pointers to the functions required by the @@ -3001,24 +2970,42 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. + **Thread safety:** + + In the :term:`free-threaded build`, implementations must ensure: + + * The export counter increment in step (3) is atomic. + + * The underlying buffer data remains valid and at a stable memory + location for the lifetime of all exports. + + * For objects that support resizing or reallocation (such as + :class:`bytearray`), the export counter is checked atomically before + such operations, and :exc:`BufferError` is raised if exports exist. + + * The function is safe to call concurrently from multiple threads. + + See also :ref:`thread-safety-memoryview` for the Python-level + thread safety guarantees of :class:`memoryview` objects. + If *exporter* is part of a chain or tree of buffer providers, two main schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -3056,13 +3043,23 @@ Buffer Object Structures (2) If the counter is ``0``, free all memory associated with *view*. + **Thread safety:** + + In the :term:`free-threaded build`: + + * The export counter decrement in step (1) must be atomic. + + * Resource cleanup when the counter reaches zero must be done atomically, + as the final release may race with concurrent releases from other + threads and dellocation must only happen once. + The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain constant, while a consumer MAY pass a copy of the original buffer as the *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). @@ -3077,8 +3074,6 @@ Buffer Object Structures Async Object Structures ----------------------- -.. sectionauthor:: Yury Selivanov - .. versionadded:: 3.5 .. c:type:: PyAsyncMethods diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index ca7c8bb11a5d78a..9bf801ad608c773 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -5,9 +5,6 @@ Unicode Objects and Codecs -------------------------- -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Georg Brandl - Unicode Objects ^^^^^^^^^^^^^^^ @@ -65,6 +62,27 @@ Python: .. versionadded:: 3.3 + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL + + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 + + The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: @@ -658,7 +676,8 @@ APIs: Append the string *right* to the end of *p_left*. *p_left* must point to a :term:`strong reference` to a Unicode object; - :c:func:`!PyUnicode_Append` releases ("steals") this reference. + :c:func:`!PyUnicode_Append` releases (":term:`steals `") + this reference. On error, set *\*p_left* to ``NULL`` and set an exception. @@ -744,7 +763,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1156,7 +1175,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1178,7 +1197,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs @@ -1837,8 +1856,6 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. - .. versionadded:: 3.14 - .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) Write the wide string *str* into *writer*. @@ -1849,7 +1866,7 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. -.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, const Py_UCS4 *str, Py_ssize_t size) Writer the UCS4 string *str* into *writer*. @@ -1865,13 +1882,23 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + To write a :class:`str` subclass which overrides the :meth:`~object.__str__` + method, :c:func:`PyUnicode_FromObject` can be used to get the original + string. + .. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + If *obj* is ``NULL``, write the string ``""`` into *writer*. + On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + .. versionchanged:: 3.14.4 + + Added support for ``NULL``. + .. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) Write the substring ``str[start:end]`` into *writer*. @@ -1927,7 +1954,7 @@ The following API is deprecated. whether you selected a "narrow" or "wide" Unicode version of Python at build time. - .. deprecated-removed:: 3.13 3.15 + .. deprecated-removed:: 3.13 3.16 .. c:function:: int PyUnicode_READY(PyObject *unicode) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 3b07b5fbed5959b..6256bf7a1454a9a 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -191,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start symbol and must one of the :ref:`available start symbols `. + the start symbol and must be one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -396,3 +396,43 @@ Available start symbols * :pep:`484` .. versionadded:: 3.8 + + +Stack Effects +^^^^^^^^^^^^^ + +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index db6ae0a9d4ea3d7..8762a003c5218d3 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -43,7 +43,11 @@ as much as it can. should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly @@ -59,7 +63,11 @@ as much as it can. collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly diff --git a/Doc/conf.py b/Doc/conf.py index 0f1412d1007dc27..9b103a594b235cf 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -33,6 +31,7 @@ 'issue_role', 'lexers', 'misc_news', + 'profiling_trace', 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', @@ -42,8 +41,10 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( + 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', + 'sphinxcontrib.rsvgconverter', ) for optional_ext in _OPTIONAL_EXTENSIONS: try: @@ -70,11 +71,12 @@ # General substitutions. project = 'Python' copyright = "2001 Python Software Foundation" +_doc_authors = 'Python documentation authors' # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -174,6 +176,7 @@ ('c:type', '__int64'), ('c:type', 'unsigned __int64'), ('c:type', 'double'), + ('c:type', '_Float16'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), @@ -226,10 +229,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot - # be resolved, as the method is currently undocumented. For context, see - # https://github.com/python/cpython/pull/103289. - ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: ('py:attr', '__wrapped__'), @@ -348,8 +347,6 @@ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} } -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', # The paper size ('letterpaper' or 'a4paper'). @@ -361,69 +358,74 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _doc_authors, 'manual'), ( 'extending/index', 'extending.tex', 'Extending and Embedding Python', - _stdauthor, + _doc_authors, 'manual', ), ( 'installing/index', 'installing.tex', 'Installing Python Modules', - _stdauthor, + _doc_authors, 'manual', ), ( 'library/index', 'library.tex', 'The Python Library Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'reference/index', 'reference.tex', 'The Python Language Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'tutorial/index', 'tutorial.tex', 'Python Tutorial', - _stdauthor, + _doc_authors, 'manual', ), ( 'using/index', 'using.tex', 'Python Setup and Usage', - _stdauthor, + _doc_authors, 'manual', ), ( 'faq/index', 'faq.tex', 'Python Frequently Asked Questions', - _stdauthor, + _doc_authors, 'manual', ), ( 'whatsnew/' + version, 'whatsnew.tex', 'What\'s New in Python', - 'A. M. Kuchling', + _doc_authors, 'howto', ), ] # Collect all HOWTOs individually latex_documents.extend( - ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + ( + 'howto/' + fn[:-4], + 'howto-' + fn[:-4] + '.tex', + '', + _doc_authors, + 'howto', + ) for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst' ) @@ -434,9 +436,14 @@ # Options for Epub output # ----------------------- -epub_author = 'Python Documentation Authors' +epub_author = _doc_authors epub_publisher = 'Python Software Foundation' -epub_exclude_files = ('index.xhtml', 'download.xhtml') +epub_exclude_files = ( + 'index.xhtml', + 'download.xhtml', + '_static/tachyon-example-flamegraph.html', + '_static/tachyon-example-heatmap.html', +) # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 @@ -546,15 +553,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True @@ -564,6 +576,18 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' +threadsafety_file = 'data/threadsafety.dat' + +# Options for notfound.extension +# ------------------------------- + +if not os.getenv("READTHEDOCS"): + if language_code: + notfound_urls_prefix = ( + f'/{language_code.replace("_", "-").lower()}/{version}/' + ) + else: + notfound_urls_prefix = f'/{version}/' # Options for sphinxext-opengraph # ------------------------------- diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 48f4f4919e89661..60c02aabeb89c51 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1472,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1485,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1506,6 +1513,14 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: PyModule_New:char*:name:: @@ -2022,6 +2037,11 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: +PySentinel_New:const char*:repr:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2412,6 +2432,20 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + +PyType_GetModule_DuringGC:PyObject*::0: +PyType_GetModule_DuringGC:PyTypeObject*:type:0: + +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + +PyType_GetModuleByToken_DuringGC:PyObject*::0: +PyType_GetModuleByToken_DuringGC:PyTypeObject*:type:0: +PyType_GetModuleByToken_DuringGC:PyModuleDef*:mod_token:: + PyType_GetModuleByDef:PyObject*::0: PyType_GetModuleByDef:PyTypeObject*:type:0: PyType_GetModuleByDef:PyModuleDef*:def:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 5cbf3771950fc07..86080fac7163838 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,7 @@ role,name,added,ifdef_note,struct_abi_kind macro,METH_CLASS,3.2,, macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.7,, +macro,METH_FASTCALL,3.10,, macro,METH_METHOD,3.7,, macro,METH_NOARGS,3.2,, macro,METH_O,3.2,, @@ -129,6 +129,12 @@ func,PyComplex_FromDoubles,3.2,, func,PyComplex_ImagAsDouble,3.2,, func,PyComplex_RealAsDouble,3.2,, data,PyComplex_Type,3.2,, +type,PyCriticalSection,3.15,,full-abi +type,PyCriticalSection2,3.15,,full-abi +func,PyCriticalSection2_Begin,3.15,, +func,PyCriticalSection2_End,3.15,, +func,PyCriticalSection_Begin,3.15,, +func,PyCriticalSection_End,3.15,, func,PyDescr_NewClassMethod,3.2,, func,PyDescr_NewGetSet,3.2,, func,PyDescr_NewMember,3.2,, @@ -160,6 +166,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, @@ -362,6 +369,10 @@ func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, +type,PyInterpreterGuard,3.15,,opaque +func,PyInterpreterGuard_Close,3.15,, +func,PyInterpreterGuard_FromCurrent,3.15,, +func,PyInterpreterGuard_FromView,3.15,, type,PyInterpreterState,3.2,,opaque func,PyInterpreterState_Clear,3.2,, func,PyInterpreterState_Delete,3.2,, @@ -369,6 +380,10 @@ func,PyInterpreterState_Get,3.9,, func,PyInterpreterState_GetDict,3.8,, func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, +type,PyInterpreterView,3.15,,opaque +func,PyInterpreterView_Close,3.15,, +func,PyInterpreterView_FromCurrent,3.15,, +func,PyInterpreterView_FromMain,3.15,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, func,PyIter_NextItem,3.14,, @@ -388,8 +403,14 @@ func,PyList_SetSlice,3.2,, func,PyList_Size,3.2,, func,PyList_Sort,3.2,, data,PyList_Type,3.2,, +type,PyLongExport,3.15,,full-abi +type,PyLongLayout,3.15,,full-abi type,PyLongObject,3.2,,opaque data,PyLongRangeIter_Type,3.2,, +type,PyLongWriter,3.15,,opaque +func,PyLongWriter_Create,3.15,, +func,PyLongWriter_Discard,3.15,, +func,PyLongWriter_Finish,3.15,, func,PyLong_AsDouble,3.2,, func,PyLong_AsInt,3.13,, func,PyLong_AsInt32,3.14,, @@ -408,6 +429,8 @@ func,PyLong_AsUnsignedLongLong,3.2,, func,PyLong_AsUnsignedLongLongMask,3.2,, func,PyLong_AsUnsignedLongMask,3.2,, func,PyLong_AsVoidPtr,3.2,, +func,PyLong_Export,3.15,, +func,PyLong_FreeExport,3.15,, func,PyLong_FromDouble,3.2,, func,PyLong_FromInt32,3.14,, func,PyLong_FromInt64,3.14,, @@ -424,6 +447,7 @@ func,PyLong_FromUnsignedLongLong,3.2,, func,PyLong_FromUnsignedNativeBytes,3.14,, func,PyLong_FromVoidPtr,3.2,, func,PyLong_GetInfo,3.2,, +func,PyLong_GetNativeLayout,3.15,, data,PyLong_Type,3.2,, macro,PyMODEXPORT_FUNC,3.15,, data,PyMap_Type,3.2,, @@ -460,9 +484,10 @@ func,PyMemoryView_GetContiguous,3.2,, data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi data,PyMethodDescr_Type,3.2,, -type,PyModuleDef,3.2,,full-abi -type,PyModuleDef_Base,3.2,,full-abi +type,PyModuleDef,3.2,,abi3t-opaque +type,PyModuleDef_Base,3.2,,abi3t-opaque func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -484,7 +509,9 @@ func,PyModule_GetName,3.2,, func,PyModule_GetNameObject,3.7,, func,PyModule_GetState,3.2,, func,PyModule_GetStateSize,3.15,, +func,PyModule_GetState_DuringGC,3.15,, func,PyModule_GetToken,3.15,, +func,PyModule_GetToken_DuringGC,3.15,, func,PyModule_New,3.2,, func,PyModule_NewObject,3.7,, func,PyModule_SetDocString,3.7,, @@ -552,6 +579,7 @@ func,PyObject_ASCII,3.2,, func,PyObject_AsFileDescriptor,3.2,, func,PyObject_Bytes,3.2,, func,PyObject_Call,3.2,, +func,PyObject_CallFinalizerFromDealloc,3.15,, func,PyObject_CallFunction,3.2,, func,PyObject_CallFunctionObjArgs,3.2,, func,PyObject_CallMethod,3.2,, @@ -587,6 +615,7 @@ func,PyObject_GetIter,3.2,, func,PyObject_GetOptionalAttr,3.13,, func,PyObject_GetOptionalAttrString,3.13,, func,PyObject_GetTypeData,3.12,, +func,PyObject_GetTypeData_DuringGC,3.15,, func,PyObject_HasAttr,3.2,, func,PyObject_HasAttrString,3.2,, func,PyObject_HasAttrStringWithError,3.13,, @@ -655,6 +684,19 @@ func,PySlice_GetIndicesEx,3.2,, func,PySlice_New,3.2,, data,PySlice_Type,3.2,, func,PySlice_Unpack,3.7,, +type,PySlot,3.15,,full-abi +macro,PySlot_DATA,3.15,, +macro,PySlot_END,3.15,, +macro,PySlot_FUNC,3.15,, +macro,PySlot_INT64,3.15,, +macro,PySlot_INTPTR,3.15,, +macro,PySlot_OPTIONAL,3.15,, +macro,PySlot_PTR,3.15,, +macro,PySlot_PTR_STATIC,3.15,, +macro,PySlot_SIZE,3.15,, +macro,PySlot_STATIC,3.15,, +macro,PySlot_STATIC_DATA,3.15,, +macro,PySlot_UINT64,3.15,, func,PyState_AddModule,3.3,, func,PyState_FindModule,3.2,, func,PyState_RemoveModule,3.3,, @@ -682,14 +724,18 @@ func,PySys_SetObject,3.2,, func,PySys_WriteStderr,3.2,, func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque +type,PyThreadStateToken,3.15,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.15,, +func,PyThreadState_EnsureFromView,3.15,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.15,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, @@ -734,18 +780,23 @@ func,PyType_ClearCache,3.2,, func,PyType_Freeze,3.14,, func,PyType_FromMetaclass,3.12,, func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSlots,3.15,, func,PyType_FromSpec,3.2,, func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, func,PyType_GenericNew,3.2,, func,PyType_GetBaseByToken,3.14,, +func,PyType_GetBaseByToken_DuringGC,3.15,, func,PyType_GetFlags,3.2,, func,PyType_GetFullyQualifiedName,3.13,, func,PyType_GetModule,3.10,, func,PyType_GetModuleByDef,3.13,, func,PyType_GetModuleByToken,3.15,, +func,PyType_GetModuleByToken_DuringGC,3.15,, func,PyType_GetModuleName,3.13,, func,PyType_GetModuleState,3.10,, +func,PyType_GetModuleState_DuringGC,3.15,, +func,PyType_GetModule_DuringGC,3.15,, func,PyType_GetName,3.11,, func,PyType_GetQualName,3.11,, func,PyType_GetSlot,3.4,, @@ -887,6 +938,8 @@ macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, +macro,Py_BEGIN_CRITICAL_SECTION,3.15,, +macro,Py_BEGIN_CRITICAL_SECTION2,3.15,, macro,Py_BLOCK_THREADS,3.2,, func,Py_BuildValue,3.2,, func,Py_BytesMain,3.8,, @@ -894,6 +947,8 @@ func,Py_CompileString,3.2,, func,Py_DecRef,3.2,, func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, +macro,Py_END_CRITICAL_SECTION,3.15,, +macro,Py_END_CRITICAL_SECTION2,3.15,, func,Py_EncodeLocale,3.7,, func,Py_EndInterpreter,3.2,, func,Py_EnterRecursiveCall,3.9,, @@ -914,6 +969,7 @@ func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, func,Py_InitializeEx,3.2,, @@ -935,6 +991,8 @@ func,Py_REFCNT,3.14,, macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, +func,Py_SET_SIZE,3.15,, +func,Py_SIZE,3.15,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, @@ -979,9 +1037,14 @@ macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, macro,Py_mod_abi,3.15,, +macro,Py_mod_create,3.5,, macro,Py_mod_doc,3.15,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, macro,Py_mod_methods,3.15,, +macro,Py_mod_multiple_interpreters,3.12,, macro,Py_mod_name,3.15,, +macro,Py_mod_slots,3.15,, macro,Py_mod_state_clear,3.15,, macro,Py_mod_state_free,3.15,, macro,Py_mod_state_size,3.15,, @@ -1025,6 +1088,9 @@ macro,Py_nb_rshift,3.2,, macro,Py_nb_subtract,3.2,, macro,Py_nb_true_divide,3.2,, macro,Py_nb_xor,3.2,, +macro,Py_slot_end,3.15,, +macro,Py_slot_invalid,3.15,, +macro,Py_slot_subslots,3.15,, macro,Py_sq_ass_item,3.2,, macro,Py_sq_concat,3.2,, macro,Py_sq_contains,3.2,, @@ -1037,6 +1103,7 @@ type,Py_ssize_t,3.2,, macro,Py_tp_alloc,3.2,, macro,Py_tp_base,3.2,, macro,Py_tp_bases,3.2,, +macro,Py_tp_basicsize,3.15,, macro,Py_tp_call,3.2,, macro,Py_tp_clear,3.2,, macro,Py_tp_dealloc,3.2,, @@ -1044,7 +1111,9 @@ macro,Py_tp_del,3.2,, macro,Py_tp_descr_get,3.2,, macro,Py_tp_descr_set,3.2,, macro,Py_tp_doc,3.2,, +macro,Py_tp_extra_basicsize,3.15,, macro,Py_tp_finalize,3.5,, +macro,Py_tp_flags,3.15,, macro,Py_tp_free,3.2,, macro,Py_tp_getattr,3.2,, macro,Py_tp_getattro,3.2,, @@ -1052,15 +1121,20 @@ macro,Py_tp_getset,3.2,, macro,Py_tp_hash,3.2,, macro,Py_tp_init,3.2,, macro,Py_tp_is_gc,3.2,, +macro,Py_tp_itemsize,3.15,, macro,Py_tp_iter,3.2,, macro,Py_tp_iternext,3.2,, macro,Py_tp_members,3.2,, +macro,Py_tp_metaclass,3.15,, macro,Py_tp_methods,3.2,, +macro,Py_tp_module,3.15,, +macro,Py_tp_name,3.15,, macro,Py_tp_new,3.2,, macro,Py_tp_repr,3.2,, macro,Py_tp_richcompare,3.2,, macro,Py_tp_setattr,3.2,, macro,Py_tp_setattro,3.2,, +macro,Py_tp_slots,3.15,, macro,Py_tp_str,3.2,, macro,Py_tp_token,3.14,, macro,Py_tp_traverse,3.2,, diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat new file mode 100644 index 000000000000000..ea5a24a5505e208 --- /dev/null +++ b/Doc/data/threadsafety.dat @@ -0,0 +1,284 @@ +# Thread safety annotations for C API functions. +# +# Each line has the form: +# function_name : level +# +# Where level is one of: +# incompatible -- not safe even with external locking +# compatible -- safe if the caller serializes all access with external locks +# distinct -- safe on distinct objects without external synchronization +# shared -- safe for concurrent use on the same object +# atomic -- atomic +# +# Lines beginning with '#' are ignored. +# The function name must match the C domain identifier used in the documentation. + +# Synchronization primitives (Doc/c-api/synchronization.rst) +PyMutex_Lock:atomic: +PyMutex_Unlock:atomic: +PyMutex_IsLocked:atomic: + + +# Dictionary objects (Doc/c-api/dict.rst) + +# Type checks - read ob_type pointer, always safe +PyDict_Check:atomic: +PyDict_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyDict_New:atomic: + +# Lock-free lookups - use _Py_dict_lookup_threadsafe(), no locking. +# Atomic with simple types. +PyDict_Contains:shared: +PyDict_ContainsString:atomic: +PyDict_GetItemRef:shared: +PyDict_GetItemStringRef:atomic: +PyDict_Size:atomic: +PyDict_GET_SIZE:atomic: + +# Borrowed-reference lookups - lock-free dict access but returned +# borrowed reference is unsafe in free-threaded builds without +# external synchronization +PyDict_GetItem:compatible: +PyDict_GetItemWithError:compatible: +PyDict_GetItemString:compatible: +PyDict_SetDefault:compatible: + +# Iteration - no locking; returns borrowed refs +PyDict_Next:compatible: + +# Single-item mutations - protected by per-object critical section +PyDict_SetItem:shared: +PyDict_SetItemString:atomic: +PyDict_DelItem:shared: +PyDict_DelItemString:atomic: +PyDict_SetDefaultRef:shared: +PyDict_Pop:shared: +PyDict_PopString:atomic: + +# Bulk reads - hold per-object lock for duration +PyDict_Clear:atomic: +PyDict_Copy:atomic: +PyDict_Keys:atomic: +PyDict_Values:atomic: +PyDict_Items:atomic: + +# Merge/update - lock target dict; also lock source when it is a dict +PyDict_Update:shared: +PyDict_Merge:shared: +PyDict_MergeFromSeq2:shared: + +# Watcher registration - no synchronization on interpreter state +PyDict_AddWatcher:compatible: +PyDict_ClearWatcher:compatible: + +# Per-dict watcher tags - non-atomic RMW on _ma_watcher_tag; +# safe on distinct dicts only +PyDict_Watch:distinct: +PyDict_Unwatch:distinct: + + +# List objects (Doc/c-api/list.rst) + +# Type checks - read ob_type pointer, always safe +PyList_Check:atomic: +PyList_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyList_New:atomic: + +# Size - uses atomic load on free-threaded builds +PyList_Size:atomic: +PyList_GET_SIZE:atomic: + +# Strong-reference lookup - lock-free with atomic ops +PyList_GetItemRef:atomic: + +# Borrowed-reference lookups - no locking; returned borrowed +# reference is unsafe in free-threaded builds without +# external synchronization +PyList_GetItem:compatible: +PyList_GET_ITEM:compatible: + +# Single-item mutations - hold per-object lock for duration; +# appear atomic to lock-free readers +PyList_SetItem:atomic: +PyList_Append:atomic: + +# Insert - protected by per-object critical section; shifts +# elements so lock-free readers may observe intermediate states +PyList_Insert:shared: + +# Initialization macro - no synchronization; normally only used +# to fill in new lists where there is no previous content +PyList_SET_ITEM:compatible: + +# Bulk operations - hold per-object lock for duration +PyList_GetSlice:atomic: +PyList_AsTuple:atomic: +PyList_Clear:atomic: + +# Reverse - protected by per-object critical section; swaps +# elements so lock-free readers may observe intermediate states +PyList_Reverse:shared: + +# Slice assignment - lock target list; also lock source when it +# is a list +PyList_SetSlice:shared: + +# Sort - per-object lock held; the list is emptied before sorting +# so other threads may observe an empty list, but they won't see the +# intermediate states of the sort +PyList_Sort:shared: + +# Extend - lock target list; also lock source when it is a +# list, set, or dict +PyList_Extend:shared: + +# Creation - pure allocation, no shared state +PyBytes_FromString:atomic: +PyBytes_FromStringAndSize:atomic: +PyBytes_DecodeEscape:atomic: + +# Creation from formatting C primitives - pure allocation, no shared state +PyBytes_FromFormat:atomic: +PyBytes_FromFormatV:atomic: + +# Creation from object - uses buffer protocol so may call arbitrary code; +# safe as long as the buffer is not mutated by another thread during the operation +PyBytes_FromObject:shared: + +# Size - uses atomic load on free-threaded builds +PyBytes_Size:atomic: +PyBytes_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytes object is shared between threads +PyBytes_AsString:compatible: +PyBytes_AS_STRING:compatible: +PyBytes_AsStringAndSize:compatible: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyBytes_Concat:shared: +PyBytes_ConcatAndDel:shared: +PyBytes_Join:shared: + +# Resizing - safe if the object is unique +_PyBytes_Resize:distinct: + +# Repr - atomic as bytes are immutable +PyBytes_Repr:atomic: + +# Creation from object - may call arbitrary code +PyByteArray_FromObject:shared: + +# Creation - pure allocation, no shared state +PyByteArray_FromStringAndSize:atomic: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyByteArray_Concat:shared: + +# Size - uses atomic load on free-threaded builds +PyByteArray_Size:atomic: +PyByteArray_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytearray object is shared between threads +PyByteArray_AsString:compatible: +PyByteArray_AS_STRING:compatible: + +# Creation - may iterate the iterable argument, calling arbitrary code. +# Atomic for sets, frozensets, dicts, and frozendicts. +PySet_New:shared: +PyFrozenSet_New:shared: + +# Size - uses atomic load on free-threaded builds +PySet_Size:atomic: +PySet_GET_SIZE:atomic: + +# Contains - lock-free, atomic with simple types +PySet_Contains:shared: + +# Mutations - hold per-object lock for duration +# atomic with simple types +PySet_Add:shared: +PySet_Discard:shared: + +# Pop - hold per-object lock for duration +PySet_Pop:atomic: + +# Clear - empties the set before clearing +PySet_Clear:atomic: + +# Capsule objects (Doc/c-api/capsule.rst) + +# Type check - read ob_type pointer, always safe +PyCapsule_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyCapsule_New:atomic: + +# Validation - reads pointer and name fields; safe on distinct objects +PyCapsule_IsValid:distinct: + +# Getters - read struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_GetPointer:distinct: +PyCapsule_GetName:distinct: +PyCapsule_GetDestructor:distinct: +PyCapsule_GetContext:distinct: + +# Setters - write struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_SetPointer:distinct: +PyCapsule_SetName:distinct: +PyCapsule_SetDestructor:distinct: +PyCapsule_SetContext:distinct: + +# Import - looks up a capsule from a module attribute and +# calls PyCapsule_GetPointer; may call arbitrary code +PyCapsule_Import:compatible: + +# Tuple objects + +# Creation - pure allocation, no shared state +PyTuple_New:atomic: +PyTuple_FromArray:atomic: +PyTuple_Pack:atomic: + +# Size - tuples are immutable so size never changes +PyTuple_Size:atomic: +PyTuple_GET_SIZE:atomic: + +# Borrowed-reference lookups - tuples are immutable so items +# never change, however the tuple must be kept alive while using the borrowed reference +PyTuple_GetItem:compatible: +PyTuple_GET_ITEM:compatible: + +# Slice - creates a new tuple from an existing tuple +PyTuple_GetSlice:atomic: + +# SetItem - only usable on tuples with refcount 1 +PyTuple_SetItem:compatible: +PyTuple_SET_ITEM:compatible: + +# Resize - only usable on tuples with refcount 1 +_PyTuple_Resize:compatible: + +# Struct Sequence objects + +# Creation +PyStructSequence_NewType:atomic: +PyStructSequence_New:atomic: + +# Initialization - modifies the type object in place +PyStructSequence_InitType:distinct: +PyStructSequence_InitType2:distinct: + +# Borrowed-reference lookups - same as tuple items +PyStructSequence_GetItem:compatible: +PyStructSequence_GET_ITEM:compatible: + +# SetItem - only for filling in brand new instances +PyStructSequence_SetItem:compatible: +PyStructSequence_SET_ITEM:compatible: + diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index 9927b876760d344..6d0d47403ff2ed4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -7,8 +7,6 @@ Pending removal in Python 3.15 Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project `__ can be used to get :c:func:`PyWeakref_GetRef` on Python 3.12 and older. -* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: - Use :c:type:`wchar_t` instead. * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: @@ -48,88 +46,3 @@ Pending removal in Python 3.15 The `pythoncapi-compat project `__ can be used to get :c:func:`PyConfig_Get` on Python 3.13 and older. - -* Functions to configure Python's initialization, deprecated in Python 3.11: - - * :c:func:`!PySys_SetArgvEx()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!PySys_SetArgv()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!Py_SetProgramName()`: - Set :c:member:`PyConfig.program_name` instead. - * :c:func:`!Py_SetPythonHome()`: - Set :c:member:`PyConfig.home` instead. - * :c:func:`!PySys_ResetWarnOptions`: - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: - Use :c:member:`PyConfig.parser_debug` or - :c:func:`PyConfig_Get("parser_debug") ` instead. - * :c:var:`Py_VerboseFlag`: - Use :c:member:`PyConfig.verbose` or - :c:func:`PyConfig_Get("verbose") ` instead. - * :c:var:`Py_QuietFlag`: - Use :c:member:`PyConfig.quiet` or - :c:func:`PyConfig_Get("quiet") ` instead. - * :c:var:`Py_InteractiveFlag`: - Use :c:member:`PyConfig.interactive` or - :c:func:`PyConfig_Get("interactive") ` instead. - * :c:var:`Py_InspectFlag`: - Use :c:member:`PyConfig.inspect` or - :c:func:`PyConfig_Get("inspect") ` instead. - * :c:var:`Py_OptimizeFlag`: - Use :c:member:`PyConfig.optimization_level` or - :c:func:`PyConfig_Get("optimization_level") ` instead. - * :c:var:`Py_NoSiteFlag`: - Use :c:member:`PyConfig.site_import` or - :c:func:`PyConfig_Get("site_import") ` instead. - * :c:var:`Py_BytesWarningFlag`: - Use :c:member:`PyConfig.bytes_warning` or - :c:func:`PyConfig_Get("bytes_warning") ` instead. - * :c:var:`Py_FrozenFlag`: - Use :c:member:`PyConfig.pathconfig_warnings` or - :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: - Use :c:member:`PyConfig.use_environment` or - :c:func:`PyConfig_Get("use_environment") ` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: - Use :c:member:`PyConfig.write_bytecode` or - :c:func:`PyConfig_Get("write_bytecode") ` instead. - * :c:var:`Py_NoUserSiteDirectory`: - Use :c:member:`PyConfig.user_site_directory` or - :c:func:`PyConfig_Get("user_site_directory") ` instead. - * :c:var:`Py_UnbufferedStdioFlag`: - Use :c:member:`PyConfig.buffered_stdio` or - :c:func:`PyConfig_Get("buffered_stdio") ` instead. - * :c:var:`Py_HashRandomizationFlag`: - Use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` or - :c:func:`PyConfig_Get("hash_seed") ` instead. - * :c:var:`Py_IsolatedFlag`: - Use :c:member:`PyConfig.isolated` or - :c:func:`PyConfig_Get("isolated") ` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: - Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or - :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: - Use :c:member:`PyConfig.legacy_windows_stdio` or - :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` or - :c:func:`PyConfig_Get("filesystem_encoding") ` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: - Use :c:member:`PyConfig.filesystem_errors` or - :c:func:`PyConfig_Get("filesystem_errors") ` instead. - * :c:var:`!Py_UTF8Mode`: - Use :c:member:`PyPreConfig.utf8_mode` or - :c:func:`PyConfig_Get("utf8_mode") ` instead. - (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be - used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst index 9453f83799c43d7..fe2d91cf316b18f 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.16.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -1,4 +1,87 @@ Pending removal in Python 3.16 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdec``. +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. + +* :c:type:`Py_UNICODE` which was deprecated by :pep:`393`. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 022aee93aa70c47..820334ee43c82e4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -40,6 +40,10 @@ Pending removal in Python 3.18 * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + * :c:func:`PyGen_New`: (no replacement). + * :c:func:`PyGen_NewWithQualName`: (no replacement). + * :c:func:`PyCoro_New`: (no replacement). + * :c:func:`PyAsyncGen_New`: (no replacement). The `pythoncapi-compat project `__ can be used to get diff --git a/Doc/deprecations/c-api-pending-removal-in-3.19.rst b/Doc/deprecations/c-api-pending-removal-in-3.19.rst new file mode 100644 index 000000000000000..ac9dcb8b424a17d --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.19.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.19 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* :pep:`456` embedders support for the string hashing scheme definition. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.20.rst b/Doc/deprecations/c-api-pending-removal-in-3.20.rst index 18623b19a2ab8da..8de55bbe7e695c3 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.20.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.20.rst @@ -1,6 +1,13 @@ Pending removal in Python 3.20 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + * The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`). Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` to convert a Python complex number to/from the C :c:type:`Py_complex` diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index f3cecb321d633fd..6176169a6f1eba6 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -1,25 +1,27 @@ Deprecations ============ -.. include:: pending-removal-in-3.15.rst - -.. include:: pending-removal-in-3.16.rst - .. include:: pending-removal-in-3.17.rst +.. include:: pending-removal-in-3.18.rst + .. include:: pending-removal-in-3.19.rst .. include:: pending-removal-in-3.20.rst +.. include:: pending-removal-in-3.21.rst + .. include:: pending-removal-in-future.rst +.. include:: soft-deprecations.rst + C API deprecations ------------------ -.. include:: c-api-pending-removal-in-3.15.rst - .. include:: c-api-pending-removal-in-3.18.rst +.. include:: c-api-pending-removal-in-3.19.rst + .. include:: c-api-pending-removal-in-3.20.rst .. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a2c42..d5b8c80e8f9aa0f 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 09cbd6f01a0580f..5c0e592f4caeda1 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -3,14 +3,9 @@ Pending removal in Python 3.15 * The import system: - * Setting :attr:`~module.__cached__` on a module while + * Setting ``__cached__`` on a module while failing to set :attr:`__spec__.cached ` - is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or - take into consideration by the import system or standard library. (:gh:`97879`) - - * Setting :attr:`~module.__package__` on a module while - failing to set :attr:`__spec__.parent ` - is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) * :mod:`ctypes`: @@ -33,16 +28,6 @@ Pending removal in Python 3.15 * ``load_module()`` method: use ``exec_module()`` instead. -* :class:`locale`: - - * The :func:`~locale.getdefaultlocale` function - has been deprecated since Python 3.11. - Its removal was originally planned for Python 3.13 (:gh:`90817`), - but has been postponed to Python 3.15. - Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, - and :func:`~locale.getencoding` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) - * :mod:`pathlib`: * :meth:`!.PurePath.is_reserved` @@ -64,13 +49,13 @@ Pending removal in Python 3.15 * :func:`~threading.RLock` will take no arguments in Python 3.15. Passing any arguments has been deprecated since Python 3.14, - as the Python version does not permit any arguments, + as the Python version does not permit any arguments, but the C version allows any number of positional or keyword arguments, ignoring every argument. * :mod:`types`: - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + * :class:`types.CodeType`: Accessing :attr:`!codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. @@ -92,7 +77,7 @@ Pending removal in Python 3.15 Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` to create a TypedDict with zero field. - * The :func:`!typing.no_type_check_decorator` decorator function + * The :deco:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index b00c7002b03772d..9b19ec3236f92fb 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -8,6 +8,13 @@ Pending removal in Python 3.16 is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or taken into consideration by the import system or the standard library. + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.16, :attr:`!__package__` will cease to be + taken into consideration by the import system or standard library. (:gh:`97879`) + +* The bundled copy of ``libmpdec``. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) @@ -26,12 +33,12 @@ Pending removal in Python 3.16 * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` + * :class:`!asyncio.AbstractEventLoopPolicy` + * :class:`!asyncio.DefaultEventLoopPolicy` + * :class:`!asyncio.WindowsSelectorEventLoopPolicy` + * :class:`!asyncio.WindowsProactorEventLoopPolicy` + * :func:`!asyncio.get_event_loop_policy` + * :func:`!asyncio.set_event_loop_policy` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with *loop_factory* to use the desired event loop implementation. @@ -84,12 +91,12 @@ Pending removal in Python 3.16 * :mod:`symtable`: - * The :meth:`Class.get_methods ` method + * The :meth:`!symtable.Class.get_methods` method has been deprecated since Python 3.14. * :mod:`sys`: - * The :func:`~sys._enablelegacywindowsfsencoding` function + * The :func:`!_enablelegacywindowsfsencoding` function has been deprecated since Python 3.13. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. @@ -101,5 +108,5 @@ Pending removal in Python 3.16 * :mod:`tarfile`: - * The undocumented and unused :attr:`!TarFile.tarfile` attribute + * The undocumented and unused :attr:`!TarInfo.tarfile` attribute has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index e769c9d371e133c..d6e4e81afbbd639 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,14 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`datetime`: + + * :meth:`~datetime.datetime.strptime` calls using a format string containing + ``%e`` (day of month) without a year. + This has been deprecated since Python 3.15. + (Contributed by Stan Ulbrych in :gh:`70647`.) + + * :mod:`collections.abc`: - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. @@ -27,7 +35,12 @@ Pending removal in Python 3.17 - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` is deprecated and scheduled for removal in Python 3.17. - (Contributed by Stan Ulbrych in :gh:`136702`) + (Contributed by Stan Ulbrych in :gh:`136702`.) + +* :mod:`webbrowser`: + + - :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS`. (:gh:`137586`) * :mod:`typing`: @@ -55,3 +68,13 @@ Pending removal in Python 3.17 See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`tkinter`: + + - The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, + :meth:`!trace` (an alias of :meth:`!trace_variable`), + :meth:`!trace_vdelete` and :meth:`!trace_vinfo`, deprecated since + Python 3.14, are scheduled for removal in Python 3.17. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` + instead. + (Contributed by Serhiy Storchaka in :gh:`120220`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst new file mode 100644 index 000000000000000..19113aab981bbc6 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -0,0 +1,18 @@ +Pending removal in Python 3.18 +------------------------------ + +* No longer accept a boolean value when a file descriptor is expected. + (Contributed by Serhiy Storchaka in :gh:`82626`.) + +* :mod:`decimal`: + + * The non-standard and undocumented :class:`~decimal.Decimal` format + specifier ``'N'``, which is only supported in the :mod:`!decimal` module's + C implementation, has been deprecated since Python 3.13. + (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* Deprecations defined by :pep:`829`: + + * ``import`` lines in :file:`{name}.pth` files are silently ignored. + + (Contributed by Barry Warsaw in :gh:`148641`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 25f9cba390de68e..4a58c606ab7596e 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -22,3 +22,21 @@ Pending removal in Python 3.19 supported depending on the backend implementation of hash functions. Prefer passing the initial data as a positional argument for maximum backwards compatibility. + +* :mod:`http.cookies`: + + * :meth:`http.cookies.Morsel.js_output` is deprecated and will be + removed in Python 3.19. + + * :meth:`http.cookies.BaseCookie.js_output` is deprecated and will be + removed in Python 3.19. + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index c0feda1968258dc..011565dfbb090d4 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -1,14 +1,23 @@ Pending removal in Python 3.20 ------------------------------ -* The ``__version__`` attribute has been deprecated in these standard library - modules and will be removed in Python 3.20. - Use :py:data:`sys.version_info` instead. +* Calling the ``__new__()`` method of :class:`struct.Struct` without the + *format* argument is deprecated and will be removed in Python 3.20. Calling + :meth:`~object.__init__` method on initialized :class:`~struct.Struct` + objects is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. - :mod:`argparse` - :mod:`csv` + - :mod:`ctypes` - :mod:`!ctypes.macholib` - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` - :mod:`imaplib` - :mod:`ipaddress` - :mod:`json` @@ -19,8 +28,28 @@ Pending removal in Python 3.20 - :mod:`re` - :mod:`socketserver` - :mod:`tabnanny` + - :mod:`tarfile` - :mod:`tkinter.font` - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` - :mod:`zlib` (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +* Deprecations defined by :pep:`829`: + + * Warnings are produced for ``import`` lines found in :file:`{name}.pth` + files. + + * :file:`{name}.pth` files are no longer decoded in the locale encoding by + default. They **MUST** be encoded in ``utf-8-sig``. + + (Contributed by Barry Warsaw in :gh:`148641`.) + +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. diff --git a/Doc/deprecations/pending-removal-in-3.21.rst b/Doc/deprecations/pending-removal-in-3.21.rst new file mode 100644 index 000000000000000..dbd73313b4f7779 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.21.rst @@ -0,0 +1,25 @@ +Pending removal in Python 3.21 +------------------------------ + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, will be removed in Python 3.21. These types + are not generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` will be removed in Python 3.21. Use + the ``ast.Tuple.elts`` property instead. + +* :mod:`struct`: + + * Soft-deprecated since Python 3.15, using ``'F'`` and ``'D'`` type codes are now + deprecated. These codes will be removed in Python 3.21. Use instead + two-letter forms ``'Zf'`` and ``'Zd'``. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 7ed430625f305e9..74f98d33a4b61f1 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -35,7 +35,6 @@ although there is currently no date scheduled for their removal. * Support for ``__complex__()`` method returning a strict subclass of :class:`complex`: these methods will be required to return an instance of :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. * Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. @@ -48,7 +47,7 @@ although there is currently no date scheduled for their removal. * :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method +* :attr:`!codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. * :mod:`datetime`: @@ -76,7 +75,15 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. + +* :mod:`os.path`: :func:`os.path.commonprefix` is deprecated, use + :func:`os.path.commonpath` for path prefixes. The :func:`os.path.commonprefix` + function is being deprecated due to having a misleading name and module. + The function is not safe to use for path prefixes despite being included in a + module about path manipulation, meaning it is easy to accidentally + introduce path traversal vulnerabilities into Python programs by using this + function. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/deprecations/soft-deprecations.rst b/Doc/deprecations/soft-deprecations.rst new file mode 100644 index 000000000000000..a270052788ef2a1 --- /dev/null +++ b/Doc/deprecations/soft-deprecations.rst @@ -0,0 +1,21 @@ +Soft deprecations +----------------- + +There are no plans to remove :term:`soft deprecated` APIs. + +* :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index dee92312169a276..110dfea8cb98abe 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -3,154 +3,20 @@ .. _extending-intro: -****************************** -Extending Python with C or C++ -****************************** +******************************** +Using the C API: Assorted topics +******************************** -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - `_ library rather than writing - custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first two lines of our file can be:: - - #define PY_SSIZE_T_CLEAN - #include - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - ``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be - used in some APIs instead of ``int``. - It is not necessary since Python 3.13, but we keep it here for backward compatibility. - See :ref:`arg-parsing-string-and-buffers` for a description of this macro. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. - -.. tip:: - - For backward compatibility, :file:`Python.h` includes several standard header files. - C extensions should include the standard headers that they use, - and should not rely on these implicit includes. - If using the limited C API version 3.13 or newer, the implicit includes are: - - * ```` - * ```` (on Windows) - * ```` - * ```` - * ```` - * ```` - * ```` - * ```` (if present) - - If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, - the headers below are also included: - - * ```` - * ```` (on POSIX) - - If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, - the headers below are also included: - - * ```` - * ```` - * ```` - * ```` - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return ``NULL`` immediately (as we saw in the example). +The :ref:`tutorial ` walked you through +creating a C API extension module, but left many areas unexplained. +This document looks at several concepts that you'll need to learn +in order to write more complex extensions. .. _extending-errors: -Intermezzo: Errors and Exceptions -================================= +Errors and Exceptions +===================== An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value @@ -321,194 +187,14 @@ call to :c:func:`PyErr_SetString` as shown below:: } -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns ``NULL`` (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`!command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`!command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:expr:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a ``NULL`` pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`!spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef spam_methods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spam_module = { - ... - .m_methods = spam_methods, - ... - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -:c:func:`!PyInit_spam` is called when each interpreter imports its module -:mod:`!spam` for the first time. (See below for comments about embedding Python.) -A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, -so that the import machinery can create the module and store it in ``sys.modules``. - -When embedding Python, the :c:func:`!PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`!PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - #define PY_SSIZE_T_CLEAN - #include - - int - main(int argc, char *argv[]) - { - PyStatus status; - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Add a built-in module, before Py_Initialize */ - if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { - fprintf(stderr, "Error: could not extend in-built modules table\n"); - exit(1); - } - - /* Pass argv[0] to the Python interpreter */ - status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]); - if (PyStatus_Exception(status)) { - goto exception; - } - - /* Initialize the Python interpreter. Required. - If this step fails, it will be a fatal error. */ - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyObject *pmodule = PyImport_ImportModule("spam"); - if (!pmodule) { - PyErr_Print(); - fprintf(stderr, "Error: could not import module 'spam'\n"); - } - - // ... use Python C API here ... - - return 0; - - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - -.. note:: - - If you declare a global variable or a local static one, the module may - experience unintended side-effects on re-initialisation, for example when - removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process - (or following a :c:func:`fork` without an intervening :c:func:`exec`). - If module state is not yet fully :ref:`isolated `, - authors should consider marking the module as having no support for subinterpreters - (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxlimited.c`. This file may be used as a template or simply -read as an example. - - .. _compilation: -Compilation and Linkage -======================= +Embedding an extension +====================== -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. - -If you can't use dynamic loading, or if you want to make your module a permanent +If you want to make your module a permanent part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place +and rebuild the interpreter. On Unix, place your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory of an unpacked source distribution, add a line to the file :file:`Modules/Setup.local` describing your file: @@ -536,7 +222,7 @@ on the line in the configuration file as well, for instance: Calling Python Functions from C =============================== -So far we have concentrated on making C functions callable from Python. The +The tutorial concentrated on making C functions callable from Python. The reverse is also useful: calling Python functions from C. This is especially the case for libraries that support so-called "callback" functions. If a C interface makes use of callbacks, the equivalent Python often needs to provide a @@ -545,10 +231,8 @@ calling the Python callback functions from a C callback. Other uses are also imaginable. Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) +standard interface to call a Python function. (If you're interested in how to call the +Python parser with a particular string as input, see :ref:`veryhigh`.) Calling a Python function is easy. First, the Python program must somehow pass you the Python function object. You should provide a function (or some other @@ -581,7 +265,7 @@ be part of a module definition:: } This function must be registered with the interpreter using the -:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The +:c:macro:`METH_VARARGS` flag in :c:type:`PyMethodDef.ml_flags`. The :c:func:`PyArg_ParseTuple` function and its arguments are documented in section :ref:`parsetuple`. @@ -676,14 +360,21 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Py_DECREF(result); +.. index:: single: PyArg_ParseTuple (C function) + .. _parsetuple: Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple (C function) +The :ref:`tutorial ` uses a ":c:data:`METH_O`" +function, which is limited to a single Python argument. +If you want more, you can use :c:data:`METH_VARARGS` instead. +With this flag, the C function will receive a :py:class:`tuple` of arguments +instead of a single object. -The :c:func:`PyArg_ParseTuple` function is declared as follows:: +For unpacking the tuple, CPython provides the :c:func:`PyArg_ParseTuple` +function, declared as follows:: int PyArg_ParseTuple(PyObject *arg, const char *format, ...); @@ -693,6 +384,19 @@ whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference Manual. The remaining arguments must be addresses of variables whose type is determined by the format string. +For example, to receive a single Python :py:class:`str` object and turn it +into a C buffer, you would use ``"s"`` as the format string:: + + const char *command; + if (!PyArg_ParseTuple(args, "s", &command)) { + return NULL; + } + +If an error is detected in the argument list, :c:func:`!PyArg_ParseTuple` +returns ``NULL`` (the error indicator for functions returning object pointers); +your function may return ``NULL``, relying on the exception set by +:c:func:`PyArg_ParseTuple`. + Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have the required types, it cannot check the validity of the addresses of C variables passed to the call: if you make mistakes there, your code will probably crash or @@ -703,7 +407,6 @@ Note that any Python object references which are provided to the caller are Some example calls:: - #define PY_SSIZE_T_CLEAN #include :: @@ -773,6 +476,17 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== +If you also want your function to accept +:term:`keyword arguments `, use the :c:data:`METH_KEYWORDS` +flag in combination with :c:data:`METH_VARARGS`. +(:c:data:`!METH_KEYWORDS` can also be used with other flags; see its +documentation for the allowed combinations.) + +In this case, the C function should accept a third ``PyObject *`` parameter +which will be a dictionary of keywords. +Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a +function. + .. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: @@ -833,19 +547,6 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdarg_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "keywdarg", - .m_size = 0, - .m_methods = keywdarg_methods, - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModuleDef_Init(&keywdarg_module); - } - .. _buildvalue: @@ -938,7 +639,7 @@ and the object is freed. An alternative strategy is called :dfn:`automatic garbage collection`. (Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big +strategy, hence the use of "automatic" to distinguish the two.) The big advantage of automatic garbage collection is that the user doesn't need to call :c:func:`free` explicitly. (Another claimed advantage is an improvement in speed or memory usage --- this is no hard fact however.) The disadvantage is that for @@ -986,11 +687,11 @@ needed. Ownership of a reference can be transferred. There are three ways to dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. Forgetting to dispose of an owned reference creates a memory leak. -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The +It is also possible to :dfn:`borrow` [#borrow]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. +memory and should be avoided completely [#dont-check-refcount]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1169,7 +870,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees -that it is always a tuple [#]_. +that it is always a tuple [#old-calling-convention]_. It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. @@ -1200,9 +901,6 @@ define this symbol). Providing a C API for an Extension Module ========================================= -.. sectionauthor:: Konrad Hinsen - - Many extension modules just provide new functions and types to be used from Python, but sometimes the code in an extension module can be useful for other extension modules. For example, an extension module could implement a type @@ -1226,8 +924,8 @@ the module whose functions one wishes to call might not have been loaded yet! Portability therefore requires not to make any assumptions about symbol visibility. This means that all symbols in extension modules should be declared ``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from +avoid name clashes with other extension modules. And it means that symbols +that *should* be accessible from other extension modules must be exported in a different way. Python provides a special mechanism to pass C-level information (pointers) from @@ -1269,8 +967,9 @@ file corresponding to the module provides a macro that takes care of importing the module and retrieving its C API pointers; client modules only have to call this macro before accessing the C API. -The exporting module is a modification of the :mod:`!spam` module from section -:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call +The exporting module is a modification of the :mod:`!spam` module from the +:ref:`tutorial `. +The function :func:`!spam.system` does not call the C library function :c:func:`system` directly, but a function :c:func:`!PySpam_System`, which would of course do something more complicated in reality (such as adding "spam" to every command). This function @@ -1412,15 +1111,14 @@ code distribution). .. rubric:: Footnotes -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. - -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. +.. [#borrow] The metaphor of "borrowing" a reference is not completely correct: + the owner still has a copy of the reference. -.. [#] Checking that the reference count is at least 1 **does not work** --- the +.. [#dont-check-refcount] Checking that the reference count is at least 1 + **does not work** --- the reference count itself could be in freed memory and may thus be reused for another object! -.. [#] These guarantees don't hold when you use the "old" style calling convention --- +.. [#old-calling-convention] These guarantees don't hold when you use the + "old" style calling convention --- this is still found in much existing code. diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst new file mode 100644 index 000000000000000..55a772e2aca24f5 --- /dev/null +++ b/Doc/extending/first-extension-module.rst @@ -0,0 +1,699 @@ +.. highlight:: c + + +.. _extending-simpleexample: +.. _first-extension-module: + +********************************* +Your first C API extension module +********************************* + +This tutorial will take you through creating a simple +Python extension module written in C or C++. + +We will use the low-level Python C API directly. +For easier ways to create extension modules, see +the :ref:`recommended third party tools `. + +The tutorial assumes basic knowledge about Python: you should be able to +define functions in Python code before starting to write them in C. +See :ref:`tutorial-index` for an introduction to Python itself. + +The tutorial should be approachable for anyone who can write a basic C library. +While we will mention several concepts that a C beginner would not be expected +to know, like ``static`` functions or linkage declarations, understanding these +is not necessary for success. + +We will focus on giving you a "feel" of what Python's C API is like. +It will not teach you important concepts, like error handling +and reference counting, which are covered in later chapters. + +We will assume that you use a Unix-like system (including macOS and +Linux), or Windows. +On other systems, you might need to adjust some details -- for example, +a system command name. + +You need to have a suitable C compiler and Python development headers installed. +On Linux, headers are often in a package like ``python3-dev`` +or ``python3-devel``. + +You need to be able to install Python packages. +This tutorial uses `pip `__ (``pip install``), but you +can substitute any tool that can build and install ``pyproject.toml``-based +projects, like `uv `_ (``uv pip install``). +Preferably, have a :ref:`virtual environment ` activated. + + +.. note:: + + This tutorial uses APIs that were added in CPython 3.15. + To create an extension that's compatible with earlier versions of CPython, + please follow an earlier version of this documentation. + + This tutorial uses C syntax added in C11 and C++20. + If your extension needs to be compatible with earlier standards, + please follow tutorials in documentation for Python 3.14 or below. + + +What we'll do +============= + +Let's create an extension module called ``spam`` [#why-spam]_, +which will include a Python interface to the C +standard library function :c:func:`system`. +This function is defined in ``stdlib.h``. +It takes a C string as argument, runs the argument as a system +command, and returns a result value as an integer. +A manual page for :c:func:`system` might summarize it this way:: + + #include + int system(const char *command); + +Note that like many functions in the C standard library, +this function is already exposed in Python. +In production, use :py:func:`os.system` or :py:func:`subprocess.run` +rather than the module you'll write here. + +We want this function to be callable from Python as follows: + +.. code-block:: pycon + + >>> import spam + >>> status = spam.system("whoami") + User Name + >>> status + 0 + +.. note:: + + The system command ``whoami`` prints out your username. + It's useful in tutorials like this one because it has the same name on + both Unix and Windows. + + +Start with the headers +====================== + +Begin by creating a directory for this tutorial, and switching to it +on the command line. +Then, create a file named :file:`spammodule.c` in your directory. +[#why-spammodule]_ + +In this file, we'll include two headers: :file:`Python.h` to pull in +all declarations of the Python C API, and :file:`stdlib.h` for the +:c:func:`system` function. [#stdlib-h]_ + +Add the following lines to :file:`spammodule.c`: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: + :end-at: + +Be sure to put :file:`stdlib.h`, and any other standard library includes, +*after* :file:`Python.h`. +On some systems, Python may define some pre-processor definitions +that affect the standard headers. + + +Running your build tool +======================= + +With only the includes in place, your extension won't do anything. +Still, it's a good time to compile it and try to import it. +This will ensure that your build tool works, so that you can make +and test incremental changes as you follow the rest of the text. + +CPython itself does not come with a tool to build extension modules; +it is recommended to use a third-party project for this. +In this tutorial, we'll use `meson-python`_. +(If you want to use another one, see :ref:`first-extension-other-tools`.) + +.. at the time of writing, meson-python has the least overhead for a + simple extension using PyModExport. + Change this if another tool makes things easier. + +``meson-python`` requires defining a "project" using two extra files. + +First, add ``pyproject.toml`` with these contents: + +.. code-block:: toml + + [build-system] + build-backend = 'mesonpy' + requires = ['meson-python'] + + [project] + # Placeholder project information + # (change this before distributing the module) + name = 'sampleproject' + version = '0' + +Then, create ``meson.build`` containing the following: + +.. code-block:: meson + + project('sampleproject', 'c') + + py = import('python').find_installation(pure: false) + + py.extension_module( + 'spam', # name of the importable Python module + 'spammodule.c', # the C source file + install: true, + ) + +.. note:: + + See the `meson-python documentation `_ for details on + configuration. + +Now, build install the *project in the current directory* (``.``) via ``pip``: + +.. code-block:: sh + + python -m pip -v install . + +The ``-v`` (``--verbose``) option causes ``pip`` to show the output from +the compiler, which is often useful during development. + +.. tip:: + + If you don't have ``pip`` installed, run ``python -m ensurepip``, + preferably in a :ref:`virtual environment `. + (Or, if you prefer another tool that can build and install + ``pyproject.toml``-based projects, use that.) + +.. _meson-python: https://mesonbuild.com/meson-python/ +.. _virtual environment: https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments + +Note that you will need to run this command again every time you change your +extension. +Unlike Python, C has an explicit compilation step. + +When your extension is compiled and installed, start Python and try to +import it. +This should fail with the following exception: + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + ImportError: dynamic module does not define module export function (PyModExport_spam or PyInit_spam) + + +Module export hook +================== + +The exception you got when you tried to import the module told you that Python +is looking for a "module export function", also known as a +:ref:`module export hook `. +Let's define one. + +First, add a prototype below the ``#include`` lines: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Export hook prototype + :end-before: /// + +.. tip:: + The prototype is not strictly necessary, but some modern compilers emit + warnings without it. + It's generally better to add the prototype than to disable the warning. + +The :c:macro:`PyMODEXPORT_FUNC` macro declares the function's +return type, and adds any special linkage declarations needed +to make the function visible and usable when CPython loads it. + +After the prototype, add the function itself. +For now, make it return ``NULL``: + +.. code-block:: c + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return NULL; + } + +Compile and load the module again. +You should get a different error this time. + +.. code-block:: pycon + + >>> import spam + Traceback (most recent call last): + ... + SystemError: module export hook for module 'spam' failed without setting an exception + +Simply returning ``NULL`` is *not* correct behavior for an export hook, +and CPython complains about it. +That's good -- it means that CPython found the function! +Let's now make it do something useful. + + +The slot table +============== + +Rather than ``NULL``, the export hook should return the information needed to +create a module. +Let's start with the basics: the name and docstring. + +The information should be defined in an array of +:c:type:`PySlot` entries, which are essentially key-value pairs. +Define this array just before your export hook: + +.. code-block:: c + + PyABIInfo_VAR(abi_info); + + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_END + }; + +The :c:macro:`PySlot_STATIC_DATA` macro is used when the slot value +(here: ``&abi_info``, ``"spam"``, and the docstring) is a pointer to constant, +statically allocated data. + +The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot +are a bit of boilerplate that helps prevent extensions compiled for +a different version of Python from crashing the interpreter. + +For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C +strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. + +Note ``PySlot_END`` sentinel entry at the end. +This marks the end of the array. +If you forget it, you'll trigger undefined behavior. + +The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. +This will be a common theme. +CPython only needs to access the export hook; all global variables +and all other functions should generally be ``static``, so that they don't +clash with other extensions. + +Return this array from your export hook instead of ``NULL``: + +.. code-block:: c + :emphasize-lines: 4 + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +Now, recompile and try it out: + +.. code-block:: pycon + + >>> import spam + >>> print(spam) + + +You have an extension module! +Try ``help(spam)`` to see the docstring. + +The next step will be adding a function. + + +.. _backtoexample: + +Exposing a function +=================== + +To expose the :c:func:`system` C function directly to Python, +we'll need to write a layer of glue code to convert arguments from Python +objects to C values, and the C return value back to Python. + +One of the simplest ways to write glue code is a ":c:data:`METH_O`" function, +which takes two Python objects and returns one. +All Python objects -- regardless of the Python type -- are represented in C +as pointers to the :c:type:`PyObject` structure. + +Add such a function above the slots array:: + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + Py_RETURN_NONE; + } + +For now, we ignore the arguments, and use the :c:macro:`Py_RETURN_NONE` +macro, which expands to a ``return`` statement that properly returns +a Python :py:data:`None` object. + +Recompile your extension to make sure you don't have syntax errors. +We haven't yet added ``spam_system`` to the module, so you might get a +warning that ``spam_system`` is unused. + +.. _methodtable: + +Method definitions +------------------ + +To expose the C function to Python, you will need to provide several pieces of +information in a structure called +:c:type:`PyMethodDef` [#why-pymethoddef]_: + +* ``ml_name``: the name of the Python function; +* ``ml_doc``: a docstring; +* ``ml_meth``: the C function to be called; and +* ``ml_flags``: a set of flags describing details like how Python arguments are + passed to the C function. + We'll use :c:data:`METH_O` here -- the flag that matches our + ``spam_system`` function's signature. + +Because modules typically create several functions, these definitions +need to be collected in an array, with a zero-filled sentinel at the end. +Add this array just below the ``spam_system`` function: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module method table + :end-before: /// + +As with module slots, a zero-filled sentinel marks the end of the array. + +Next, we'll add the method to the module. +Add a :c:data:`Py_mod_methods` slot to your :c:type:`PyMethodDef` array: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-after: /// Module slot table + :end-before: /// + :emphasize-lines: 5 + +Recompile your extension again, and test it. +Be sure to restart the Python interpreter, so that ``import spam`` picks +up the new version of the module. + +You should now be able to call the function: + +.. code-block:: pycon + + >>> import spam + >>> print(spam.system) + + >>> print(spam.system('whoami')) + None + +Note that our ``spam.system`` does not yet run the ``whoami`` command; +it only returns ``None``. + +Check that the function accepts exactly one argument, as specified by +the :c:data:`METH_O` flag: + +.. code-block:: pycon + + >>> print(spam.system('too', 'many', 'arguments')) + Traceback (most recent call last): + ... + TypeError: spam.system() takes exactly one argument (3 given) + + +Returning an integer +==================== + +Now, let's take a look at the return value. +Instead of ``None``, we'll want ``spam.system`` to return a number -- that is, +a Python :py:type:`int` object. +Eventually this will be the exit code of a system command, +but let's start with a fixed value, say, ``3``. + +The Python C API provides a function to create a Python :py:type:`int` object +from a C ``int`` value: :c:func:`PyLong_FromLong`. [#why-pylongfromlong]_ + +To call it, replace the ``Py_RETURN_NONE`` with the following 3 lines: + +.. this could be a one-liner, but we want to show the data types here + +.. code-block:: c + :emphasize-lines: 4-6 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + + +Recompile, restart the Python interpreter again, +and check that the function now returns 3: + +.. code-block:: pycon + + >>> import spam + >>> spam.system('whoami') + 3 + + +Accepting a string +================== + +Finally, let's handle the function argument. + +Our C function, :c:func:`!spam_system`, takes two arguments. +The first one, ``PyObject *self``, will be set to the ``spam`` module +object. +This isn't useful in our case, so we'll ignore it. + +The other one, ``PyObject *arg``, will be set to the object that the user +passed from Python. +We expect that it should be a Python string. +In order to use the information in it, we will need +to convert it to a C value -- in this case, a C string (``const char *``). + +There's a slight type mismatch here: Python's :py:class:`str` objects store +Unicode text, but C strings are arrays of bytes. +So, we'll need to *encode* the data, and we'll use the UTF-8 encoding for it. +(UTF-8 might not always be correct for system commands, but it's what +:py:meth:`str.encode` uses by default, +and the C API has special support for it.) + +The function to encode a Python string into a UTF-8 buffer is named +:c:func:`PyUnicode_AsUTF8AndSize` [#why-pyunicodeasutf8]_. +Call it like this: + +.. code-block:: c + :emphasize-lines: 4 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +If :c:func:`PyUnicode_AsUTF8AndSize` is successful, *command* will point to the +resulting C string -- a zero-terminated array of bytes [#embedded-nul]_. +This buffer is managed by the *arg* object, which means we don't need to free +it, but we must follow some rules: + +* We should only use the buffer inside the ``spam_system`` function. + After ``spam_system`` returns, *arg* and the buffer it manages might be + garbage-collected. +* We must not modify it. This is why we use ``const``. + +If :c:func:`PyUnicode_AsUTF8AndSize` was *not* successful, it returns a ``NULL`` +pointer. +When calling *any* Python C API, we always need to handle such error cases. +The way to do this in general is left for later chapters of this documentation. +For now, be assured that we are already handling errors from +:c:func:`PyLong_FromLong` correctly. + +For the :c:func:`PyUnicode_AsUTF8AndSize` call, the correct way to handle +errors is returning ``NULL`` from ``spam_system``. +Add an ``if`` block for this: + + +.. code-block:: c + :emphasize-lines: 5-7 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = 3; + PyObject *result = PyLong_FromLong(status); + return result; + } + +To test that error handling works, compile again, restart Python so that +``import spam`` picks up the new version of your module, and try passing +a non-string value to your function: + +.. code-block:: pycon + + >>> import spam + >>> spam.system(3) + Traceback (most recent call last): + ... + TypeError: bad argument type for built-in operation + +Now, all that is left is calling the C library function :c:func:`system` with +the ``char *`` buffer, and using its result instead of the ``3``: + +.. code-block:: c + :emphasize-lines: 8 + + static PyObject * + spam_system(PyObject *self, PyObject *arg) + { + const char *command = PyUnicode_AsUTF8AndSize(arg); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; + } + +Compile your module, restart Python, and test. +This time, you should see your username -- the output of the ``whoami`` +system command: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('whoami') + User Name + >>> result + 0 + +You can also test with other commands, like ``ls``, ``dir``, or one +that doesn't exist: + +.. code-block:: pycon + + >>> import spam + >>> result = spam.system('nonexistent-command') + sh: line 1: nonexistent-command: command not found + >>> result + 32512 + + +The result +========== + + +Congratulations! +You have written a complete Python C API extension module, +and completed this tutorial! + +Here is the entire source file, for your convenience: + +.. _extending-spammodule-source: + +.. literalinclude:: ../includes/capi-extension/spammodule-01.c + :start-at: /// + + +.. _first-extension-other-tools: + +Appendix: Other build tools +=========================== + +You should be able to follow this tutorial -- except the +*Running your build tool* section itself -- with a build tool other +than ``meson-python``. + +The Python Packaging User Guide has a `list of recommended tools `_; +be sure to choose one for the C language. + + +Workaround for missing PyInit function +-------------------------------------- + +If your build tool output complains about missing ``PyInit_spam``, +add the following function to your module for now: + +.. code-block:: c + + // A workaround + void *PyInit_spam(void) { return NULL; } + +This is a shim for an old-style :ref:`initialization function `, +which was required in extension modules for CPython 3.14 and below. +Current CPython does not need it, but some build tools may still assume that +all extension modules need to define it. + +If you use this workaround, you will get the exception +``SystemError: initialization of spam failed without raising an exception`` +instead of +``ImportError: dynamic module does not define module export function``. + + +Compiling directly +------------------ + +Using a third-party build tool is heavily recommended, +as it will take care of various details of your platform and Python +installation, of naming the resulting extension, and, later, of distributing +your work. + +If you are building an extension for as *specific* system, or for yourself +only, you might instead want to run your compiler directly. +The way to do this is system-specific; be prepared for issues you will need +to solve yourself. + +Linux +^^^^^ + +On Linux, the Python development package may include a ``python3-config`` +command that prints out the required compiler flags. +If you use it, check that it corresponds to the CPython interpreter you'll use +to load the module. +Then, start with the following command: + +.. code-block:: sh + + gcc --shared $(python3-config --cflags --ldflags) spammodule.c -o spam.so + +This should generate a ``spam.so`` file that you need to put in a directory +on :py:attr:`sys.path`. + + +.. rubric:: Footnotes + +.. [#why-spam] ``spam`` is the favorite food of Monty Python fans... +.. [#why-spammodule] The source file name is entirely up to you, + though some tools can be picky about the ``.c`` extension. + This tutorial uses the traditional ``*module.c`` suffix. + Some people would just use :file:`spam.c` to implement a module + named ``spam``, + projects where Python isn't the primary language might use ``py_spam.c``, + and so on. +.. [#stdlib-h] Including :file:`stdlib.h` is technically not necessary, + since :file:`Python.h` includes it and + :ref:`several other standard headers ` for its own use + or for backwards compatibility. + However, it is good practice to explicitly include what you need. +.. [#why-pymethoddef] The :c:type:`!PyMethodDef` structure is also used + to create methods of classes, so there's no separate + ":c:type:`!PyFunctionDef`". +.. [#why-pylongfromlong] The name :c:func:`PyLong_FromLong` + might not seem obvious. + ``PyLong`` refers to a the Python :py:class:`int`, which was originally + called ``long``; the ``FromLong`` refers to the C ``long`` (or ``long int``) + type. +.. [#why-pyunicodeasutf8] Here, ``PyUnicode`` refers to the original name of + the Python :py:class:`str` class: ``unicode``. + + The ``AndSize`` part of the name refers to the fact that this function can + also retrieve the size of the buffer, using an output argument. + We don't need this, so we set the second argument to NULL. +.. [#embedded-nul] We're ignoring the fact that Python strings can also + contain NUL bytes, which terminate a C string. + In other words, our function will treat ``spam.system("foo\0bar")`` as + ``spam.system("foo")``. + This possibility can lead to security issues, so the real ``os.system`` + function size checks for this case and raises an error. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 4cc2c96d8d5b47e..c0c494c3059d99c 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,15 +5,17 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how +interpreter with new modules. Those modules can do what Python code does -- +define functions, object types and methods -- but also interact with native +libraries or achieve better performance by avoiding the overhead of an +interpreter. The document also describes how to embed the Python interpreter in another application, for use as an extension language. Finally, it shows how to compile and link extension modules so that they can be loaded dynamically (at run time) into the interpreter, if the underlying operating system supports this feature. -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` +This document assumes basic knowledge about C and Python. For an informal +introduction to Python, see :ref:`tutorial-index`. :ref:`reference-index` gives a more formal definition of the language. :ref:`library-index` documents the existing object types, functions and modules (both built-in and written in Python) that give the language its wide application range. @@ -21,37 +23,75 @@ Python) that give the language its wide application range. For a detailed description of the whole Python/C API, see the separate :ref:`c-api-index`. +To support extensions, Python's C API (Application Programmers Interface) +defines a set of functions, macros and variables that provide access to most +aspects of the Python run-time system. The Python API is incorporated in a C +source file by including the header ``"Python.h"``. + +.. note:: + + The C extension interface is specific to CPython, and extension modules do + not work on other Python implementations. In many cases, it is possible to + avoid writing C extensions and preserve portability to other implementations. + For example, if your use case is calling C library functions or system calls, + you should consider using the :mod:`ctypes` module or the `cffi + `_ library rather than writing + custom C code. + These modules let you write Python code to interface with C code and are more + portable between implementations of Python than writing and compiling a C + extension module. + + +.. toctree:: + :hidden: + + first-extension-module.rst + extending.rst + newtypes_tutorial.rst + newtypes.rst + building.rst + windows.rst + embedding.rst + Recommended third party tools ============================= -This guide only covers the basic tools for creating extensions provided +This document only covers the basic tools for creating extensions provided as part of this version of CPython. Some :ref:`third party tools ` offer both simpler and more sophisticated approaches to creating C and C++ extensions for Python. +While this document is aimed at extension authors, it should also be helpful to +the authors of such tools. +For example, the tutorial module can serve as a simple test case for a build +tool or sample expected output of a code generator. + + +C API Tutorial +============== + +This tutorial describes how to write a simple module in C or C++, +using the Python C API -- that is, using the basic tools provided +as part of this version of CPython. -Creating extensions without third party tools -============================================= + +#. :ref:`first-extension-module` + + +Guides for intermediate topics +============================== This section of the guide covers creating C and C++ extensions without assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. -.. seealso:: - - :pep:`489` -- Multi-phase extension module initialization - -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst +* :ref:`extending-intro` +* :ref:`defining-new-types` +* :ref:`new-types-topics` +* :ref:`building` +* :ref:`building-on-windows` Embedding the CPython runtime in a larger application ===================================================== @@ -61,8 +101,4 @@ interpreter as the main application, it is desirable to instead embed the CPython runtime inside a larger application. This section covers some of the details involved in doing that successfully. -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst +* :ref:`embedding` diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 3bbee33bd50698f..9f3cd1d6f4cf33a 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -6,11 +6,6 @@ Defining Extension Types: Tutorial ********************************** -.. sectionauthor:: Michael Hudson -.. sectionauthor:: Dave Kuhlman -.. sectionauthor:: Jim Fulton - - Python allows the writer of a C extension module to define new types that can be manipulated from Python code, much like the built-in :class:`str` and :class:`list` types. The code for all extension types follows a diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index a97c6182553c302..cd81b443603d174 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -47,9 +47,6 @@ things manually, it may be instructive to study the project file for the Differences Between Unix and Windows ==================================== -.. sectionauthor:: Chris Phoenix - - Unix and Windows use completely different paradigms for run-time loading of code. Before you try to build a module that can be dynamically loaded, be aware of how your system works. @@ -109,9 +106,6 @@ separate copy. Using DLLs in Practice ====================== -.. sectionauthor:: Chris Phoenix - - Windows Python is built in Microsoft Visual C++; using other compilers may or may not work. The rest of this section is MSVC++ specific. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ac0aa81e56bb07a..c914089e9806ec6 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -263,6 +263,8 @@ In general, structured switch statements execute one block of code when an expression has a particular value or set of values. Since Python 3.10 one can easily match literal values, or constants within a namespace, with a ``match ... case`` statement. +See :ref:`the specification ` and :ref:`the tutorial ` +for more information about :keyword:`match` statements. An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 6f9dfa8616ed442..c2f8f72ee1f2c4b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1226,13 +1229,13 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. -How do you remove multiple items from a list --------------------------------------------- +How do you remove multiple items from a list? +--------------------------------------------- As with removing duplicates, explicitly iterating in reverse with a delete condition is one possibility. However, it is easier and faster to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: +Here are three variations:: mylist[:] = filter(keep_function, mylist) mylist[:] = (x for x in mylist if keep_condition) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1852,6 +1858,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- @@ -1883,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1916,7 +1924,7 @@ correctly using identity tests: .. code-block:: python - _sentinel = object() + _sentinel = sentinel('_sentinel') def pop(self, key, default=_sentinel): if key in self: @@ -1954,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -1995,11 +2003,11 @@ How do I cache method calls? ---------------------------- The two principal tools for caching methods are -:func:`functools.cached_property` and :func:`functools.lru_cache`. The +:deco:`functools.cached_property` and :deco:`functools.lru_cache`. The former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2008,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2042,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2092,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2114,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2219,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4066d42927f64e..bb00a4f02f0efd5 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -39,10 +39,11 @@ Glossary ABCs with the :mod:`abc` module. annotate function - A function that can be called to retrieve the :term:`annotations ` - of an object. This function is accessible as the :attr:`~object.__annotate__` - attribute of functions, classes, and modules. Annotate functions are a - subset of :term:`evaluate functions `. + A callable that can be called to retrieve the :term:`annotations ` of + an object. Annotate functions are usually :term:`functions `, + automatically generated as the :attr:`~object.__annotate__` attribute of functions, + classes, and modules. Annotate functions are a subset of + :term:`evaluate functions `. annotation A label associated with a variable, a class @@ -95,21 +96,24 @@ Glossary :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. + Informally used to mean either an :term:`asynchronous generator + function` or an :term:`asynchronous generator iterator`, depending on + context. The formal terms :term:`asynchronous generator function` and + :term:`asynchronous generator iterator` are uncommon in practice; + "asynchronous generator" alone is almost always sufficient. + + asynchronous generator function + A function which returns an :term:`asynchronous generator iterator`. + It looks like a coroutine function defined with :keyword:`async def` + except that it contains :keyword:`yield` expressions for producing a + series of values usable in an :keyword:`async for` loop. See :pep:`525`. An asynchronous generator function may contain :keyword:`await` expressions as well as :keyword:`async for`, and :keyword:`async with` statements. asynchronous generator iterator - An object created by an :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator function`. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -134,6 +138,14 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + atomic operation + An operation that appears to execute as a single, indivisible step: no + other thread can observe it half-done, and its effects become visible all + at once. Python does not guarantee that high-level statements are atomic + (for example, ``x += 1`` performs multiple bytecode operations and is not + atomic). Atomicity is only guaranteed where explicitly documented. See + also :term:`race condition` and :term:`data race`. + attached thread state A :term:`thread state` that is active for the current OS thread. @@ -152,9 +164,9 @@ Glossary On most builds of Python, having an attached thread state implies that the caller holds the :term:`GIL` for the current interpreter, so only one OS thread can have an attached thread state at a given moment. In - :term:`free-threaded ` builds of Python, threads can concurrently - hold an attached thread state, allowing for true parallelism of the bytecode - interpreter. + :term:`free-threaded builds ` of Python, threads can + concurrently hold an attached thread state, allowing for true parallelism of + the bytecode interpreter. attribute A value associated with an object which is usually referenced by name @@ -289,6 +301,22 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + concurrency + The ability of a computer program to perform multiple tasks at the same + time. Python provides libraries for writing programs that make use of + different forms of concurrency. :mod:`asyncio` is a library for dealing + with asynchronous tasks and coroutines. :mod:`threading` provides + access to operating system threads and :mod:`multiprocessing` to + operating system processes. Multi-core processors can execute threads and + processes on different CPU cores at the same time (see + :term:`parallelism`). + + concurrent modification + When multiple threads modify shared data at the same time. Concurrent + modification without proper synchronization can cause + :term:`race conditions `, and might also trigger a + :term:`data race `, data corruption, or both. + context This term has different meanings depending on where and how it is used. Some common meanings: @@ -363,10 +391,32 @@ Glossary the :term:`cyclic garbage collector ` is to identify these groups and break the reference cycles so that the memory can be reclaimed. + data race + A situation where multiple threads access the same memory location + concurrently, at least one of the accesses is a write, and the threads + do not use any synchronization to control their access. Data races + lead to :term:`non-deterministic` behavior and can cause data corruption. + Proper use of :term:`locks ` and other :term:`synchronization primitives + ` prevents data races. Note that data races + can only happen in native code, but that :term:`native code` might be + exposed in a Python API. See also :term:`race condition` and + :term:`thread-safe`. + + deadlock + A situation in which two or more tasks (threads, processes, or coroutines) + wait indefinitely for each other to release resources or complete actions, + preventing any from making progress. For example, if thread A holds lock + 1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1, + both threads will wait indefinitely. In Python this often arises from + acquiring multiple locks in conflicting orders or from circular + join/await dependencies. Deadlocks can be avoided by always acquiring + multiple :term:`locks ` in a consistent order. See also + :term:`lock` and :term:`reentrant`. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. + decorators are :deco:`classmethod` and :deco:`staticmethod`. The decorator syntax is merely syntactic sugar, the following two function definitions are semantically equivalent:: @@ -534,6 +584,13 @@ Glossary the :term:`global interpreter lock` which allows only one thread to execute Python bytecode at a time. See :pep:`703`. + free-threaded build + + A build of :term:`CPython` that supports :term:`free threading`, + configured using the :option:`--disable-gil` option before compilation. + + See :ref:`freethreading-python-howto`. + free variable Formally, as defined in the :ref:`language execution model `, a free variable is any variable used in a namespace which is not a local variable in that @@ -587,23 +644,33 @@ Glossary .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. + Informally used to mean either a :term:`generator function` or a + :term:`generator iterator`, depending on context. The formal terms + :term:`generator function` and :term:`generator iterator` are uncommon + in practice; "generator" alone is almost always sufficient. + + .. index:: single: generator function - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. + generator function + A function which returns a :term:`generator` object. It looks like a + normal function except that it contains :keyword:`yield` expressions + for producing a series of values usable in a :keyword:`for`\-loop or + that can be retrieved one at a time with the :func:`next` function. + See :ref:`yieldexpr`. generator iterator - An object created by a :term:`generator` function. + An object created by a :term:`generator function` or a + :term:`generator expression`. Each :keyword:`yield` temporarily suspends processing, remembering the - execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). + execution state (including local variables and pending try-statements). + When the *generator iterator* resumes, it picks up where it left off + (in contrast to functions which start fresh on every invocation). + + Generator iterators also implement the :meth:`~generator.send` method + to send a value into the suspended generator, and the + :meth:`~generator.throw` method to raise an exception at the point + where the generator was paused. See :ref:`generator-methods`. .. index:: single: generator expression @@ -622,7 +689,7 @@ Glossary determined by the dispatch algorithm. See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. + :deco:`functools.singledispatch` decorator, and :pep:`443`. generic type A :term:`type` that can be parameterized; typically a @@ -662,6 +729,14 @@ Glossary requires the GIL to be held in order to use it. This refers to having an :term:`attached thread state`. + global state + Data that is accessible throughout a program, such as module-level + variables, class variables, or C static variables in :term:`extension modules + `. In multi-threaded programs, global state shared + between threads typically requires synchronization to avoid + :term:`race conditions ` and + :term:`data races `. + hash-based pyc A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See @@ -706,7 +781,9 @@ Glossary tuples. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key - in a dictionary. + in a dictionary. Immutable objects are inherently :term:`thread-safe` + because their state cannot be modified after creation, eliminating concerns + about improperly synchronized :term:`concurrent modification`. import path A list of locations (or :term:`path entries `) that are @@ -723,6 +800,19 @@ Glossary An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. + index + A numeric value that represents the position of an element in + a :term:`sequence`. + + In Python, indexing starts at zero. + For example, ``things[0]`` names the *first* element of ``things``; + ``things[1]`` names the second one. + + In some contexts, Python allows negative indexes for counting from the + end of a sequence, and indexing using :term:`slices `. + + See also :term:`subscript`. + interactive Python has an interactive interpreter which means you can enter statements and expressions at the interpreter prompt, immediately @@ -796,9 +886,13 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. - And also please note that the free-threading CPython does not guarantee - the thread-safety of iterator operations. + And also please note that :term:`free-threaded ` + CPython does not guarantee :term:`thread-safe` behavior of iterator + operations. + key + A value that identifies an entry in a :term:`mapping`. + See also :term:`subscript`. key function A key function or collation function is a callable that returns a value @@ -813,7 +907,7 @@ Glossary :func:`itertools.groupby`. There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive + :meth:`str.casefold` method can serve as a key function for case insensitive sorts. Alternatively, a key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, :func:`operator.attrgetter`, :func:`operator.itemgetter`, and @@ -835,10 +929,11 @@ Glossary :keyword:`if` statements. In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another + :term:`race condition` between "the looking" and "the leaping". For example, + the code, ``if key in mapping: return mapping[key]`` can fail if another thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. + This issue can be solved with :term:`locks ` or by using the + :term:`EAFP` approach. See also :term:`thread-safe`. lexical analyzer @@ -857,6 +952,29 @@ Glossary clause is optional. If omitted, all elements in ``range(256)`` are processed. + lock + A :term:`synchronization primitive` that allows only one thread at a + time to access a shared resource. A thread must acquire a lock before + accessing the protected resource and release it afterward. If a thread + attempts to acquire a lock that is already held by another thread, it + will block until the lock becomes available. Python's :mod:`threading` + module provides :class:`~threading.Lock` (a basic lock) and + :class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used + to prevent :term:`race conditions ` and ensure + :term:`thread-safe` access to shared data. Alternative design patterns + to locks exist such as queues, producer/consumer patterns, and + thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + + lock-free + An operation that does not acquire any :term:`lock` and uses atomic CPU + instructions to ensure correctness. Lock-free operations can execute + concurrently without blocking each other and cannot be blocked by + operations that hold locks. In :term:`free-threaded ` + Python, built-in types like :class:`dict` and :class:`list` provide + lock-free read operations, which means other threads may observe + intermediate states during multi-step modifications even when those + modifications hold the :term:`per-object lock`. + loader An object that loads a module. It must define the :meth:`!exec_module` and :meth:`!create_module` methods @@ -942,8 +1060,11 @@ Glossary See :term:`method resolution order`. mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. + An :term:`object` with state that is allowed to change during the course + of the program. In multi-threaded programs, mutable objects that are + shared between threads require careful synchronization to avoid + :term:`race conditions `. See also :term:`immutable`, + :term:`thread-safe`, and :term:`concurrent modification`. named tuple The term "named tuple" applies to any type or class that inherits from @@ -995,6 +1116,13 @@ Glossary See also :term:`module`. + native code + Code that is compiled to machine instructions and runs directly on the + processor, as opposed to code that is interpreted or runs in a virtual + machine. In the context of Python, native code typically refers to + C, C++, Rust or Fortran code in :term:`extension modules ` + that can be called from Python. See also :term:`extension module`. + nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -1011,6 +1139,15 @@ Glossary properties, :meth:`~object.__getattribute__`, class methods, and static methods. + non-deterministic + Behavior where the outcome of a program can vary between executions with + the same inputs. In multi-threaded programs, non-deterministic behavior + often results from :term:`race conditions ` where the + relative timing or interleaving of threads affects the result. + Proper synchronization using :term:`locks ` and other + :term:`synchronization primitives ` helps + ensure deterministic behavior. + object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style @@ -1041,6 +1178,16 @@ Glossary See also :term:`regular package` and :term:`namespace package`. + parallelism + Executing multiple operations at the same time (e.g. on multiple CPU + cores). In Python builds with the + :term:`global interpreter lock (GIL) `, only one + thread runs Python bytecode at a time, so taking advantage of multiple + CPU cores typically involves multiple processes + (e.g. :mod:`multiprocessing`) or native extensions that release the GIL. + In :term:`free-threaded ` Python, multiple Python threads + can run Python code simultaneously on different cores. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -1094,6 +1241,16 @@ Glossary `, the :class:`inspect.Parameter` class, the :ref:`function` section, and :pep:`362`. + per-object lock + A :term:`lock` associated with an individual object instance rather than + a global lock shared across all objects. In :term:`free-threaded + ` Python, built-in types like :class:`dict` and + :class:`list` use per-object locks to allow concurrent operations on + different objects while serializing operations on the same object. + Operations that hold the per-object lock prevent other locking operations + on the same object from proceeding, but do not block :term:`lock-free` + operations. + path entry A single location on the :term:`import path` which the :term:`path based finder` consults to find modules for importing. @@ -1215,6 +1372,18 @@ Glossary >>> email.mime.text.__name__ 'email.mime.text' + race condition + A condition of a program where the behavior + depends on the relative timing or ordering of events, particularly in + multi-threaded programs. Race conditions can lead to + :term:`non-deterministic` behavior and bugs that are difficult to + reproduce. A :term:`data race` is a specific type of race condition + involving unsynchronized access to shared memory. The :term:`LBYL` + coding style is particularly susceptible to race conditions in + multi-threaded code. Using :term:`locks ` and other + :term:`synchronization primitives ` + helps prevent race conditions. + reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are @@ -1236,6 +1405,25 @@ Glossary See also :term:`namespace package`. + reentrant + A property of a function or :term:`lock` that allows it to be called or + acquired multiple times by the same thread without causing errors or a + :term:`deadlock`. + + For functions, reentrancy means the function can be safely called again + before a previous invocation has completed, which is important when + functions may be called recursively or from signal handlers. Thread-unsafe + functions may be :term:`non-deterministic` if they're called reentrantly in a + multithreaded program. + + For locks, Python's :class:`threading.RLock` (reentrant lock) is + reentrant, meaning a thread that already holds the lock can acquire it + again without blocking. In contrast, :class:`threading.Lock` is not + reentrant - attempting to acquire it twice from the same thread will cause + a deadlock. + + See also :term:`lock` and :term:`deadlock`. + REPL An acronym for the "read–eval–print loop", another name for the :term:`interactive` interpreter shell. @@ -1279,10 +1467,11 @@ Glossary chosen based on the type of a single argument. slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. + An object of type :class:`slice`, used to describe a portion of + a :term:`sequence`. + A slice object is created when using the :ref:`slicing ` form + of :ref:`subscript notation `, with colons inside square + brackets, such as in ``variable_name[1:3:5]``. soft deprecated A soft deprecated API should not be used in new code, @@ -1326,6 +1515,14 @@ Glossary stdlib An abbreviation of :term:`standard library`. + steal + In Python's C API, "*stealing*" an argument means that ownership of the + argument is transferred to the called function. + The caller must not use that reference after the call. + Generally, functions that "steal" an argument do so even if they fail. + + See :ref:`api-refcountdetails` for a full explanation. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1340,6 +1537,26 @@ Glossary See also :term:`borrowed reference`. + subscript + The expression in square brackets of a + :ref:`subscription expression `, for example, + the ``3`` in ``items[3]``. + Usually used to select an element of a container. + Also called a :term:`key` when subscripting a :term:`mapping`, + or an :term:`index` when subscripting a :term:`sequence`. + + synchronization primitive + A basic building block for coordinating (synchronizing) the execution of + multiple threads to ensure :term:`thread-safe` access to shared resources. + Python's :mod:`threading` module provides several synchronization primitives + including :class:`~threading.Lock`, :class:`~threading.RLock`, + :class:`~threading.Semaphore`, :class:`~threading.Condition`, + :class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally, + the :mod:`queue` module provides multi-producer, multi-consumer queues + that are especially useful in multithreaded programs. These + primitives help prevent :term:`race conditions ` and + coordinate thread execution. See also :term:`lock`. + t-string t-strings String literals prefixed with ``t`` or ``T`` are commonly called @@ -1392,6 +1609,19 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. + thread-safe + A module, function, or class that behaves correctly when used by multiple + threads concurrently. Thread-safe code uses appropriate + :term:`synchronization primitives ` like + :term:`locks ` to protect shared mutable state, or is designed + to avoid shared mutable state entirely. In the + :term:`free-threaded ` build, built-in types like + :class:`dict`, :class:`list`, and :class:`set` use internal locking + to make many operations thread-safe, although thread safety is not + necessarily guaranteed. Code that is not thread-safe may experience + :term:`race conditions ` and :term:`data races ` + when used in multi-threaded programs. + token A small unit of source code, generated by the diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst index af1e39480cc1f6f..7a7a87cb9584001 100644 --- a/Doc/howto/a-conceptual-overview-of-asyncio.rst +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -1,7 +1,7 @@ .. _a-conceptual-overview-of-asyncio: **************************************** -A Conceptual Overview of :mod:`!asyncio` +A conceptual overview of :mod:`!asyncio` **************************************** This :ref:`HOWTO ` article seeks to help you build a sturdy mental @@ -37,15 +37,15 @@ In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: the event loop, coroutine functions, coroutine objects, tasks, and ``await``. ========== -Event Loop +Event loop ========== Everything in :mod:`!asyncio` happens relative to the event loop. -It's the star of the show. +It's the star of the show, but prefers to work behind the scenes, managing +and coordinating resources. It's like an orchestra conductor. -It's behind the scenes managing resources. Some power is explicitly granted to it, but a lot of its ability to get things -done comes from the respect and cooperation of its worker bees. +done comes from the respect and cooperation of its band members. In more technical terms, the event loop contains a collection of jobs to be run. Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. @@ -59,7 +59,7 @@ This process repeats indefinitely, with the event loop cycling endlessly onwards. If there are no more jobs pending execution, the event loop is smart enough to rest and avoid needlessly wasting CPU cycles, and will come back when there's -more work to be done. +more work to be done, such as when I/O operations complete or timers expire. Effective execution relies on jobs sharing well and cooperating; a greedy job could hog control and leave the other jobs to starve, rendering the overall @@ -115,7 +115,7 @@ The terms "coroutine function" and "coroutine object" are often conflated as coroutine. That can be confusing! In this article, coroutine specifically refers to a coroutine object, or more -precisely, an instance of :data:`types.CoroutineType` (native coroutine). +precisely, an instance of :class:`types.CoroutineType` (native coroutine). Note that coroutines can also exist as instances of :class:`collections.abc.Coroutine` -- a distinction that matters for type checking. @@ -170,14 +170,17 @@ Roughly speaking, :ref:`tasks ` are coroutines (not coroutine functions) tied to an event loop. A task also maintains a list of callback functions whose importance will become clear in a moment when we discuss :keyword:`await`. -The recommended way to create tasks is via :func:`asyncio.create_task`. Creating a task automatically schedules it for execution (by adding a callback to run it in the event loop's to-do list, that is, collection of jobs). +The recommended way to create tasks is via :func:`asyncio.create_task`. -Since there's only one event loop (in each thread), :mod:`!asyncio` takes care of -associating the task with the event loop for you. As such, there's no need -to specify the event loop. +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. :: @@ -250,6 +253,10 @@ different ways:: In a crucial way, the behavior of ``await`` depends on the type of object being awaited. +^^^^^^^^^^^^^^ +Awaiting tasks +^^^^^^^^^^^^^^ + Awaiting a task will cede control from the current task or coroutine to the event loop. In the process of relinquishing control, a few important things happen. @@ -281,6 +288,10 @@ This is a basic, yet reliable mental model. In practice, the control handoffs are slightly more complex, but not by much. In part 2, we'll walk through the details that make this possible. +^^^^^^^^^^^^^^^^^^^ +Awaiting coroutines +^^^^^^^^^^^^^^^^^^^ + **Unlike tasks, awaiting a coroutine does not hand control back to the event loop!** Wrapping a coroutine in a task first, then awaiting that would cede @@ -347,8 +358,10 @@ The design intentionally trades off some conceptual clarity around usage of ``await`` for improved performance. Each time a task is awaited, control needs to be passed all the way up the call stack to the event loop. -That might sound minor, but in a large program with many ``await`` statements and a deep -call stack, that overhead can add up to a meaningful performance drag. +Then, the event loop needs to manage its internal state and work through +its processing logic to resume the next job. +That might sound minor, but in a large program with many ``await``\ s, that +overhead can add up to a non-negligible performance drag. ------------------------------------------------ A conceptual overview part 2: the nuts and bolts @@ -364,7 +377,8 @@ and how to make your own asynchronous operators. The inner workings of coroutines ================================ -:mod:`!asyncio` leverages four components to pass around control. +:mod:`!asyncio` leverages four components of Python to pass +around control. :meth:`coroutine.send(arg) ` is the method used to start or resume a coroutine. @@ -448,9 +462,9 @@ That might sound odd to you. You might be thinking: That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` This was intentionally designed for the sake of simplicity -- mandating only one way of using coroutines. + Despite that, ``yield from`` and ``await`` effectively do the same thing. Initially ``yield`` was barred as well, but was re-accepted to allow for async generators. - Despite that, ``yield from`` and ``await`` effectively do the same thing. ======= Futures diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst new file mode 100644 index 000000000000000..c542efbdea8decc --- /dev/null +++ b/Doc/howto/abi3t-migration.rst @@ -0,0 +1,714 @@ +.. highlight:: c + +.. _abi3t-migration-howto: + +****************************************************** +Migrating to Stable ABI for free threading (``abi3t``) +****************************************************** + +Starting with the 3.15 release, CPython supports a variant of the Stable ABI +that supports :term:`free-threaded ` Python: +Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +This document describes how to adapt C API extensions to support free threading. + +Why do this +=========== + +The typical reason to use Stable ABI is to reduce the number of artifacts that +you need to build and distribute for each version of your library. + +Without the Stable ABI, you must build a separate shared library, and typically +a *wheel* distribution, for each feature version of CPython you wish +to support. +For example, each tag in the following table represents a separate +library/wheel: + ++-----------------+-----------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+=======================+========================+ +| 3.12 | ``cpython-312`` | --- | ++-----------------+-----------------------+------------------------+ +| 3.13 | ``cpython-313`` | ``cpython-313t`` | ++-----------------+-----------------------+------------------------+ +| 3.14 | ``cpython-314`` | ``cpython-314t`` | ++-----------------+-----------------------+------------------------+ +| 3.15 | ``cpython-315`` | ``cpython-315t`` | ++-----------------+-----------------------+------------------------+ +| 3.16 | ``cpython-316`` | ``cpython-316t`` | ++-----------------+-----------------------+------------------------+ +| Later versions | :samp:`cpython-3{XX}` | :samp:`cpython-3{XX}t` | ++-----------------+-----------------------+------------------------+ + +That's a lot of builds, especially when multiplied by the number +of supported platforms. + +With the Stable ABI (``abi3``, introduced in CPython 3.2), a single extension +(per platform) can cover all *non-free-threaded* builds of CPython: + ++-----------------+-------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+========================+ +| 3.12 | ``abi3`` | --- | ++-----------------+ +------------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+ +------------------------+ +| 3.15 | | ``cpython-315t`` | ++-----------------+ +------------------------+ +| 3.16 | | ``cpython-316t`` | ++-----------------+ +------------------------+ +| Later versions | | :samp:`cpython-3{XX}t` | ++-----------------+-------------------+------------------------+ + +The Stable ABI for free-threaded builds (``abi3t``), introduced in +CPython 3.15, does the same for free-threaded builds. +And it's compatible with non-free-threaded ones as well: + ++-----------------+-------------------+------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+==================+ +| 3.12 | ``abi3`` * | --- | ++-----------------+ +------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+-------------------+------------------+ +| 3.15 | ``abi3t`` | ++-----------------+ + +| 3.16 | | ++-----------------+ + +| Later versions | | ++-----------------+-------------------+------------------+ + +\* (As above, the ``abi3`` extension is compatible with all non-free-threaded +builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) + +Why *not* do this +----------------- + +There are two main downsides to Stable ABI. + +First, you extension may become slower, since Stable ABI prioritizes +compatibility over performance. +The difference is usually not noticeable, and often can be mitigated by +using the same source to build both a Stable ABI build and a few +version-specific ones for "tier 1" CPython versions. + +Second, not all of the C API is available. +Extensions need to be ported to build for Stable ABI, which may be difficult +or, in rare cases, impossible. + +Specifically, ``abi3t`` requires APIs added in CPython 3.15. +If you want to build your extension for older versions of CPython from the +same source, you have two main options: + +- Use preprocessor conditionals. + + When following this guide, use ``#ifdef Py_TARGET_ABI3T`` blocks whenever + you are told to do a change that breaks the build on CPython versions you + care about. Keep the pre-existing code in ``#else`` blocks. + + For hand-written C extensions, this approach is reasonable down to + CPython 3.12, due to additions introduced in :pep:`697`. + Keeping compatibility with 3.11 and below may be worth it for code + generators (for example, Cython). + +- Do not port to ``abi3t``, and continue building separate extensions for + each version of CPython, until you can drop support for the older versions. + + This is a valid approach. Not all extensions need to switch to ``abi3t`` + right now. + + +Prerequisites +============= + +This guide assumes that you have an extension written directly in C (or C++), +which you want to port to ``abi3t``. + +If your extenstion uses a code generator (like Cython) or language binding +(like PyO3), it's best to wait until that tool has support for ``abi3t``. +If you maintain such a tool, you might be able to adapt the instructions +here for your tool. + +Non-free-threaded Stable ABI +---------------------------- + +Your extension should support the Stable ABI (``abi3t``). +If not, either port it first, or follow this guide but be prepared to fix +issues it does not mention. + +Free-threading support +---------------------- + +While it's technically not a hard prerequisite, you will most likely want to +prepare your extension for free threading before you port it to ``abi3t``. +See :ref:`freethreading-extensions-howto` for instructions. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `__: + A community-maintained porting guide for extension authors. + +Isolating extension modules +--------------------------- + +Your module should use :ref:`multi-phase initialization `, +and it should either be isolated or limit itself to be loaded at most once +per process. +If it is not your case, follow :ref:`isolating-extensions-howto` first. +(See the :ref:`opt-out section ` for a shortcut.) + +Avoiding variable-sized types +----------------------------- + +If your extension defines variable-sized types (using :c:macro:`Py_tp_itemsize` +or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to +``abi3t`` 3.15. + + +Setting up the build +==================== + +If you use a build tool (such as setuptools, meson-python, scikit-build-core), +search its documentation for a way to select ``abi3t``. +At the time of writing, not all of them have this; but if your tool does, +use it. +You may want to verify that it set the right flag by temporarily adding the +following just after ``#include ``:: + + #if Py_TARGET_ABI3T+0 <= 0x30f0000 + #error "abi3t define is not set!" + #endif + +This should result in a different error than "``abt3t`` define is not set". + +.. note:: + + If your build tool doesn't support ``abi3t`` yet, set the following macro + before including ``Python.h``:: + + #define Py_TARGET_ABI3T 0x30f0000 + + or specify it as a compiler flag, for example:: + + -DPy_TARGET_ABI3T=0x30f0000 + + Once your extension builds with this setting, it will be compatible with + CPython 3.15 and above. + + If you set this macro manually, you will later need to name and tag the + resulting extension manually as well. + This is covered in :ref:`abi3t-migration-tagging` below. + +This guide will ask you to make a series of changes. +After each one, verify that your extension still builds in the original +(non-``abi3t``) configuration, and ideally run tests on all Python +versions you support. +This will ensure that nothing breaks as you are porting. + + +.. _abi3t-howto-modexport: + +Module export hook +================== + +Unless you've done this step already, your extension module defines a +:ref:`module initialization function ` +named :samp:`PyInit_{}`. +You will need to port it to a :ref:`module export hook `, +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:pep:`793`. + +Your existing init function should look like this (with your own names +for ```` and ````): + +.. code-block:: + :class: bad + + PyMODINIT_FUNC + PyInit_(void) + { + return PyModuleDef_Init(&); + } + +If there is some code before the ``return``, move it to +a :c:macro:`Py_mod_create` or :c:macro:`Py_mod_exec` slot function. +See the :ref:`PyInit documentation ` for related information. + +The function references a ``PyModuleDef`` object (```` in the code +above). +Its definition should be similar to the following, with different values +and perhaps some fields unnnamed or left out: + +.. code-block:: + :class: bad + + static PyModuleDef = { + PyModuleDef_HEAD_INIT, + .m_name = "my_module", + .m_doc = "my docstring", + .m_size = sizeof(my_state_struct), + .m_methods = my_methods, + .m_slots = my_slots, + .m_traverse = my_traverse, + .m_clear = my_clear, + .m_free = my_free, + }; + +Remove this definition and the ``PyInit`` function (or put them in +an ``#ifndef Py_TARGET_ABI3T`` block, to retain backwards compatibility), +and replace them with the following: + +.. code-block:: + :class: good + + PyABIInfo_VAR(abi_info); + + static PySlot my_slot_array[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "my_module"), + PySlot_STATIC_DATA(Py_mod_doc, "my docstring"), + PySlot_SIZE(Py_mod_state_size, sizeof(my_state_struct)), + PySlot_STATIC_DATA(Py_mod_methods, my_methods), + PySlot_STATIC_DATA(Py_mod_slots, my_slots), + PySlot_FUNC(Py_mod_state_traverse, my_traverse), + PySlot_FUNC(Py_mod_state_clear, my_clear), + PySlot_FUNC(Py_mod_state_free, my_free), + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_(void) + { + return my_slot_array; + } + +Leave out any fields that were missing (except the new :c:macro:`Py_mod_abi`), +and substitute your own values. + +See the :c:type:`PySlot` and :c:ref:`export hook ` +documentation for details on this API. + +As in the example, your ``PyModExport_`` function should *only* return a +pointer to static data. +If you cannot avoid additional code, refer to the +:ref:`caveats in PyModExport documentation `. + + +Existing slots +-------------- + +If you have a ``Py_mod_slots`` slot, check the array it refers to. +It should be a :c:type:`PyModuleDef_Slot` array like the following: + +.. code-block:: + :class: bad + + static PyObject *create_module(PyObject *spec, PyModuleDef *def) { ... } + static int my_first_module_exec(PyObject *module) { ... } + static int my_second_module_exec(PyObject *module) { ... } + + static PyModuleDef_Slot my_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_create, my_module_create}, + {Py_mod_exec, my_first_module_exec}, + {Py_mod_exec, my_second_module_exec}, + {0, NULL} + }; + +``py_mod_create`` +................. + + +If you have a :c:macro:`Py_mod_create` entry, make sure the function can be +called with ``NULL`` as its second argument (instead of the +:c:type:`PyModuleDef`, which you are removing). +Often, this argument isn't used at all; you can check by renaming it: + +.. code-block:: + :class: good + + static PyObject *create_module(PyObject *spec, PyModuleDef *_unused) { ... } + +If the argument is used, find a different way to pass in the data. +Commonly, the information is static and you can refer to it directly. +(If you're reusing a single function for several different modules, consider +defining several functions instead.) + + +Multiple ``py_mod_exec`` +........................ + +If you have *more than one* :c:macro:`Py_mod_exec` entry, consolidate them: +create a new function that calls the others, and replace existing slots +with it. + +.. code-block:: + :class: good + + static int my_module_exec(PyObject *module) { + if (my_first_module_exec(module) < 0) return -1; + if (my_second_module_exec(module) < 0) return -1; + } + + static PyModuleDef_Slot my_slots[] = { + ... + /* (remove other Py_mod_exec slots) */ + ... + {Py_mod_exec, my_module_exec}, + {0, NULL} + }; + +If the functions aren't used elsewhere, you can combine their bodies instead. + + +Merging slot arrays +................... + +Optionally, when you break compatibility with Python 3.14, you may clean up +the code by moving slots into the :c:type:`PySlot` array, and converting the +definitions to :c:macro:`PySlot_DATA` and :c:macro:`PySlot_FUNC`: + +.. code-block:: + :class: good + + static PySlot my_slot_array[] = { + ... + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + PySlot_FUNC(Py_mod_create, my_module_create), + PySlot_FUNC(Py_mod_exec, my_module_exec), + PySlot_END + }; + +If you do this, delete the original :c:type:`PyModuleDef_Slot` array and +its ``Py_mod_slots`` entry. + + +Associated ``PyModuleDef`` +-------------------------- + +Since the new API does not use a :c:type:`!PyModuleDef` structure, a definition +will not be associated with the resulting module. +This changes the behavior of the following functions: + +- :c:func:`PyModule_GetDef` +- :c:func:`PyType_GetModuleByDef` + +Check your code for these. +If you do not use them, you can skip this section. + +These functions are typically used for two purposes: + +1. To get the definition the module was created with. + This is no longer possible using the new API. + Modules no longer keep a reference to the definition, so you will need to + figure out a different way to pass the relevant data around. + +.. _abi3t-migration-module-token: + +2. To check if a given module object is “yours”. + This use case is now served by :ref:`module tokens ` -- + opaque pointers that identify a module. + To use a token, declare (or reuse) a unique static variable, for example: + + .. code-block:: + :class: good + + static char my_token; + + and add a pointer to it in a new entry to your module's ``PySlot`` array: + + .. code-block:: + :class: good + :emphasize-lines: 3 + + static PySlot my_slot_array[] = { + ... + PySlot_STATIC_DATA(Py_mod_token, &my_token), + PySlot_END + } + + Then, switch from :c:func:`PyModule_GetDef` calls such as: + + .. code-block:: + :class: bad + + PyModuleDef *def = PyModule_GetDef(module); + + to :c:func:`PyModule_GetToken` (which uses an output argument and may fail + with an exception): + + .. code-block:: + :class: good + + void *token; + if (PyModule_GetToken(module, &token) < 0) { + /* handle error */ + } + + and from :c:func:`PyType_GetModuleByDef` calls such as: + + .. code-block:: + :class: bad + + PyObject *module = PyType_GetModuleByDef(type, my_def); + /* handle error; use module */ + + to :c:func:`PyType_GetModuleByToken` (which returns a strong reference): + + .. code-block:: + :class: good + + PyObject *module = PyType_GetModuleByToken(type, my_token); + /* handle error; use module */ + Py_XDECREF(module); + +``PyObject`` opaqueness +======================= + +The :c:type:`PyObject` and :c:type:`PyVarObject` structures are opaque +in ``abi3t``. + +Accessing their members is prohibited. +If you do this, switch to getter/setter functions mentioned in +their documentation: + +- :c:member:`PyObject.ob_type` +- :c:member:`PyObject.ob_refcnt` +- :c:member:`PyVarObject.ob_size` + +Also, the *size* of the :c:type:`PyObject` structures is +unknown to the compiler. +It can -- and *does* -- change between different CPython builds. + +.. note:: + + While the size is available at runtime (for example as + ``sys.getsizeof(object())`` in Python code), you should resist the + temptation to calculate pointer offsets from it. + The object memory layout is subject to change in future + ``abi3t`` implementations. + + +Custom type definitions +----------------------- + +Since :c:type:`!PyObject` is opaque, the traditional way of defining +custom types no longer works: + +.. code-block:: + :class: bad + + typedef struct { + PyObject_HEAD // expands to `PyObject ob_base;` which has unknown size + + int my_data; + } CustomObject; + + static PyType_Spec CustomType_spec = { + ... + .basicsize = sizeof(CustomObject), + ... + }; + +Most likely, all your class definitions, *and* all code that accesses +your classes' data, will need to be rewritten. +This will probably be the biggest change you need to support ``abi3t``. + +For each such type, instead of defining a ``struct`` for the entire instance, +define one with only the “additional” fields -- ones specific to your class, +not its superclasses: + +.. code-block:: + :class: good + + typedef struct { + int my_data; + } CustomObjectData; + +Change the name. +Almost all code that uses the struct will need to change +(notably, pointers to the new structure cannot be cast to/from ``PyObject*``), +and changing the name will highlight the usages as compiler errors. +(If you use ``typeof``, C++ ``auto``, or similar ways to avoid +typing the type name, this won't work. Be extra careful, and consider running +tools to detect undefined behavior.) + +Then, to create the class, use *negative* ``basicsize`` to indicate +“extra” storage space rather than *total* instance size: + +.. code-block:: + :class: good + + static PyType_Spec CustomType_spec = { + ... + .basicsize = -sizeof(CustomObjectData), /* note the minus sign */ + ... + }; + +If you use :c:macro:`Py_tp_members`, set the :c:macro:`Py_RELATIVE_OFFSET` +flag on each member and specify the :c:member:`~PyMemberDef.offset` +relative to your new struct. + + +Custom type data access +----------------------- + +Then comes the hard part: in all code that needs to access this struct, +you will need an additional :c:func:`PyObject_GetTypeData` call to +retrieve a ``CustomObjectData *`` pointer from ``PyObject *``: + +.. code-block:: + :class: good + + PyObject *obj = ...; + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + +Note that this call requires the *type object* for your class (``cls``). + +If your class is not subclassable (that is, it does not use the +:c:macro:`Py_TPFLAGS_BASETYPE` flag), ``cls`` will be ``Py_TYPE(obj)``. +Otherwise, **DO NOT USE** ``Py_TYPE`` with :c:func:`!PyObject_GetTypeData`: +it might return memory reserved to an unrelated subclass! +For example, if a user makes a subclass like this: + +.. code-block:: python + + class Sub(YourCustomClass): + __slots__ = ('a', 'b') + +then ``Py_TYPE(obj)`` is ``Sub``, and the underlying memory may +look like this: + +.. code-block:: text + + ╭─ PyObject *obj + │ ╭─ the pointer you want + │ │ ╭─ PyObject_GetTypeData(obj, Py_TYPE(obj)) + ▼ ▼ ▼ + ┌──────────┬───┬────────────────┬───┬─────────────┬───┬─────────────┐ + │ PyObject │...│ CustomTypeData │...│ PyObject *a │...│ PyObject *b │ + └──────────┴───┴────────────────┴───┴─────────────┴───┴─────────────┘ + +(Ellipses indicate possible padding. +Note that this memory layout is not guaranteed: future versions of Python may +add different padding or even switch the order of the structures.) + +There are two main ways to get the right class: + +- In instance methods, your implementation may use the :c:type:`PyCMethod` + signature (and the :c:macro:`METH_METHOD` bit in + :c:member:`PyMethodDef.ml_flags`), + and get the class as the ``defining_class`` argument. +- Otherwise, give your class a unique static token using the + :c:macro:`Py_tp_token` slot, and use: + + .. code-block:: + :class: good + + PyTypeObject cls; + if (PyType_GetBaseByToken(Py_TYPE(obj), my_tp_token, &cls) < 0) { + /* handle error */ + } + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + + Type tokens work similarly to module tokens covered :ref:`earlier in this + guide `. + + + +Avoid build-time conditionals +============================= + +Check your code for API that identifies the version of Python used to +*build* your extension. +This no longer corresponds to the Python your extension runs on, so code +that uses this information often needs changing. +The macros to check for are: + +- :c:macro:`PY_VERSION_HEX`, :c:macro:`PY_MAJOR_VERSION`, + :c:macro:`PY_MINOR_VERSION`: + + - to get the run-time version, use :c:data:`Py_Version`; + - to determine what C API is available, use :c:macro:`Py_TARGET_ABI3T`. + This macro is set to the minimum supported version. + +- :c:macro:`Py_GIL_DISABLED`: under ``abi3t``, this macro is always defined. + Code that works with free-threaded Python *should* also work with + the GIL enabled (since the GIL can be enabled at run time), + and usually *does* (unless it, for some reason, requires more than one + :term:`attached thread state ` at one time). + + +Further code changes +==================== + +If you are still left with compiler errors or warnings, find a way to fix them. +Alas, this guide is limited, and cannot cover all possible code +changes extensions may need. + +If you find a problem that other extension authors might run into, +consider :ref:`reporting an issue ` (or sending +a pull request) for this guide. + +It is possible your issue cannot be fixed for the current version of ``abi3t``. +In that case, reporting it may help it get prioritized for the next version +of CPython. + + +.. _abi3t-migration-tagging: + +Tagging and distribution +======================== + +If you are using a build tool with ``abi3t`` support, your extension is ready, +but you might want to check that it was built correctly. + +Extensions built with ``abi3t`` should have the following extension: + +- On Windows: ``.pyd`` (like any other extension); +- Linux, macOS, and other systems that use the ``.so`` suffix: ``.abi3t.so`` + (**not** ``.cpython-315t.so`` or ``.abi3.so``). + Note that both free-threaded and non-free-threaded builds will + load ``.abi3t.so`` extensions; +- Other systems: consult your distributor, and perhaps update this guide. + +If you distribute the extension as a *wheel*, use the following tags: + +* Python tag: :samp:`cp3{XX}`, where *XX* is the minimum Python version + the extension is built for. + (For example, ``cp315`` if you set ``Py_TARGET_ABI3T`` to ``0x30f0000``. + See :ref:`abi3-compiling` for more values.) +* ABI tag: ``abi3.abi3t``. This is a *compressed tag set* that indicates + support for both non-free-threaded and free-threaded builds. + +For example, the wheel filename may look like this: + +.. code-block:: text + + myproject-1.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl + +.. seealso:: `Platform Compatibility Tags `__ in the PyPA package distribution metadata. + +If the filename or tags are incorrect, fix them. + + +Testing +======= + +Note that when you build an extension compatible with multiple versions of +CPython, you should always *test* it with each version it supports (for +example, 3.15, 3.16, and so on). +Stable ABI only guarantees *ABI* compatibility; there may also be behavior +changes -- both intentional ones (covered by :pep:`387`) and bugs. + +Be sure to run tests on both free-threaded and non-free-threaded builds +of CPython. + +If they pass, congratulations! You have an ``abi3t`` extension. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index d7deb6c6bc1768f..fcc4fe838c783ba 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -4,8 +4,6 @@ Annotations Best Practices ************************** -:author: Larry Hastings - .. topic:: Abstract This document is designed to encapsulate the best practices @@ -154,7 +152,7 @@ on an arbitrary object ``o``: as the ``globals``, and ``dict(vars(o))`` as the ``locals``, when calling :func:`eval`. * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively + :deco:`functools.wraps`, or :func:`functools.partial`, iteratively unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 902c50de00803c5..0cb8c5cc3ebd369 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -4,8 +4,6 @@ Argparse Tutorial ***************** -:author: Tshepang Mbambo - .. currentmodule:: argparse This tutorial is intended to be a gentle introduction to :mod:`argparse`, the diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 816639552d7cd6d..7bcf63c0b458962 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -6,9 +6,6 @@ .. currentmodule:: curses -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - .. topic:: Abstract @@ -55,8 +52,8 @@ code, all the functions described here will probably be available. The older versions of curses carried by some proprietary Unixes may not support everything, though. -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called :pypi:`UniCurses` is available. +The Windows version of Python doesn't include the :mod:`curses` module. +The third-party :pypi:`windows-curses` package provides the same interface on Windows. The Python curses module diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 9d5a9ac8b718cb0..7233898860ada14 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -4,9 +4,6 @@ Descriptor Guide ================ -:Author: Raymond Hettinger -:Contact: - .. Contents:: @@ -28,7 +25,7 @@ This guide has four major sections: 4) The last section has pure Python equivalents for built-in descriptors that are written in C. Read this if you're curious about how functions turn into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :deco:`classmethod`, :deco:`staticmethod`, :deco:`property`, and :term:`__slots__`. @@ -317,8 +314,8 @@ Descriptors invert that relationship and allow the data being looked-up to have a say in the matter. Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as +bound methods. Common tools like :deco:`classmethod`, :deco:`staticmethod`, +:deco:`property`, and :deco:`functools.cached_property` are all implemented as descriptors. @@ -594,7 +591,7 @@ a pure Python equivalent: def object_getattribute(obj, name): "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() + null = sentinel('null') objtype = type(obj) cls_var = find_name_in_mro(objtype, name, null) descr_get = getattr(type(cls_var), '__get__', null) @@ -1326,7 +1323,7 @@ example calls are unexciting: 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: +:deco:`staticmethod` would look like this: .. testcode:: @@ -1466,7 +1463,7 @@ Now a new dictionary of unique keys can be constructed like this: {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: +:deco:`classmethod` would look like this: .. testcode:: @@ -1604,7 +1601,7 @@ matters when a large number of instances are going to be created. 4. Improves speed. Reading instance variables is 35% faster with ``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). -5. Blocks tools like :func:`functools.cached_property` which require an +5. Blocks tools like :deco:`functools.cached_property` which require an instance dictionary to function correctly: .. testcode:: @@ -1635,12 +1632,12 @@ by member descriptors: .. testcode:: - null = object() + null = sentinel('null') class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 7713aede6d564a4..adb9dc9a4879eb5 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -256,7 +256,7 @@ Ensuring unique enumeration values ---------------------------------- By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, you can use the :func:`unique` decorator:: +When this behavior isn't desired, you can use the :deco:`unique` decorator:: >>> from enum import Enum, unique >>> @unique @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: @@ -509,7 +509,7 @@ to use the standard :func:`repr`. .. note:: - Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + Adding :deco:`~dataclasses.dataclass` decorator to :class:`Enum` and its subclasses is not supported. It will not raise any errors, but it will produce very strange results at runtime, such as members being equal to each other:: @@ -965,75 +965,16 @@ want one of them to be the value:: Finer Points -^^^^^^^^^^^^ - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" +------------ -- :attr:`~Enum._name_` -- name of the member -- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` -- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; - may be overridden -- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a - :class:`str`, that will not be transformed into members, and will be removed - from the final class -- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for - an enum member; may be overridden -- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an - existing member. See `MultiValueEnum`_ for an example. +Supported ``__dunder__`` and ``_sunder_`` names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - .. versionchanged:: 3.13 - Prior versions would use the last seen value instead of the highest value. - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_: - ['RED', 'BLUE', 'GREEN'] - ['RED', 'GREEN', 'BLUE'] - -.. note:: - - In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition - order is lost before it can be recorded. +The supported ``__dunder__`` and ``_sunder_`` names can be found in the :ref:`Enum API documentation `. _Private__names -""""""""""""""" +^^^^^^^^^^^^^^^ :ref:`Private names ` are not converted to enum members, but remain normal attributes. @@ -1042,7 +983,7 @@ but remain normal attributes. ``Enum`` member type -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Enum members are instances of their enum class, and are normally accessed as ``EnumClass.member``. In certain situations, such as writing custom enum @@ -1055,7 +996,7 @@ recommended. Creating members that are mixed with other data types -""""""""""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When subclassing other data types, such as :class:`int` or :class:`str`, with an :class:`Enum`, all values after the ``=`` are passed to that data type's @@ -1069,7 +1010,7 @@ constructor. For example:: Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enum classes that are mixed with non-:class:`Enum` types (such as :class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in @@ -1084,7 +1025,7 @@ Plain :class:`Enum` classes always evaluate as :data:`True`. ``Enum`` classes with methods -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you give your enum subclass extra methods, like the `Planet`_ class below, those methods will show up in a :func:`dir` of the member, @@ -1097,7 +1038,7 @@ but not of the class:: Combining members of ``Flag`` -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Iterating over a combination of :class:`Flag` members will only return the members that are comprised of a single bit:: @@ -1117,7 +1058,7 @@ are comprised of a single bit:: ``Flag`` and ``IntFlag`` minutia -"""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Using the following snippet for our examples:: @@ -1478,6 +1419,7 @@ alias:: behaviors as well as disallowing aliases. If the only desired change is disallowing aliases, the :func:`unique` decorator can be used instead. +.. _multi-value-enum: MultiValueEnum ^^^^^^^^^^^^^^^^^ @@ -1538,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 5647ab2d87c79c3..ad0578df0a27029 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -45,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -60,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. @@ -221,13 +218,15 @@ Thread State and GIL APIs Python provides a set of functions and macros to manage thread state and the GIL, such as: +* :c:func:`PyThreadState_Ensure`, :c:func:`PyThreadState_EnsureFromView`, + and :c:func:`PyThreadState_Release` * :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` * :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` These functions should still be used in the free-threaded build to manage thread state even when the :term:`GIL` is disabled. For example, if you -create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +create a thread outside of Python, you must call :c:func:`PyThreadState_Ensure` before calling into the Python API to ensure that the thread has a valid Python thread state. @@ -387,6 +386,30 @@ Important Considerations internal extension state, standard mutexes or other synchronization primitives might be more appropriate. +.. _per-object-locks: + +Per-Object Locks (``ob_mutex``) +............................... + +In the free-threaded build, each Python object contains a :c:member:`~PyObject.ob_mutex` +field of type :c:type:`PyMutex`. This mutex is **reserved for use by the +critical section API** (:c:macro:`Py_BEGIN_CRITICAL_SECTION` / +:c:macro:`Py_END_CRITICAL_SECTION`). + +.. warning:: + + Do **not** lock ``ob_mutex`` directly with ``PyMutex_Lock(&obj->ob_mutex)``. + Mixing direct ``PyMutex_Lock`` calls with the critical section API on the + same mutex can cause deadlocks. + +Even if your own code never uses critical sections on a particular object type, +**CPython internals may use the critical section API on any Python object**. + +If your extension type needs its own lock, add a separate :c:type:`PyMutex` +field (or another synchronization primitive) to your object struct. +:c:type:`PyMutex` is very lightweight, so there is negligible cost to having +an additional one. + Building Extensions for the Free-Threaded Build =============================================== @@ -395,11 +418,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded - wheels will be built by default. On Python 3.13, you will need to set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index e4df7a787a2b172..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -11,9 +11,7 @@ available processing power by running threads in parallel on available CPU cores While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -The free-threaded mode is working and continues to be improved, but -there is some additional overhead in single-threaded workloads compared -to the regular build. Additionally, third-party packages, in particular ones +Some third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. @@ -101,63 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage under Python 3.13. This -has been addressed in the 3.14 release, where the aforementioned objects use -deferred reference counting to avoid reference count contention. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized in the 3.13 release. This -behavior is part of the 3.14 release as well and it is expected to remain in -future free-threaded builds. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. - -The specializing adaptive interpreter has been re-enabled in a thread-safe way -in the 3.14 release. The performance penalty on single-threaded code in -free-threaded mode is now roughly 5-10%, depending on the platform and C -compiler used. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes @@ -188,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 053558e389030a8..a61fdaee27f6b18 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,9 +4,6 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling -:Release: 0.32 - In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the concepts of functional programming, we'll look at language features such as @@ -1042,7 +1039,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) @@ -1185,7 +1182,8 @@ about whether this lambda-free style is better. Revision History and Acknowledgements ===================================== -The author would like to thank the following people for offering suggestions, +This HOWTO was originally written by A. M. Kuchling. The author would like to +thank the following people for offering suggestions, corrections and assistance with various drafts of this article: Ian Bicking, Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro Lameiro, Jussi Salmela, Collin Winter, Blake Winton. @@ -1239,9 +1237,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__. Python documentation diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 98ce813ca4ab024..33d1fbf8cd9e9ea 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -136,7 +136,7 @@ enabled:: at Objects/unicodeobject.c:551 #7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569 #8 0x0000000000584abd in PyDict_GetItemString (v= - {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': , '__doc__': None}, key= + {'Yuck': , '__builtins__': , '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': , 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__name__': '__main__', 'z': , '__doc__': None}, key= 0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171 Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 81fc7e63f35bd79..57e2d6e0752447f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -37,6 +37,7 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + abi3t-migration.rst remote_debugging.rst General: @@ -61,6 +62,7 @@ Advanced development: * :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` +* :ref:`abi3t-migration-howto` * :ref:`python_2.3_mro` * :ref:`socket-howto` * :ref:`timerfd-howto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index b3db1189e5dcbcd..8f0b0c41ea48617 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -6,9 +6,6 @@ Instrumenting CPython with DTrace and SystemTap =============================================== -:author: David Malcolm -:author: Łukasz Langa - DTrace and SystemTap are monitoring tools, each providing a way to inspect what the processes on a computer system are doing. They both use domain-specific languages allowing a user to write scripts which: @@ -341,6 +338,84 @@ Available static markers .. versionadded:: 3.8 +C Entry Points +^^^^^^^^^^^^^^ + +To simplify triggering of DTrace markers, Python's C API comes with a number +of helper functions that mirror each static marker. On builds of Python without +DTrace enabled, these do nothing. + +In general, it is not necessary to call these yourself, as Python will do +it for you. + +.. list-table:: + :widths: 50 25 25 + :header-rows: 1 + + * * C API Function + * Static Marker + * Notes + * * .. c:function:: void PyDTrace_LINE(const char *arg0, const char *arg1, int arg2) + * :c:func:`!line` + * + * * .. c:function:: void PyDTrace_FUNCTION_ENTRY(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__entry` + * + * * .. c:function:: void PyDTrace_FUNCTION_RETURN(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__return` + * + * * .. c:function:: void PyDTrace_GC_START(int arg0) + * :c:func:`!gc__start` + * + * * .. c:function:: void PyDTrace_GC_DONE(Py_ssize_t arg0) + * :c:func:`!gc__done` + * + * * .. c:function:: void PyDTrace_INSTANCE_NEW_START(int arg0) + * :c:func:`!instance__new__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_NEW_DONE(int arg0) + * :c:func:`!instance__new__done` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_START(int arg0) + * :c:func:`!instance__delete__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_DONE(int arg0) + * :c:func:`!instance__delete__done` + * Not used by Python + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) + * :c:func:`!import__find__load__start` + * + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) + * :c:func:`!import__find__load__done` + * + * * .. c:function:: void PyDTrace_AUDIT(const char *arg0, void *arg1) + * :c:func:`!audit` + * + + +C Probing Checks +^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDTrace_LINE_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_ENTRY_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_RETURN_ENABLED(void) +.. c:function:: int PyDTrace_GC_START_ENABLED(void) +.. c:function:: int PyDTrace_GC_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) +.. c:function:: int PyDTrace_AUDIT_ENABLED(void) + + All calls to ``PyDTrace`` functions must be guarded by a call to one + of these functions. This allows Python to minimize performance impact + when probing is disabled. + + On builds without DTrace enabled, these functions do nothing and return + ``0``. + SystemTap Tapsets ----------------- diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst index e852db98156facc..646c4c4d9e7ab14 100644 --- a/Doc/howto/ipaddress.rst +++ b/Doc/howto/ipaddress.rst @@ -8,9 +8,6 @@ An introduction to the ipaddress module *************************************** -:author: Peter Moody -:author: Nick Coghlan - .. topic:: Overview This document aims to provide a gentle introduction to the diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a39795e8..6092c75f48fdef0 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 52537a91df542c1..87025814aafb9ab 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4,8 +4,6 @@ Logging Cookbook ================ -:Author: Vinay Sajip - This page contains a number of recipes related to logging, which have been found useful in the past. For links to tutorial and reference information, please see :ref:`cookbook-ref-links`. @@ -229,7 +227,7 @@ messages should not. Here's how you can achieve this:: # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger - logging.getLogger('').addHandler(console) + logging.getLogger().addHandler(console) # Now, we can log to the root logger, or any other logger. First the root... logging.info('Jackdaws love my big sphinx of quartz.') @@ -650,7 +648,7 @@ the receiving end. A simple way of doing this is attaching a import logging, logging.handlers - rootLogger = logging.getLogger('') + rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) socketHandler = logging.handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) @@ -1549,10 +1547,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -1564,9 +1562,6 @@ process. This can be set up using a process management tool such as Supervisor - Using file rotation ------------------- -.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see ) - Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of @@ -3619,7 +3614,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3854,7 +3848,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3862,7 +3856,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3880,8 +3874,8 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() - m = self.tz_offset.match(time.strftime('%z')) + asctime = dt.datetime.fromtimestamp(record.created).isoformat() + m = self.tz_offset.prefixmatch(time.strftime('%z')) has_offset = False if m and time.timezone: hrs, mins = m.groups() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index b7225ff1c2cbfc8..c8ce0df9e937f8c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -4,8 +4,6 @@ Logging HOWTO ============= -:Author: Vinay Sajip - .. _logging-basic-tutorial: .. currentmodule:: logging @@ -28,7 +26,7 @@ When to use logging ^^^^^^^^^^^^^^^^^^^ You can access logging functionality by creating a logger via ``logger = -getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +logging.getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and :meth:`~Logger.critical` methods. To determine when to use logging, and to see which logger methods to use when, see the table below. It states, for each of a diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index 0872bedcd3a2d34..72cc311e30d669c 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -10,7 +10,7 @@ The Python 2.3 Method Resolution Order The Method Resolution Order discussed here was *introduced* in Python 2.3, but it is still used in later versions -- including Python 3. -By `Michele Simionato `__. +By `Michele Simionato `__. :Abstract: diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index fc4772bbccab57a..5565f99b244f11b 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -6,8 +6,6 @@ Python support for the ``perf map`` compatible profilers ======================================================== -:author: Pablo Galindo - `The Linux perf profiler `_ and `samply `_ are powerful tools that allow you to profile and obtain information about the performance of your application. @@ -217,8 +215,9 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For best results, Python should be compiled with -``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows +For best results, keep frame pointers enabled. On supported GCC-compatible +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and similar +flags (see :option:`--without-frame-pointers` for details). These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 9f73c811cfcbc0d..f19f6006dbac9e7 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -6,8 +6,6 @@ How to port Python 2 Code to Python 3 ************************************* -:author: Brett Cannon - Python 2 reached its official end-of-life at the start of 2020. This means that no new bug reports, fixes, or changes will be made to Python 2 - it's no longer supported: see :pep:`373` and diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 7486a378dbb06f6..ecdb35136a99f3b 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1,11 +1,9 @@ .. _regex-howto: **************************** - Regular Expression HOWTO + Regular expression HOWTO **************************** -:Author: A.M. Kuchling - .. TODO: Document lookbehind assertions Better way of displaying a RE, a string, and what it matches @@ -47,7 +45,7 @@ Python code to do the processing; while Python code will be slower than an elaborate regular expression, it will also probably be more understandable. -Simple Patterns +Simple patterns =============== We'll start by learning about the simplest possible regular expressions. Since @@ -59,7 +57,7 @@ expressions (deterministic and non-deterministic finite automata), you can refer to almost any textbook on writing compilers. -Matching Characters +Matching characters ------------------- Most letters and characters will simply match themselves. For example, the @@ -159,7 +157,7 @@ match even a newline. ``.`` is often used where you want to match "any character". -Repeating Things +Repeating things ---------------- Being able to match varying sets of characters is the first thing regular @@ -210,7 +208,7 @@ this RE against the string ``'abcbd'``. | | | ``[bcd]*`` is only matching | | | | ``bc``. | +------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | +| 7 | ``abcb`` | Try ``b`` again. This time | | | | the character at the | | | | current position is ``'b'``, so | | | | it succeeds. | @@ -255,7 +253,7 @@ is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use to read. -Using Regular Expressions +Using regular expressions ========================= Now that we've looked at some simple regular expressions, how do we actually use @@ -264,7 +262,7 @@ expression engine, allowing you to compile REs into objects and then perform matches with them. -Compiling Regular Expressions +Compiling regular expressions ----------------------------- Regular expressions are compiled into pattern objects, which have @@ -295,7 +293,7 @@ disadvantage which is the topic of the next section. .. _the-backslash-plague: -The Backslash Plague +The backslash plague -------------------- As stated earlier, regular expressions use the backslash character (``'\'``) to @@ -335,7 +333,7 @@ expressions will often be written in Python code using this raw string notation. In addition, special escape sequences that are valid in regular expressions, but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`, which means the sequences will be invalid if raw string notation or escaping the backslashes isn't used. @@ -351,7 +349,7 @@ the backslashes isn't used. +-------------------+------------------+ -Performing Matches +Performing matches ------------------ Once you have an object representing a compiled regular expression, what do you @@ -362,20 +360,21 @@ for a complete listing. +------------------+-----------------------------------------------+ | Method/Attribute | Purpose | +==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ | ``search()`` | Scan through a string, looking for any | | | location where this RE matches. | +------------------+-----------------------------------------------+ +| ``prefixmatch()``| Determine if the RE matches at the beginning | +| | of the string. Previously named :ref:`match() | +| | `. | ++------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | +| | return them as a list. | +------------------+-----------------------------------------------+ | ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | +| | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If +:meth:`~re.Pattern.search` and :meth:`~re.Pattern.prefixmatch` return ``None`` if no match can be found. If they're successful, a :ref:`match object ` instance is returned, containing information about the match: where it starts and ends, the substring it matched, and more. @@ -393,19 +392,19 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the +:meth:`~re.Pattern.search` should return ``None`` in this case, which will cause the interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: +:meth:`!search` to make this clear. :: - >>> p.match("") - >>> print(p.match("")) + >>> p.search("") + >>> print(p.search("")) None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.Pattern.match` will return a :ref:`match object `, so you +case, :meth:`~re.Pattern.search` will return a :ref:`match object `, so you should store the result in a variable for later use. :: - >>> m = p.match('tempo') + >>> m = p.search('tempo') >>> m @@ -437,27 +436,28 @@ Trying these methods will soon clarify their meaning:: :meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start` and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.Pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: +returns both start and end indexes in a single tuple. +The :meth:`~re.Pattern.search` method of patterns +scans through the string, so the match may not start at zero. +However, the :meth:`~re.Pattern.prefixmatch` +method only checks if the RE matches at the start of a string, so :meth:`!start` +will always be zero in that case. :: - >>> print(p.match('::: message')) - None >>> m = p.search('::: message'); print(m) >>> m.group() 'message' >>> m.span() (4, 11) + >>> print(p.prefixmatch('::: message')) + None In actual programs, the most common style is to store the :ref:`match object ` in a variable, and then check if it was ``None``. This usually looks like:: p = re.compile( ... ) - m = p.match( 'string goes here' ) + m = p.search( 'string goes here' ) if m: print('Match found: ', m.group()) else: @@ -473,7 +473,7 @@ Two pattern methods return all of the matches for a pattern. The ``r`` prefix, making the literal a raw string literal, is needed in this example because escape sequences in a normal "cooked" string literal that are not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`. See :ref:`the-backslash-plague`. :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the @@ -491,19 +491,19 @@ result. The :meth:`~re.Pattern.finditer` method returns a sequence of (29, 31) -Module-Level Functions +Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions +:mod:`re` module also provides top-level functions called :func:`~re.search`, +:func:`~re.prefixmatch`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions take the same arguments as the corresponding pattern method with the RE string added as the first argument, and still return either ``None`` or a :ref:`match object ` instance. :: - >>> print(re.match(r'From\s+', 'Fromage amk')) + >>> print(re.prefixmatch(r'From\s+', 'Fromage amk')) None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS + >>> re.prefixmatch(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS Under the hood, these functions simply create a pattern object for you @@ -518,7 +518,7 @@ Outside of loops, there's not much difference thanks to the internal cache. -Compilation Flags +Compilation flags ----------------- .. currentmodule:: re @@ -642,7 +642,7 @@ of each one. whitespace is in a character class or preceded by an unescaped backslash; this lets you organize and indent the RE more clearly. This flag also lets you put comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped + a ``'#'`` that's neither in a character class nor preceded by an unescaped backslash. For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it @@ -669,7 +669,7 @@ of each one. to understand than the version using :const:`re.VERBOSE`. -More Pattern Power +More pattern power ================== So far we've only covered a part of the features of regular expressions. In @@ -679,7 +679,7 @@ retrieve portions of the text that was matched. .. _more-metacharacters: -More Metacharacters +More metacharacters ------------------- There are some metacharacters that we haven't covered yet. Most of them will be @@ -812,7 +812,7 @@ of a group with a quantifier, such as ``*``, ``+``, ``?``, or ``ab``. :: >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) + >>> print(p.search('ababababab').span()) (0, 10) Groups indicated with ``'('``, ``')'`` also capture the starting and ending @@ -825,7 +825,7 @@ argument. Later we'll see how to express groups that don't capture the span of text that they match. :: >>> p = re.compile('(a)b') - >>> m = p.match('ab') + >>> m = p.search('ab') >>> m.group() 'ab' >>> m.group(0) @@ -836,7 +836,7 @@ to determine the number, just count the opening parenthesis characters, going from left to right. :: >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') + >>> m = p.search('abcd') >>> m.group(0) 'abcd' >>> m.group(1) @@ -875,7 +875,7 @@ Backreferences like this aren't often useful for just searching through a string find out that they're *very* useful when performing string substitutions. -Non-capturing and Named Groups +Non-capturing and named groups ------------------------------ Elaborate REs may use many groups, both to capture substrings of interest, and @@ -912,10 +912,10 @@ but aren't interested in retrieving the group's contents. You can make this fact explicit by using a non-capturing group: ``(?:...)``, where you can replace the ``...`` with any other regular expression. :: - >>> m = re.match("([abc])+", "abc") + >>> m = re.search("([abc])+", "abc") >>> m.groups() ('c',) - >>> m = re.match("(?:[abc])+", "abc") + >>> m = re.search("(?:[abc])+", "abc") >>> m.groups() () @@ -949,7 +949,7 @@ given numbers, so you can retrieve information about a group in two ways:: Additionally, you can retrieve named groups as a dictionary with :meth:`~re.Match.groupdict`:: - >>> m = re.match(r'(?P\w+) (?P\w+)', 'Jane Doe') + >>> m = re.search(r'(?P\w+) (?P\w+)', 'Jane Doe') >>> m.groupdict() {'first': 'Jane', 'last': 'Doe'} @@ -979,7 +979,7 @@ current point. The regular expression for finding doubled words, 'the the' -Lookahead Assertions +Lookahead assertions -------------------- Another zero-width assertion is the lookahead assertion. Lookahead assertions @@ -1061,7 +1061,7 @@ end in either ``bat`` or ``exe``: ``.*[.](?!bat$|exe$)[^.]*$`` -Modifying Strings +Modifying strings ================= Up to this point, we've simply performed searches against a static string. @@ -1083,7 +1083,7 @@ using the following pattern methods: +------------------+-----------------------------------------------+ -Splitting Strings +Splitting strings ----------------- The :meth:`~re.Pattern.split` method of a pattern splits a string apart @@ -1137,7 +1137,7 @@ argument, but is otherwise the same. :: ['Words', 'words, words.'] -Search and Replace +Search and replace ------------------ Another common task is to find all the matches for a pattern, and replace them @@ -1236,7 +1236,7 @@ pattern object as the first parameter, or use embedded modifiers in the pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. -Common Problems +Common problems =============== Regular expressions are a powerful tool for some applications, but in some ways @@ -1244,7 +1244,7 @@ their behaviour isn't intuitive and at times they don't behave the way you may expect them to. This section will point out some of the most common pitfalls. -Use String Methods +Use string methods ------------------ Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed @@ -1274,21 +1274,26 @@ In short, before turning to the :mod:`re` module, consider whether your problem can be solved with a faster and simpler string method. -match() versus search() ------------------------ +.. _match-versus-search: + +prefixmatch() (aka match) versus search() +----------------------------------------- -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: +:func:`~re.prefixmatch` was added in Python 3.15 as the :ref:`preferred name +` for :func:`~re.match`. Before this, it was only known +as :func:`!match` and the distinction with :func:`~re.search` was often +misunderstood. - >>> print(re.match('super', 'superstition').span()) +:func:`!prefixmatch` aka :func:`!match` only checks if the RE matches at the +beginning of the string while :func:`!search` scans forward through the +string for a match. :: + + >>> print(re.prefixmatch('super', 'superstition').span()) (0, 5) - >>> print(re.match('super', 'insuperable')) + >>> print(re.prefixmatch('super', 'insuperable')) None -On the other hand, :func:`~re.search` will scan forward through the string, +On the other hand, :func:`~re.search` scans forward through the string, reporting the first match it finds. :: >>> print(re.search('super', 'superstition').span()) @@ -1296,21 +1301,11 @@ reporting the first match it finds. :: >>> print(re.search('super', 'insuperable').span()) (2, 7) -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. +This distinction is important to remember when using the old :func:`~re.match` +name in code requiring compatibility with older Python versions. -Greedy versus Non-Greedy +Greedy versus non-greedy ------------------------ When repeating a regular expression, as in ``a*``, the resulting action is to @@ -1322,9 +1317,9 @@ doesn't work because of the greedy nature of ``.*``. :: >>> s = 'Title' >>> len(s) 32 - >>> print(re.match('<.*>', s).span()) + >>> print(re.prefixmatch('<.*>', s).span()) (0, 32) - >>> print(re.match('<.*>', s).group()) + >>> print(re.prefixmatch('<.*>', s).group()) Title The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of @@ -1340,7 +1335,7 @@ example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and when it fails, the engine advances a character at a time, retrying the ``'>'`` at every step. This produces just the right result:: - >>> print(re.match('<.*?>', s).group()) + >>> print(re.prefixmatch('<.*?>', s).group()) (Note that parsing HTML or XML with regular expressions is painful. @@ -1388,9 +1383,9 @@ Feedback ======== Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you +understand them? Were there parts that were unclear, or problems you encountered that weren't covered here? If so, please send suggestions for -improvements to the author. +improvements to the :ref:`issue tracker `. The most complete book on regular expressions is almost certainly Jeffrey Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index 78b40bcdf7127b2..1d5cf24d0628432 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -8,6 +8,16 @@ execute Python code remotely. Most platforms require elevated privileges to attach to another Python process. +Disabling remote debugging +-------------------------- + +To disable remote debugging support, use any of the following: + +* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before + starting the interpreter. +* Use the :option:`-X disable_remote_debug` command-line option. +* Compile Python with the :option:`--without-remote-debug` build flag. + .. _permission-requirements: Permission requirements @@ -615,3 +625,57 @@ To inject and execute a Python script in a remote process: 7. Resume the process (if suspended). The script will execute at the next safe evaluation point. +.. _remote-debugging-threat-model: + +Security and threat model +========================= + +The remote debugging protocol relies on the same operating system primitives +used by native debuggers such as GDB and LLDB. Attaching to a process +requires the **same privileges** that those debuggers require, for example +``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and +``SeDebugPrivilege`` on Windows. Python does not introduce any new privilege +escalation path; if an attacker already possesses the permissions needed to +attach to a process, they could equally use GDB to read memory or inject +code. + +The following principles define what is, and is not, considered a security +vulnerability in this feature: + +Attaching requires OS-level privileges + On every supported platform the operating system gates cross-process + memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or + administrator rights). A report that demonstrates an issue only after + these privileges have already been obtained is **not** a vulnerability in + CPython, since the OS security boundary was already crossed. + +Crashes or memory errors when reading a compromised process are not vulnerabilities + A tool that reads internal interpreter state from a target process must + trust that memory to be well-formed. If the target process has been + corrupted or is controlled by an attacker, the debugger or profiler may + crash, produce garbage output, or behave unpredictably. This is the same + risk accepted by every ``ptrace``-based debugger. Bugs in this category + (buffer overflows, segmentation faults, or undefined behaviour triggered + by reading corrupted state) are **not** treated as security issues, though + fixes that improve robustness are welcome. + +Vulnerabilities in the target process are not in scope + If the Python process being debugged has already been compromised, the + attacker already controls execution in that process. Demonstrating further + impact from that starting point does not constitute a vulnerability in the + remote debugging protocol. + +When to use ``PYTHON_DISABLE_REMOTE_DEBUG`` +------------------------------------------- + +The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the +equivalent :option:`-X disable_remote_debug` flag) allows operators to disable +the in-process side of the protocol as a **defence-in-depth** measure. This +may be useful in hardened or sandboxed deployment environments where no +debugging or profiling of the process is expected and reducing attack surface +is a priority, even though the OS-level privilege checks already prevent +unprivileged access. + +Setting this variable does **not** affect other OS-level debugging interfaces +(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available +according to their own permission models. diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index cbc49d15a0771b9..b17ab3f4391dad0 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -4,9 +4,6 @@ Socket Programming HOWTO **************************** -:Author: Gordon McMillan - - .. topic:: Abstract Sockets are used nearly everywhere, but are one of the most severely diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 70c34cde8a06592..b511aa4e59fe1e7 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -3,9 +3,6 @@ Sorting Techniques ****************** -:Author: Andrew Dalke and Raymond Hettinger - - Python lists have a built-in :meth:`list.sort` method that modifies the list in-place. There is also a :func:`sorted` built-in function that builds a new sorted list from an iterable. @@ -375,7 +372,7 @@ Odds and Ends :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` for details on the mechanics). To avoid surprises, :pep:`8` recommends that all six comparison methods be implemented. - The :func:`~functools.total_ordering` decorator is provided to make that + The :deco:`~functools.total_ordering` decorator is provided to make that task easier. * Key functions need not depend directly on the objects being sorted. A key diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe7293553535..cbd53d5b619f5a3 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -4,8 +4,6 @@ Unicode HOWTO ***************** -:Release: 1.12 - This HOWTO discusses Python's support for the Unicode specification for representing textual data, and explains various problems that people commonly encounter when trying to work with Unicode. @@ -352,6 +350,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index d79d1abe8d05770..e4f218f088ba89d 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -4,9 +4,6 @@ HOWTO Fetch Internet Resources Using The urllib Package *********************************************************** -:Author: `Michael Foord `_ - - Introduction ============ @@ -15,7 +12,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. diff --git a/Doc/improve-page-nojs.rst b/Doc/improve-page-nojs.rst new file mode 100644 index 000000000000000..91b3a88b95d38b2 --- /dev/null +++ b/Doc/improve-page-nojs.rst @@ -0,0 +1,29 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the no-javascript version of this page. The one most people + will see (with JavaScript enabled) is improve-page.rst. If you edit + this page, please also edit that one, and vice versa. + +.. only:: html and not epub + +We are always interested to hear ideas about improvements to the documentation. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a topic in the Documentation category: + `New Documentation topic `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new issue with the "docs" label: + `New docs issue `_. diff --git a/Doc/improve-page.rst b/Doc/improve-page.rst new file mode 100644 index 000000000000000..dc89fcb22fbb59c --- /dev/null +++ b/Doc/improve-page.rst @@ -0,0 +1,65 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the JavaScript-enabled version of this page. Another version + (for those with JavaScript disabled) is improve-page-nojs.rst. If you + edit this page, please also edit that one, and vice versa. + +.. only:: html and not epub + + .. raw:: html + + + +We are always interested to hear ideas about improvements to the documentation. + +You were reading "PAGETITLE" at ``_. The source for that page is on +`GitHub `_. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a pre-populated topic: + `Question about page "PAGETITLE" `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new pre-populated issue: + `Docs: problem with page "PAGETITLE" `_. + +- You can `edit the page on GitHub `_ + to open a pull request and begin the contribution process. diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c new file mode 100644 index 000000000000000..8ddb416e7d6cd08 --- /dev/null +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -0,0 +1,58 @@ +/* This file needs to be kept in sync with the tutorial + * at Doc/extending/first-extension-module.rst + */ + +/// Includes + +#include +#include // for system() + +/// Implementation of spam.system + +static PyObject * +spam_system(PyObject *self, PyObject *arg) +{ + const char *command = PyUnicode_AsUTF8AndSize(arg, NULL); + if (command == NULL) { + return NULL; + } + int status = system(command); + PyObject *result = PyLong_FromLong(status); + return result; +} + +/// Module method table + +static PyMethodDef spam_methods[] = { + { + .ml_name="system", + .ml_meth=spam_system, + .ml_flags=METH_O, + .ml_doc="Execute a shell command.", + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +/// Module slot table + +PyABIInfo_VAR(abi_info); + +static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_STATIC_DATA(Py_mod_methods, spam_methods), + PySlot_END +}; + +/// Export hook prototype + +PyMODEXPORT_FUNC PyModExport_spam(void); + +/// Module export hook + +PyMODEXPORT_FUNC +PyModExport_spam(void) +{ + return spam_slots; +} diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83fc08..bc4bd58ff3e3f11 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py index 1fa6e615e46a763..762b1b62fc871d4 100644 --- a/Doc/includes/tzinfo_examples.py +++ b/Doc/includes/tzinfo_examples.py @@ -1,68 +1,70 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) +import datetime as dt # A class capturing the platform's idea of local time. # (May result in wrong values on historical times in # timezones where UTC offset and/or the DST rules had # changed in the past.) -import time as _time +import time + +ZERO = dt.timedelta(0) +HOUR = dt.timedelta(hours=1) +SECOND = dt.timedelta(seconds=1) -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) +STDOFFSET = dt.timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = dt.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET -class LocalTimezone(tzinfo): - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] +class LocalTimezone(dt.tzinfo): + + def fromutc(self, when): + assert when.tzinfo is self + stamp = (when - dt.datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] dst_diff = DSTDIFF // SECOND # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) + fold = (args == time.localtime(stamp - dst_diff)) + return dt.datetime(*args, microsecond=when.microsecond, + tzinfo=self, fold=fold) - def utcoffset(self, dt): - if self._isdst(dt): + def utcoffset(self, when): + if self._isdst(when): return DSTOFFSET else: return STDOFFSET - def dst(self, dt): - if self._isdst(dt): + def dst(self, when): + if self._isdst(when): return DSTDIFF else: return ZERO - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] + def tzname(self, when): + return time.tzname[self._isdst(when)] - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) + def _isdst(self, when): + tt = (when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() # A complete implementation of current DST rules for major US time zones. -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() +def first_sunday_on_or_after(when): + days_to_go = 6 - when.weekday() if days_to_go: - dt += timedelta(days_to_go) - return dt + when += dt.timedelta(days_to_go) + return when # US DST Rules @@ -75,21 +77,22 @@ def first_sunday_on_or_after(dt): # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) +DSTSTART_2007 = dt.datetime(1, 3, 8, 2) # and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) +DSTEND_2007 = dt.datetime(1, 11, 1, 2) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) +DSTSTART_1987_2006 = dt.datetime(1, 4, 1, 2) +DSTEND_1987_2006 = dt.datetime(1, 10, 25, 2) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time) # on the last Sunday of October, which is the first Sunday # on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) +DSTSTART_1967_1986 = dt.datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + def us_dst_range(year): # Find start and end times for US DST. For years before 1967, return # start = end for no DST. @@ -100,17 +103,17 @@ def us_dst_range(year): elif 1966 < year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: - return (datetime(year, 1, 1), ) * 2 + return (dt.datetime(year, 1, 1), ) * 2 start = first_sunday_on_or_after(dststart.replace(year=year)) end = first_sunday_on_or_after(dstend.replace(year=year)) return start, end -class USTimeZone(tzinfo): +class USTimeZone(dt.tzinfo): def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) + self.stdoffset = dt.timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname @@ -118,45 +121,45 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): - if self.dst(dt): + def tzname(self, when): + if self.dst(when): return self.dstname else: return self.stdname - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) + def utcoffset(self, when): + return self.stdoffset + self.dst(when) - def dst(self, dt): - if dt is None or dt.tzinfo is None: + def dst(self, when): + if when is None or when.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. + # implementation) passes a datetime with when.tzinfo is self. return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + assert when.tzinfo is self + start, end = us_dst_range(when.year) # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: + # when first. + when = when.replace(tzinfo=None) + if start + HOUR <= when < end - HOUR: # DST is in effect. return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: + if end - HOUR <= when < end: + # Fold (an ambiguous hour): use when.fold to disambiguate. + return ZERO if when.fold else HOUR + if start <= when < start + HOUR: # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO + return HOUR if when.fold else ZERO # DST is off. return ZERO - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + def fromutc(self, when): + assert when.tzinfo is self + start, end = us_dst_range(when.year) start = start.replace(tzinfo=self) end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset + std_time = when + self.stdoffset dst_time = std_time + HOUR if end <= dst_time < end + HOUR: # Repeated hour diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 3a485a43a5a7518..c372d9f47418001 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,16 +1,14 @@ -.. highlight:: none +.. highlight:: shell .. _installing-index: ************************* -Installing Python Modules +Installing Python modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. +available for other Python developers to use under open-source license terms. This allows Python users to share and collaborate effectively, benefiting from the solutions others have already created to common (and sometimes @@ -34,34 +32,24 @@ creating and sharing your own Python projects, refer to the Key terms ========= -* ``pip`` is the preferred installer program. Starting with Python 3.4, it +* :program:`pip` is the preferred installer program. It is included by default with the Python binary installers. * A *virtual environment* is a semi-isolated Python environment that allows packages to be installed for use by a particular application, rather than being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Package Index `__ is a public +* ``venv`` is the standard tool for creating virtual environments. + It defaults to installing :program:`pip` into all created virtual environments. +* ``virtualenv`` is a third-party alternative (and predecessor) to + ``venv``. +* The `Python Package Index (PyPI) `__ is a public repository of open source licensed packages made available for use by other Python users. -* the `Python Packaging Authority +* The `Python Packaging Authority `__ is the group of developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. @@ -79,7 +67,7 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Package Index:: +dependencies from PyPI:: python -m pip install SomePackage @@ -106,7 +94,7 @@ explicitly:: python -m pip install --upgrade SomePackage -More information and resources regarding ``pip`` and its capabilities can be +More information and resources regarding :program:`pip` and its capabilities can be found in the `Python Packaging User Guide `__. Creation of virtual environments is done through the :mod:`venv` module. @@ -124,19 +112,6 @@ How do I ...? These are quick answers or links for some common tasks. -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - .. installing-per-user-installation: ... install packages just for the current user? @@ -150,10 +125,10 @@ package just for the current user, rather than for all users of the system. --------------------------------------- A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by +aren't currently easy to install using :program:`pip` directly. +It will often be easier for users to install these packages by `other means `__ -rather than attempting to install them with ``pip``. +rather than attempting to install them with :program:`pip`. .. seealso:: @@ -166,22 +141,18 @@ rather than attempting to install them with ``pip``. On Linux, macOS, and other POSIX systems, use the versioned Python commands in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: +:program:`pip`:: - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 + python3 -m pip install SomePackage # default Python 3 + python3.14 -m pip install SomePackage # specifically Python 3.14 -Appropriately versioned ``pip`` commands may also be available. +Appropriately versioned :program:`pip` commands may also be available. -On Windows, use the ``py`` Python launcher in combination with the ``-m`` +On Windows, use the :program:`py` Python launcher in combination with the ``-m`` switch:: - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 + py -3 -m pip install SomePackage # default Python 3 + py -3.14 -m pip install SomePackage # specifically Python 3.14 .. other questions: @@ -201,39 +172,38 @@ On Linux systems, a Python installation will typically be included as part of the distribution. Installing into this Python installation requires root access to the system, and may interfere with the operation of the system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. +is unexpectedly upgraded using :program:`pip`. On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. +per-user installation when installing packages with :program:`pip`. Pip not installed ----------------- -It is possible that ``pip`` does not get installed by default. One potential fix is:: +It is possible that :program:`pip` does not get installed by default. One potential fix is:: python -m ensurepip --default-pip -There are also additional resources for `installing pip. -`__ +There are also additional resources for `installing pip +`__. Installing binary extensions ---------------------------- -Python has typically relied heavily on source based distribution, with end +Python once relied heavily on source-based distribution, with end users being expected to compile extension modules from source as part of the installation process. -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and macOS through the -Python Package Index, this problem is expected to diminish over time, +With the introduction of the binary wheel format, and the +ability to publish wheels through PyPI, this problem is diminishing, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. Some of the solutions for installing `scientific software `__ -that are not yet available as pre-built ``wheel`` files may also help with +that are not yet available as pre-built wheel files may also help with obtaining other binary extensions without needing to build them locally. .. seealso:: diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 5d916b30112d3c8..749e4543c5b8235 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -15,7 +15,7 @@ before the release in which the feature becomes standard. While these future statements are given additional special meaning by the Python compiler, they are still executed like any other import statement and -the :mod:`__future__` exists and is handled by the import system the same way +the :mod:`!__future__` exists and is handled by the import system the same way any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to @@ -23,17 +23,17 @@ any other Python module would be. This design serves three purposes: * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining + can be inspected programmatically via importing :mod:`!__future__` and examining its contents. * To ensure that :ref:`future statements ` run under releases prior to - Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + Python 2.1 at least yield runtime exceptions (the import of :mod:`!__future__` will fail, because there was no module of that name prior to 2.1). Module Contents --------------- -No feature description will ever be deleted from :mod:`__future__`. Since its +No feature description will ever be deleted from :mod:`!__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 47f5eabb6f2180f..13f463a1e95340e 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -162,58 +157,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 49e541a9d9b1cb0..be25a94e7e94f6f 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -4,10 +4,6 @@ .. module:: abc :synopsis: Abstract base classes according to :pep:`3119`. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Georg Brandl -.. much of the content adapted from docstrings - **Source code:** :source:`Lib/abc.py` -------------- @@ -170,17 +166,17 @@ The :mod:`!abc` module also provides the following decorator: or is derived from it. A class that has a metaclass derived from :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used + of the normal 'super' call mechanisms. :deco:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`!abstractmethod` only affects subclasses derived using regular + :deco:`!abstractmethod` only affects subclasses derived using regular inheritance; "virtual subclasses" registered with the ABC's :meth:`~ABCMeta.register` method are not affected. - When :func:`!abstractmethod` is applied in combination with other method + When :deco:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -218,7 +214,7 @@ The :mod:`!abc` module also provides the following decorator: the descriptor must identify itself as abstract using :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: + example, Python's built-in :deco:`property` does the equivalent of:: class Descriptor: ... @@ -241,14 +237,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`classmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`classmethod`, indicating an abstract + classmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`classmethod` decorator + This special case is deprecated, as the :deco:`classmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -262,14 +258,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractstaticmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`staticmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`staticmethod`, indicating an abstract + staticmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`staticmethod` decorator + This special case is deprecated, as the :deco:`staticmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -282,15 +278,15 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractproperty - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`property`, :deco:`property.getter`, + :deco:`property.setter` and :deco:`property.deleter` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`property`, indicating an abstract + A subclass of the built-in :class:`property`, indicating an abstract property. - This special case is deprecated, as the :func:`property` decorator + This special case is deprecated, as the :deco:`property` decorator is now correctly identified as abstract when applied to an abstract method:: diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 40f2a6dc30460b7..af28fe0e2fde2f8 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -510,6 +510,81 @@ annotations from the class and puts them in a separate attribute: return typ +Creating a custom callable annotate function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Custom :term:`annotate functions ` may be literal functions like those +automatically generated for functions, classes, and modules. Or, they may wish to utilise +the encapsulation provided by classes, in which case any :term:`callable` can be used as +an :term:`annotate function`. + +To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or +:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide +the following attribute: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with a supported format. + +To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to +automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are +not supported directly, :term:`annotate functions ` must provide the +following attributes: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with + :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. +* A :ref:`code object ` ``__code__`` containing the compiled code for the + annotate function. +* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the + function represented by ``__code__`` uses any positional defaults. +* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function + represented by ``__code__`` uses any keyword defaults. +* Optional: All other :ref:`function attributes `. + +.. code-block:: python + + class Annotate: + called_formats = [] + + def __call__(self, format=None, /, *, _self=None): + # When called with fake globals, `_self` will be the + # actual self value, and `self` will be the format. + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format <= 2: # VALUE or VALUE_WITH_FAKE_GLOBALS + return {"x": MyType} + raise NotImplementedError + + __code__ = __call__.__code__ + __defaults__ = (None,) + __kwdefaults__ = property(lambda self: dict(_self=self)) + + __globals__ = {} + __builtins__ = {} + __closure__ = None + +This can then be called with: + +.. code-block:: pycon + + >>> from annotationlib import call_annotate_function, Format + >>> call_annotate_function(Annotate(), format=Format.STRING) + {'x': 'MyType'} + +Or used as the annotate function for an object: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> class C: + ... pass + >>> C.__annotate__ = Annotate() + >>> get_annotations(Annotate(), format=Format.STRING) + {'x': 'MyType'} + + Limitations of the ``STRING`` format ------------------------------------ diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 5a8f0bde2e385d1..e4a5f4d109b4992 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -4,16 +4,13 @@ .. module:: argparse :synopsis: Command-line option and argument parsing library. -.. moduleauthor:: Steven Bethard -.. sectionauthor:: Steven Bethard - .. versionadded:: 3.2 **Source code:** :source:`Lib/argparse.py` .. note:: - While :mod:`argparse` is the default recommended standard library module + While :mod:`!argparse` is the default recommended standard library module for implementing basic command line applications, authors with more exacting requirements for exactly how their command line applications behave may find it doesn't provide the necessary level of control. @@ -445,9 +442,8 @@ is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. .. note:: - Empty lines are treated as empty strings (``''``), which are allowed as values but - not as arguments. Empty lines that are read as arguments will result in an - "unrecognized arguments" error. + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -605,18 +601,13 @@ choices (if specified) or subparser names, along with a "maybe you meant" suggestion if a close match is found. Note that this only applies for arguments when the choices specified are strings:: - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=True) - >>> parser.add_argument('--action', choices=['sum', 'max']) - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) - tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') - -You can disable suggestions by setting ``suggest_on_error`` to ``False``:: + >>> parser = argparse.ArgumentParser(suggest_on_error=True) + >>> parser.add_argument('--action', choices=['debug', 'dryrun']) + >>> parser.parse_args(['--action', 'debugg']) + usage: tester.py [-h] [--action {debug,dryrun}] + tester.py: error: argument --action: invalid choice: 'debugg', maybe you meant 'debug'? (choose from debug, dryrun) - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=False) +You can disable suggestions by setting ``suggest_on_error`` to ``False``. .. versionadded:: 3.14 .. versionchanged:: 3.15 @@ -645,6 +636,24 @@ are set. .. versionadded:: 3.14 +To highlight inline code in your description, epilog, or argument ``help`` +text, you can use single or double backticks:: + + >>> parser = argparse.ArgumentParser( + ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... description='Run ``python -m myapp`` to start.', + ... epilog='''Examples: + ... `python -m myapp --verbose` + ... ``python -m myapp --config settings.json`` + ... ''') + >>> parser.add_argument('--foo', help='set the `foo` value') + +When colors are enabled, the text inside backticks will be displayed in a +distinct color to help examples stand out. When colors are disabled, backticks +are preserved as-is, which is readable in plain text. + +.. versionadded:: 3.15 + The add_argument() method ------------------------- @@ -685,6 +694,8 @@ The add_argument() method * deprecated_ - Whether or not use of the argument is deprecated. + The method returns an :class:`Action` object representing the argument. + The following sections describe how each of these are used. @@ -726,13 +737,13 @@ By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. -The dest_ parameter defaults to the argument name with underscores ``_`` -replacing hyphens ``-`` . The metavar_ parameter defaults to the -upper-cased name. For example:: +For optional arguments, the dest_ parameter defaults to the argument name, with +underscores ``_`` replacing hyphens ``-``. The metavar_ parameter defaults to +the upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') - >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() usage: [-h] [--foo-bar FOO-BAR] @@ -767,9 +778,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -789,8 +800,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo 1 --foo 2'.split()) Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -801,8 +812,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -816,7 +827,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -1040,6 +1051,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1103,7 +1118,15 @@ User defined functions can be used as well: The :func:`bool` function is not recommended as a type converter. All it does is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. +This is usually not what is desired:: + + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--verbose', type=bool) + >>> parser.parse_args(['--verbose', 'False']) + Namespace(verbose=True) + +See :class:`BooleanOptionalAction` or ``action='store_true'`` for common +alternatives. In general, the ``type`` keyword is a convenience that should only be used for simple conversions that can only raise one of the three supported exceptions. @@ -1322,8 +1345,12 @@ attribute is determined by the ``dest`` keyword argument of For optional argument actions, the value of ``dest`` is normally inferred from the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from +taking the first double-dash long option string and stripping away the initial +``-`` characters. +If no double-dash long option strings were supplied, ``dest`` will be derived +from the first single-dash long option string by stripping the initial ``-`` +character. +If no long option strings were supplied, ``dest`` will be derived from the first short option string by stripping the initial ``-`` character. Any internal ``-`` characters will be converted to ``_`` characters to make sure the string is a valid attribute name. The examples below illustrate this @@ -1331,11 +1358,12 @@ behavior:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('-f', '--foo-bar', '--foo') + >>> parser.add_argument('-q', '-quz') >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') + >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) + Namespace(foo_bar='1', quz='2', x='3') + >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) + Namespace(foo_bar='1', quz='2', x='2') ``dest`` allows a custom attribute name to be provided:: @@ -1344,6 +1372,14 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. + +.. versionchanged:: 3.15 + Single-dash long option now takes precedence over short options. + .. _deprecated: @@ -1437,8 +1473,18 @@ this API may be passed as the ``action`` parameter to >>> parser.parse_args(['--no-foo']) Namespace(foo=False) + Single-dash long options are also supported. + For example, negative option ``-nofoo`` is automatically added for + positive option ``-foo``. + But no additional options are added for short options such as ``-f``. + .. versionadded:: 3.9 + .. versionchanged:: 3.15 + Added support for single-dash options. + + Added support for alternate prefix_chars_. + The parse_args() method ----------------------- @@ -1661,7 +1707,7 @@ The Namespace object Other utilities --------------- -Sub-commands +Subcommands ^^^^^^^^^^^^ .. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ @@ -1690,7 +1736,7 @@ Sub-commands * *description* - description for the sub-parser group in help output, by default ``None`` - * *prog* - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with subcommand help, by default the name of the program and any positional arguments before the subparser argument @@ -1700,7 +1746,7 @@ Sub-commands * action_ - the basic type of action to be taken when this argument is encountered at the command line - * dest_ - name of the attribute under which sub-command name will be + * dest_ - name of the attribute under which subcommand name will be stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default @@ -1732,13 +1778,18 @@ Sub-commands >>> parser.parse_args(['--foo', 'b', '--baz', 'Z']) Namespace(baz='Z', foo=True) - Note that the object returned by :meth:`parse_args` will only contain + Note that the object returned by :meth:`~ArgumentParser.parse_args` will only contain attributes for the main parser and the subparser that was selected by the command line (and not any other subparsers). So in the example above, when the ``a`` command is specified, only the ``foo`` and ``bar`` attributes are present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -1775,7 +1826,7 @@ Sub-commands -h, --help show this help message and exit --baz {X,Y,Z} baz help - The :meth:`add_subparsers` method also supports ``title`` and ``description`` + The :meth:`~ArgumentParser.add_subparsers` method also supports ``title`` and ``description`` keyword arguments. When either is present, the subparser's commands will appear in their own group in the help output. For example:: @@ -1796,34 +1847,8 @@ Sub-commands {foo,bar} additional help - Furthermore, :meth:`~_SubParsersAction.add_parser` supports an additional - *aliases* argument, - which allows multiple strings to refer to the same subparser. This example, - like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> checkout = subparsers.add_parser('checkout', aliases=['co']) - >>> checkout.add_argument('foo') - >>> parser.parse_args(['co', 'bar']) - Namespace(foo='bar') - - :meth:`~_SubParsersAction.add_parser` supports also an additional - *deprecated* argument, which allows to deprecate the subparser. - - >>> import argparse - >>> parser = argparse.ArgumentParser(prog='chicken.py') - >>> subparsers = parser.add_subparsers() - >>> run = subparsers.add_parser('run') - >>> fly = subparsers.add_parser('fly', deprecated=True) - >>> parser.parse_args(['fly']) # doctest: +SKIP - chicken.py: warning: command 'fly' is deprecated - Namespace() - - .. versionadded:: 3.13 - One particularly effective way of handling subcommands is to combine the use - of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so + of the :meth:`~ArgumentParser.add_subparsers` method with calls to :meth:`~ArgumentParser.set_defaults` so that each subparser knows which Python function it should execute. For example:: @@ -1859,12 +1884,12 @@ Sub-commands >>> args.func(args) ((XYZYX)) - This way, you can let :meth:`parse_args` do the job of calling the + This way, you can let :meth:`~ArgumentParser.parse_args` do the job of calling the appropriate function after argument parsing is complete. Associating functions with actions like this is typically the easiest way to handle the different actions for each of your subparsers. However, if it is necessary to check the name of the subparser that was invoked, the ``dest`` keyword - argument to the :meth:`add_subparsers` call will work:: + argument to the :meth:`~ArgumentParser.add_subparsers` call will work:: >>> parser = argparse.ArgumentParser() >>> subparsers = parser.add_subparsers(dest='subparser_name') @@ -1883,6 +1908,43 @@ Sub-commands the main parser. +.. method:: _SubParsersAction.add_parser(name, *, help=None, aliases=None, \ + deprecated=False, **kwargs) + + Create and return a new :class:`ArgumentParser` object for the + subcommand *name*. + + The *name* argument is the name of the sub-command. + + The *help* argument provides a short description for this sub-command. + + The *aliases* argument allows providing alternative names for this + sub-command. For example:: + + >>> parser = argparse.ArgumentParser() + >>> subparsers = parser.add_subparsers() + >>> checkout = subparsers.add_parser('checkout', aliases=['co']) + >>> checkout.add_argument('foo') + >>> parser.parse_args(['co', 'bar']) + Namespace(foo='bar') + + The *deprecated* argument, if ``True``, marks the sub-command as + deprecated and will issue a warning when used. For example:: + + >>> parser = argparse.ArgumentParser(prog='chicken.py') + >>> subparsers = parser.add_subparsers() + >>> fly = subparsers.add_parser('fly', deprecated=True) + >>> args = parser.parse_args(['fly']) + chicken.py: warning: command 'fly' is deprecated + Namespace() + + All other keyword arguments are passed directly to the + :class:`!ArgumentParser` constructor. + + .. versionadded:: 3.13 + Added the *deprecated* parameter. + + FileType objects ^^^^^^^^^^^^^^^^ @@ -1918,7 +1980,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 @@ -1980,6 +2042,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. deprecated-removed:: 3.11 3.14 Calling :meth:`add_argument_group` on an argument group now raises an exception. This nesting was never supported, often failed to work @@ -2180,6 +2245,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 1f04f697c7507f5..c688d54318e707f 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,10 +9,10 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are +:dfn:`type code`. The following type codes are defined: +-----------+--------------------+-------------------+-----------------------+-------+ @@ -22,9 +22,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -42,26 +40,47 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'e'`` | _Float16 | float | 2 | \(2) | ++-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ +| ``'Zf'`` | float complex | complex | 8 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ +| ``'Zd'`` | double complex | complex | 16 | \(3) | ++-----------+--------------------+-------------------+-----------------------+-------+ + Notes: (1) - It can be 16 bits or 32 bits depending on the platform. + .. versionadded:: 3.13 - .. versionchanged:: 3.9 - ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. +(2) + The IEEE 754 binary16 "half precision" type was introduced in the 2008 + revision of the `IEEE 754 standard `_. + This type is not widely supported by C compilers. It's available + as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. - .. deprecated-removed:: 3.3 3.16 - Please migrate to ``'w'`` typecode. + .. versionadded:: 3.15 -(2) - .. versionadded:: 3.13 +(3) + Complex types (``Zf`` and ``Zd``) are available unconditionally, + regardless on support for complex types (the Annex G of the C11 standard) + by the C compiler. + As specified in the C11 standard, each complex type is represented by a + two-element C array containing, respectively, the real and imaginary parts. + + .. versionadded:: 3.15 + +.. seealso:: + + The :ref:`ctypes ` and + :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. The actual representation of values is determined by the machine architecture @@ -73,7 +92,10 @@ The module defines the following item: .. data:: typecodes - A string with all available type codes. + A tuple with all available type codes. + + .. versionchanged:: 3.15 + The type changed from :class:`str` to :class:`tuple`. The module defines the following type: @@ -93,12 +115,14 @@ The module defines the following type: otherwise, the initializer's iterator is passed to the :meth:`extend` method to add initial items to the array. - Array objects support the ordinary sequence operations of indexing, slicing, + Array objects support the ordinary :ref:`mutable ` :term:`sequence` operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array @@ -112,9 +136,9 @@ The module defines the following type: The length in bytes of one array item in the internal representation. - .. method:: append(x) + .. method:: append(value, /) - Append a new item with value *x* to the end of the array. + Append a new item with the specified value to the end of the array. .. method:: buffer_info() @@ -139,17 +163,18 @@ The module defines the following type: .. method:: byteswap() "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is + 1, 2, 4, 8 or 16 bytes in size; for other types of values, :exc:`RuntimeError` is raised. It is useful when reading data from a file written on a machine with a - different byte order. + different byte order. Note, that for complex types the order of + components (the real part, followed by imaginary part) is preserved. - .. method:: count(x) + .. method:: count(value, /) - Return the number of occurrences of *x* in the array. + Return the number of occurrences of *value* in the array. - .. method:: extend(iterable) + .. method:: extend(iterable, /) Append items from *iterable* to the end of the array. If *iterable* is another array, it must have *exactly* the same type code; if not, :exc:`TypeError` will @@ -157,7 +182,7 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(buffer) + .. method:: frombytes(buffer, /) Appends items from the :term:`bytes-like object`, interpreting its content as an array of machine values (as if it had been read @@ -167,7 +192,7 @@ The module defines the following type: :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - .. method:: fromfile(f, n) + .. method:: fromfile(f, n, /) Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, @@ -175,47 +200,47 @@ The module defines the following type: inserted into the array. - .. method:: fromlist(list) + .. method:: fromlist(list, /) Append items from the list. This is equivalent to ``for x in list: a.append(x)`` except that if there is a type error, the array is unchanged. - .. method:: fromunicode(s) + .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. - The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. + The array must have type code ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. + *value* in the array. The optional arguments *start* and *stop* can be + specified to search for *value* within a subsection of the array. Raise + :exc:`ValueError` if *value* is not found. .. versionchanged:: 3.10 Added optional *start* and *stop* parameters. - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert a new item with value *x* in the array before position *i*. Negative + Insert a new item *value* in the array before position *index*. Negative values are treated as being relative to the end of the array. - .. method:: pop([i]) + .. method:: pop(index=-1, /) Removes the item with the index *i* from the array and returns it. The optional argument defaults to ``-1``, so that by default the last item is removed and returned. - .. method:: remove(x) + .. method:: remove(value, /) - Remove the first occurrence of *x* from the array. + Remove the first occurrence of *value* from the array. .. method:: clear() @@ -240,7 +265,7 @@ The module defines the following type: :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - .. method:: tofile(f) + .. method:: tofile(f, /) Write all items (as machine values) to the :term:`file object` *f*. @@ -252,7 +277,7 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to obtain a Unicode string from an array of some other type. @@ -260,7 +285,7 @@ The module defines the following type: The string representation of array objects has the form ``array(typecode, initializer)``. The *initializer* is omitted if the array is empty, otherwise it is -a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a Unicode string if the *typecode* is ``'w'``, otherwise it is a list of numbers. The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the @@ -282,3 +307,5 @@ Examples:: `NumPy `_ The NumPy package defines another array type. + +.. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0ea3c3c59a660db..4809fdb42bf3d77 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -4,9 +4,6 @@ .. module:: ast :synopsis: Abstract Syntax Tree classes and manipulation. -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - .. testsetup:: import ast @@ -15,7 +12,7 @@ -------------- -The :mod:`ast` module helps Python applications to process trees of the Python +The :mod:`!ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like. @@ -38,15 +35,17 @@ The abstract grammar is currently defined as follows: :language: asdl +.. _ast_nodes: + Node classes ------------ .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C - module and re-exported in :mod:`ast`. + module and re-exported in :mod:`!ast`. There is one class defined for each left-hand side symbol in the abstract grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, @@ -134,17 +133,26 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.13 + + AST node constructors were changed to provide sensible defaults for omitted + fields: optional fields now default to ``None``, list fields default to an + empty list, and fields of type :class:`!ast.expr_context` default to + :class:`Load() `. Previously, omitted attributes would not exist on constructed + nodes (accessing them raised :exc:`AttributeError`). + .. versionchanged:: 3.14 The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes the values of the node fields. -.. deprecated:: 3.8 +.. deprecated-removed:: 3.8 3.14 - Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, - :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. + Previous versions of Python provided the AST classes :class:`!ast.Num`, + :class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and + :class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes + were removed in Python 3.14, and their functionality has been replaced with + :class:`ast.Constant`. .. deprecated:: 3.9 @@ -158,8 +166,16 @@ Node classes Previous versions of Python allowed the creation of AST nodes that were missing required fields. Similarly, AST node constructors allowed arbitrary keyword arguments that were set as attributes of the AST node, even if they did not - match any of the fields of the AST node. This behavior is deprecated and will - be removed in Python 3.15. + match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. + +.. deprecated-removed:: 3.15 3.20 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + .. note:: The descriptions of the specific node classes displayed here @@ -264,18 +280,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the Python object it represents. The values represented can be instances of :class:`str`, :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, and the constants :data:`None` and :data:`Ellipsis`. + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. + .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) @@ -1113,7 +1136,8 @@ Imports names=[ alias(name='x'), alias(name='y'), - alias(name='z')])]) + alias(name='z')], + is_lazy=0)]) .. class:: ImportFrom(module, names, level) @@ -1134,7 +1158,8 @@ Imports alias(name='x'), alias(name='y'), alias(name='z')], - level=0)]) + level=0, + is_lazy=0)]) .. class:: alias(name, asname) @@ -1152,7 +1177,8 @@ Imports names=[ alias(name='a', asname='b'), alias(name='c')], - level=2)]) + level=2, + is_lazy=0)]) Control flow ^^^^^^^^^^^^ @@ -2199,10 +2225,10 @@ Async and await occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` helpers ------------------- +:mod:`!ast` helpers +------------------- -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, the :mod:`!ast` module defines these utility functions and classes for traversing abstract syntax trees: .. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) @@ -2261,7 +2287,7 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. @@ -2419,12 +2445,12 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. - .. deprecated:: 3.8 + .. deprecated-removed:: 3.8 3.14 Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, - :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called + in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle + all constant nodes. .. class:: NodeTransformer() @@ -2471,7 +2497,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2481,6 +2507,10 @@ and classes for traversing abstract syntax trees: numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to true. + If *color* is ``True``, the returned string is syntax highlighted using + ANSI escape sequences. + If ``False`` (the default), colored output is always disabled. + If *indent* is a non-negative integer or string, then the tree will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) @@ -2518,6 +2548,23 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. + .. versionchanged:: 3.15 + Added the *color* parameter. + + +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + .. _ast-compiler-flags: @@ -2554,20 +2601,6 @@ effects on the compilation of a program: .. versionadded:: 3.8 -.. function:: compare(a, b, /, *, compare_attributes=False) - - Recursively compares two ASTs. - - *compare_attributes* affects whether AST attributes are considered - in the comparison. If *compare_attributes* is ``False`` (default), then - attributes are ignored. Otherwise they must all be equal. This - option is useful to check whether the ASTs are structurally equal but - differ in whitespace or similar details. Attributes include line numbers - and column offsets. - - .. versionadded:: 3.14 - - .. _ast-cli: Command-line usage @@ -2575,7 +2608,11 @@ Command-line usage .. versionadded:: 3.9 -The :mod:`ast` module can be executed as a script from the command line. +.. versionchanged:: 3.15 + The output is now syntax highlighted by default. This can be + :ref:`controlled using environment variables `. + +The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 7831b613bd4a605..713b40d746680af 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -248,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # imitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 72f484fd1cbe771..65bc349e2a727d3 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -48,10 +48,10 @@ an event loop: running event loop. If there is no running event loop set, the function will return - the result of the ``get_event_loop_policy().get_event_loop()`` call. + the loop set by :func:`set_event_loop`, or raise a :exc:`RuntimeError` + if no loop has been set. - Because this function has rather complex behavior (especially - when custom event loop policies are in use), using the + Because this function has rather complex behavior, using the :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. @@ -62,13 +62,6 @@ an event loop: .. versionchanged:: 3.14 Raises a :exc:`RuntimeError` if there is no current event loop. - .. note:: - - The :mod:`!asyncio` policy system is deprecated and will be removed - in Python 3.16; from there on, this function will return the current - running event loop if present else it will return the - loop set by :func:`set_event_loop`. - .. function:: set_event_loop(loop) Set *loop* as the current event loop for the current OS thread. @@ -77,10 +70,6 @@ an event loop: Create and return a new event loop object. -Note that the behaviour of :func:`get_event_loop`, :func:`set_event_loop`, -and :func:`new_event_loop` functions can be altered by -:ref:`setting a custom event loop policy `. - .. rubric:: Contents @@ -105,7 +94,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -297,8 +286,9 @@ clocks to track time. are called is undefined. The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. + it is called. Use :func:`functools.partial` + :ref:`to pass keyword arguments ` to + *callback*. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *callback* to run in. @@ -360,7 +350,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -961,7 +951,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -1034,8 +1024,8 @@ Watching file descriptors .. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. + invoke *callback* with the specified arguments *args* once *fd* is + available for writing. Any preexisting callback registered for *fd* is cancelled and replaced by *callback*. @@ -1308,7 +1298,8 @@ Unix signals .. method:: loop.add_signal_handler(signum, callback, *args) - Set *callback* as the handler for the *signum* signal. + Set *callback* as the handler for the *signum* signal, + passing *args* as positional arguments. The callback will be invoked by *loop*, along with other queued callbacks and runnable coroutines of that event loop. Unlike signal handlers @@ -1343,7 +1334,8 @@ Executing code in thread or process pools .. awaitablemethod:: loop.run_in_executor(executor, func, *args) - Arrange for *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor + passing *args* as positional arguments. The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. @@ -1428,7 +1420,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1531,7 +1523,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1669,7 +1661,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1712,7 +1704,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1855,7 +1847,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1968,10 +1960,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: @@ -2052,7 +2044,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 4b69e569523c58b..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -101,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -196,6 +198,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index f5af888f31f1863..faf67209a2b0ef5 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,10 +19,10 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (running or current via the current policy). + - Get the running event loop or the event loop set for the current thread. * - :func:`asyncio.set_event_loop` - - Set the event loop as current via the current policy. + - Set the event loop for the current thread. * - :func:`asyncio.new_event_loop` - Create a new event loop. @@ -497,27 +497,3 @@ Protocol classes can implement the following **callback methods**: - Called when the child process has exited. It can be called before :meth:`~SubprocessProtocol.pipe_data_received` and :meth:`~SubprocessProtocol.pipe_connection_lost` methods. - - -Event Loop Policies -=================== - -Policies is a low-level mechanism to alter the behavior of -functions like :func:`asyncio.get_event_loop`. See also -the main :ref:`policies section ` for more -details. - - -.. rubric:: Accessing Policies -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`asyncio.get_event_loop_policy` - - Return the current process-wide policy. - - * - :meth:`asyncio.set_event_loop_policy` - - Set a new process-wide policy. - - * - :class:`AbstractEventLoopPolicy` - - Base class for policy objects. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst deleted file mode 100644 index 57f964912dd6ea2..000000000000000 --- a/Doc/library/asyncio-policy.rst +++ /dev/null @@ -1,173 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-policies: - -======== -Policies -======== - -.. warning:: - - Policies are deprecated and will be removed in Python 3.16. - Users are encouraged to use the :func:`asyncio.run` function - or the :class:`asyncio.Runner` with *loop_factory* to use - the desired loop implementation. - - -An event loop policy is a global object -used to get and set the current :ref:`event loop `, -as well as create new event loops. -The default policy can be :ref:`replaced ` with -:ref:`built-in alternatives ` -to use different event loop implementations, -or substituted by a :ref:`custom policy ` -that can override these behaviors. - -The :ref:`policy object ` -gets and sets a separate event loop per *context*. -This is per-thread by default, -though custom policies could define *context* differently. - -Custom event loop policies can control the behavior of -:func:`get_event_loop`, :func:`set_event_loop`, and :func:`new_event_loop`. - -Policy objects should implement the APIs defined -in the :class:`AbstractEventLoopPolicy` abstract base class. - - -.. _asyncio-policy-get-set: - -Getting and Setting the Policy -============================== - -The following functions can be used to get and set the policy -for the current process: - -.. function:: get_event_loop_policy() - - Return the current process-wide policy. - - .. deprecated:: 3.14 - The :func:`get_event_loop_policy` function is deprecated and - will be removed in Python 3.16. - -.. function:: set_event_loop_policy(policy) - - Set the current process-wide policy to *policy*. - - If *policy* is set to ``None``, the default policy is restored. - - .. deprecated:: 3.14 - The :func:`set_event_loop_policy` function is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-policy-objects: - -Policy Objects -============== - -The abstract event loop policy base class is defined as follows: - -.. class:: AbstractEventLoopPolicy - - An abstract base class for asyncio policies. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Return an event loop object implementing the - :class:`AbstractEventLoop` interface. - - This method should never return ``None``. - - .. versionchanged:: 3.6 - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object. - - This method should never return ``None``. - - .. deprecated:: 3.14 - The :class:`AbstractEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-policy-builtin: - -asyncio ships with the following built-in policies: - - -.. class:: DefaultEventLoopPolicy - - The default asyncio policy. Uses :class:`SelectorEventLoop` - on Unix and :class:`ProactorEventLoop` on Windows. - - There is no need to install the default policy manually. asyncio - is configured to use the default policy automatically. - - .. versionchanged:: 3.8 - - On Windows, :class:`ProactorEventLoop` is now used by default. - - .. versionchanged:: 3.14 - The :meth:`get_event_loop` method of the default asyncio policy now - raises a :exc:`RuntimeError` if there is no set event loop. - - .. deprecated:: 3.14 - The :class:`DefaultEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. class:: WindowsSelectorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`SelectorEventLoop` event loop implementation. - - .. availability:: Windows. - - .. deprecated:: 3.14 - The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. class:: WindowsProactorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`ProactorEventLoop` event loop implementation. - - .. availability:: Windows. - - .. deprecated:: 3.14 - The :class:`WindowsProactorEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-custom-policies: - -Custom Policies -=============== - -To implement a new event loop policy, it is recommended to subclass -:class:`DefaultEventLoopPolicy` and override the methods for which -custom behavior is wanted, e.g.:: - - class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - loop = super().get_event_loop() - # Do something with loop ... - return loop - - asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 5208f14c94a50f0..58f77feb3119841 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d481a1921d532bb..a9735ae80652df6 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -107,7 +107,7 @@ Queue The queue can no longer grow. Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. Currently blocked callers of :meth:`~Queue.put` will be unblocked - and will raise :exc:`QueueShutDown` in the formerly blocked thread. + and will raise :exc:`QueueShutDown` in the formerly awaiting task. If *immediate* is false (the default), the queue can be wound down normally with :meth:`~Queue.get` calls to extract tasks diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 48d78099fd3ce7e..cd125b30703c796 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -43,9 +43,7 @@ Running an asyncio Program otherwise :func:`asyncio.new_event_loop` is used. The loop is closed at the end. This function should be used as a main entry point for asyncio programs, and should ideally only be called once. It is recommended to use - *loop_factory* to configure the event loop instead of policies. - Passing :class:`asyncio.EventLoop` allows running asyncio without the - policy system. + *loop_factory* to configure the event loop. The executor is given a timeout duration of 5 minutes to shutdown. If the executor hasn't finished within that duration, a warning is @@ -76,12 +74,6 @@ Running an asyncio Program *coro* can be any awaitable object. - .. note:: - - The :mod:`!asyncio` policy system is deprecated and will be removed - in Python 3.16; from there on, an explicit *loop_factory* is needed - to configure the event loop. - Runner context manager ====================== diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 9416c758e51d95d..a6514649bf9a0a8 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -311,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -351,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 863b3e336572aa5..64f0810777e41b9 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,7 +231,7 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` @@ -300,7 +300,7 @@ Creating Tasks Added the *eager_start* parameter by passing on all *kwargs*. -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -324,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -355,6 +355,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: 3.15 + Example:: async def main(): @@ -366,7 +394,8 @@ Example:: The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group (for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). +and calling ``tg.create_task()`` in that coroutine). There is also opportunity to +request termination of the entire task group with ``tg.cancel()``, based on some condition. Once the last task has finished and the ``async with`` block is exited, no new tasks may be added to the group. @@ -427,53 +456,6 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass - - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done - Sleeping ======== @@ -498,13 +480,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -519,7 +501,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -557,7 +539,7 @@ Running Tasks Concurrently provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks: if a task (or a subtask, a task scheduled by a task) raises an exception, *TaskGroup* will, while *gather* will not, - cancel the remaining scheduled tasks). + cancel the remaining scheduled tasks. .. _asyncio_example_gather: @@ -621,7 +603,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -664,7 +646,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -771,6 +753,9 @@ Timeouts An :ref:`asynchronous context manager ` for cancelling overdue coroutines. + Prefer using :func:`asyncio.timeout` or :func:`asyncio.timeout_at` + rather than instantiating :class:`!Timeout` directly. + ``when`` should be an absolute time at which the context should time out, as measured by the event loop's clock: @@ -891,7 +876,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -1011,7 +996,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1071,7 +1056,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1195,7 +1180,7 @@ Introspection .. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) @@ -1245,6 +1230,9 @@ Task Object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-threading.rst b/Doc/library/asyncio-threading.rst new file mode 100644 index 000000000000000..526901a2e7eb206 --- /dev/null +++ b/Doc/library/asyncio-threading.rst @@ -0,0 +1,154 @@ +.. currentmodule:: asyncio + +.. _asyncio-threading: + +asyncio and free-threaded Python +================================ + +asyncio uses an event loop as a scheduler to enable highly efficient +concurrency by switching between tasks to allow non-blocking I/O +operations. This results in better performance for I/O-bound use +cases. It also allows off-loading CPU-bound work to a thread or +process pool, but that is still limited by the :term:`global +interpreter lock` in CPython. + +However, in :ref:`free-threaded Python `, +the GIL is disabled and Python can run true multi-threaded code. This +means that asyncio can now take advantage of multiple CPU cores without +the limitations imposed by the GIL. + +Since Python 3.14, asyncio has first-class support for free-threaded +Python, and the implementation of asyncio is safe to use in a +multi-threaded environment. + +A single event loop on one core can handle many connections +concurrently, but the Python code that runs to handle each one still +executes serially. Once requests involve a non-trivial amount of +per-request computation, that handling becomes the bottleneck, and a +single core can no longer keep up. Combining asyncio with threads is +most useful here: by running an event loop per thread, the handling of +different requests can run in parallel across multiple CPU cores. It is +also useful when you need to run blocking or CPU-bound code from an +asyncio application. + + +.. seealso:: + + `Scaling asyncio on Free-Threaded Python + `__, + a blog post by Kumar Aditya which explains the internal changes + that make asyncio safe and efficient under free-threaded Python, + together with benchmarks of the resulting improvements. + + +Thread safety considerations +---------------------------- + +While asyncio is designed to be thread-safe in a free-threaded Python +environment, there are still some considerations to keep in mind when +using asyncio with threads: + +1. **Event loop**: Each thread should have its own event loop which + should not be shared across threads. This ensures that the event loop + can manage its own tasks and callbacks without interference from + other threads. + +2. **Task management**: Tasks and futures created in one thread should + not be awaited or manipulated from another thread. + +3. **Thread-safe APIs**: When interacting with asyncio from multiple + threads, it's important to use thread-safe APIs provided by asyncio, + such as :func:`asyncio.run_coroutine_threadsafe` for submitting + coroutines to an event loop from another thread. If you need to + call a callback from a different thread, you can use + :meth:`loop.call_soon_threadsafe` to schedule it safely. + +4. **Synchronization**: The synchronization primitives provided by + asyncio (like :class:`asyncio.Lock` and :class:`asyncio.Event`) + are not designed to be used across threads. If you need to + synchronize between threads, you should use the synchronization + primitives from the :mod:`threading` module instead. + + +Using asyncio with threads +-------------------------- + +asyncio supports running one event loop per thread, which allows you to +take advantage of multiple CPU cores in a free-threaded Python +environment. Each thread can run its own event loop, and tasks can be +scheduled on those loops independently. + +Here's an example of how to use asyncio with threads:: + + import asyncio + import threading + + async def worker(name: str) -> None: + print(f"Worker {name} starting") + await asyncio.sleep(1) + print(f"Worker {name} done") + + def run_loop(name: str) -> None: + asyncio.run(worker(name)) + + threads = [ + threading.Thread(target=run_loop, args=(f"T{i}",)) + for i in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + +In this example, each thread creates its own event loop with +:func:`asyncio.run` and runs a coroutine on it. The threads execute +concurrently, and in a free-threaded build they can run on separate +CPU cores in parallel. + + +Producer/consumer across threads +-------------------------------- + +When a regular (non-asyncio) thread needs to hand work to an asyncio +event loop running in another thread, use a thread-safe primitive such +as :class:`queue.Queue` rather than :class:`asyncio.Queue`, which is +only safe within a single event loop.:: + + import asyncio + import queue + import threading + + def producer(q: queue.Queue[int]) -> None: + for i in range(5): + print(f"Producing {i}") + q.put(i) + q.shutdown() + + async def consumer(q: queue.Queue[int]) -> None: + while True: + try: + item = q.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + except queue.ShutDown: + break + print(f"Consumed {item}") + await asyncio.sleep(item) + + q: queue.Queue[int] = queue.Queue() + consumer_thread = threading.Thread( + target=lambda: asyncio.run(consumer(q)) + ) + consumer_thread.start() + producer(q) + consumer_thread.join() + +The producer runs on the main thread while the consumer runs inside an +event loop on its own thread, yet they communicate safely through +``queue.Queue``. When the queue is empty the consumer sleeps briefly +and tries again. When the producer is done it calls +:meth:`~queue.Queue.shutdown`, which causes subsequent +:meth:`~queue.Queue.get_nowait` calls to raise :exc:`queue.ShutDown` +so the consumer can exit cleanly. + diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 0f72e31dee5f1d1..4ae6d1e43f2ac7b 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -117,7 +117,6 @@ for full functionality and the latest features. asyncio-eventloop.rst asyncio-future.rst asyncio-protocol.rst - asyncio-policy.rst asyncio-platforms.rst asyncio-extending.rst @@ -128,6 +127,7 @@ for full functionality and the latest features. asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + asyncio-threading.rst .. note:: The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 02d2f0807df8f69..b5caf5502d0e1c6 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -4,14 +4,11 @@ .. module:: atexit :synopsis: Register and execute cleanup functions. -.. moduleauthor:: Skip Montanaro -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`atexit` module defines functions to register and unregister cleanup +The :mod:`!atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* +interpreter termination. :mod:`!atexit` runs these functions in the *reverse* order in which they were registered; if you register ``A``, ``B``, and ``C``, at interpreter termination time they will be run in the order ``C``, ``B``, ``A``. @@ -64,7 +61,7 @@ a cleanup function is undefined. Remove *func* from the list of functions to be run at interpreter shutdown. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality + of that function in the :mod:`!atexit` call stack will be removed. Equality comparisons (``==``) are used internally during unregistration, so function references do not need to have matching identities. @@ -72,14 +69,14 @@ a cleanup function is undefined. .. seealso:: Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history + Useful example of :mod:`!atexit` to read and write :mod:`readline` history files. .. _atexit-example: -:mod:`atexit` Example ---------------------- +:mod:`!atexit` Example +---------------------- The following simple example demonstrates how a module can initialize a counter from a file when it is imported and save the counter's updated value diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 529a72424438203..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -51,7 +53,7 @@ The :rfc:`4648` encodings are suitable for encoding binary data so that it can b safely sent by email, used as parts of URLs, or included as part of an HTTP POST request. -.. function:: b64encode(s, altchars=None) +.. function:: b64encode(s, altchars=None, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. @@ -61,11 +63,20 @@ POST request. This allows an application to e.g. generate URL or filesystem safe Base64 strings. The default is ``None``, for which the standard Base64 alphabet is used. - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a - :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. -.. function:: b64decode(s, altchars=None, validate=False) + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + + +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -74,18 +85,47 @@ POST request. of length 2 which specifies the alternative alphabet used instead of the ``+`` and ``/`` characters. + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *validate* is false, + or causes an :exc:`~binascii.Error` when *validate* is true unless + b'=' is included in *ignorechars*. + A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. - If *validate* is ``False`` (the default), characters that are neither - in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *validate* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *validate* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + + If *validate* is false, characters that are neither + in the normal base-64 alphabet nor (if *ignorechars* is not specified) + the alternative alphabet are + discarded prior to the padding check, but the ``+`` and ``/`` characters + keep their meaning if they are not in *altchars* (they will be discarded + in future Python versions). + + If *validate* is true, these non-alphabet characters in the input + result in a :exc:`binascii.Error`. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base64` for details. For more information about the strict base64 check, see :func:`binascii.a2b_base64` - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. + + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters with an alternative alphabet + is now deprecated. + .. function:: standard_b64encode(s) @@ -99,16 +139,19 @@ POST request. Base64 alphabet and return the decoded :class:`bytes`. -.. function:: urlsafe_b64encode(s) +.. function:: urlsafe_b64encode(s, *, padded=True) Encode :term:`bytes-like object` *s* using the URL- and filesystem-safe alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. + can still contain ``=`` if *padded* is true (default). + + .. versionchanged:: 3.15 + Added the *padded* parameter. -.. function:: urlsafe_b64decode(s) +.. function:: urlsafe_b64decode(s, *, padded=False) Decode :term:`bytes-like object` or ASCII string *s* using the URL- and filesystem-safe @@ -116,14 +159,32 @@ POST request. ``/`` in the standard Base64 alphabet, and return the decoded :class:`bytes`. + .. versionchanged:: 3.15 + Added the *padded* parameter. + Padding of input is no longer required by default. + + .. deprecated:: 3.15 + Accepting the ``+`` and ``/`` characters is now deprecated. + -.. function:: b32encode(s) +.. function:: b32encode(s, *, padded=True, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base32 and return the encoded :class:`bytes`. + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. -.. function:: b32decode(s, casefold=False, map01=None) + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + + +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False) Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -139,20 +200,39 @@ POST request. digit 0 is always mapped to the letter O). For security purposes the default is ``None``, so that 0 and 1 are not allowed in the input. + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it raises an :exc:`~binascii.Error` unless + b'=' is included in *ignorechars*. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base32` for details. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. -.. function:: b32hexencode(s) + +.. function:: b32hexencode(s, *, padded=True, wrapcol=0) Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *padded* and *wrapcol* parameters. + -.. function:: b32hexdecode(s, casefold=False) +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False) Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -164,14 +244,24 @@ POST request. .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Added the *canonical*, *ignorechars*, and *padded* parameters. + -.. function:: b16encode(s) +.. function:: b16encode(s, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Base16 and return the encoded :class:`bytes`. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. + + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: b16decode(s, casefold=False) +.. function:: b16decode(s, casefold=False, *, ignorechars=b'') Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -180,28 +270,44 @@ POST request. lowercase alphabet is acceptable as input. For security purposes, the default is ``False``. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + + .. _base64-base-85: Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -212,78 +318,137 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long, excluding the trailing newline. + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False) Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_ascii85` for details. + .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* parameter. + Single-character final groups are now always rejected as encoding + violations. + -.. function:: b85encode(b, pad=False) +.. function:: b85encode(b, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: b85decode(b) +.. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. -.. function:: z85encode(s) + +.. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. + + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not add any newlines. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + The *pad* parameter was added. + + .. versionchanged:: 3.15 + Added the *wrapcol* parameter. + -.. function:: z85decode(s) +.. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. .. versionadded:: 3.13 + .. versionchanged:: 3.15 + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. + .. _base64-legacy: @@ -353,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index c7a3e0c596b9d03..c8b48901901f98c 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints +The :mod:`!bdb` module handles basic debugger functions, like setting breakpoints or managing execution via the debugger. The following exception is defined: @@ -18,7 +18,7 @@ The following exception is defined: Exception raised by the :class:`Bdb` class for quitting the debugger. -The :mod:`bdb` module also defines two classes: +The :mod:`!bdb` module also defines two classes: .. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) @@ -236,7 +236,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684bbabe..ceb80a35a1a76bb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -10,10 +10,10 @@ -------------- -The :mod:`binascii` module contains a number of methods to convert between +The :mod:`!binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not use these functions directly but use wrapper modules like -:mod:`base64` instead. The :mod:`binascii` module contains +:mod:`base64` instead. The :mod:`!binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -28,7 +28,7 @@ higher-level modules. ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. -The :mod:`binascii` module defines the following functions: +The :mod:`!binascii` module defines the following functions: .. function:: a2b_uu(string) @@ -48,34 +48,240 @@ The :mod:`binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, strict_mode=False) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. + Optional *alphabet* must be a :class:`bytes` object of length 64 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 4 base 64 alphabet characters must + be padded with the '=' character. + If *padded* is false, padding is neither required nor recognized: + the '=' character is not treated as padding but as a non-alphabet + character, which means it is silently discarded when *strict_mode* is false, + or causes an :exc:`~binascii.Error` when *strict_mode* is true unless + b'=' is included in *ignorechars*. + + If *ignorechars* is specified, it should be a :term:`bytes-like object` + containing characters to ignore from the input when *strict_mode* is true. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + The default value of *strict_mode* is ``True`` if *ignorechars* is specified, + ``False`` otherwise. + If *strict_mode* is true, only valid base64 data will be converted. Invalid base64 data will raise :exc:`binascii.Error`. Valid base64: - * Conforms to :rfc:`3548`. + * Conforms to :rfc:`4648`. * Contains only characters from the base64 alphabet. * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. This check is independent of *strict_mode*. + .. versionchanged:: 3.11 Added the *strict_mode* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *canonical*, *ignorechars*, and *padded* parameters. + + +.. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) + + Convert binary data to a line(s) of ASCII characters in base64 coding, + as specified in :rfc:`4648`. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 4. + If *padded* is false, do not add the pad characters. -.. function:: b2a_base64(data, *, newline=True) + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. + If *newline* is true (default), a newline character will be added + at the end of the output. .. versionchanged:: 3.6 Added the *newline* parameter. + .. versionchanged:: 3.15 + Added the *alphabet*, *padded* and *wrapcol* parameters. + + +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False) + + Convert Ascii85 data back to binary and return the binary data. + + Valid Ascii85 data contains characters from the Ascii85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is + accepted as a short form of the group ``!!!!!``, which encodes four + consecutive null bytes. A single-character final group is always rejected + as an encoding violation. + + *foldspaces* is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). + This feature is not supported by the "standard" Ascii85 encoding. + + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + This should only contain whitespace characters. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_ascii85` would produce: the ``z`` abbreviation must be used + for all-zero groups (rather than ``!!!!!``), and partial final groups + must use the same padding digits as the encoder. + + Invalid Ascii85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_ascii85(data, /, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) + + Convert binary data to a formatted sequence of ASCII characters in Ascii85 + coding. The return value is the converted data. + + *foldspaces* is an optional flag that uses the special short sequence 'y' + instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This + feature is not supported by the "standard" Ascii85 encoding. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. + + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. + + .. versionadded:: 3.15 + + +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False) + + Convert Base85 data back to binary and return the binary data. + More than one line may be passed at a time. + + Valid Base85 data contains characters from the Base85 alphabet in groups + of five (except for the final group, which may have from two to five + characters). Each group encodes 32 bits of binary data in the range from + ``0`` to ``2 ** 32 - 1``, inclusive. A single-character final group is + always rejected as an encoding violation. + + Optional *alphabet* must be a :class:`bytes` object of length 85 which + specifies an alternative alphabet. + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_base85` would produce: partial final groups must use the + same padding digits as the encoder. + + Invalid Base85 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + + +.. function:: b2a_base85(data, /, *, alphabet=BASE85_ALPHABET, wrapcol=0, pad=False) + + Convert binary data to a line of ASCII characters in Base85 coding. + The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 85 which + specifies an alternative alphabet. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. + + .. versionadded:: 3.15 + + +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False) + + Convert base32 data back to binary and return the binary data. + + Valid base32 data contains characters from the base32 alphabet specified + in :rfc:`4648` in groups of eight (if necessary, the final group is padded + to eight characters with ``=``). Each group encodes 40 bits of binary data + in the range from ``0`` to ``2 ** 40 - 1``, inclusive. + + .. note:: + This function does not map lowercase characters (which are invalid in + standard base32) to their uppercase counterparts, nor does it + contextually map ``0`` to ``O`` and ``1`` to ``I``/``L`` as :rfc:`4648` + allows. + + Optional *alphabet* must be a :class:`bytes` object of length 32 which + specifies an alternative alphabet. + + If *padded* is true, the last group of 8 base 32 alphabet characters must + be padded with the '=' character. + If *padded* is false, the '=' character is treated as other non-alphabet + characters (depending on the value of *ignorechars*). + + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + If *ignorechars* contains the pad character ``'='``, the pad characters + presented before the end of the encoded data and the excess pad characters + will be ignored. + + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. + + Invalid base32 data will raise :exc:`binascii.Error`. + + .. versionadded:: 3.15 + +.. function:: b2a_base32(data, /, *, padded=True, alphabet=BASE32_ALPHABET, wrapcol=0) + + Convert binary data to a line of ASCII characters in base32 coding, + as specified in :rfc:`4648`. The return value is the converted line. + + Optional *alphabet* must be a :term:`bytes-like object` of length 32 which + specifies an alternative alphabet. + + If *padded* is true (default), pad the encoded data with the '=' + character to a size multiple of 8. + If *padded* is false, do not add the pad characters. + + If *wrapcol* is non-zero, insert a newline (``b'\n'``) character + after at most every *wrapcol* characters. + If *wrapcol* is zero (default), do not insert any newlines. + + .. versionadded:: 3.15 .. function:: a2b_qp(data, header=False) @@ -150,17 +356,23 @@ The :mod:`binascii` module defines the following functions: .. versionchanged:: 3.8 The *sep* and *bytes_per_sep* parameters were added. -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) +.. function:: a2b_hex(hexstr, *, ignorechars=b'') + unhexlify(hexstr, *, ignorechars=b'') Return the binary data represented by the hexadecimal string *hexstr*. This function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + *ignorechars* should be a :term:`bytes-like object` containing characters + to ignore from the input. + + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. + + .. versionchanged:: 3.15 + Added the *ignorechars* parameter. + .. exception:: Error @@ -173,6 +385,69 @@ The :mod:`binascii` module defines the following functions: but may be handled by reading a little more data and trying again. +.. data:: BASE64_ALPHABET + + The Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: URLSAFE_BASE64_ALPHABET + + The "URL and filename safe" Base 64 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: UU_ALPHABET + + The uuencoding alphabet. + + .. versionadded:: 3.15 + +.. data:: CRYPT_ALPHABET + + The Base 64 alphabet used in the :manpage:`crypt(3)` routine and in the GEDCOM format. + + .. versionadded:: 3.15 + +.. data:: BINHEX_ALPHABET + + The Base 64 alphabet used in BinHex 4 (HQX) within the classic Mac OS. + + .. versionadded:: 3.15 + +.. data:: BASE85_ALPHABET + + The Base85 alphabet. + + .. versionadded:: 3.15 + +.. data:: ASCII85_ALPHABET + + The Ascii85 alphabet. + + .. versionadded:: 3.15 + +.. data:: Z85_ALPHABET + + The `Z85 `_ alphabet. + + .. versionadded:: 3.15 + +.. data:: BASE32_ALPHABET + + The Base 32 alphabet according to :rfc:`4648`. + + .. versionadded:: 3.15 + +.. data:: BASE32HEX_ALPHABET + + The "Extended Hex" Base 32 alphabet according to :rfc:`4648`. + Data encoded with this alphabet maintains its sort order during bitwise + comparisons. + + .. versionadded:: 3.15 + + .. seealso:: Module :mod:`base64` diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index d5ec4212c1f9f4c..39ab75c0904df9c 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -3,9 +3,6 @@ .. module:: bisect :synopsis: Array bisection algorithms for binary searching. -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger -.. example based on the PyModules FAQ entry by Aaron Watters **Source code:** :source:`Lib/bisect.py` @@ -16,7 +13,7 @@ having to sort the list after each insertion. For long lists of items with expensive comparison operations, this can be an improvement over linear searches or frequent resorting. -The module is called :mod:`bisect` because it uses a basic bisection +The module is called :mod:`!bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` @@ -27,9 +24,9 @@ point between values in an array. .. note:: The functions in this module are not thread-safe. If multiple threads - concurrently use :mod:`bisect` functions on the same sequence, this + concurrently use :mod:`!bisect` functions on the same sequence, this may result in undefined behaviour. Likewise, if the provided sequence - is mutated by a different thread while a :mod:`bisect` function + is mutated by a different thread while a :mod:`!bisect` function is operating on it, the result is undefined. For example, using :py:func:`~bisect.insort_left` on the same list from multiple threads may result in the list becoming unsorted. @@ -132,7 +129,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :py:func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:deco:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -203,9 +200,9 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] + >>> def grade(score): + ... i = bisect([60, 70, 80, 90], score) + ... return "FDCBA"[i] ... >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 12650861c0fb5da..6c20e9c94a3eaee 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -4,11 +4,6 @@ .. module:: bz2 :synopsis: Interfaces for bzip2 compression and decompression. -.. moduleauthor:: Gustavo Niemeyer -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Gustavo Niemeyer -.. sectionauthor:: Nadeem Vawda - **Source code:** :source:`Lib/bz2.py` -------------- @@ -16,7 +11,7 @@ This module provides a comprehensive interface for compressing and decompressing data using the bzip2 compression algorithm. -The :mod:`bz2` module contains: +The :mod:`!bz2` module contains: * The :func:`.open` function and :class:`BZ2File` class for reading and writing compressed files. @@ -317,7 +312,7 @@ One-shot (de)compression Examples of usage ----------------- -Below are some examples of typical usage of the :mod:`bz2` module. +Below are some examples of typical usage of the :mod:`!bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index f76b1013dfbc663..31faa8c4fb43dc6 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -5,8 +5,6 @@ :synopsis: Functions for working with calendars, including some emulation of the Unix cal program. -.. sectionauthor:: Drew Csillag - **Source code:** :source:`Lib/calendar.py` -------------- @@ -56,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -88,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -158,6 +156,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: prweek(theweek, width) + + Print a week's calendar as returned by :meth:`formatweek` and without a + final newline. + .. method:: formatday(theday, weekday, width) @@ -405,7 +408,7 @@ For simple text calendars this module provides the following functions. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -443,11 +446,11 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. -The :mod:`calendar` module exports the following data attributes: +The :mod:`!calendar` module exports the following data attributes: .. data:: day_name @@ -573,13 +576,18 @@ The :mod:`calendar` module exports the following data attributes: .. versionadded:: 3.12 -The :mod:`calendar` module defines the following exceptions: +The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. @@ -612,7 +620,7 @@ Command-line usage .. versionadded:: 2.5 -The :mod:`calendar` module can be executed as a script from the command line +The :mod:`!calendar` module can be executed as a script from the command line to interactively print a calendar. .. code-block:: shell @@ -748,6 +756,11 @@ The following options are accepted: By default, today's date is highlighted in color and can be :ref:`controlled using environment variables `. +.. versionchanged:: 3.15 + By default, the month is now also highlighted in color, and + the days of the week are also in color. This behavior can be + :ref:`controlled using environment variables `. + *HTML-mode options:* .. option:: --css CSS, -c CSS diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index b6d5dbee21dcd5a..f602003e49b8218 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -124,7 +124,7 @@ rectangular coordinates to polar coordinates and back. The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. + separate :mod:`!cmath` module function for this operation. .. function:: polar(z) @@ -357,7 +357,7 @@ Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't interested in complex numbers, and perhaps don't even know what they are. They would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a +number. Also note that the functions defined in :mod:`!cmath` always return a complex number, even if the answer can be expressed as a real number (in which case the complex number has an imaginary part of zero). diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 66544f82f6ff3f2..c988fcebd68a019 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -4,8 +4,6 @@ .. module:: cmd :synopsis: Build line-oriented command interpreters. -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/cmd.py` -------------- @@ -243,9 +241,7 @@ Instances of :class:`Cmd` subclasses have some public instance variables: Cmd Example ----------- -.. sectionauthor:: Raymond Hettinger - -The :mod:`cmd` module is mainly useful for building custom shells that let a +The :mod:`!cmd` module is mainly useful for building custom shells that let a user work with a program interactively. This section presents a simple example of how to build a shell around a few of diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7cec28..6418706269f1eda 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -12,27 +12,28 @@ The following modules have a command-line interface. * :ref:`calendar ` * :mod:`code` * :ref:`compileall ` -* :mod:`cProfile`: see :ref:`profile ` +* ``cProfile``: see :ref:`profiling.tracing ` * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` * :mod:`poplib` -* :ref:`profile ` -* :mod:`pstats` +* :ref:`profiling.sampling ` +* :ref:`profiling.tracing ` +* :ref:`pstats ` * :ref:`py_compile ` * :mod:`pyclbr` * :mod:`pydoc` @@ -52,8 +53,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 305e5d07a3529e6..99fcf35aa893e42 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -4,10 +4,6 @@ .. module:: codecs :synopsis: Encode and decode data and streams. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/codecs.py` .. index:: @@ -317,7 +313,7 @@ and writing to platform dependent files: Codec Base Classes ------------------ -The :mod:`codecs` module defines a set of base classes which define the +The :mod:`!codecs` module defines a set of base classes which define the interfaces for working with codec objects, and can also be used as the basis for custom codec implementations. @@ -1159,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1196,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1403,6 +1399,14 @@ encodings. | punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +| | | | +| | | .. warning:: | +| | | | +| | | The decoding and | +| | | encoding algorithms | +| | | scale poorly, so | +| | | limit the length of | +| | | untrusted input. | +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | | | | :samp:`\\u{XXXX}` and | @@ -1551,8 +1555,8 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. -:mod:`encodings` --- Encodings package --------------------------------------- +:mod:`!encodings` --- Encodings package +--------------------------------------- .. module:: encodings :synopsis: Encodings package @@ -1611,12 +1615,11 @@ This module implements the following exception: Raised when a codec is invalid or incompatible. -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- +:mod:`!encodings.idna` --- Internationalized Domain Names in Applications +------------------------------------------------------------------------- .. module:: encodings.idna :synopsis: Internationalized Domain Names implementation -.. moduleauthor:: Martin v. Löwis This module implements :rfc:`3490` (Internationalized Domain Names in Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for @@ -1654,7 +1657,7 @@ When receiving host names from the wire (such as in reverse name lookup), no automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. -The module :mod:`encodings.idna` also implements the nameprep procedure, which +The module :mod:`!encodings.idna` also implements the nameprep procedure, which performs certain normalizations on host names, to achieve case-insensitivity of international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. @@ -1677,8 +1680,8 @@ functions can be used directly if desired. Convert a label to Unicode, as specified in :rfc:`3490`. -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ +:mod:`!encodings.mbcs` --- Windows ANSI codepage +------------------------------------------------ .. module:: encodings.mbcs :synopsis: Windows ANSI codepage @@ -1695,12 +1698,11 @@ This module implements the ANSI codepage (CP_ACP). Support any error handler. -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- +:mod:`!encodings.utf_8_sig` --- UTF-8 codec with BOM signature +-------------------------------------------------------------- .. module:: encodings.utf_8_sig :synopsis: UTF-8 codec with BOM signature -.. moduleauthor:: Walter Dörwald This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 16f674adb4b22bb..622e57d2ee63dbd 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -4,14 +4,11 @@ .. module:: codeop :synopsis: Compile (possibly incomplete) Python code. -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Michael Hudson - **Source code:** :source:`Lib/codeop.py` -------------- -The :mod:`codeop` module provides utilities upon which the Python +The :mod:`!codeop` module provides utilities upon which the Python read-eval-print loop can be emulated, as is done in the :mod:`code` module. As a result, you probably don't want to use the module directly; if you want to include such a loop in your program you probably want to use the :mod:`code` @@ -25,7 +22,7 @@ There are two parts to this job: #. Remembering which future statements the user has entered, so subsequent input can be compiled with these in effect. -The :mod:`codeop` module provides a way of doing each of these things, and a way +The :mod:`!codeop` module provides a way of doing each of these things, and a way of doing them both. To do just the former: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index e6daccb91f2b4e2..10e3790717ed6ed 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -4,9 +4,6 @@ .. module:: collections.abc :synopsis: Abstract base classes for containers -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. versionadded:: 3.3 Formerly, this module was part of the :mod:`collections` module. @@ -459,7 +456,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 9a8108d882e02ff..d09a6c92bbd37dc 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -4,9 +4,6 @@ .. module:: collections :synopsis: Container datatypes -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/collections/__init__.py` .. testsetup:: * @@ -240,7 +237,9 @@ For example:: [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] -.. class:: Counter([iterable-or-mapping]) +.. class:: Counter(**kwargs) + Counter(iterable, /, **kwargs) + Counter(mapping, /, **kwargs) A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys @@ -290,7 +289,7 @@ For example:: >>> sorted(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common(n=None) Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, @@ -300,7 +299,9 @@ For example:: >>> Counter('abracadabra').most_common(3) [('a', 5), ('b', 2), ('r', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract(**kwargs) + subtract(iterable, /, **kwargs) + subtract(mapping, /, **kwargs) Elements are subtracted from an *iterable* or from another *mapping* (or counter). Like :meth:`dict.update` but subtracts counts instead @@ -325,13 +326,15 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update(**kwargs) + update(iterable, /, **kwargs) + update(mapping, /, **kwargs) Elements are counted from an *iterable* or added-in from another *mapping* (or counter). Like :meth:`dict.update` but adds counts @@ -481,17 +484,19 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: - .. method:: append(x) + .. method:: append(item, /) - Add *x* to the right side of the deque. + Add *item* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(item, /) - Add *x* to the left side of the deque. + Add *item* to the left side of the deque. .. method:: clear() @@ -506,38 +511,38 @@ or subtracting from an empty counter. .. versionadded:: 3.5 - .. method:: count(x) + .. method:: count(value, /) - Count the number of deque elements equal to *x*. + Count the number of deque elements equal to *value*. .. versionadded:: 3.2 - .. method:: extend(iterable) + .. method:: extend(iterable, /) Extend the right side of the deque by appending elements from the iterable argument. - .. method:: extendleft(iterable) + .. method:: extendleft(iterable, /) Extend the left side of the deque by appending elements from *iterable*. Note, the series of left appends results in reversing the order of elements in the iterable argument. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) - Return the position of *x* in the deque (at or after index *start* + Return the position of *value* in the deque (at or after index *start* and before index *stop*). Returns the first match or raises :exc:`ValueError` if not found. .. versionadded:: 3.5 - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert *x* into the deque at position *i*. + Insert *value* into the deque at position *index*. If the insertion would cause a bounded deque to grow beyond *maxlen*, an :exc:`IndexError` is raised. @@ -557,7 +562,7 @@ or subtracting from an empty counter. elements are present, raises an :exc:`IndexError`. - .. method:: remove(value) + .. method:: remove(value, /) Remove the first occurrence of *value*. If not found, raises a :exc:`ValueError`. @@ -570,7 +575,7 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n=1) + .. method:: rotate(n=1, /) Rotate the deque *n* steps to the right. If *n* is negative, rotate to the left. @@ -722,7 +727,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict(default_factory=None, /, [...]) +.. class:: defaultdict(default_factory=None, /, **kwargs) + defaultdict(default_factory, mapping, /, **kwargs) + defaultdict(default_factory, iterable, /, **kwargs) Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable @@ -734,11 +741,14 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key, /) If the :attr:`default_factory` attribute is ``None``, this raises a :exc:`KeyError` exception with the *key* as argument. @@ -944,7 +954,7 @@ In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with field names, the method and attribute names start with an underscore. -.. classmethod:: somenamedtuple._make(iterable) +.. classmethod:: somenamedtuple._make(iterable, /) Class method that makes a new instance from an existing sequence or iterable. @@ -1141,7 +1151,9 @@ Some differences from :class:`dict` still remain: * Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. -.. class:: OrderedDict([items]) +.. class:: OrderedDict(**kwargs) + OrderedDict(mapping, /, **kwargs) + OrderedDict(iterable, /, **kwargs) Return an instance of a :class:`dict` subclass that has methods specialized for rearranging dictionary order. @@ -1209,19 +1221,19 @@ If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' + 'Store items in the order that the keys were last updated.' def __setitem__(self, key, value): super().__setitem__(key, value) self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`: +variants of :deco:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1236,10 +1248,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result @@ -1322,23 +1334,31 @@ subclass directly from :class:`dict`; however, this class can be easier to work with because the underlying dictionary is accessible as an attribute. -.. class:: UserDict([initialdata]) +.. class:: UserDict(**kwargs) + UserDict(mapping, /, **kwargs) + UserDict(iterable, /, **kwargs) Class that simulates a dictionary. The instance's contents are kept in a regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. + :class:`!UserDict` instances. If arguments are provided, they are used to + initialize :attr:`data`, like a regular dictionary. In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + :class:`!UserDict` instances provide the following attribute: .. attribute:: data A real dictionary used to store the contents of the :class:`UserDict` class. + :class:`!UserDict` instances also override the following method: + + .. method:: popitem + Remove and return a ``(key, value)`` pair from the wrapped dictionary. Pairs are + returned in the same order as ``data.popitem()``. (For the default + :meth:`dict.popitem`, this order is :abbr:`LIFO (last-in, first-out)`.) If the + dictionary is empty, raises a :exc:`KeyError`. :class:`UserList` objects ------------------------- diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index ffebf4e40dd609a..dffc16ae8b7d475 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -4,13 +4,11 @@ .. module:: colorsys :synopsis: Conversion functions between RGB and other color systems. -.. sectionauthor:: David Ascher - **Source code:** :source:`Lib/colorsys.py` -------------- -The :mod:`colorsys` module defines bidirectional conversions of color values +The :mod:`!colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color @@ -24,7 +22,7 @@ spaces, the coordinates are all between 0 and 1. https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. -The :mod:`colorsys` module defines the following functions: +The :mod:`!colorsys` module defines the following functions: .. function:: rgb_to_yiq(r, g, b) diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 89b6fe540f5ba7d..6d99e36e1e5bb65 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -73,7 +73,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be non-None. @@ -117,7 +117,7 @@ Reading and writing compressed files argument is not None, a :exc:`!TypeError` will be raised. When writing, the *options* argument can be a dictionary - providing advanced decompression parameters; see + providing advanced compression parameters; see :class:`CompressionParameter` for detailed information about supported parameters. The *level* argument is the compression level to use when writing compressed data. Only one of *level* or *options* may be passed. The @@ -331,10 +331,14 @@ Compressing and decompressing data in memory If *max_length* is non-negative, the method returns at most *max_length* bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to + output can be produced (or EOF is reached), the :attr:`~.needs_input` + attribute will be set to ``False``. In this case, the next call to :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + more of the output. The full content can thus be read like:: + + process_output(d.decompress(data, max_length)) + while not d.eof and not d.needs_input: + process_output(d.decompress(b"", max_length)) If all of the input data was decompressed and returned (either because this was less than *max_length* bytes, or because diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index c2e2f7f820f4ef1..cadf841b43537e5 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -12,7 +12,7 @@ and :source:`Lib/concurrent/futures/interpreter.py` -------------- -The :mod:`concurrent.futures` module provides a high-level interface for +The :mod:`!concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using @@ -21,6 +21,11 @@ or separate processes, using :class:`ProcessPoolExecutor`. Each implements the same interface, which is defined by the abstract :class:`Executor` class. +:class:`concurrent.futures.Future` must not be confused with +:class:`asyncio.Future`, which is designed for use with :mod:`asyncio` +tasks and coroutines. See the :doc:`asyncio's Future ` +documentation for a detailed comparison of the two. + .. include:: ../includes/wasm-notavail.rst Executor Objects @@ -151,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) @@ -308,7 +315,7 @@ the bytes over a shared :mod:`socket ` or .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -320,11 +327,11 @@ likewise serializes the return value when sending it back. When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -414,6 +421,11 @@ in a REPL or a lambda should not be expected to work. require the *fork* start method for :class:`ProcessPoolExecutor` you must explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. versionchanged:: next + Fixed a deadlock (:gh:`115634`) where the executor could hang after + a worker process exited upon reaching its *max_tasks_per_child* + limit while tasks remained queued. + .. method:: terminate_workers() Attempt to terminate all living worker processes immediately by calling @@ -715,15 +727,6 @@ Exception classes .. versionadded:: 3.14 -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: 3.14 - .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 55036090e8d5b86..a7b115e5f6307dc 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -4,9 +4,6 @@ .. module:: concurrent.interpreters :synopsis: Multiple interpreters in the same process -.. moduleauthor:: Eric Snow -.. sectionauthor:: Eric Snow - .. versionadded:: 3.14 **Source code:** :source:`Lib/concurrent/interpreters` diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb109a9b742cb77..4d720176fcc334a 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -4,13 +4,6 @@ .. module:: configparser :synopsis: Configuration file parser. -.. moduleauthor:: Ken Manheimer -.. moduleauthor:: Barry Warsaw -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Christopher G. Petrilli -.. sectionauthor:: Łukasz Langa - **Source code:** :source:`Lib/configparser.py` .. index:: @@ -31,6 +24,11 @@ can be customized by end users easily. This library does *not* interpret or write the value-type prefixes used in the Windows Registry extended version of INI syntax. +.. warning:: + Be cautious when parsing data from untrusted sources. A malicious + INI file may cause the decoder to consume considerable CPU and memory + resources. Limiting the size of data to be parsed is recommended. + .. seealso:: Module :mod:`tomllib` @@ -80,7 +78,7 @@ Let's take a very basic configuration file that looks like this: The structure of INI files is described `in the following section <#supported-ini-file-structure>`_. Essentially, the file consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by +:mod:`!configparser` classes can read and write such files. Let's start by creating the above configuration file programmatically. .. doctest:: @@ -449,7 +447,7 @@ Mapping Protocol Access .. versionadded:: 3.2 Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, +custom objects as if they were dictionaries. In case of :mod:`!configparser`, the mapping interface implementation is using the ``parser['section']['option']`` notation. @@ -459,7 +457,7 @@ the original parser on demand. What's even more important is that when values are changed on a section proxy, they are actually mutated in the original parser. -:mod:`configparser` objects behave as close to actual dictionaries as possible. +:mod:`!configparser` objects behave as close to actual dictionaries as possible. The mapping interface is complete and adheres to the :class:`~collections.abc.MutableMapping` ABC. However, there are a few differences that should be taken into account: @@ -507,7 +505,7 @@ Customizing Parser Behaviour ---------------------------- There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible +:mod:`!configparser` goes a long way to provide support for the largest sensible set of INI styles available. The default functionality is mainly dictated by historical background and it's very likely that you will want to customize some of the features. @@ -560,7 +558,7 @@ the :meth:`!__init__` options: * *allow_no_value*, default value: ``False`` Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The + which otherwise conform to the syntax supported by :mod:`!configparser`. The *allow_no_value* parameter to the constructor can be used to indicate that such values should be accepted: @@ -615,7 +613,7 @@ the :meth:`!__init__` options: prefixes for whole line comments. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. Please note that config parsers don't support escaping of comment prefixes so @@ -672,7 +670,7 @@ the :meth:`!__init__` options: parsers in new applications. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``strict=False``. * *empty_lines_in_values*, default value: ``True`` @@ -842,7 +840,7 @@ be overridden by subclasses or by attribute assignment. Legacy API Examples ------------------- -Mainly because of backwards compatibility concerns, :mod:`configparser` +Mainly because of backwards compatibility concerns, :mod:`!configparser` provides also a legacy API with explicit ``get``/``set`` methods. While there are valid use cases for the methods outlined below, mapping protocol access is preferred for new projects. The legacy API is at times more advanced, @@ -1378,7 +1376,7 @@ Exceptions .. exception:: Error - Base class for all other :mod:`configparser` exceptions. + Base class for all other :mod:`!configparser` exceptions. .. exception:: NoSectionError diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index d058ba206c6cd62..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -68,7 +68,7 @@ A small number of constants live in the built-in namespace. They are: The same as the ellipsis literal "``...``", an object frequently used to indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but assignment to ``...`` raises a :exc:`SyntaxError`. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index d0fa645093a3e96..666f85997448881 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -21,9 +21,9 @@ Functions and classes provided: .. class:: AbstractContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default + :meth:`~object.__enter__` and :meth:`~object.__exit__`. A default + implementation for :meth:`~object.__enter__` is provided which returns + ``self`` while :meth:`~object.__exit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`typecontextmanager`. .. versionadded:: 3.6 @@ -32,9 +32,9 @@ Functions and classes provided: .. class:: AbstractAsyncContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default + :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. A default + implementation for :meth:`~object.__aenter__` is provided which returns + ``self`` while :meth:`~object.__aexit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`async-context-managers`. @@ -88,11 +88,11 @@ Functions and classes provided: the exception has been handled, and execution will resume with the statement immediately following the :keyword:`!with` statement. - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers + :deco:`contextmanager` uses :class:`ContextDecorator` so the context managers it creates can be used as decorators as well as in :keyword:`with` statements. When used as a decorator, a new generator instance is implicitly created on each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context + created by :deco:`contextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators). .. versionchanged:: 3.2 @@ -101,7 +101,7 @@ Functions and classes provided: .. decorator:: asynccontextmanager - Similar to :func:`~contextlib.contextmanager`, but creates an + Similar to :deco:`~contextlib.contextmanager`, but creates an :ref:`asynchronous context manager `. This function is a :term:`decorator` that can be used to define a factory @@ -128,7 +128,7 @@ Functions and classes provided: .. versionadded:: 3.7 - Context managers defined with :func:`asynccontextmanager` can be used + Context managers defined with :deco:`asynccontextmanager` can be used either as decorators or with :keyword:`async with` statements:: import time @@ -148,11 +148,11 @@ Functions and classes provided: When used as a decorator, a new generator instance is implicitly created on each function call. This allows the otherwise "one-shot" context managers - created by :func:`asynccontextmanager` to meet the requirement that context + created by :deco:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can + Async context managers created with :deco:`asynccontextmanager` can be used as decorators. @@ -228,7 +228,7 @@ Functions and classes provided: .. function:: nullcontext(enter_result=None) - Return a context manager that returns *enter_result* from ``__enter__``, but + Return a context manager that returns *enter_result* from :meth:`~object.__enter__`, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: @@ -335,7 +335,7 @@ Functions and classes provided: For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the + :meth:`~object.__enter__` method and so is available as the target of the :keyword:`with` statement:: with redirect_stdout(io.StringIO()) as f: @@ -396,10 +396,11 @@ Functions and classes provided: A base class that enables a context manager to also be used as a decorator. Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional + :meth:`~object.__enter__` and :meth:`~object.__exit__` as normal. + ``__exit__`` retains its optional exception handling even when used as a decorator. - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this + ``ContextDecorator`` is used by :deco:`contextmanager`, so you get this functionality automatically. Example of ``ContextDecorator``:: @@ -466,12 +467,40 @@ Functions and classes provided: statements. If this is not the case, then the original construct with the explicit :keyword:`!with` statement inside the function should be used. + When the decorated callable is a generator function, coroutine function, or + asynchronous generator function, the returned wrapper is of the same kind + and keeps the context manager open for the lifetime of the iteration or + await rather than only for the call that creates the generator or coroutine + object. Wrapped generators and asynchronous generators are explicitly + closed when iteration ends, as if by :func:`closing` or :func:`aclosing`. + + .. note:: + For asynchronous generators the wrapper re-yields each value with + ``async for``; values sent with :meth:`~agen.asend` and exceptions + thrown with :meth:`~agen.athrow` are not forwarded to the wrapped + generator. + .. versionadded:: 3.2 + .. versionchanged:: 3.15 + Decorating a generator function, coroutine function, or asynchronous + generator function now keeps the context manager open across iteration + or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + .. class:: AsyncContextDecorator - Similar to :class:`ContextDecorator` but only for asynchronous functions. + Similar to :class:`ContextDecorator`, but the context manager is entered + and exited with :keyword:`async with`. Decorate coroutine functions and + asynchronous generator functions with this class; the returned wrapper is + of the same kind. + + .. note:: + Synchronous functions and generators are accepted, but the wrapper is + always asynchronous, so the decorated callable must then be awaited or + iterated with ``async for``. If that change of calling convention is + not intended, use :class:`ContextDecorator` instead. Example of ``AsyncContextDecorator``:: @@ -509,6 +538,13 @@ Functions and classes provided: .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Decorating an asynchronous generator function now keeps the context + manager open across iteration. Previously the context manager exited + as soon as the generator object was created. Synchronous functions + and synchronous generator functions are also now accepted, with an + asynchronous wrapper returned. + .. class:: ExitStack() @@ -564,6 +600,10 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not a context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__enter__` and + :meth:`!__exit__`. + .. method:: push(exit) Adds a context manager's :meth:`~object.__exit__` method to the callback stack. @@ -582,6 +622,9 @@ Functions and classes provided: The passed in object is returned from the function, allowing this method to be used as a function decorator. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__exit__`. + .. method:: callback(callback, /, *args, **kwds) Accepts an arbitrary callback function and arguments and adds it to @@ -639,11 +682,17 @@ Functions and classes provided: Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* is not an asynchronous context manager. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aenter__` and :meth:`!__aexit__`. + .. method:: push_async_exit(exit) Similar to :meth:`ExitStack.push` but expects either an asynchronous context manager or a coroutine function. + .. versionchanged:: 3.15 + Added support for arbitrary descriptors :meth:`!__aexit__`. + .. method:: push_async_callback(callback, /, *args, **kwds) Similar to :meth:`ExitStack.callback` but expects a coroutine function. @@ -653,7 +702,7 @@ Functions and classes provided: Similar to :meth:`ExitStack.close` but properly handles awaitables. - Continuing the example for :func:`asynccontextmanager`:: + Continuing the example for :deco:`asynccontextmanager`:: async with AsyncExitStack() as stack: connections = [await stack.enter_async_context(get_connection()) @@ -668,7 +717,7 @@ Examples and Recipes -------------------- This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. +the tools provided by :mod:`!contextlib`. Supporting a variable number of context managers @@ -697,9 +746,9 @@ context management protocol. Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is occasionally desirable to catch exceptions from an ``__enter__`` +It is occasionally desirable to catch exceptions from an :meth:`~object.__enter__` method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` +the :keyword:`with` statement body or the context manager's :meth:`~object.__exit__` method. By using :class:`ExitStack` the steps in the context management protocol can be separated slightly in order to allow this:: @@ -911,7 +960,7 @@ Files are an example of effectively single use context managers, since the first :keyword:`with` statement will close the file, preventing any further IO operations using that file object. -Context managers created using :func:`contextmanager` are also single use +Context managers created using :deco:`contextmanager` are also single use context managers, and will complain about the underlying generator failing to yield if an attempt is made to use them a second time:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index b218468a084db1b..b0cc0be8e911bf0 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -4,8 +4,6 @@ .. module:: contextvars :synopsis: Context Variables -.. sectionauthor:: Yury Selivanov - -------------- This module provides APIs to manage, store, and access context-local @@ -44,6 +42,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -77,6 +78,32 @@ Context Variables to restore the variable to its previous value via the :meth:`ContextVar.reset` method. + For convenience, the token object can be used as a context manager + to avoid calling :meth:`ContextVar.reset` manually:: + + var = ContextVar('var', default='default value') + + with var.set('new value'): + assert var.get() == 'new value' + + assert var.get() == 'default value' + + It is a shorthand for:: + + var = ContextVar('var', default='default value') + + token = var.set('new value') + try: + assert var.get() == 'new value' + finally: + var.reset(token) + + assert var.get() == 'default value' + + .. versionadded:: 3.14 + + Added support for using tokens as context managers. + .. method:: reset(token) Reset the context variable to the value it had before the @@ -93,24 +120,21 @@ Context Variables # After the reset call the var has no value again, so # var.get() would raise a LookupError. + The same *token* cannot be used twice. + .. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert the value of the variable to what it was before the corresponding - *set*. - - The token supports :ref:`context manager protocol ` - to restore the corresponding context variable value at the exit from - :keyword:`with` block:: - - var = ContextVar('var', default='default value') + *set*. A single token cannot reset a context variable more than once. - with var.set('new value'): - assert var.get() == 'new value' + Tokens support the :ref:`context manager protocol ` + to automatically reset context variables. See :meth:`ContextVar.set`. - assert var.get() == 'default value' + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. .. versionadded:: 3.14 diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 210ad7188003e66..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,15 +72,19 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered +methods. In fact, the :mod:`!copy` module uses the registered pickle functions from the :mod:`copyreg` module. .. index:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 6e3144824ebe91f..d59936029da69df 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`copyreg` module offers a way to define functions used while pickling +The :mod:`!copyreg` module offers a way to define functions used while pickling specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions when pickling/copying those objects. The module provides configuration information about object constructors which are not classes. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4a033d823e6a7ee..87c2d4702e74086 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -4,8 +4,6 @@ .. module:: csv :synopsis: Write and read tabular data to and from delimited files. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/csv.py` .. index:: @@ -25,14 +23,14 @@ similar enough that it is possible to write a single module which can efficiently manipulate such data, hiding the details of reading and writing the data from the programmer. -The :mod:`csv` module implements classes to read and write tabular data in CSV +The :mod:`!csv` module implements classes to read and write tabular data in CSV format. It allows programmers to say, "write this data in the format preferred by Excel," or "read data from this file which was generated by Excel," without knowing the precise details of the CSV format used by Excel. Programmers can also describe the CSV formats understood by other applications or define their own special-purpose CSV formats. -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and +The :mod:`!csv` module's :class:`reader` and :class:`writer` objects read and write sequences. Programmers can also read and write data in dictionary form using the :class:`DictReader` and :class:`DictWriter` classes. @@ -47,7 +45,7 @@ using the :class:`DictReader` and :class:`DictWriter` classes. Module Contents --------------- -The :mod:`csv` module defines the following functions: +The :mod:`!csv` module defines the following functions: .. index:: @@ -83,6 +81,13 @@ The :mod:`csv` module defines the following functions: Spam, Spam, Spam, Spam, Spam, Baked Beans Spam, Lovely Spam, Wonderful Spam + where :file:`eggs.csv` contains: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: writer(csvfile, /, dialect='excel', **fmtparams) @@ -112,6 +117,13 @@ The :mod:`csv` module defines the following functions: spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) + which writes :file:`eggs.csv` containing: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: register_dialect(name, /, dialect='excel', **fmtparams) @@ -146,7 +158,7 @@ The :mod:`csv` module defines the following functions: given, this becomes the new limit. -The :mod:`csv` module defines the following classes: +The :mod:`!csv` module defines the following classes: .. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ dialect='excel', *args, **kwds) @@ -193,6 +205,14 @@ The :mod:`csv` module defines the following classes: >>> print(row) {'first_name': 'John', 'last_name': 'Cleese'} + where :file:`names.csv` contains: + + .. code-block:: text + + first_name,last_name + Eric,Idle + John,Cleese + .. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ dialect='excel', *args, **kwds) @@ -230,6 +250,15 @@ The :mod:`csv` module defines the following classes: writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) + which writes :file:`names.csv` containing: + + .. code-block:: text + + first_name,last_name + Baked,Beans + Lovely,Spam + Wonderful,Spam + .. class:: Dialect @@ -314,7 +343,7 @@ An example for :class:`Sniffer` use:: .. _csv-constants: -The :mod:`csv` module defines the following constants: +The :mod:`!csv` module defines the following constants: .. data:: QUOTE_ALL @@ -375,7 +404,7 @@ The :mod:`csv` module defines the following constants: .. versionadded:: 3.12 -The :mod:`csv` module defines the following exception: +The :mod:`!csv` module defines the following exception: .. exception:: Error diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 9c0b246c095483c..51f08fdc0544c96 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -4,28 +4,30 @@ .. module:: ctypes :synopsis: A foreign function library for Python. -.. moduleauthor:: Thomas Heller - **Source code:** :source:`Lib/ctypes` -------------- -:mod:`ctypes` is a foreign function library for Python. It provides C compatible +:mod:`!ctypes` is a foreign function library for Python. It provides C compatible data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. .. include:: ../includes/optional-module.rst +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -36,13 +38,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -72,11 +77,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -85,7 +92,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -182,7 +196,7 @@ handle (passing ``None`` as single argument to call it with a ``NULL`` pointer): To find out the correct calling convention you have to look into the C header file or the documentation for the function you want to call. -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent +On Windows, :mod:`!ctypes` uses win32 structured exception handling to prevent crashes from general protection faults when functions are called with invalid argument values:: @@ -192,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -205,7 +217,7 @@ as pointer to the memory block that contains their data (:c:expr:`char *` or :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. +more about :mod:`!ctypes` data types. .. _ctypes-fundamental-data-types: @@ -213,89 +225,170 @@ more about :mod:`ctypes` data types. Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: - -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int8` | :c:type:`int8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int16` | :c:type:`int16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int32` | :c:type:`int32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int64` | :c:type:`int64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint8` | :c:type:`uint8_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint16` | :c:type:`uint16_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint32` | :c:type:`uint32_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint64` | :c:type:`uint64_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_time_t` | :c:type:`time_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. +:mod:`!ctypes` defines a number of primitive C compatible data types: + +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_bool` + - :c:expr:`_Bool` + - :py:class:`bool` + - ``'?'`` + * - :class:`c_char` + - :c:expr:`char` + - 1-character :py:class:`bytes` + - ``'c'`` + * - :class:`c_wchar` + - :c:type:`wchar_t` + - 1-character :py:class:`str` + - ``'u'`` + * - :class:`c_byte` + - :c:expr:`char` + - :py:class:`int` + - ``'b'`` + * - :class:`c_ubyte` + - :c:expr:`unsigned char` + - :py:class:`int` + - ``'B'`` + * - :class:`c_short` + - :c:expr:`short` + - :py:class:`int` + - ``'h'`` + * - :class:`c_ushort` + - :c:expr:`unsigned short` + - :py:class:`int` + - ``'H'`` + * - :class:`c_int` + - :c:expr:`int` + - :py:class:`int` + - ``'i'`` \* + * - :class:`c_int8` + - :c:type:`int8_t` + - :py:class:`int` + - \* + * - :class:`c_int16` + - :c:type:`int16_t` + - :py:class:`int` + - \* + * - :class:`c_int32` + - :c:type:`int32_t` + - :py:class:`int` + - \* + * - :class:`c_int64` + - :c:type:`int64_t` + - :py:class:`int` + - \* + * - :class:`c_uint` + - :c:expr:`unsigned int` + - :py:class:`int` + - ``'I'`` \* + * - :class:`c_uint8` + - :c:type:`uint8_t` + - :py:class:`int` + - \* + * - :class:`c_uint16` + - :c:type:`uint16_t` + - :py:class:`int` + - \* + * - :class:`c_uint32` + - :c:type:`uint32_t` + - :py:class:`int` + - \* + * - :class:`c_uint64` + - :c:type:`uint64_t` + - :py:class:`int` + - \* + * - :class:`c_long` + - :c:expr:`long` + - :py:class:`int` + - ``'l'`` + * - :class:`c_ulong` + - :c:expr:`unsigned long` + - :py:class:`int` + - ``'L'`` + * - :class:`c_longlong` + - :c:expr:`long long` + - :py:class:`int` + - ``'q'`` \* + * - :class:`c_ulonglong` + - :c:expr:`unsigned long long` + - :py:class:`int` + - ``'Q'`` \* + * - :class:`c_size_t` + - :c:type:`size_t` + - :py:class:`int` + - \* + * - :class:`c_ssize_t` + - :c:type:`Py_ssize_t` + - :py:class:`int` + - \* + * - :class:`c_time_t` + - :c:type:`time_t` + - :py:class:`int` + - \* + * - :class:`c_float` + - :c:expr:`float` + - :py:class:`float` + - ``'f'`` + * - :class:`c_double` + - :c:expr:`double` + - :py:class:`float` + - ``'d'`` + * - :class:`c_longdouble` + - :c:expr:`long double` + - :py:class:`float` + - ``'g'`` \* + * - :class:`c_char_p` + - :c:expr:`char *` (NUL terminated) + - :py:class:`bytes` or ``None`` + - ``'z'`` + * - :class:`c_wchar_p` + - :c:expr:`wchar_t *` (NUL terminated) + - :py:class:`str` or ``None`` + - ``'Z'`` + * - :class:`c_void_p` + - :c:expr:`void *` + - :py:class:`int` or ``None`` + - ``'P'`` + * - :class:`py_object` + - :c:expr:`PyObject *` + - :py:class:`object` + - ``'O'`` + * - :ref:`VARIANT_BOOL ` + - :c:expr:`short int` + - :py:class:`bool` + - ``'v'`` Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported in both C and ``libffi``, the following complex types are available: -+----------------------------------+---------------------------------+-----------------+ -| ctypes type | C type | Python type | -+==================================+=================================+=================+ -| :class:`c_float_complex` | :c:expr:`float complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_double_complex` | :c:expr:`double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_float_complex` + - :c:expr:`float complex` + - :py:class:`complex` + - ``'Zf'`` + * - :class:`c_double_complex` + - :c:expr:`double complex` + - :py:class:`complex` + - ``'Zd'`` + * - :class:`c_longdouble_complex` + - :c:expr:`long double complex` + - :py:class:`complex` + - ``'Zg'`` + +.. versionchanged:: 3.15 + The :py:attr:`~_SimpleCData._type_` types ``F``, ``D`` and ``G`` have been + replaced with ``Zf``, ``Zd`` and ``Zg``. All these types can be created by calling them with an optional initializer of @@ -309,6 +402,16 @@ the correct type and value:: c_ushort(65533) >>> +The constructors for numeric types will convert input using +:py:meth:`~object.__bool__`, +:py:meth:`~object.__index__` (for ``int``), +:py:meth:`~object.__float__` or :py:meth:`~object.__complex__`. +This means :py:class:`~ctypes.c_bool` accepts any object with a truth value:: + + >>> empty_list = [] + >>> c_bool(empty_list) + c_bool(False) + Since these types are mutable, their value can also be changed afterwards:: >>> i = c_int(42) @@ -397,7 +500,7 @@ from within *IDLE* or *PythonWin*:: >>> As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so +bytes objects have to be wrapped in their corresponding :mod:`!ctypes` type, so that they can be converted to the required C data type:: >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) @@ -431,10 +534,10 @@ specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an +You can also customize :mod:`!ctypes` argument conversion to allow instances of +your own classes be used as function arguments. :mod:`!ctypes` looks for an :attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +attribute must be an integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: @@ -448,7 +551,7 @@ object with an :attr:`!_as_parameter_` attribute:: >>> If you don't want to store the instance's data in the :attr:`!_as_parameter_` -instance variable, you could define a :class:`property` which makes the +instance variable, you could define a :deco:`property` which makes the attribute available on request. @@ -490,7 +593,7 @@ the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an +integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute. @@ -600,7 +703,7 @@ Sometimes a C api function expects a *pointer* to a data type as parameter, probably to write into the corresponding location, or if the data is too large to be passed by value. This is also known as *passing parameters by reference*. -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters +:mod:`!ctypes` exports the :func:`byref` function which is used to pass parameters by reference. The same effect can be achieved with the :func:`pointer` function, although :func:`pointer` does a lot more work since it constructs a real pointer object, so it is faster to use :func:`byref` if you don't need the pointer @@ -625,12 +728,12 @@ Structures and unions ^^^^^^^^^^^^^^^^^^^^^ Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must +base classes which are defined in the :mod:`!ctypes` module. Each subclass must define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. +The field type must be a :mod:`!ctypes` type like :class:`c_int`, or any other +derived :mod:`!ctypes` type: structure, union, array, pointer. Here is a simple example of a POINT structure, which contains two integers named *x* and *y*, and also shows how to initialize a structure in the constructor:: @@ -689,7 +792,7 @@ See :class:`CField`:: .. warning:: - :mod:`ctypes` does not support passing unions or structures with bit-fields + :mod:`!ctypes` does not support passing unions or structures with bit-fields to functions by value. While this may work on 32-bit x86, it's not guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. @@ -707,7 +810,7 @@ structure itself by setting the class attributes :attr:`~Structure._pack_` and/or :attr:`~Structure._align_`, respectively. See the attribute documentation for details. -:mod:`ctypes` uses the native byte order for Structures and Unions. To build +:mod:`!ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, :class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These @@ -796,7 +899,7 @@ Pointers ^^^^^^^^ Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: +:mod:`!ctypes` type:: >>> from ctypes import * >>> i = c_int(42) @@ -810,7 +913,7 @@ returns the object to which the pointer points, the ``i`` object above:: c_long(42) >>> -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a +Note that :mod:`!ctypes` does not have OOR (original object return), it constructs a new, equivalent object each time you retrieve an attribute:: >>> pi.contents is i @@ -854,7 +957,7 @@ item. Behind the scenes, the :func:`pointer` function does more than simply create pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a +:func:`POINTER` function, which accepts any :mod:`!ctypes` type, and returns a new type:: >>> PI = POINTER(c_int) @@ -876,7 +979,7 @@ Calling the pointer type without an argument creates a ``NULL`` pointer. False >>> -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing +:mod:`!ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing invalid non-\ ``NULL`` pointers would crash Python):: >>> null_ptr[0] @@ -896,7 +999,7 @@ invalid non-\ ``NULL`` pointers would crash Python):: Thread safety without the GIL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -From Python 3.13 onward, the :term:`GIL` can be disabled on :term:`free threaded ` builds. +From Python 3.13 onward, the :term:`GIL` can be disabled on the :term:`free-threaded build`. In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: .. code-block:: pycon @@ -961,7 +1064,7 @@ To set a POINTER type field to ``NULL``, you can assign ``None``:: .. XXX list other conversions... Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be +into another type. :mod:`!ctypes` provides a :func:`cast` function which can be used in the same way. The ``Bar`` structure defined above accepts ``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, but not instances of other types:: @@ -1025,7 +1128,7 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the +In :mod:`!ctypes`, we can define the ``cell`` class and set the :attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * @@ -1059,7 +1162,7 @@ other, and finally follow the pointer chain a few times:: Callback functions ^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` allows creating C callable function pointers from Python callables. +:mod:`!ctypes` allows creating C callable function pointers from Python callables. These are sometimes called *callback functions*. First, you must create a class for the callback function. The class knows the @@ -1158,7 +1261,7 @@ write:: .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be + are used from C code. :mod:`!ctypes` doesn't, and if you don't, they may be garbage collected, crashing your program when a callback is made. Also, note that if the callback function is called in a thread created @@ -1177,7 +1280,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of +:mod:`!ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1196,7 +1299,7 @@ Quoting the docs for that value: tricks with this to provide a dynamically created collection of frozen modules. So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: +size, we show only how this table can be read with :mod:`!ctypes`:: >>> from ctypes import * >>> @@ -1242,7 +1345,7 @@ for testing. Try it out with ``import __hello__`` for example. Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other +There are some edges in :mod:`!ctypes` where you might expect something other than what actually happens. Consider the following example:: @@ -1310,7 +1413,7 @@ constructs a new Python object each time! Variable-sized data types ^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` provides some support for variable-sized arrays and structures. +:mod:`!ctypes` provides some support for variable-sized arrays and structures. The :func:`resize` function can be used to resize the memory buffer of an existing ctypes object. The function takes the object as first argument, and @@ -1344,7 +1447,7 @@ get errors accessing other elements:: IndexError: invalid index >>> -Another way to use variable-sized data types with :mod:`ctypes` is to use the +Another way to use variable-sized data types with :mod:`!ctypes` is to use the dynamic nature of Python, and (re-)define the data type after the required size is already known, on a case by case basis. @@ -1354,115 +1457,95 @@ is already known, on a case by case basis. ctypes reference ---------------- - -.. _ctypes-finding-shared-libraries: - -Finding shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -When programming in a compiled language, shared libraries are accessed when -compiling/linking a program, and when the program is run. - -The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way -similar to what the compiler or runtime loader does (on platforms with several -versions of a shared library the most recent should be loaded), while the ctypes -library loaders act like when a program is run, and call the runtime loader -directly. - -The :mod:`!ctypes.util` module provides a function which can help to determine -the library to load. - - -.. data:: find_library(name) - :module: ctypes.util - :noindex: - - Try to find a library and return a pathname. *name* is the library name without - any prefix like *lib*, suffix like ``.so``, ``.dylib`` or version number (this - is the form used for the posix linker option :option:`!-l`). If no library can - be found, returns ``None``. - -The exact functionality is system dependent. - -On Linux, :func:`~ctypes.util.find_library` tries to run external programs -(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. -It returns the filename of the library file. - -.. versionchanged:: 3.6 - On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used - when searching for libraries, if a library cannot be found by any other means. - -Here are some examples:: - - >>> from ctypes.util import find_library - >>> find_library("m") - 'libm.so.6' - >>> find_library("c") - 'libc.so.6' - >>> find_library("bz2") - 'libbz2.so.1.0' - >>> - -On macOS and Android, :func:`~ctypes.util.find_library` uses the system's -standard naming schemes and paths to locate the library, and returns a full -pathname if successful:: - - >>> from ctypes.util import find_library - >>> find_library("c") - '/usr/lib/libc.dylib' - >>> find_library("m") - '/usr/lib/libm.dylib' - >>> find_library("bz2") - '/usr/lib/libbz2.dylib' - >>> find_library("AGL") - '/System/Library/Frameworks/AGL.framework/AGL' - >>> - -On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and -returns the full pathname, but since there is no predefined naming scheme a call -like ``find_library("c")`` will fail and return ``None``. - -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. - - -.. _ctypes-listing-loaded-shared-libraries: - -Listing loaded shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When writing code that relies on code loaded from shared libraries, it can be -useful to know which shared libraries have already been loaded into the current -process. - -The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, -which calls the different APIs provided by the various platforms to help determine -which shared libraries have already been loaded into the current process. - -The exact output of this function will be system dependent. On most platforms, -the first entry of this list represents the current process itself, which may -be an empty string. -For example, on glibc-based Linux, the return may look like:: - - >>> from ctypes.util import dllist - >>> dllist() - ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] - .. _ctypes-loading-shared-libraries: Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: +way is to instantiate :py:class:`CDLL` or one of its subclasses: .. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + `LoadLibraryExW`_ function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On Windows, the ``.DLL`` suffix may be missing. (For details, see + `LoadLibraryExW`_ documentation.) + Other platform-specific prefixes and suffixes (for example, ``lib``, + ``.so``, ``.dylib``, or version numbers) must be present in *name*; + they are not added automatically. + See :ref:`ctypes-finding-shared-libraries` for more information. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + `LoadLibraryExW`_ flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. On Windows creating a :class:`CDLL` instance may fail even if the DLL name exists. When a dependent DLL of the loaded DLL is not found, a @@ -1474,20 +1557,49 @@ way is to instantiate one of the following classes: DLLs and determine which one is not found using Windows debugging and tracing tools. + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. -.. seealso:: + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle + + The system handle used to access the library. - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. + .. attribute:: _name + The name of the library passed in the constructor. -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) +.. _LoadLibraryExW: https://learn.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexw - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are +.. class:: OleDLL + + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in this library use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a @@ -1499,133 +1611,51 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: WinDLL -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. .. availability:: Windows - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: PyDLL + See :py:class:`~ctypes.CDLL`, the superclass, for common information. -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the + When functions in this library are called, the Python GIL is *not* released during the function call, and after the function execution the Python error flag is checked. If the error flag is set, a Python exception is raised. - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. + Thus, this is only useful to call Python C API functions directly. .. data:: RTLD_GLOBAL - :noindex: Flag to use as *mode* parameter. On platforms where this flag is not available, it is defined as the integer zero. .. data:: RTLD_LOCAL - :noindex: Flag to use as *mode* parameter. On platforms where this is not available, it is the same as *RTLD_GLOBAL*. .. data:: DEFAULT_MODE - :noindex: The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the :meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as attribute of the loader instance. - .. class:: LibraryLoader(dlltype) Class which loads shared libraries. *dlltype* should be one of the @@ -1643,44 +1673,36 @@ attribute of the loader instance. These prefabricated library loaders are available: -.. data:: cdll - :noindex: - - Creates :class:`CDLL` instances. + .. data:: cdll + Creates :class:`CDLL` instances. -.. data:: windll - :noindex: - Creates :class:`WinDLL` instances. + .. data:: windll - .. availability:: Windows + Creates :class:`WinDLL` instances. + .. availability:: Windows -.. data:: oledll - :noindex: - Creates :class:`OleDLL` instances. + .. data:: oledll - .. availability:: Windows + Creates :class:`OleDLL` instances. + .. availability:: Windows -.. data:: pydll - :noindex: - Creates :class:`PyDLL` instances. + .. data:: pydll + Creates :class:`PyDLL` instances. -For accessing the C Python api directly, a ready-to-use Python shared library -object is available: -.. data:: pythonapi - :noindex: + .. data:: pythonapi - An instance of :class:`PyDLL` that exposes Python C API functions as - attributes. Note that all these functions are assumed to return C - :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`!restype` attribute to use these functions. + An instance of :class:`PyDLL` that exposes Python C API functions as + attributes. Note that all these functions are assumed to return C + :c:expr:`int`, which is of course not always the truth, so you have to assign + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1700,6 +1722,134 @@ object is available: accessing a function raises an auditing event ``ctypes.dlsym/handle`` with arguments ``handle`` (the raw library handle) and ``name``. + +.. _ctypes-finding-shared-libraries: + +Finding shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^ + +When programming in a compiled language, shared libraries are accessed when +compiling/linking a program, and when the program is run. +The programmer specifies a short name; the C compiler, linker, and +runtime dynamic library loader then interact in system-specific ways to find +the filename of the library to load. + +While the mapping from short names to filenames is not consistently exposed +by platforms, the :mod:`!ctypes.util` module provides a function, +:func:`!find_library`, that attempts to match it. +However, as backwards compatibility concerns make it difficult to adjust +its behavior for new platforms and configurations, the function +is :term:`soft deprecated`. + +If wrapping a shared library with :mod:`!ctypes`, consider determining the +shared library name at development time, and hardcoding it into the wrapper +module instead of using :func:`!find_library` to locate the library +at runtime. +Also consider adding a configuration option or environment variable to let +users select a library to use, and then perhaps use :func:`!find_library` +as a default or fallback. + +.. function:: find_library(name) + :module: ctypes.util + + Try to find a library and return a pathname. + + *name* is the "short" library name without any prefix like ``lib``, + suffix like ``.so``, ``.dylib`` or version number. + (This is the form used for the posix linker option :option:`!-l`.) + The result is in a format suitable for passing to :py:class:`~ctypes.CDLL`. + + If no library can be found, return ``None``. + + The exact functionality is system dependent, and is *not guaranteed* + to match the behavior of the compiler, linker, and loader used for + (or by) Python. + It is recommended to only use this function as a default or fallback, + + .. soft-deprecated:: 3.15 + + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. + +On Linux, :func:`!find_library` tries to run external +programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the +library file. +If the output of these programs does not correspond to the dynamic +linker used by Python, the result of this function may be misleading. + +.. versionchanged:: 3.6 + On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used + when searching for libraries, if a library cannot be found by any other means. + +Here are some examples:: + + >>> from ctypes.util import find_library + >>> find_library("m") + 'libm.so.6' + >>> find_library("c") + 'libc.so.6' + >>> find_library("bz2") + 'libbz2.so.1.0' + >>> + +On macOS and Android, :func:`!find_library` uses the system's +standard naming schemes and paths to locate the library, and returns a full +pathname if successful:: + + >>> from ctypes.util import find_library + >>> find_library("c") + '/usr/lib/libc.dylib' + >>> find_library("m") + '/usr/lib/libm.dylib' + >>> find_library("bz2") + '/usr/lib/libbz2.dylib' + >>> find_library("AGL") + '/System/Library/Frameworks/AGL.framework/AGL' + >>> + +On Windows, :func:`!find_library` searches along the system search path, and +returns the full pathname, but since there is no predefined naming scheme a call +like ``find_library("c")`` will fail and return ``None``. + +.. function:: find_msvcrt() + :module: ctypes.util + + Returns the filename of the VC runtime library used by Python, + and by the extension modules. + + If the name of the library cannot be determined, ``None`` is returned. + Notably, this will happen for recent versions of the VC runtime library, + which are not directly loadable. + + If you need to free memory, for example, allocated by an extension module + with a call to the ``free(void *)``, it is important that you use the + function in the same library that allocated the memory. + + .. availability:: Windows + + +.. _ctypes-listing-loaded-shared-libraries: + +Listing loaded shared libraries +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When writing code that relies on code loaded from shared libraries, it can be +useful to know which shared libraries have already been loaded into the current +process. + +The :mod:`!ctypes.util` module provides the :func:`~ctypes.util.dllist` function, +which calls the different APIs provided by the various platforms to help determine +which shared libraries have already been loaded into the current process. + +The exact output of this function will be system dependent. On most platforms, +the first entry of this list represents the current process itself, which may +be an empty string. +For example, on glibc-based Linux, the return may look like:: + + >>> from ctypes.util import dllist + >>> dllist() + ['', 'linux-vdso.so.1', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libc.so.6', ... ] + .. _ctypes-foreign-functions: Foreign functions @@ -1926,7 +2076,7 @@ the windows header file is this:: LPCWSTR lpCaption, UINT uType); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import c_int, WINFUNCTYPE, windll >>> from ctypes.wintypes import HWND, LPCWSTR, UINT @@ -1949,7 +2099,7 @@ function retrieves the dimensions of a specified window by copying them into HWND hWnd, LPRECT lpRect); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError >>> from ctypes.wintypes import BOOL, HWND, RECT @@ -1977,7 +2127,7 @@ do the error checking, and raises an exception when the api call failed:: >>> If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output +unchanged, :mod:`!ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them instead, the normal processing will no longer take place:: @@ -2122,31 +2272,6 @@ Utility functions .. availability:: Windows -.. function:: find_library(name) - :module: ctypes.util - - Try to find a library and return a pathname. *name* is the library name - without any prefix like ``lib``, suffix like ``.so``, ``.dylib`` or version - number (this is the form used for the posix linker option :option:`!-l`). If - no library can be found, returns ``None``. - - The exact functionality is system dependent. - - -.. function:: find_msvcrt() - :module: ctypes.util - - Returns the filename of the VC runtime library used by Python, - and by the extension modules. If the name of the library cannot be - determined, ``None`` is returned. - - If you need to free memory, for example, allocated by an extension module - with a call to the ``free(void *)``, it is important that you use the - function in the same library that allocated the memory. - - .. availability:: Windows - - .. function:: dllist() :module: ctypes.util @@ -2445,10 +2570,32 @@ Fundamental data types Python bytes object or string. When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement + a new object is returned each time. :mod:`!ctypes` does *not* implement original object return, always a new object is constructed. The same is true for all other ctypes object instances. + Each subclass has a class attribute: + + .. attribute:: _type_ + + Class attribute that contains an internal type code, as a string. + See :ref:`ctypes-fundamental-data-types` for a summary. + + Types marked \* in the summary may be (or always are) aliases of a + different :class:`_SimpleCData` subclass, and will not necessarily + use the listed type code. + For example, if the platform's :c:expr:`long`, :c:expr:`long long` + and :c:expr:`time_t` C types are the same, then :class:`c_long`, + :class:`c_longlong` and :class:`c_time_t` all refer to a single class, + :class:`c_long`, whose :attr:`_type_` code is ``'l'``. + The ``'L'`` code will be unused. + + .. seealso:: + + The :mod:`array` and :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently @@ -2570,6 +2717,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`signed long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_short @@ -2581,11 +2730,15 @@ These are the fundamental ctypes data types: .. class:: c_size_t Represents the C :c:type:`size_t` datatype. + Usually an alias for another unsigned integer type. .. class:: c_ssize_t - Represents the C :c:type:`ssize_t` datatype. + Represents the :c:type:`Py_ssize_t` datatype. + This is a signed version of :c:type:`size_t`; + that is, the POSIX :c:type:`ssize_t` type. + Usually an alias for another integer type. .. versionadded:: 3.2 @@ -2593,6 +2746,7 @@ These are the fundamental ctypes data types: .. class:: c_time_t Represents the C :c:type:`time_t` datatype. + Usually an alias for another integer type. .. versionadded:: 3.12 @@ -2645,6 +2799,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`unsigned long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_ushort @@ -2696,8 +2852,11 @@ These are the fundamental ctypes data types: .. versionchanged:: 3.14 :class:`!py_object` is now a :term:`generic type`. +.. _ctypes-wintypes: + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, +:c:type:`!VARIANT_BOOL` or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. @@ -2744,7 +2903,7 @@ fields, or any other data types containing pointer type fields. Abstract base class for structures in *native* byte order. Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will + types, and at least define a :attr:`_fields_` class variable. :mod:`!ctypes` will create :term:`descriptor`\s which allow reading and writing the fields by direct attribute accesses. These are the @@ -2798,7 +2957,7 @@ fields, or any other data types containing pointer type fields. Setting :attr:`!_pack_` to 0 is the same as not setting it at all. Otherwise, the value must be a positive power of two. The effect is equivalent to ``#pragma pack(N)`` in C, except - :mod:`ctypes` may allow larger *n* than what the compiler accepts. + :mod:`!ctypes` may allow larger *n* than what the compiler accepts. :attr:`!_pack_` must already be defined when :attr:`_fields_` is assigned, otherwise it will have no effect. @@ -2819,7 +2978,7 @@ fields, or any other data types containing pointer type fields. The value must not be negative. The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC - or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + or ``#pragma align(N)`` on MSVC, except :mod:`!ctypes` may allow values that the compiler would reject. :attr:`!_align_` can only *increase* a structure's alignment @@ -2868,7 +3027,7 @@ fields, or any other data types containing pointer type fields. assigned, otherwise it will have no effect. The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows + :mod:`!ctypes` will create descriptors in the structure type that allows accessing the nested fields directly, without the need to create the structure or union field. @@ -3012,12 +3171,14 @@ Arrays and pointers Abstract base class for arrays. The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass + :mod:`!ctypes` data type with a non-negative integer. Alternatively, you can subclass this type and define :attr:`_length_` and :attr:`_type_` class variables. Array elements can be read and written using standard subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ @@ -3038,10 +3199,10 @@ Arrays and pointers Create an array. Equivalent to ``type * length``, where *type* is a - :mod:`ctypes` data type and *length* an integer. + :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index cb895664ff1b11c..3ac76edd38dcc8f 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -4,14 +4,11 @@ .. module:: curses.ascii :synopsis: Constants and set-membership functions for ASCII characters. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/curses/ascii.py` -------------- -The :mod:`curses.ascii` module supplies name constants for ASCII characters and +The :mod:`!curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: @@ -117,7 +114,7 @@ C library: .. function:: isblank(c) - Checks for an ASCII whitespace character; space or horizontal tab. + Checks for an ASCII blank character; space or horizontal tab. .. function:: iscntrl(c) @@ -133,7 +130,7 @@ C library: .. function:: isgraph(c) - Checks for ASCII any printable character except space. + Checks for any ASCII printable character except space. .. function:: islower(c) @@ -148,7 +145,7 @@ C library: .. function:: ispunct(c) - Checks for any printable ASCII character which is not a space or an alphanumeric + Checks for any ASCII printable character which is not a space or an alphanumeric character. @@ -171,7 +168,8 @@ C library: .. function:: isctrl(c) - Checks for an ASCII control character (ordinal values 0 to 31). + Checks for an ASCII control character (ordinal values 0 to 31). Unlike + :func:`iscntrl`, this does not include the delete character (0x7f). .. function:: ismeta(c) diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 11fd841d381f69f..dd345bff428ad68 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -4,8 +4,6 @@ .. module:: curses.panel :synopsis: A panel stack extension that adds depth to curses windows. -.. sectionauthor:: A.M. Kuchling - -------------- Panels are windows with the added feature of depth, so they can be stacked on @@ -18,7 +16,15 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- -The module :mod:`curses.panel` defines the following functions: +The module :mod:`!curses.panel` defines the following exception: + + +.. exception:: error + + Exception raised when a curses panel library function returns an error. + + +The module :mod:`!curses.panel` defines the following functions: .. function:: bottom_panel() @@ -28,7 +34,8 @@ The module :mod:`curses.panel` defines the following functions: .. function:: new_panel(win) - Returns a panel object, associating it with the given window *win*. Be aware + Returns a panel object, associating it with the given window *win* and + placing the new panel on top of the panel stack. Be aware that you need to keep the returned panel object referenced explicitly. If you don't, the panel object is garbage collected and removed from the panel stack. @@ -46,75 +53,94 @@ The module :mod:`curses.panel` defines the following functions: .. _curses-panel-objects: -Panel Objects +Panel objects ------------- -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. +.. raw:: html + + + + + + + + + + + + + + + +.. class:: panel + + Panel objects, as returned by :func:`new_panel` above, are windows with a + stacking order. There's always a window associated with a panel which + determines the content, while the panel methods are responsible for the + window's depth in the panel stack. -Panel objects have the following methods: + Panel objects have the following methods: -.. method:: Panel.above() +.. method:: panel.above() Returns the panel above the current panel. -.. method:: Panel.below() +.. method:: panel.below() Returns the panel below the current panel. -.. method:: Panel.bottom() +.. method:: panel.bottom() Push the panel to the bottom of the stack. -.. method:: Panel.hidden() +.. method:: panel.hidden() Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. -.. method:: Panel.hide() +.. method:: panel.hide() Hide the panel. This does not delete the object, it just makes the window on screen invisible. -.. method:: Panel.move(y, x) +.. method:: panel.move(y, x) Move the panel to the screen coordinates ``(y, x)``. -.. method:: Panel.replace(win) +.. method:: panel.replace(win) Change the window associated with the panel to the window *win*. -.. method:: Panel.set_userptr(obj) +.. method:: panel.set_userptr(obj) Set the panel's user pointer to *obj*. This is used to associate an arbitrary piece of data with the panel, and can be any Python object. -.. method:: Panel.show() +.. method:: panel.show() - Display the panel (which might have been hidden). + Display the panel (which might have been hidden), placing it on top of + the panel stack. -.. method:: Panel.top() +.. method:: panel.top() Push panel to the top of the stack. -.. method:: Panel.userptr() +.. method:: panel.userptr() Returns the user pointer for the panel. This might be any Python object. -.. method:: Panel.window() +.. method:: panel.window() Returns the window object associated with the panel. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 057d338edda92a4..e413cd9d2bef2d9 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -4,16 +4,12 @@ .. module:: curses :synopsis: An interface to the curses library, providing portable terminal handling. - :platform: Unix - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Eric Raymond **Source code:** :source:`Lib/curses` -------------- -The :mod:`curses` module provides an interface to the curses library, the +The :mod:`!curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available @@ -25,6 +21,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/optional-module.rst +.. availability:: Unix. + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -54,7 +52,7 @@ Linux and the BSD variants of Unix. Functions --------- -The module :mod:`curses` defines the following exception: +The module :mod:`!curses` defines the following exception: .. exception:: error @@ -67,77 +65,88 @@ The module :mod:`curses` defines the following exception: default to the current cursor location. Whenever *attr* is optional, it defaults to :const:`A_NORMAL`. -The module :mod:`curses` defines the following functions: +The module :mod:`!curses` defines the following functions: -.. function:: assume_default_colors(fg, bg, /) +Initialization and termination +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Allow use of default values for colors on terminals supporting this feature. - Use this to support transparency in your application. +.. function:: initscr() - * Assign terminal default foreground/background colors to color number ``-1``. - So ``init_pair(x, COLOR_RED, -1)`` will initialize pair *x* as red - on default background and ``init_pair(x, -1, COLOR_BLUE)`` will - initialize pair *x* as default foreground on blue. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. - * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + See :func:`setupterm` for a caveat about calling it before this function. - .. versionadded:: 3.14 + .. note:: + If there is an error opening the terminal, the underlying curses library may + cause the interpreter to exit. -.. function:: baudrate() +.. function:: endwin() - Return the output speed of the terminal in bits per second. On software - terminal emulators it will have a fixed high value. Included for historical - reasons; in former times, it was used to write output loops for time delays and - occasionally to change interfaces depending on the line speed. + De-initialize the library, and return terminal to normal status. +.. function:: isendwin() -.. function:: beep() + Return ``True`` if :func:`endwin` has been called (that is, the curses library has + been deinitialized). - Emit a short attention sound. +.. function:: newterm(type=None, fd=None, infd=None, /) + Initialize a new terminal in addition to the one initialized by + :func:`initscr`, + and return a :ref:`screen ` for it. + This allows a program to drive more than one terminal. -.. function:: can_change_color() + *type* is the terminal name, as in :func:`setupterm`; + if ``None``, the value of the :envvar:`TERM` environment variable is used. + *fd* and *infd* are the output and input files for the terminal: + either a file object or a file descriptor. + They default to :data:`sys.stdout` and :data:`sys.stdin`. - Return ``True`` or ``False``, depending on whether the programmer can change the colors - displayed by the terminal. + The new screen becomes the current one. + Use :func:`set_term` to switch between screens. + See :func:`setupterm` for a caveat about calling it before this function. -.. function:: cbreak() + .. versionadded:: next - Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty - line buffering is turned off and characters are available to be read one by one. - However, unlike raw mode, special characters (interrupt, quit, suspend, and flow - control) retain their effects on the tty driver and calling program. Calling - first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. +.. function:: new_prescr() + Return a new :ref:`screen ` + that can be used to call functions that affect global state + before :func:`initscr` or :func:`newterm` is called. -.. function:: color_content(color_number) + Availability: if the underlying curses library provides ``new_prescr()``. - Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, - containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). + .. versionadded:: next +.. function:: set_term(screen, /) -.. function:: color_pair(pair_number) + Make *screen*, a :ref:`screen ` returned by + :func:`newterm`, the current terminal, + and return the previously current screen. + Returns ``None`` if the previous screen was the one created by + :func:`initscr`. - Return the attribute value for displaying text in the specified color pair. - Only the first 256 color pairs are supported. This - attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart - to this function. + .. versionadded:: next +.. function:: wrapper(func, /, *args, **kwargs) -.. function:: curs_set(visibility) + Initialize curses and call another callable object, *func*, which should be the + rest of your curses-using application. If the application raises an exception, + this function will restore the terminal to a sane state before re-raising the + exception and generating a traceback. The callable object *func* is then passed + the main window 'stdscr' as its first argument, followed by any other arguments + passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on + cbreak mode, turns off echo, enables the terminal keypad, and initializes colors + if the terminal has color support. On exit (whether normally or by exception) + it restores cooked mode, turns on echo, and disables the terminal keypad. - Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, - normal, or very visible. If the terminal supports the visibility requested, return the - previous cursor state; otherwise raise an exception. On many - terminals, the "visible" mode is an underline cursor and the "very visible" mode - is a block cursor. +Terminal mode control +~~~~~~~~~~~~~~~~~~~~~ .. function:: def_prog_mode() @@ -146,7 +155,6 @@ The module :mod:`curses` defines the following functions: program is not in curses.) Subsequent calls to :func:`reset_prog_mode` will restore this mode. - .. function:: def_shell_mode() Save the current terminal mode as the "shell" mode, the mode when the running @@ -154,17 +162,49 @@ The module :mod:`curses` defines the following functions: program is using curses capabilities.) Subsequent calls to :func:`reset_shell_mode` will restore this mode. +.. function:: reset_prog_mode() -.. function:: delay_output(ms) + Restore the terminal to "program" mode, as previously saved by + :func:`def_prog_mode`. - Insert an *ms* millisecond pause in output. +.. function:: reset_shell_mode() + + Restore the terminal to "shell" mode, as previously saved by + :func:`def_shell_mode`. + +.. function:: savetty() + + Save the current state of the terminal modes in a buffer, usable by + :func:`resetty`. + +.. function:: resetty() + + Restore the state of the terminal modes to what it was at the last call to + :func:`savetty`. +.. function:: curs_set(visibility) + + Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, + normal, or very visible. If the terminal supports the visibility requested, return the + previous cursor state; otherwise raise an exception. On many + terminals, the "visible" mode is an underline cursor and the "very visible" mode + is a block cursor. + +.. function:: getsyx() + + Return the current coordinates of the virtual screen cursor as a tuple + ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. + +.. function:: setsyx(y, x) + + Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then + :meth:`leaveok ` is set ``True``. .. function:: doupdate() Update the physical screen. The curses library keeps two data structures, one representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the + representing the desired next state. The :func:`doupdate` function updates the physical screen to match the virtual screen. The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write @@ -175,170 +215,175 @@ The module :mod:`curses` defines the following functions: followed by a single :func:`!doupdate`. -.. function:: echo() - - Enter echo mode. In echo mode, each character input is echoed to the screen as - it is entered. +Input options +~~~~~~~~~~~~~ +.. function:: cbreak() -.. function:: endwin() + Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty + line buffering is turned off and characters are available to be read one by one. + However, unlike raw mode, special characters (interrupt, quit, suspend, and flow + control) retain their effects on the tty driver and calling program. Calling + first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. - De-initialize the library, and return terminal to normal status. +.. function:: nocbreak() + Leave cbreak mode. Return to normal "cooked" mode with line buffering. -.. function:: erasechar() +.. function:: echo() - Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this - is a property of the controlling tty of the curses program, and is not set by - the curses library itself. + Enter echo mode. In echo mode, each character input is echoed to the screen as + it is entered. +.. function:: noecho() -.. function:: filter() + Leave echo mode. Echoing of input characters is turned off. - The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the - capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` - string is set to the value of ``cr``. The effect is that the cursor is confined to - the current line, and so are screen updates. This may be used for enabling - character-at-a-time line editing without touching the rest of the screen. +.. function:: raw() + Enter raw mode. In raw mode, normal line buffering and processing of + interrupt, quit, suspend, and flow control keys are turned off; characters are + presented to curses input functions one by one. -.. function:: flash() +.. function:: noraw() - Flash the screen. That is, change it to reverse-video and then change it back - in a short interval. Some people prefer such as 'visible bell' to the audible - attention signal produced by :func:`beep`. + Leave raw mode. Return to normal "cooked" mode with line buffering. +.. function:: halfdelay(tenths) -.. function:: flushinp() + Used for half-delay mode, which is similar to cbreak mode in that characters + typed by the user are immediately available to the program. However, after + blocking for *tenths* tenths of seconds, raise an exception if nothing has + been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use + :func:`nocbreak` to leave half-delay mode. - Flush all input buffers. This throws away any typeahead that has been typed - by the user and has not yet been processed by the program. +.. function:: meta(flag) + If *flag* is ``True``, allow 8-bit characters to be input. If + *flag* is ``False``, allow only 7-bit chars. -.. function:: getmouse() +.. function:: nl(flag=True) - After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be called to retrieve the queued mouse event, represented as a - 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish - multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is - currently unused.) *bstate* is an integer value whose bits will be set to - indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 5: - :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, - :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, - :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + Enter newline mode. This mode translates the return key into newline on input, + and translates newline into return and line-feed on output. Newline mode is + initially on. - .. versionchanged:: 3.10 - The ``BUTTON5_*`` constants are now exposed if they are provided by the - underlying curses library. + If *flag* is ``False``, the effect is the same as calling :func:`nonl`. +.. function:: nonl() -.. function:: getsyx() + Leave newline mode. Disable translation of return into newline on input, and + disable low-level translation of newline into newline/return on output (but this + does not change the behavior of ``addch('\n')``, which always does the + equivalent of return and line feed on the virtual screen). With translation + off, curses can sometimes speed up vertical motion a little; also, it will be + able to detect the return key on input. - Return the current coordinates of the virtual screen cursor as a tuple - ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. +.. function:: intrflush(flag) + If *flag* is ``True``, pressing an interrupt key (interrupt, break, or quit) + will flush all output in the terminal driver queue. If *flag* is ``False``, + no flushing is done. -.. function:: getwin(file) +.. function:: qiflush([flag]) - Read window related data stored in the file by an earlier :func:`window.putwin` call. - The routine then creates and initializes a new window using that data, returning - the new window object. + If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If + *flag* is ``True``, or no argument is provided, the queues will be flushed when + these control characters are read. +.. function:: noqiflush() -.. function:: has_colors() + When the :func:`!noqiflush` routine is used, normal flush of input and output queues + associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may + want to call :func:`!noqiflush` in a signal handler if you want output to + continue as though the interrupt had not occurred, after the handler exits. - Return ``True`` if the terminal can display colors; otherwise, return ``False``. +.. function:: typeahead(fd) -.. function:: has_extended_color_support() + Specify that the file descriptor *fd* be used for typeahead checking. If *fd* + is ``-1``, then no typeahead checking is done. - Return ``True`` if the module supports extended colors; otherwise, return - ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). + The curses library does "line-breakout optimization" by looking for typeahead + periodically while updating the screen. If input is found, and it is coming + from a tty, the current update is postponed until refresh or doupdate is called + again, allowing faster response to commands typed in advance. This function + allows specifying a different file descriptor for typeahead checking. - Extended color support requires ncurses version 6.1 or later. +.. function:: is_cbreak() - .. versionadded:: 3.10 + Return ``True`` if cbreak mode (see :func:`cbreak`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. -.. function:: has_ic() + .. versionadded:: next - Return ``True`` if the terminal has insert- and delete-character capabilities. - This function is included for historical reasons only, as all modern software - terminal emulators have such capabilities. +.. function:: is_echo() + Return ``True`` if echo mode (see :func:`echo`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. -.. function:: has_il() + .. versionadded:: next - Return ``True`` if the terminal has insert- and delete-line capabilities, or can - simulate them using scrolling regions. This function is included for - historical reasons only, as all modern software terminal emulators have such - capabilities. +.. function:: is_nl() + Return ``True`` if nl mode (see :func:`nl`) is enabled, ``False`` otherwise. + Availability: ncurses 6.5 or later. -.. function:: has_key(ch) + .. versionadded:: next - Take a key value *ch*, and return ``True`` if the current terminal type recognizes - a key with that value. +.. function:: is_raw() + Return ``True`` if raw mode (see :func:`raw`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. -.. function:: halfdelay(tenths) + .. versionadded:: next - Used for half-delay mode, which is similar to cbreak mode in that characters - typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, raise an exception if nothing has - been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use - :func:`nocbreak` to leave half-delay mode. +Keyboard input +~~~~~~~~~~~~~~ -.. function:: init_color(color_number, r, g, b) +.. function:: ungetch(ch) - Change the definition of a color, taking the number of the color to be changed - followed by three RGB values (for the amounts of red, green, and blue - components). The value of *color_number* must be between ``0`` and - ``COLORS - 1``. Each of *r*, *g*, *b*, must be a value between ``0`` and - ``1000``. When :func:`init_color` is used, all occurrences of that color on the - screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``True``. + Push *ch* so the next :meth:`~window.getch` or :meth:`~window.get_wch` will + return it. + *ch* may be an integer (a key code or character code), a byte, or a string of + length 1. A one-character string is pushed like :func:`unget_wch`; on a + narrow build it must encode to a single byte. -.. function:: init_pair(pair_number, fg, bg) + .. note:: - Change the definition of a color-pair. It takes three arguments: the number of - the color-pair to be changed, the foreground color number, and the background - color number. The value of *pair_number* must be between ``1`` and - ``COLOR_PAIRS - 1`` (the ``0`` color pair can only be changed by - :func:`use_default_colors` and :func:`assume_default_colors`). - The value of *fg* and *bg* arguments must be between ``0`` and - ``COLORS - 1``, or, after calling :func:`!use_default_colors` or - :func:`!assume_default_colors`, ``-1``. - If the color-pair was previously initialized, the screen is - refreshed and all occurrences of that color-pair are changed to the new - definition. + Only one *ch* can be pushed before :meth:`!getch` is called. + .. versionchanged:: next + A one-character string argument is no longer required to encode to a single + byte, except on a narrow build. -.. function:: initscr() +.. function:: unget_wch(ch) - Initialize the library. Return a :ref:`window ` object - which represents the whole screen. + Push *ch* so the next :meth:`~window.get_wch` will return it. .. note:: - If there is an error opening the terminal, the underlying curses library may - cause the interpreter to exit. - + Only one *ch* can be pushed before :meth:`!get_wch` is called. -.. function:: is_term_resized(nlines, ncols) + .. versionadded:: 3.3 - Return ``True`` if :func:`resize_term` would modify the window structure, - ``False`` otherwise. + .. versionchanged:: next + Also available on a narrow build, where *ch* must encode to a single byte + (an 8-bit locale). +.. function:: flushinp() -.. function:: isendwin() + Flush all input buffers. This throws away any typeahead that has been typed + by the user and has not yet been processed by the program. - Return ``True`` if :func:`endwin` has been called (that is, the curses library has - been deinitialized). +.. function:: has_key(ch) + Take a key value *ch*, and return ``True`` if the current terminal type recognizes + a key with that value. .. function:: keyname(k) @@ -349,316 +394,551 @@ The module :mod:`curses` defines the following functions: bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. + Raise a :exc:`ValueError` if *k* is negative. -.. function:: killchar() +.. function:: define_key(definition, keycode) - Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems - this is a property of the controlling tty of the curses program, and is not set - by the curses library itself. + Define an escape sequence *definition*, a string, as a key that generates + the key code *keycode*, so that :mod:`curses` interprets it like one of the + keys predefined in the terminal database. + If *definition* is ``None``, any existing binding for *keycode* is removed. + If *keycode* is zero or negative, any existing binding for *definition* is + removed. -.. function:: longname() + .. versionadded:: next - Return a bytes object containing the terminfo long name field describing the current - terminal. The maximum length of a verbose description is 128 characters. It is - defined only after the call to :func:`initscr`. +.. function:: key_defined(definition) + Return the key code bound to the escape sequence *definition*, a string, + ``0`` if no key code is bound to it, or ``-1`` if *definition* is a prefix + of a longer bound sequence (and so is ambiguous). -.. function:: meta(flag) + .. versionadded:: next - If *flag* is ``True``, allow 8-bit characters to be input. If - *flag* is ``False``, allow only 7-bit chars. +.. function:: keyok(keycode, enable) + Enable (if *enable* is true) or disable (otherwise) interpretation of the + key code *keycode*. Unlike :meth:`window.keypad`, this affects a single + key code rather than all of them. -.. function:: mouseinterval(interval) + .. versionadded:: next - Set the maximum time in milliseconds that can elapse between press and release - events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. +Mouse +~~~~~ -.. function:: mousemask(mousemask) +.. function:: getmouse() - Set the mouse events to be reported, and return a tuple ``(availmask, - oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse - events are ever reported. + After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this + method should be called to retrieve the queued mouse event, represented as a + 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish + multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is + currently unused.) *bstate* is an integer value whose bits will be set to + indicate the type of event, and will be the bitwise OR of one or more of the + following constants, where *n* is the button number from 1 to 5: + :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, + :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, + :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. -.. function:: napms(ms) +.. function:: ungetmouse(id, x, y, z, bstate) - Sleep for *ms* milliseconds. + Push a :const:`KEY_MOUSE` event onto the input queue, associating the given + state data with it. +.. function:: mousemask(mousemask) -.. function:: newpad(nlines, ncols) + Set the mouse events to be reported, and return a tuple ``(availmask, + oldmask)``. *availmask* indicates which of the specified mouse events can be + reported; on complete failure it returns ``0``. *oldmask* is the previous value of + the mouse event mask. If this function is never called, no mouse + events are ever reported. - Create and return a pointer to a new pad data structure with the given number - of lines and columns. Return a pad as a window object. +.. function:: mouseinterval(interval) - A pad is like a window, except that it is not restricted by the screen size, and - is not necessarily associated with a particular part of the screen. Pads can be - used when a large window is needed, and only a part of the window will be on the - screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` - methods of a pad require 6 arguments to specify the part of the pad to be - displayed and the location on the screen to be used for the display. The - arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and - the *s* arguments define a clipping box on the screen within which the pad region - is to be displayed. + Set the maximum time in milliseconds that can elapse between press and release + events in order for them to be recognized as a click, and return the previous + interval value. The default value is 166 milliseconds, or one sixth of a second. + Use a negative *interval* to obtain the interval value without changing it. +.. function:: has_mouse() -.. function:: newwin(nlines, ncols) - newwin(nlines, ncols, begin_y, begin_x) + Return ``True`` if the mouse driver has been successfully initialized. - Return a new :ref:`window `, whose left-upper corner - is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. + Availability: if the underlying curses library provides ``has_mouse()``. - By default, the window will extend from the specified position to the lower - right corner of the screen. + .. versionadded:: next -.. function:: nl() +Color +~~~~~ - Enter newline mode. This mode translates the return key into newline on input, - and translates newline into return and line-feed on output. Newline mode is - initially on. +.. function:: start_color() + Must be called if the programmer wants to use colors, and before any other color + manipulation routine is called. It is good practice to call this routine right + after :func:`initscr`. -.. function:: nocbreak() + :func:`start_color` initializes eight basic colors (black, red, green, yellow, + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` + module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number + of colors and color-pairs the terminal can support. It also restores the colors + on the terminal to the values they had when the terminal was just turned on. - Leave cbreak mode. Return to normal "cooked" mode with line buffering. +.. function:: has_colors() + Return ``True`` if the terminal can display colors; otherwise, return ``False``. -.. function:: noecho() +.. function:: has_extended_color_support() - Leave echo mode. Echoing of input characters is turned off. + Return ``True`` if the module supports extended colors; otherwise, return + ``False``. Extended color support allows more than 256 color pairs for + terminals that support more than 16 colors (for example, xterm-256color). + Extended color support requires ncurses version 6.1 or later. -.. function:: nonl() + .. versionadded:: 3.10 - Leave newline mode. Disable translation of return into newline on input, and - disable low-level translation of newline into newline/return on output (but this - does not change the behavior of ``addch('\n')``, which always does the - equivalent of return and line feed on the virtual screen). With translation - off, curses can sometimes speed up vertical motion a little; also, it will be - able to detect the return key on input. +.. function:: can_change_color() + Return ``True`` or ``False``, depending on whether the programmer can change the colors + displayed by the terminal. -.. function:: noqiflush() +.. function:: init_color(color_number, r, g, b) - When the :func:`!noqiflush` routine is used, normal flush of input and output queues - associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may - want to call :func:`!noqiflush` in a signal handler if you want output to - continue as though the interrupt had not occurred, after the handler exits. + Change the definition of a color, taking the number of the color to be changed + followed by three RGB values (for the amounts of red, green, and blue + components). The value of *color_number* must be between ``0`` and + ``COLORS - 1``. Each of *r*, *g*, *b*, must be a value between ``0`` and + ``1000``. When :func:`init_color` is used, all occurrences of that color on the + screen immediately change to the new definition. This function is a no-op on + most terminals; it is active only if :func:`can_change_color` returns ``True``. +.. function:: color_content(color_number) -.. function:: noraw() + Return the intensity of the red, green, and blue (RGB) components in the color + *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, + containing the R,G,B values for the given color, which will be between + ``0`` (no component) and ``1000`` (maximum amount of component). Raise an + exception if the color is not supported. - Leave raw mode. Return to normal "cooked" mode with line buffering. +.. function:: init_pair(pair_number, fg, bg) + Change the definition of a color-pair. It takes three arguments: the number of + the color-pair to be changed, the foreground color number, and the background + color number. The value of *pair_number* must be between ``1`` and + ``COLOR_PAIRS - 1`` (the ``0`` color pair can only be changed by + :func:`use_default_colors` and :func:`assume_default_colors`). + The value of *fg* and *bg* arguments must be between ``0`` and + ``COLORS - 1``, or, after calling :func:`!use_default_colors` or + :func:`!assume_default_colors`, ``-1``. + If the color-pair was previously initialized, the screen is + refreshed and all occurrences of that color-pair are changed to the new + definition. .. function:: pair_content(pair_number) Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. +.. function:: color_pair(pair_number) + + Return the attribute value for displaying text in the specified color pair. + Only color pairs that fit in the color-pair field of the returned value can + be represented (usually the first 256); a larger *pair_number* raises + :exc:`OverflowError` rather than being silently masked to a different pair. + Use :meth:`~window.color_set` or :meth:`~window.attr_set` to display higher + pairs. This attribute value can be combined with :const:`A_STANDOUT`, + :const:`A_REVERSE`, and the other :const:`!A_\*` attributes. + :func:`pair_number` is the counterpart to this function. .. function:: pair_number(attr) Return the number of the color-pair set by the attribute value *attr*. :func:`color_pair` is the counterpart to this function. +.. function:: use_default_colors() -.. function:: putp(str) + Equivalent to ``assume_default_colors(-1, -1)``. - Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified - terminfo capability for the current terminal. Note that the output of :func:`putp` - always goes to standard output. +.. function:: assume_default_colors(fg, bg, /) + Allow use of default values for colors on terminals supporting this feature. + Use this to support transparency in your application. -.. function:: qiflush([flag]) + * Assign terminal default foreground/background colors to color number ``-1``. + So ``init_pair(x, COLOR_RED, -1)`` will initialize pair *x* as red + on default background and ``init_pair(x, -1, COLOR_BLUE)`` will + initialize pair *x* as default foreground on blue. - If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If - *flag* is ``True``, or no argument is provided, the queues will be flushed when - these control characters are read. + * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + This is an ncurses extension. -.. function:: raw() + .. versionadded:: 3.14 - Enter raw mode. In raw mode, normal line buffering and processing of - interrupt, quit, suspend, and flow control keys are turned off; characters are - presented to curses input functions one by one. +.. function:: alloc_pair(fg, bg) + Allocate a color pair for foreground color *fg* and background color *bg*, + and return its number. If a color pair for the same combination of colors + already exists, return its number. Otherwise allocate a new color pair and + return its number. -.. function:: reset_prog_mode() + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). - Restore the terminal to "program" mode, as previously saved by - :func:`def_prog_mode`. + .. versionadded:: next +.. function:: free_pair(pair_number) -.. function:: reset_shell_mode() + Free the color pair *pair_number*, which must have been allocated by + :func:`alloc_pair`. The pair must not be in use. - Restore the terminal to "shell" mode, as previously saved by - :func:`def_shell_mode`. + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + .. versionadded:: next -.. function:: resetty() +.. function:: find_pair(fg, bg) - Restore the state of the terminal modes to what it was at the last call to - :func:`savetty`. + Return the number of a color pair for foreground color *fg* and background + color *bg*, or ``-1`` if no color pair for this combination of colors has + been allocated. + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). -.. function:: resize_term(nlines, ncols) + .. versionadded:: next - Backend function used by :func:`resizeterm`, performing most of the work; - when resizing the windows, :func:`resize_term` blank-fills the areas that are - extended. The calling application should fill in these areas with - appropriate data. The :func:`!resize_term` function attempts to resize all - windows. However, due to the calling convention of pads, it is not possible - to resize these without additional interaction with the application. +.. function:: reset_color_pairs() + Discard all color-pair definitions, releasing the color pairs allocated by + :func:`init_pair` and :func:`alloc_pair`. -.. function:: resizeterm(nlines, ncols) + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). - Resize the standard and current windows to the specified dimensions, and - adjusts other bookkeeping data used by the curses library that record the - window dimensions (in particular the SIGWINCH handler). + .. versionadded:: next -.. function:: savetty() +Windows and pads +~~~~~~~~~~~~~~~~ - Save the current state of the terminal modes in a buffer, usable by - :func:`resetty`. +.. function:: newwin(nlines, ncols) + newwin(nlines, ncols, begin_y, begin_x) -.. function:: get_escdelay() + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. - Retrieves the value set by :func:`set_escdelay`. + By default, the window will extend from the specified position to the lower + right corner of the screen. - .. versionadded:: 3.9 +.. function:: newpad(nlines, ncols) -.. function:: set_escdelay(ms) + Create and return a pointer to a new pad data structure with the given number + of lines and columns. Return a pad as a window object. - Sets the number of milliseconds to wait after reading an escape character, - to distinguish between an individual escape character entered on the - keyboard from escape sequences sent by cursor and function keys. + A pad is like a window, except that it is not restricted by the screen size, and + is not necessarily associated with a particular part of the screen. Pads can be + used when a large window is needed, and only a part of the window will be on the + screen at one time. Automatic refreshes of pads (such as from scrolling or + echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` + methods of a pad require 6 arguments to specify the part of the pad to be + displayed and the location on the screen to be used for the display. The + arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* + arguments refer to the upper-left corner of the pad region to be displayed and + the *s* arguments define a clipping box on the screen within which the pad region + is to be displayed. - .. versionadded:: 3.9 -.. function:: get_tabsize() +Soft labels +~~~~~~~~~~~ - Retrieves the value set by :func:`set_tabsize`. +.. _curses-slk: - .. versionadded:: 3.9 +The following functions manage *soft-label keys*, a row of labels displayed +along the bottom line of the screen, typically used to label a row of function +keys. :func:`slk_init` must be called before :func:`initscr` or +:func:`newterm`; it takes one screen line away from the standard window for the +labels. -.. function:: set_tabsize(size) - Sets the number of columns used by the curses library when converting a tab - character to spaces as it adds the tab to a window. +.. function:: slk_init(fmt=0) - .. versionadded:: 3.9 + Reserve a screen line for the soft labels and choose their layout. *fmt* + selects the arrangement: ``0`` for 3-2-3 (eight labels), ``1`` for 4-4 + (eight labels). Where the underlying curses library supports them, ``2`` + gives 4-4-4 (twelve labels) and ``3`` gives 4-4-4 with an index line. -.. function:: setsyx(y, x) + Must be called before :func:`initscr` or :func:`newterm`. - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then - :meth:`leaveok ` is set ``True``. + .. versionadded:: next -.. function:: setupterm(term=None, fd=-1) +.. function:: slk_set(labnum, label, justify) - Initialize the terminal. *term* is a string giving - the terminal name, or ``None``; if omitted or ``None``, the value of the - :envvar:`TERM` environment variable will be used. *fd* is the - file descriptor to which any initialization sequences will be sent; if not - supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + Set the text of soft label number *labnum*, in the range ``1`` through ``8`` + (or ``12`` in a twelve-label layout). *justify* controls how *label* is + placed within the label: ``0`` for left, ``1`` for centered, ``2`` for right. + .. versionadded:: next -.. function:: start_color() - Must be called if the programmer wants to use colors, and before any other color - manipulation routine is called. It is good practice to call this routine right - after :func:`initscr`. +.. function:: slk_label(labnum) - :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` - module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number - of colors and color-pairs the terminal can support. It also restores the colors - on the terminal to the values they had when the terminal was just turned on. + Return the current text of soft label number *labnum*, justified as it was + set, or an empty string if it has no label. + .. versionadded:: next -.. function:: termattrs() - Return a logical OR of all video attributes supported by the terminal. This - information is useful when a curses program needs complete control over the - appearance of the screen. +.. function:: slk_refresh() + Update the soft labels on the physical screen, like + :meth:`~curses.window.refresh` for a window. -.. function:: termname() + .. versionadded:: next - Return the value of the environment variable :envvar:`TERM`, as a bytes object, - truncated to 14 characters. +.. function:: slk_noutrefresh() -.. function:: tigetflag(capname) + Update the soft labels on the virtual screen, like + :meth:`window.noutrefresh`. Use it together with :func:`doupdate` to batch + screen updates. - Return the value of the Boolean capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a - Boolean capability, or ``0`` if it is canceled or absent from the terminal - description. + .. versionadded:: next -.. function:: tigetnum(capname) +.. function:: slk_clear() - Return the value of the numeric capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a - numeric capability, or ``-1`` if it is canceled or absent from the terminal - description. + Remove the soft labels from the screen. + .. versionadded:: next -.. function:: tigetstr(capname) - Return the value of the string capability corresponding to the terminfo - capability name *capname* as a bytes object. Return ``None`` if *capname* - is not a terminfo "string capability", or is canceled or absent from the - terminal description. +.. function:: slk_restore() + Restore the soft labels to the screen after a :func:`slk_clear`. -.. function:: tparm(str[, ...]) + .. versionadded:: next - Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. - ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. +.. function:: slk_touch() -.. function:: typeahead(fd) + Force all the soft labels to be redrawn by the next :func:`slk_refresh` or + :func:`slk_noutrefresh`. - Specify that the file descriptor *fd* be used for typeahead checking. If *fd* - is ``-1``, then no typeahead checking is done. + .. versionadded:: next - The curses library does "line-breakout optimization" by looking for typeahead - periodically while updating the screen. If input is found, and it is coming - from a tty, the current update is postponed until refresh or doupdate is called - again, allowing faster response to commands typed in advance. This function - allows specifying a different file descriptor for typeahead checking. +.. function:: slk_attron(attr) + slk_attroff(attr) + slk_attrset(attr) -.. function:: unctrl(ch) + Add, remove, or set the attributes used to display the soft labels, given as + packed ``A_*`` attributes. - Return a bytes object which is a printable representation of the character *ch*. - Control characters are represented as a caret followed by the character, for - example as ``b'^C'``. Printing characters are left as they are. + .. versionadded:: next -.. function:: ungetch(ch) +.. function:: slk_attr() - Push *ch* so the next :meth:`~window.getch` will return it. + Return the current attributes of the soft labels as packed ``A_*`` + attributes. Availability depends on the underlying curses library. - .. note:: + .. versionadded:: next - Only one *ch* can be pushed before :meth:`!getch` is called. +.. function:: slk_attr_on(attr) + slk_attr_off(attr) + + Turn the given attributes on or off without affecting any others. Like the + ``attr_*`` window methods, these work with the + :ref:`WA_* attributes ` rather than packed ``A_*`` + attributes. + + .. versionadded:: next + + +.. function:: slk_attr_set(attr, pair=0) + + Set the attributes and color pair of the soft labels. *attr* is given as + :ref:`WA_* attributes ` and *pair* as a color pair + number. + + .. versionadded:: next + + +.. function:: slk_color(pair) + + Set the color pair of the soft labels to color pair number *pair*. + + .. versionadded:: next + + +Saving and restoring +~~~~~~~~~~~~~~~~~~~~ + +.. function:: getwin(file) + + Read window-related data stored in the file by an earlier :meth:`window.putwin` call. + The routine then creates and initializes a new window using that data, returning + the new window object. The *file* argument must be a file object opened for + reading in binary mode. + +.. function:: scr_dump(filename) + + Write the current contents of the virtual screen to *filename*, which may be + a string or a :term:`path-like object`. The file can later be read by + :func:`scr_restore`, :func:`scr_init` or :func:`scr_set`. This is the + whole-screen counterpart of :meth:`window.putwin`. + + .. versionadded:: next + +.. function:: scr_restore(filename) + + Set the virtual screen to the contents of *filename*, which must have been + written by :func:`scr_dump`. The next call to :func:`doupdate` or + :meth:`window.refresh` restores the screen to those contents. + + .. versionadded:: next + +.. function:: scr_init(filename) + + Initialize the assumed contents of the terminal from *filename*, which must + have been written by :func:`scr_dump`. Use it when the terminal already + displays those contents, for example after another program has drawn the + screen, so that curses does not redraw what is already there. + + .. versionadded:: next + +.. function:: scr_set(filename) + + Use *filename*, which must have been written by :func:`scr_dump`, as both + the virtual screen and the assumed terminal contents. This combines the + effects of :func:`scr_restore` and :func:`scr_init`. + + .. versionadded:: next + + +Querying the terminal +~~~~~~~~~~~~~~~~~~~~~ + +.. function:: baudrate() + + Return the output speed of the terminal in bits per second. On software + terminal emulators it will have a fixed high value. Included for historical + reasons; in former times, it was used to write output loops for time delays and + occasionally to change interfaces depending on the line speed. + +.. function:: erasechar() + + Return the user's current erase character as a raw byte, a :class:`bytes` + object of length 1. See also :func:`erasewchar`. + +.. function:: erasewchar() + + Return the user's current erase character as a one-character :class:`str`. + Under Unix operating systems this is a property of the controlling tty of the + curses program, and is not set by the curses library itself. + + .. versionadded:: next + +.. function:: killchar() + + Return the user's current line kill character as a raw byte, a :class:`bytes` + object of length 1. See also :func:`killwchar`. + +.. function:: killwchar() + + Return the user's current line kill character as a one-character :class:`str`. + Under Unix operating systems this is a property of the controlling tty of the + curses program, and is not set by the curses library itself. + + .. versionadded:: next + +.. function:: termname() + + Return the value of the environment variable :envvar:`TERM`, as a bytes object, + truncated to 14 characters. + +.. function:: longname() + + Return a bytes object containing the terminfo long name field describing the current + terminal. The maximum length of a verbose description is 128 characters. It is + defined only after the call to :func:`initscr`. + +.. function:: termattrs() + + Return a logical OR of all video attributes supported by the terminal. This + information is useful when a curses program needs complete control over the + appearance of the screen. + +.. function:: term_attrs() + + Like :func:`termattrs`, but return the attributes as :ref:`WA_* + ` values rather than ``A_*`` values. + + .. versionadded:: next + +.. function:: has_ic() + + Return ``True`` if the terminal has insert- and delete-character capabilities. + This function is included for historical reasons only, as all modern software + terminal emulators have such capabilities. + +.. function:: has_il() + + Return ``True`` if the terminal has insert- and delete-line capabilities, or can + simulate them using scrolling regions. This function is included for + historical reasons only, as all modern software terminal emulators have such + capabilities. + + +Bell and screen flash +~~~~~~~~~~~~~~~~~~~~~ + +.. function:: beep() + + Emit a short attention sound. + +.. function:: flash() + + Flash the screen. That is, change it to reverse-video and then change it back + in a short interval. Some people prefer such as 'visible bell' to the audible + attention signal produced by :func:`beep`. + + +Terminal resizing +~~~~~~~~~~~~~~~~~ + +.. function:: resizeterm(nlines, ncols) + + Resize the standard and current windows to the specified dimensions, and + adjusts other bookkeeping data used by the curses library that record the + window dimensions (in particular the SIGWINCH handler). + +.. function:: resize_term(nlines, ncols) + + Backend function used by :func:`resizeterm`, performing most of the work; + when resizing the windows, :func:`resize_term` blank-fills the areas that are + extended. The calling application should fill in these areas with + appropriate data. The :func:`!resize_term` function attempts to resize all + windows. However, due to the calling convention of pads, it is not possible + to resize these without additional interaction with the application. + +.. function:: is_term_resized(nlines, ncols) + + Return ``True`` if :func:`resize_term` would modify the window structure, + ``False`` otherwise. .. function:: update_lines_cols() @@ -668,22 +948,112 @@ The module :mod:`curses` defines the following functions: .. versionadded:: 3.5 -.. function:: unget_wch(ch) +Terminfo database +~~~~~~~~~~~~~~~~~ - Push *ch* so the next :meth:`~window.get_wch` will return it. +.. function:: setupterm(term=None, fd=-1) + + Initialize the terminal. *term* is a string giving + the terminal name, or ``None``; if omitted or ``None``, the value of the + :envvar:`TERM` environment variable will be used. *fd* is the + file descriptor to which any initialization sequences will be sent; if not + supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + + Raise a :exc:`curses.error` if the terminal could not be found or its + terminfo database entry could not be read. If the terminal has already + been initialized, this function has no effect. .. note:: - Only one *ch* can be pushed before :meth:`!get_wch` is called. + Calling :func:`initscr` or :func:`newterm` after :func:`setupterm` + leaks the terminal that :func:`setupterm` allocated: + the curses library keeps only a single current terminal + and does not free the previously allocated one. - .. versionadded:: 3.3 +.. function:: tigetflag(capname) + Return the value of the Boolean capability corresponding to the terminfo + capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a + Boolean capability, or ``0`` if it is canceled or absent from the terminal + description. -.. function:: ungetmouse(id, x, y, z, bstate) + :func:`setupterm` (or :func:`initscr`) must be called first. - Push a :const:`KEY_MOUSE` event onto the input queue, associating the given - state data with it. +.. function:: tigetnum(capname) + + Return the value of the numeric capability corresponding to the terminfo + capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a + numeric capability, or ``-1`` if it is canceled or absent from the terminal + description. + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: tigetstr(capname) + + Return the value of the string capability corresponding to the terminfo + capability name *capname* as a bytes object. Return ``None`` if *capname* + is not a terminfo "string capability", or is canceled or absent from the + terminal description. + + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: tparm(str[, ...]) + + Instantiate the bytes object *str* with the supplied parameters, where *str* should + be a parameterized string obtained from the terminfo database. For example, + ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact + result depending on terminal type. Up to nine integer parameters may be supplied. + + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: putp(str) + + Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified + terminfo capability for the current terminal. Note that the output of :func:`putp` + always goes to standard output. + + :func:`setupterm` (or :func:`initscr`) must be called first. + + +Utilities +~~~~~~~~~ + +.. function:: unctrl(ch) + + Return a bytes object which is a printable representation of the character *ch*. + *ch* cannot be a character that does not fit in a single byte; use + :func:`wunctrl` for those. + +.. function:: wunctrl(ch) + + Return a string which is a printable representation of the character *ch*; + any attributes and color pair are ignored. + ASCII control characters are represented as a caret followed by a character, + for example as ``'^C'``. Printing characters, including non-ASCII characters + printable in the locale, are left as they are. The representation of other + characters is defined by the underlying curses library. + + .. versionadded:: next + +.. function:: filter() + + The :func:`.filter` routine, if used, must be called before :func:`initscr` is + called. The effect is that, during the initialization, :envvar:`LINES` is set to ``1``; the + capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` + string is set to the value of ``cr``. The effect is that the cursor is confined to + the current line, and so are screen updates. This may be used for enabling + character-at-a-time line editing without touching the rest of the screen. + +.. function:: nofilter() + + Undo the effect of a previous :func:`.filter` call. + Like :func:`.filter`, it must be called before :func:`initscr` (or + :func:`newterm`) so that the next initialization uses the full screen + again. + + Availability: if the underlying curses library provides ``nofilter()``. + + .. versionadded:: next .. function:: use_env(flag) @@ -694,28 +1064,44 @@ The module :mod:`curses` defines the following functions: window (in which case default behavior would be to use the window size if :envvar:`LINES` and :envvar:`COLUMNS` are not set). +.. function:: get_escdelay() -.. function:: use_default_colors() + Retrieves the value set by :func:`set_escdelay`. - Equivalent to ``assume_default_colors(-1, -1)``. + .. versionadded:: 3.9 +.. function:: set_escdelay(ms) -.. function:: wrapper(func, /, *args, **kwargs) + Sets the number of milliseconds to wait after reading an escape character, + to distinguish between an individual escape character entered on the + keyboard from escape sequences sent by cursor and function keys. - Initialize curses and call another callable object, *func*, which should be the - rest of your curses-using application. If the application raises an exception, - this function will restore the terminal to a sane state before re-raising the - exception and generating a traceback. The callable object *func* is then passed - the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on - cbreak mode, turns off echo, enables the terminal keypad, and initializes colors - if the terminal has color support. On exit (whether normally or by exception) - it restores cooked mode, turns on echo, and disables the terminal keypad. + .. versionadded:: 3.9 + +.. function:: get_tabsize() + + Retrieves the value set by :func:`set_tabsize`. + + .. versionadded:: 3.9 + +.. function:: set_tabsize(size) + + Sets the number of columns used by the curses library when converting a tab + character to spaces as it adds the tab to a window. + + .. versionadded:: 3.9 + +.. function:: napms(ms) + + Sleep for *ms* milliseconds. + +.. function:: delay_output(ms) + Insert an *ms* millisecond pause in output. .. _curses-window-objects: -Window Objects +Window objects -------------- .. class:: window @@ -724,6 +1110,9 @@ Window Objects the following methods and attributes: +Adding and inserting text +~~~~~~~~~~~~~~~~~~~~~~~~~ + .. method:: window.addch(ch[, attr]) window.addch(y, x, ch[, attr]) @@ -731,12 +1120,45 @@ Window Objects character previously painted at that location. By default, the character position and attributes are the current settings for the window object. + *ch* may be a single character, optionally followed by combining + characters, that together occupy one character cell. + .. note:: Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the character is printed. + .. versionchanged:: next + A character may now be given as a string of a base character followed + by combining characters, instead of only a single character, or as a + :class:`complexchar` cell. + +.. method:: window.addstr(str[, attr]) + window.addstr(y, x, str[, attr]) + + Paint the character string *str* at ``(y, x)`` with attributes + *attr*, overwriting anything previously on the display. + + *str* may also be a :class:`complexstr`, in which case each cell carries its + own attributes and color pair, so *attr* must not be given. A + :class:`complexstr` obtained from :meth:`in_wchstr` is written back + unchanged. + + .. note:: + + * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. + Attempting to write to the lower-right corner of a window, subwindow, + or pad will cause an exception to be raised after the string is printed. + + * A bug in ncurses, the backend for this Python module, could cause + segfaults when resizing windows. This was fixed in ncurses-6.1-20190511. + If you are stuck with an earlier ncurses, you can avoid triggering it by + not calling :meth:`!addstr` with a *str* that has embedded newlines; + instead, call :meth:`!addstr` separately for each line. + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`, as described above. .. method:: window.addnstr(str, n[, attr]) window.addnstr(y, x, str, n[, attr]) @@ -745,44 +1167,300 @@ Window Objects ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. + .. versionchanged:: next + *str* may now also be a :class:`complexstr`; see :meth:`addstr`. -.. method:: window.addstr(str[, attr]) - window.addstr(y, x, str[, attr]) +.. method:: window.echochar(ch[, attr]) - Paint the character string *str* at ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. + Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` + on the window. - .. note:: + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. - * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. +.. method:: window.insch(ch[, attr]) + window.insch(y, x, ch[, attr]) + + Insert character *ch* with attributes *attr* before the character under the + cursor, or at ``(y, x)`` if specified. All characters to the right of the + cursor are shifted one position right, with the rightmost character on the + line being lost. The cursor position does not change. + + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. + +.. method:: window.insstr(str[, attr]) + window.insstr(y, x, str[, attr]) + + Insert a character string (as many characters as will fit on the line) before + the character under the cursor. All characters to the right of the cursor are + shifted right, with the rightmost characters on the line being lost. The cursor + position does not change (after moving to *y*, *x*, if specified). + + *str* may also be a :class:`complexstr`, in which case each cell carries its + own attributes and color pair, so *attr* must not be given. + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`, as described above. + +.. method:: window.insnstr(str, n[, attr]) + window.insnstr(y, x, str, n[, attr]) + + Insert a character string (as many characters as will fit on the line) before + the character under the cursor, up to *n* characters. If *n* is zero or + negative, the entire string is inserted. All characters to the right of the + cursor are shifted right, with the rightmost characters on the line being lost. + The cursor position does not change (after moving to *y*, *x*, if specified). + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`; see :meth:`insstr`. + + +Deleting and inserting lines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.delch([y, x]) + + Delete the character under the cursor, or at ``(y, x)`` if specified. All + characters to the right on the same line are shifted one position left. + +.. method:: window.deleteln() + + Delete the line under the cursor. All following lines are moved up by one line. + +.. method:: window.insertln() + + Insert a blank line under the cursor. All following lines are moved down by one + line. + +.. method:: window.insdelln(nlines) + + Insert *nlines* lines into the specified window above the current line. The + *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines + starting with the one under the cursor, and move the remaining lines up. The + bottom *nlines* lines are cleared. The current cursor position remains the + same. + + +Reading input +~~~~~~~~~~~~~ - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. +.. method:: window.getch([y, x]) + + Get a character. Note that the integer returned does *not* have to be in ASCII + range: function keys, keypad keys and so on are represented by numbers higher + than 255. In no-delay mode, return ``-1`` if there is no input, otherwise + wait until a key is pressed. + A multibyte character is returned as its encoded bytes one at a time; use + :meth:`get_wch` to read it as a single character. + +.. method:: window.get_wch([y, x]) + + Get a wide character. Return a character for most keys, or an integer for + function keys, keypad keys, and other special keys. Unlike :meth:`getch`, an + ordinary key is returned as a one-character :class:`str`. + In no-delay mode, raise an exception if there is no input. + + .. versionadded:: 3.3 + + .. versionchanged:: next + Also available on a narrow build, where only a character representable as a + single byte (an 8-bit locale) can be returned. + +.. method:: window.getkey([y, x]) + + Get a character, returning a string instead of an integer, as :meth:`getch` + does. Function keys, keypad keys and other special keys return a multibyte + string containing the key name. In no-delay mode, raise an exception if + there is no input. + +.. method:: window.getstr() + window.getstr(n) + window.getstr(y, x) + window.getstr(y, x, n) + + Read a bytes object from the user, with primitive line editing capacity. + At most *n* characters are read; + *n* defaults to and cannot exceed 2047. + A multibyte character is returned as its encoded bytes; use :meth:`get_wstr` + to read the input as a :class:`str`. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. + +.. method:: window.get_wstr() + window.get_wstr(n) + window.get_wstr(y, x) + window.get_wstr(y, x, n) + + Read a string from the user, with primitive line editing capacity. + Unlike :meth:`getstr`, it can return characters that are not representable in + the window's encoding. + At most *n* characters are read; *n* defaults to and cannot exceed 2047. + + .. versionadded:: next + +.. method:: window.getdelay() + + Return the window's read timeout in milliseconds, + as set by :meth:`nodelay` or :meth:`timeout`: + ``-1`` for blocking, ``0`` for non-blocking, + or a positive number of milliseconds. + + .. versionadded:: next + + +Reading window contents +~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.inch([y, x]) + + Return the character at the given position in the window. + The bottom 8 bits are the character proper and the upper bits are the attributes; + extract them with the :data:`A_CHARTEXT` and :data:`A_ATTRIBUTES` bit-masks, + and the color pair with :func:`pair_number`. + It cannot represent a cell holding combining characters, a character that does + not fit in a single byte, or a color pair outside the :func:`color_pair` + range; use :meth:`in_wch` for those, which returns it as a :class:`complexchar`. + +.. method:: window.in_wch([y, x]) + + Return the complex character at the given position in the window as a + :class:`complexchar`. Unlike :meth:`inch`, the returned object carries the + cell's text (a spacing character optionally followed by combining characters) + together with its attributes and color pair. + + .. versionadded:: next + +.. method:: window.instr([n]) + window.instr(y, x[, n]) + + Return a bytes object of characters, extracted from the window starting at the + current cursor position, or at *y*, *x* if specified, and stopping at the end + of the line. Attributes and color information are stripped + from the characters. If *n* is specified, :meth:`instr` returns a string + at most *n* characters long (exclusive of the trailing NUL). + The maximum value for *n* is 2047. + A character not representable in the window's encoding cannot be returned; + use :meth:`in_wstr` for those. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. + +.. method:: window.in_wstr([n]) + window.in_wstr(y, x[, n]) + + Return a string of characters, extracted from the window starting at the + current cursor position, or at *y*, *x* if specified. Unlike :meth:`instr`, + it can return characters that are not representable in the window's encoding. + Attributes and color information are stripped from the characters. The + maximum value for *n* is 2047. + + .. versionadded:: next +.. method:: window.in_wchstr([n]) + window.in_wchstr(y, x[, n]) + + Return a :class:`complexstr` of the styled cells extracted from the window + starting at the current cursor position, or at *y*, *x* if specified, and + stopping at the end of the line. This is the variant of :meth:`instr` and + :meth:`in_wstr` that *keeps* each cell's attributes and color pair (those + methods strip the rendition). If *n* is specified, at most *n* cells are + returned. The maximum value for *n* is 2047. + + The result can be written back unchanged with :meth:`addstr` (a read and a + re-write is a round-trip that preserves every cell's rendition). + + .. versionadded:: next + + +Attributes +~~~~~~~~~~ .. method:: window.attroff(attr) Remove attribute *attr* from the "background" set applied to all writes to the current window. - .. method:: window.attron(attr) Add attribute *attr* to the "background" set applied to all writes to the current window. - .. method:: window.attrset(attr) - Set the "background" set of attributes to *attr*. This set is initially - ``0`` (no attributes). + Set the "background" set of attributes to *attr*. This set is initially + ``0`` (no attributes). + +.. method:: window.attr_get() + + Return the window's current rendition as a ``(attrs, pair)`` tuple, + where *attrs* is the set of attributes and *pair* is the color pair number. + + Unlike :meth:`attron` and friends, which take packed ``A_*`` attributes, + this method and the other ``attr_*`` methods work with the + :ref:`WA_* attributes ` and keep the color pair as a + separate number, which lets them use color pairs that do not fit alongside + the attributes in a single value. + + .. versionadded:: next + +.. method:: window.attr_set(attr, pair=0) + + Set the window's rendition to the attributes *attr* and the color pair *pair*. + + .. versionadded:: next + +.. method:: window.attr_on(attr) + + Turn on the attributes *attr* without affecting any others. + + .. versionadded:: next + +.. method:: window.attr_off(attr) + + Turn off the attributes *attr* without affecting any others. + + .. versionadded:: next + +.. method:: window.color_set(pair) + + Set the window's color pair to *pair*, leaving the other attributes unchanged. + + .. versionadded:: next + +.. method:: window.getattrs() + + Return the window's current attributes. + + .. versionadded:: next + +.. method:: window.standout() + + Turn on attribute *A_STANDOUT*. + +.. method:: window.standend() + + Turn off the standout attribute. On some terminals this has the side effect of + turning off all attributes. + +.. method:: window.chgat(attr) + window.chgat(num, attr) + window.chgat(y, x, attr) + window.chgat(y, x, num, attr) + + Set the attributes of *num* characters at the current cursor position, or at + position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, + the attribute will be set on all the characters to the end of the line. This + function moves cursor to position ``(y, x)`` if supplied. The changed line + will be touched using the :meth:`touchline` method so that the contents will + be redisplayed by the next window refresh. + +Background +~~~~~~~~~~ .. method:: window.bkgd(ch[, attr]) @@ -796,6 +1474,9 @@ Window Objects * Wherever the former background character appears, it is changed to the new background character. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. .. method:: window.bkgdset(ch[, attr]) @@ -806,6 +1487,51 @@ Window Objects characters. The background becomes a property of the character and moves with the character through any scrolling and insert/delete line/character operations. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. + +.. method:: window.getbkgd() + + Return the given window's current background character/attribute pair. + Its components can be extracted like those of :meth:`inch`. + It cannot represent a background set with a wide character or with a color + pair outside the :func:`color_pair` range; use :meth:`getbkgrnd` for those. + +.. method:: window.getbkgrnd() + + Return the given window's current background as a :class:`complexchar`. + Unlike :meth:`getbkgd`, the returned object carries the background character + together with its attributes and color pair, and the color pair is not limited + to the value that fits in a :func:`color_pair`. + + .. versionadded:: next + + +Clearing and erasing +~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.erase() + + Clear the window. + +.. method:: window.clear() + + Like :meth:`erase`, but also cause the whole window to be repainted upon next + call to :meth:`refresh`. + +.. method:: window.clrtobot() + + Erase from cursor to the end of the window: all lines below the cursor are + deleted, and then the equivalent of :meth:`clrtoeol` is performed. + +.. method:: window.clrtoeol() + + Erase from cursor to the end of the line. + + +Borders and lines +~~~~~~~~~~~~~~~~~ .. method:: window.border([ls[, rs[, ts[, bs[, tl[, tr[, bl[, br]]]]]]]]) @@ -839,311 +1565,257 @@ Window Objects | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | +-----------+---------------------+-----------------------+ + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. A single call cannot mix + them with integer or byte characters. .. method:: window.box([vertch, horch]) Similar to :meth:`border`, but both *ls* and *rs* are *vertch* and both *ts* and *bs* are *horch*. The default corner characters are always used by this function. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. A single call cannot mix + them with integer or byte characters. -.. method:: window.chgat(attr) - window.chgat(num, attr) - window.chgat(y, x, attr) - window.chgat(y, x, num, attr) - - Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, - the attribute will be set on all the characters to the end of the line. This - function moves cursor to position ``(y, x)`` if supplied. The changed line - will be touched using the :meth:`touchline` method so that the contents will - be redisplayed by the next window refresh. - - -.. method:: window.clear() - - Like :meth:`erase`, but also cause the whole window to be repainted upon next - call to :meth:`refresh`. - - -.. method:: window.clearok(flag) - - If *flag* is ``True``, the next call to :meth:`refresh` will clear the window - completely. - - -.. method:: window.clrtobot() - - Erase from cursor to the end of the window: all lines below the cursor are - deleted, and then the equivalent of :meth:`clrtoeol` is performed. - - -.. method:: window.clrtoeol() - - Erase from cursor to the end of the line. - - -.. method:: window.cursyncup() - - Update the current cursor position of all the ancestors of the window to - reflect the current cursor position of the window. - - -.. method:: window.delch([y, x]) - - Delete any character at ``(y, x)``. - - -.. method:: window.deleteln() - - Delete the line under the cursor. All following lines are moved up by one line. - - -.. method:: window.derwin(begin_y, begin_x) - window.derwin(nlines, ncols, begin_y, begin_x) - - An abbreviation for "derive window", :meth:`derwin` is the same as calling - :meth:`subwin`, except that *begin_y* and *begin_x* are relative to the origin - of the window, rather than relative to the entire screen. Return a window - object for the derived window. - - -.. method:: window.echochar(ch[, attr]) - - Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` - on the window. +.. method:: window.hline(ch, n[, attr]) + window.hline(y, x, ch, n[, attr]) + Display a horizontal line starting at ``(y, x)`` with length *n* consisting of + the character *ch* with attributes *attr*. The line stops at the right edge + of the window if fewer than *n* cells are available. -.. method:: window.enclose(y, x) - - Test whether the given pair of screen-relative character-cell coordinates are - enclosed by the given window, returning ``True`` or ``False``. It is useful for - determining what subset of the screen windows enclose the location of a mouse - event. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. - .. versionchanged:: 3.10 - Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) + Display a vertical line starting at ``(y, x)`` with length *n* consisting of the + character *ch* with attributes *attr*. -.. attribute:: window.encoding + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. - Encoding used to encode method arguments (Unicode strings and characters). - The encoding attribute is inherited from the parent window when a subwindow - is created, for example with :meth:`window.subwin`. - By default, current locale encoding is used (see :func:`locale.getencoding`). - .. versionadded:: 3.3 +Cursor position and window geometry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: window.move(new_y, new_x) -.. method:: window.erase() + Move cursor to ``(new_y, new_x)``. - Clear the window. +.. method:: window.getyx() + Return a tuple ``(y, x)`` of current cursor position relative to the window's + upper-left corner. .. method:: window.getbegyx() Return a tuple ``(y, x)`` of coordinates of upper-left corner. - -.. method:: window.getbkgd() - - Return the given window's current background character/attribute pair. - - -.. method:: window.getch([y, x]) - - Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on are represented by numbers higher - than 255. In no-delay mode, return ``-1`` if there is no input, otherwise - wait until a key is pressed. - - -.. method:: window.get_wch([y, x]) - - Get a wide character. Return a character for most keys, or an integer for - function keys, keypad keys, and other special keys. - In no-delay mode, raise an exception if there is no input. - - .. versionadded:: 3.3 - - -.. method:: window.getkey([y, x]) - - Get a character, returning a string instead of an integer, as :meth:`getch` - does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, raise an exception if - there is no input. - - .. method:: window.getmaxyx() Return a tuple ``(y, x)`` of the height and width of the window. - .. method:: window.getparyx() Return the beginning coordinates of this window relative to its parent window as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no parent. +.. method:: window.getparent() -.. method:: window.getstr() - window.getstr(n) - window.getstr(y, x) - window.getstr(y, x, n) - - Read a bytes object from the user, with primitive line editing capacity. - The maximum value for *n* is 2047. - - .. versionchanged:: 3.14 - The maximum value for *n* was increased from 1023 to 2047. - - -.. method:: window.getyx() - - Return a tuple ``(y, x)`` of current cursor position relative to the window's - upper-left corner. - - -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) - - Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. + Return the parent window of this subwindow, + or ``None`` if this window is not a subwindow. + .. versionadded:: next -.. method:: window.idcok(flag) - If *flag* is ``False``, curses no longer considers using the hardware insert/delete - character feature of the terminal; if *flag* is ``True``, use of character insertion - and deletion is enabled. When curses is first initialized, use of character - insert/delete is enabled by default. +Creating and resizing windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: window.subwin(begin_y, begin_x) + window.subwin(nlines, ncols, begin_y, begin_x) -.. method:: window.idlok(flag) + Return a sub-window, whose upper-left corner is at the screen-relative + coordinates ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. - If *flag* is ``True``, :mod:`curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. + By default, the sub-window will extend from the specified position to the lower + right corner of the window. +.. method:: window.derwin(begin_y, begin_x) + window.derwin(nlines, ncols, begin_y, begin_x) -.. method:: window.immedok(flag) + An abbreviation for "derive window", :meth:`derwin` is the same as calling + :meth:`subwin`, except that *begin_y* and *begin_x* are relative to the origin + of the window, rather than relative to the entire screen. Return a window + object for the derived window. - If *flag* is ``True``, any change in the window image automatically causes the - window to be refreshed; you no longer have to call :meth:`refresh` yourself. - However, it may degrade performance considerably, due to repeated calls to - wrefresh. This option is disabled by default. +.. method:: window.subpad(begin_y, begin_x) + window.subpad(nlines, ncols, begin_y, begin_x) + Return a sub-pad, whose upper-left corner is at ``(begin_y, begin_x)``, and + whose width/height is *ncols*/*nlines*. The coordinates are relative to the + parent pad (unlike :meth:`subwin`, which uses screen coordinates). This + method is only available for pads created with :func:`newpad`. -.. method:: window.inch([y, x]) +.. method:: window.dupwin() - Return the character at the given position in the window. The bottom 8 bits are - the character proper, and upper bits are the attributes. + Return a new window that is an exact duplicate of the window: it has the same + size, position, contents and attributes. Unlike a window created by + :meth:`subwin` or :meth:`derwin`, the duplicate is independent of the + original -- it has its own cell buffer, so later changes to one do not affect + the other. + .. versionadded:: next -.. method:: window.insch(ch[, attr]) - window.insch(y, x, ch[, attr]) +.. method:: window.mvwin(new_y, new_x) - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. + Move the window so its upper-left corner is at ``(new_y, new_x)``. + Moving the window so that any part of it would be off the screen is an error: + the window is not moved and :exc:`curses.error` is raised. -.. method:: window.insdelln(nlines) +.. method:: window.mvderwin(y, x) - Insert *nlines* lines into the specified window above the current line. The - *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines - starting with the one under the cursor, and move the remaining lines up. The - bottom *nlines* lines are cleared. The current cursor position remains the - same. + Move the window inside its parent window. The screen-relative parameters of + the window are not changed. This routine is used to display different parts of + the parent window at the same physical position on the screen. +.. method:: window.resize(nlines, ncols) -.. method:: window.insertln() + Reallocate storage for a curses window to adjust its dimensions to the + specified values. If either dimension is larger than the current values, the + window's data is filled with blanks that have the current background + rendition (as set by :meth:`bkgdset`) merged into them. - Insert a blank line under the cursor. All following lines are moved down by one - line. +Refreshing and redrawing +~~~~~~~~~~~~~~~~~~~~~~~~ -.. method:: window.insnstr(str, n[, attr]) - window.insnstr(y, x, str, n[, attr]) +.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) - Insert a character string (as many characters as will fit on the line) before - the character under the cursor, up to *n* characters. If *n* is zero or - negative, the entire string is inserted. All characters to the right of the - cursor are shifted right, with the rightmost characters on the line being lost. - The cursor position does not change (after moving to *y*, *x*, if specified). + Update the display immediately (sync actual screen with previous + drawing/deleting methods). + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`. The additional parameters are needed to indicate what part + of the pad and screen are involved. *pminrow* and *pmincol* specify the + upper-left corner of the rectangle to be displayed in the pad. *sminrow*, + *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be + displayed on the screen. The lower-right corner of the rectangle to be + displayed in the pad is calculated from the screen coordinates, since the + rectangles must be the same size. Both rectangles must be entirely contained + within their respective structures. Negative values of *pminrow*, *pmincol*, + *sminrow*, or *smincol* are treated as if they were zero. -.. method:: window.insstr(str[, attr]) - window.insstr(y, x, str[, attr]) +.. method:: window.noutrefresh() + window.noutrefresh(pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol) - Insert a character string (as many characters as will fit on the line) before - the character under the cursor. All characters to the right of the cursor are - shifted right, with the rightmost characters on the line being lost. The cursor - position does not change (after moving to *y*, *x*, if specified). + Mark for refresh but wait. This function updates the data structure + representing the desired state of the window, but does not force an update of + the physical screen. To accomplish that, call :func:`doupdate`. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`; they have the same meaning as for + :meth:`refresh`. -.. method:: window.instr([n]) - window.instr(y, x[, n]) +.. method:: window.redrawln(beg, num) - Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped - from the characters. If *n* is specified, :meth:`instr` returns a string - at most *n* characters long (exclusive of the trailing NUL). - The maximum value for *n* is 2047. + Indicate that the *num* screen lines, starting at line *beg*, are corrupted and + should be completely redrawn on the next :meth:`refresh` call. - .. versionchanged:: 3.14 - The maximum value for *n* was increased from 1023 to 2047. +.. method:: window.redrawwin() + Touch the entire window, causing it to be completely redrawn on the next + :meth:`refresh` call. -.. method:: window.is_linetouched(line) - Return ``True`` if the specified line was modified since the last call to - :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` - exception if *line* is not valid for the given window. +Output options +~~~~~~~~~~~~~~ +.. method:: window.clearok(flag) -.. method:: window.is_wintouched() + If *flag* is ``True``, the next call to :meth:`refresh` will clear the window + completely. - Return ``True`` if the specified window was modified since the last call to - :meth:`refresh`; otherwise return ``False``. +.. method:: window.idlok(flag) + If *flag* is ``True``, :mod:`!curses` will try to use hardware line + editing facilities. Otherwise, curses will not use them. -.. method:: window.keypad(flag) +.. method:: window.idcok(flag) - If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be - left as is in the input stream. + If *flag* is ``False``, curses no longer considers using the hardware insert/delete + character feature of the terminal; if *flag* is ``True``, use of character insertion + and deletion is enabled. When curses is first initialized, use of character + insert/delete is enabled by default. + +.. method:: window.immedok(flag) + If *flag* is ``True``, any change in the window image automatically causes the + window to be refreshed; you no longer have to call :meth:`refresh` yourself. + However, it may degrade performance considerably, due to repeated calls to + wrefresh. This option is disabled by default. .. method:: window.leaveok(flag) If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. + position." This reduces cursor movement where possible. If *flag* is ``False``, cursor will always be at "cursor position" after an update. +.. method:: window.scrollok(flag) -.. method:: window.move(new_y, new_x) + Control what happens when the cursor of a window is moved off the edge of the + window or scrolling region, either as a result of a newline action on the bottom + line, or typing the last character of the last line. If *flag* is ``False``, the + cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up + one line. Note that in order to get the physical scrolling effect on the + terminal, it is also necessary to call :meth:`idlok`. - Move cursor to ``(new_y, new_x)``. +.. method:: window.scroll([lines=1]) + Scroll the screen or scrolling region. Scroll upward by *lines* lines if + *lines* is positive, or downward if it is negative. Scrolling has no effect + unless it has been enabled for the window with :meth:`scrollok`. -.. method:: window.mvderwin(y, x) +.. method:: window.setscrreg(top, bottom) - Move the window inside its parent window. The screen-relative parameters of - the window are not changed. This routine is used to display different parts of - the parent window at the same physical position on the screen. + Set the scrolling region from line *top* to line *bottom*. All scrolling actions + will take place in this region. +.. method:: window.getscrreg() -.. method:: window.mvwin(new_y, new_x) + Return a tuple ``(top, bottom)`` of the window's current scrolling region, + as set by :meth:`setscrreg`. - Move the window so its upper-left corner is at ``(new_y, new_x)``. + .. versionadded:: next + +.. method:: window.syncok(flag) + + If *flag* is ``True``, then :meth:`syncup` is called automatically + whenever there is a change in the window. +Input options +~~~~~~~~~~~~~ + +.. method:: window.keypad(flag) + + If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be + left as is in the input stream. + .. method:: window.nodelay(flag) If *flag* is ``True``, :meth:`getch` will be non-blocking. - .. method:: window.notimeout(flag) If *flag* is ``True``, escape sequences will not be timed out. @@ -1151,13 +1823,18 @@ Window Objects If *flag* is ``False``, after a few milliseconds, an escape sequence will not be interpreted, and will be left in the input stream as is. +.. method:: window.timeout(delay) -.. method:: window.noutrefresh() + Set blocking or non-blocking read behavior for the window. If *delay* is + negative, blocking read is used (which will wait indefinitely for input). If + *delay* is zero, then non-blocking read is used, and :meth:`getch` will + return ``-1`` if no input is waiting. If *delay* is positive, then + :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is + still no input at the end of that time. - Mark for refresh but wait. This function updates the data structure - representing the desired state of the window, but does not force an update of - the physical screen. To accomplish that, call :func:`doupdate`. +Overlapping and touch +~~~~~~~~~~~~~~~~~~~~~ .. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) @@ -1171,7 +1848,6 @@ Window Objects coordinates of the source window, and the other variables mark a rectangle in the destination window. - .. method:: window.overwrite(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) Overwrite the window on top of *destwin*. The windows need not be the same size, @@ -1184,158 +1860,323 @@ Window Objects coordinates of the source window, the other variables mark a rectangle in the destination window. +.. method:: window.touchline(start, count[, changed]) + + Pretend *count* lines have been changed, starting with line *start*. If + *changed* is supplied, it specifies whether the affected lines are marked as + having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). + +.. method:: window.touchwin() + + Pretend the whole window has been changed, for purposes of drawing + optimizations. + +.. method:: window.untouchwin() + + Mark all lines in the window as unchanged since the last call to + :meth:`refresh`. + +.. method:: window.syncup() + + Touch all locations in ancestors of the window that have been changed in the + window. + +.. method:: window.syncdown() + + Touch each location in the window that has been touched in any of its ancestor + windows. This routine is called by :meth:`refresh`, so it should almost never + be necessary to call it manually. + +.. method:: window.cursyncup() + + Update the current cursor position of all the ancestors of the window to + reflect the current cursor position of the window. + + +Coordinate conversion +~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.enclose(y, x) + + Test whether the given pair of screen-relative character-cell coordinates are + enclosed by the given window, returning ``True`` or ``False``. It is useful for + determining what subset of the screen windows enclose the location of a mouse + event. + + .. versionchanged:: 3.10 + Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. + +.. method:: window.mouse_trafo(y, x, to_screen) + + Convert between window-relative and screen-relative (``stdscr``-relative) character-cell coordinates. + If *to_screen* is true, convert the window-relative coordinates *y*, *x* to screen-relative coordinates; + otherwise convert in the opposite direction. + The two coordinate systems differ when lines are reserved on the screen, for example for soft labels. + + Return the converted coordinates as a ``(y, x)`` tuple, or ``None`` if they lie outside the window. + + .. versionadded:: next + + +Other +~~~~~ + +.. attribute:: window.encoding + + Encoding used to encode method arguments (Unicode strings and characters). + The encoding attribute is inherited from the parent window when a subwindow + is created, for example with :meth:`window.subwin`. + By default, current locale encoding is used (see :func:`locale.getencoding`). + + .. versionadded:: 3.3 .. method:: window.putwin(file) Write all data associated with the window into the provided file object. This information can be later retrieved using the :func:`getwin` function. +.. method:: window.use(func, /, *args, **kwargs) -.. method:: window.redrawln(beg, num) + Call ``func(window, *args, **kwargs)`` with the lock of the window held, + and return its result. + This provides automatic protection for the window + against concurrent access from another thread. - Indicate that the *num* screen lines, starting at line *beg*, are corrupted and - should be completely redrawn on the next :meth:`refresh` call. + Availability: if the underlying curses library provides ``use_window()``. + .. versionadded:: next -.. method:: window.redrawwin() - Touch the entire window, causing it to be completely redrawn on the next - :meth:`refresh` call. +State queries +~~~~~~~~~~~~~ +.. method:: window.is_cleared() -.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) + Return the current value set by :meth:`clearok`. - Update the display immediately (sync actual screen with previous - drawing/deleting methods). + .. versionadded:: next - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, - *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be - displayed in the pad is calculated from the screen coordinates, since the - rectangles must be the same size. Both rectangles must be entirely contained - within their respective structures. Negative values of *pminrow*, *pmincol*, - *sminrow*, or *smincol* are treated as if they were zero. +.. method:: window.is_idcok() + Return the current value set by :meth:`idcok`. -.. method:: window.resize(nlines, ncols) + .. versionadded:: next - Reallocate storage for a curses window to adjust its dimensions to the - specified values. If either dimension is larger than the current values, the - window's data is filled with blanks that have the current background - rendition (as set by :meth:`bkgdset`) merged into them. +.. method:: window.is_idlok() + Return the current value set by :meth:`idlok`. -.. method:: window.scroll([lines=1]) + .. versionadded:: next - Scroll the screen or scrolling region upward by *lines* lines. +.. method:: window.is_immedok() + Return the current value set by :meth:`immedok`. -.. method:: window.scrollok(flag) + .. versionadded:: next - Control what happens when the cursor of a window is moved off the edge of the - window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is ``False``, the - cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up - one line. Note that in order to get the physical scrolling effect on the - terminal, it is also necessary to call :meth:`idlok`. +.. method:: window.is_keypad() + Return the current value set by :meth:`keypad`. -.. method:: window.setscrreg(top, bottom) + .. versionadded:: next - Set the scrolling region from line *top* to line *bottom*. All scrolling actions - will take place in this region. +.. method:: window.is_leaveok() + Return the current value set by :meth:`leaveok`. -.. method:: window.standend() + .. versionadded:: next - Turn off the standout attribute. On some terminals this has the side effect of - turning off all attributes. +.. method:: window.is_linetouched(line) + Return ``True`` if the specified line was modified since the last call to + :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` + exception if *line* is not valid for the given window. -.. method:: window.standout() +.. method:: window.is_nodelay() - Turn on attribute *A_STANDOUT*. + Return the current value set by :meth:`nodelay`. + .. versionadded:: next -.. method:: window.subpad(begin_y, begin_x) - window.subpad(nlines, ncols, begin_y, begin_x) +.. method:: window.is_notimeout() - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return the current value set by :meth:`notimeout`. + .. versionadded:: next -.. method:: window.subwin(begin_y, begin_x) - window.subwin(nlines, ncols, begin_y, begin_x) +.. method:: window.is_pad() - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return ``True`` if the window is a pad created by :func:`newpad`. - By default, the sub-window will extend from the specified position to the lower - right corner of the window. + .. versionadded:: next +.. method:: window.is_scrollok() -.. method:: window.syncdown() + Return the current value set by :meth:`scrollok`. - Touch each location in the window that has been touched in any of its ancestor - windows. This routine is called by :meth:`refresh`, so it should almost never - be necessary to call it manually. + .. versionadded:: next +.. method:: window.is_subwin() -.. method:: window.syncok(flag) + Return ``True`` if the window is a subwindow created by :meth:`subwin` + or :meth:`derwin`. - If *flag* is ``True``, then :meth:`syncup` is called automatically - whenever there is a change in the window. + .. versionadded:: next +.. method:: window.is_syncok() -.. method:: window.syncup() + Return the current value set by :meth:`syncok`. - Touch all locations in ancestors of the window that have been changed in the - window. + .. versionadded:: next +.. method:: window.is_wintouched() -.. method:: window.timeout(delay) + Return ``True`` if the specified window was modified since the last call to + :meth:`refresh`; otherwise return ``False``. - Set blocking or non-blocking read behavior for the window. If *delay* is - negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and :meth:`getch` will - return ``-1`` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is - still no input at the end of that time. +.. _curses-screen-objects: +Screen objects +-------------- -.. method:: window.touchline(start, count[, changed]) +.. class:: screen - Pretend *count* lines have been changed, starting with line *start*. If - *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). + A *screen* object represents a terminal initialized by :func:`newterm` + (or :func:`new_prescr`), + in addition to the default screen created by :func:`initscr`. + Screen objects are returned by those functions; + they cannot be instantiated directly. + A screen is freed automatically once it is no longer referenced, + either directly or through one of its windows. + Each window keeps its screen alive, + so a screen remains valid as long as any of its windows does. -.. method:: window.touchwin() + .. versionadded:: next - Pretend the whole window has been changed, for purposes of drawing - optimizations. +.. method:: screen.close() -.. method:: window.untouchwin() + Detach the screen's standard window, + breaking the reference cycle between them + so the screen can be reclaimed promptly instead of waiting for a + garbage collection. + Afterwards :attr:`~screen.stdscr` is ``None`` + and the window it returned earlier can no longer be used. + The screen's resources are released + once it and all its windows are no longer referenced. - Mark all lines in the window as unchanged since the last call to - :meth:`refresh`. + .. versionadded:: next -.. method:: window.vline(ch, n[, attr]) - window.vline(y, x, ch, n[, attr]) +.. attribute:: screen.stdscr - Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch* with attributes *attr*. + The standard :ref:`window ` of the screen, + covering the whole terminal, + or ``None`` for a screen created by :func:`new_prescr`. + + +.. method:: screen.use(func, /, *args, **kwargs) + + Call ``func(screen, *args, **kwargs)`` with the lock of the screen held, + and return its result. + This provides automatic protection for the screen + against concurrent access from another thread. + + Availability: if the underlying curses library provides ``use_screen()``. + + .. versionadded:: next + + +.. _curses-complexchar-objects: + +Complex character objects +------------------------- + +.. class:: complexchar(text, /, attr=0, pair=0) + + A *complex character* (or *complexchar*) is an immutable styled + character cell: a spacing character optionally followed by combining + characters, together with a set of attributes and a color pair. + + *text* is the cell's text, *attr* a combination of the + :ref:`WA_* attributes ` (equivalent to the matching + ``A_*`` constants), and *pair* a color pair number. Unlike the packed + :class:`chtype ` used by :meth:`~window.inch` and the ``A_*`` methods, + the color pair is stored separately and is not limited to the value that + fits in a :func:`color_pair`. + + Complex characters are returned by :meth:`window.in_wch` and + :meth:`window.getbkgrnd`, and are accepted (along with an integer, a byte + or a string) by the character-cell methods such as :meth:`window.addch`, + :meth:`window.insch`, :meth:`window.bkgd`, :meth:`window.border`, + :meth:`window.hline` and :meth:`window.vline`. A complex character already + carries its own rendition, so it cannot be combined with an explicit *attr* + argument. + + :func:`str` returns the cell's text; two complex characters are equal when + their text, attributes and color pair all match. + + The same code works on both wide- and narrow-character builds. On a narrow + build a cell holds a single character (no combining marks) that must encode to + one byte in the window's encoding (8-bit locales only), and *pair* is limited + to the value that fits in a :func:`color_pair`. + + .. attribute:: attr + + The attributes of the character cell (read-only). + + .. attribute:: pair + + The color pair number of the character cell (read-only). + + .. versionadded:: next + + +.. class:: complexstr(cells[, attr[, pair]]) + + A *complex character string* (or *complexstr*) is an immutable sequence of + styled character cells -- the string counterpart of + :class:`complexchar` (as :class:`str` is to a single character). + + If *cells* is a string, it is split into character cells (each a spacing + character optionally followed by combining characters), and *attr* (a + combination of the :ref:`WA_* attributes `) and *pair* + (a color pair number), if given, are applied to every cell. + + Otherwise *cells* is an iterable whose items are themselves cells, each a + :class:`complexchar` or a string; each item then carries its own rendition, + and *attr* and *pair* must be omitted. + + It is returned by :meth:`window.in_wchstr`, and accepted by + :meth:`window.addstr`, :meth:`~window.addnstr`, :meth:`~window.insstr` and + :meth:`~window.insnstr`, so a run read from a window can be written back + unchanged. + + It behaves like an immutable sequence: ``len(s)`` is the number of cells, + ``s[i]`` is the *i*-th cell as a :class:`complexchar`, slicing and + concatenation produce new :class:`!complexstr` instances, and iterating + yields the cells. :func:`str` returns the cells' text joined together, and + two complex character strings are equal when their cells all match. It is + hashable. + + To build or edit a run of cells, use an ordinary :class:`list` of + :class:`complexchar` (or strings); a :class:`!complexstr` is the immutable + form returned by a read. + + Like :class:`complexchar`, this type works on both wide- and narrow-character + builds, with the same per-cell limitations on a narrow build. + + .. versionadded:: next Constants --------- -The :mod:`curses` module defines the following data members: +General +~~~~~~~ + +The :mod:`!curses` module defines the following data members: .. data:: ERR @@ -1378,19 +2219,22 @@ The :mod:`curses` module defines the following data members: .. data:: COLS - The width of the screen, i.e., the number of columns. + The width of the screen, that is, the number of columns. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. .. data:: LINES - The height of the screen, i.e., the number of lines. + The height of the screen, that is, the number of lines. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. +Attributes +~~~~~~~~~~ + Some constants are available to specify character cell attributes. The exact constants available are system dependent. @@ -1436,6 +2280,24 @@ The exact constants available are system dependent. .. versionadded:: 3.7 ``A_ITALIC`` was added. +.. _curses-wa-constants: + +The :meth:`~window.attr_get`, :meth:`~window.attr_set`, :meth:`~window.attr_on` +and :meth:`~window.attr_off` methods use a parallel set of ``WA_*`` constants. +These have the same meaning as the corresponding ``A_*`` attributes above +(``WA_BOLD`` like :const:`A_BOLD`, and so on), but belong to the ``attr_t`` type +rather than being packed into a character. In ncurses the two sets share the +same values, but other curses implementations may give them different ones, so +use the ``WA_*`` constants with the ``attr_*`` methods. The available names are +``WA_ATTRIBUTES``, ``WA_NORMAL``, ``WA_STANDOUT``, ``WA_UNDERLINE``, +``WA_REVERSE``, ``WA_BLINK``, ``WA_DIM``, ``WA_BOLD``, ``WA_ALTCHARSET``, +``WA_INVIS``, ``WA_PROTECT``, ``WA_HORIZONTAL``, ``WA_LEFT``, ``WA_LOW``, +``WA_RIGHT``, ``WA_TOP``, ``WA_VERTICAL`` and ``WA_ITALIC`` (each available only +where the platform defines it). + +.. versionadded:: next + The ``WA_*`` constants were added. + Several constants are available to extract corresponding attributes returned by some methods. @@ -1452,11 +2314,12 @@ by some methods. | | color-pair field information | +-------------------------+-------------------------------+ +Keys +~~~~ + Keys are referred to by integer constants with names starting with ``KEY_``. The exact keycaps available are system dependent. -.. XXX this table is far too large! should it be alphabetized? - +-------------------------+--------------------------------------------+ | Key constant | Key | +=========================+============================================+ @@ -1674,6 +2537,9 @@ keys); also, the following keypad mappings are standard: | :kbd:`Page Down` | KEY_NPAGE | +------------------+-----------+ +Alternate character set +~~~~~~~~~~~~~~~~~~~~~~~~ + .. _curses-acs-codes: The following table lists characters from the alternate character set. These are @@ -1688,7 +2554,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | ACS code | Meaning | +========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | +| .. data:: ACS_BBSS | alternate name for upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_BLOCK | solid square block | +------------------------+------------------------------------------+ @@ -1696,7 +2562,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_BSBS | alternate name for horizontal line | +------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | +| .. data:: ACS_BSSB | alternate name for upper-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_BSSS | alternate name for top tee | +------------------------+------------------------------------------+ @@ -1722,9 +2588,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_LEQUAL | less-than-or-equal-to | +------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | +| .. data:: ACS_LLCORNER | lower-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | +| .. data:: ACS_LRCORNER | lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_LTEE | left tee | +------------------------+------------------------------------------+ @@ -1748,13 +2614,13 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_S9 | scan line 9 | +------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | +| .. data:: ACS_SBBS | alternate name for lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_SBSB | alternate name for vertical line | +------------------------+------------------------------------------+ | .. data:: ACS_SBSS | alternate name for right tee | +------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | +| .. data:: ACS_SSBB | alternate name for lower-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_SSBS | alternate name for bottom tee | +------------------------+------------------------------------------+ @@ -1768,13 +2634,16 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_UARROW | up arrow | +------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | +| .. data:: ACS_ULCORNER | upper-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | +| .. data:: ACS_URCORNER | upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_VLINE | vertical line | +------------------------+------------------------------------------+ +Mouse buttons +~~~~~~~~~~~~~ + The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ @@ -1794,13 +2663,16 @@ The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ | .. data:: BUTTON_CTRL | Control was down during button state change | +----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | +| .. data:: BUTTON_ALT | Alt was down during button state change | +----------------------------------+---------------------------------------------+ .. versionchanged:: 3.10 The ``BUTTON5_*`` constants are now exposed if they are provided by the underlying curses library. +Colors +~~~~~~ + The following table lists the predefined colors: +-------------------------+----------------------------+ @@ -1824,31 +2696,28 @@ The following table lists the predefined colors: +-------------------------+----------------------------+ -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== +:mod:`!curses.textpad` --- Text input widget for curses programs +================================================================ .. module:: curses.textpad :synopsis: Emacs-like input editing in a curses window. -.. moduleauthor:: Eric Raymond -.. sectionauthor:: Eric Raymond - -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles +The :mod:`!curses.textpad` module provides a :class:`Textbox` class that handles elementary text editing in a curses window, supporting a set of keybindings resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, FrameMaker, and many other programs). The module also provides a rectangle-drawing function useful for framing text boxes or for other purposes. -The module :mod:`curses.textpad` defines the following function: +The module :mod:`!curses.textpad` defines the following function: .. function:: rectangle(win, uly, ulx, lry, lrx) Draw a rectangle. The first argument must be a window object; the remaining arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the + arguments are the y and x coordinates of the upper-left corner of the rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using + coordinates of the lower-right corner. The rectangle will be drawn using VT100/IBM PC forms characters on terminals that make this possible (including xterm and most other software terminal emulators). Otherwise it will be drawn with ASCII dashes, vertical bars, and plus signs. @@ -1862,32 +2731,45 @@ Textbox objects You can instantiate a :class:`Textbox` object as follows: -.. class:: Textbox(win) +.. class:: Textbox(win, insert_mode=False) Return a textbox widget object. The *win* argument should be a curses :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. + be contained. If *insert_mode* is true, the textbox inserts typed + characters, shifting existing text to the right, rather than overwriting it. + The edit cursor of the textbox is initially located at the + upper-left corner of the containing window, with coordinates ``(0, 0)``. The instance's :attr:`stripspaces` flag is initially on. + .. versionchanged:: next + Entering and reading back the full Unicode range, including combining + characters, is now supported when curses is built with wide-character + support. + :class:`Textbox` objects have the following methods: - .. method:: edit([validator]) + .. method:: edit(validate=None) This is the entry point you will normally use. It accepts editing keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for + *validate* is supplied, it must be a function. It will be called for each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a + is done on the result. If it returns a false value, the keystroke is + ignored. This method returns the window contents as a string; whether blanks in the window are included is affected by the :attr:`stripspaces` attribute. + .. versionchanged:: next + *validate* is now called with a non-ASCII character as a string; + other keystrokes are still passed as an integer. + .. method:: do_command(ch) - Process a single command keystroke. Here are the supported special - keystrokes: + Process a single command keystroke. Returns ``1`` to continue editing, + or ``0`` if a termination keystroke was processed. Here are the supported + special keystrokes: +------------------+-------------------------------------------+ | Keystroke | Action | @@ -1910,7 +2792,8 @@ You can instantiate a :class:`Textbox` object as follows: | :kbd:`Control-H` | Delete character backward. | +------------------+-------------------------------------------+ | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | + | | otherwise move to the start of the next | + | | line. | +------------------+-------------------------------------------+ | :kbd:`Control-K` | If line is blank, delete it, otherwise | | | clear to end of line. | diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index cff36e258224d34..ce59d89843e761d 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -4,9 +4,6 @@ .. module:: dataclasses :synopsis: Generate special methods on user-defined classes. -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - **Source code:** :source:`Lib/dataclasses.py` -------------- @@ -103,12 +100,25 @@ Module contents ignored. - *eq*: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. + generated. - If the class already defines :meth:`!__eq__`, this parameter is - ignored. + This method compares the class by comparing each field in order. Both + instances in the comparison must be of the identical type. + + If the class already defines :meth:`!__eq__`, this parameter is ignored. + + .. versionchanged:: 3.13 + The generated ``__eq__`` method now compares each field individually + (for example, ``self.a == other.a and self.b == other.b``), rather than + comparing tuples of fields as in previous versions. + + This change makes the comparison faster but it may alter results in cases + where attributes compare equal by identity but not by value (such as + ``float('nan')``). + + In Python 3.12 and earlier, the comparison was performed by creating + tuples of the fields and comparing them (for example, + ``(self.a, self.b) == (other.a, other.b)``). - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be @@ -333,6 +343,10 @@ Module contents :attr:`!C.t` will be ``20``, and the class attributes :attr:`!C.x` and :attr:`!C.y` will not be set. + .. versionchanged:: 3.15 + If *metadata* is ``None``, use an empty :class:`frozendict`, instead + of a :func:`~types.MappingProxyType` of an empty :class:`dict`. + .. class:: Field :class:`!Field` objects describe each defined field. These objects @@ -370,8 +384,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -401,8 +415,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: @@ -417,7 +431,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, qualname=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -433,6 +447,9 @@ Module contents of the dataclass is set to that value. By default, it is set to the module name of the caller. + If *qualname* is defined, the :attr:`~type.__qualname__` attribute of the dataclass + is set to that value. By default, it is set to the value passed to *cls_name*. + The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments as :deco:`dataclass`. By default, the :deco:`dataclass` @@ -463,6 +480,8 @@ Module contents .. versionadded:: 3.14 Added the *decorator* parameter. + .. versionadded:: next + Added the *qualname* parameter. .. function:: replace(obj, /, **changes) @@ -497,7 +516,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 000000000000000..3c6b9b4beb7ab1d --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 000000000000000..e6b1cf877a574f2 --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 8ae1c1fb9e46bb4..f3c4ef9199075c9 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -4,16 +4,10 @@ .. module:: datetime :synopsis: Basic date and time types. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/datetime.py` -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +32,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -65,7 +60,7 @@ understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. +can be set to an instance of a subclass of the abstract :class:`!tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. @@ -77,6 +72,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +89,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +140,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +149,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: + +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. - object - timedelta - tzinfo - timezone - time - date - datetime -Common Properties +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +172,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +197,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -229,8 +230,8 @@ A :class:`timedelta` object represents a duration, the difference between two *days*, *seconds* and *microseconds* are "merged" and normalized into those three resulting attributes:: - >>> from datetime import timedelta - >>> delta = timedelta( + >>> import datetime as dt + >>> delta = dt.timedelta( ... days=50, ... seconds=27, ... microseconds=10, @@ -243,6 +244,12 @@ A :class:`timedelta` object represents a duration, the difference between two >>> delta datetime.timedelta(days=64, seconds=29156, microseconds=10) + .. tip:: + ``import datetime as dt`` instead of ``import datetime`` or + ``from datetime import datetime`` to avoid confusion between the module + and the class. See `How I Import Python’s datetime Module + `__. + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using @@ -256,8 +263,8 @@ A :class:`timedelta` object represents a duration, the difference between two Note that normalization of negative values may be surprising at first. For example:: - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) + >>> import datetime as dt + >>> d = dt.timedelta(microseconds=-1) >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) @@ -296,6 +303,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -319,13 +327,14 @@ Instance attributes (read-only): .. doctest:: - >>> from datetime import timedelta - >>> duration = timedelta(seconds=11235813) + >>> import datetime as dt + >>> duration = dt.timedelta(seconds=11235813) >>> duration.days, duration.seconds (130, 3813) >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -333,8 +342,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -396,7 +403,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -432,9 +438,9 @@ objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and + :class:`!timedelta` object are now supported, as are remainder operations and the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. + :class:`!timedelta` object by a :class:`float` object are now supported. :class:`timedelta` objects support equality and order comparisons. @@ -447,23 +453,24 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) + >>> another_year = dt.timedelta(weeks=40, days=84, hours=23, + ... minutes=50, seconds=600) >>> year == another_year True >>> year.total_seconds() @@ -471,8 +478,8 @@ An additional example of normalization:: Examples of :class:`timedelta` arithmetic:: - >>> from datetime import timedelta - >>> year = timedelta(days=365) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) >>> ten_years = 10 * year >>> ten_years datetime.timedelta(days=3650) @@ -485,10 +492,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -517,9 +525,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -541,7 +550,7 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -562,25 +571,27 @@ Other constructors, all class methods: Examples:: - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') + >>> import datetime as dt + >>> dt.date.fromisoformat('2019-12-04') datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') + >>> dt.date.fromisoformat('20191204') datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') + >>> dt.date.fromisoformat('2021-W01-1') datetime.date(2021, 1, 4) .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -595,20 +606,19 @@ Other constructors, all class methods: .. note:: - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import date + >>> import datetime as dt >>> date_string = "02/29" - >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -700,7 +710,7 @@ Notes: In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. - Order comparison between a :class:`!date` object that is not also a + Order comparison between a :class:`date` object that is not also a :class:`.datetime` instance and a :class:`!datetime` object raises :exc:`TypeError`. @@ -723,8 +733,8 @@ Instance methods: Example:: - >>> from datetime import date - >>> d = date(2002, 12, 31) + >>> import datetime as dt + >>> d = dt.date(2002, 12, 31) >>> d.replace(day=26) datetime.date(2002, 12, 26) @@ -782,23 +792,25 @@ Instance methods: For example, 2004 begins on a Thursday, so the first week of ISO year 2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() + >>> import datetime as dt + >>> dt.date(2003, 12, 29).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() + >>> dt.date(2004, 1, 4).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=7) .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -808,8 +820,8 @@ Instance methods: Return a string representing the date:: - >>> from datetime import date - >>> date(2002, 12, 4).ctime() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).ctime() 'Wed Dec 4 00:00:00 2002' ``d.ctime()`` is equivalent to:: @@ -835,19 +847,20 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: >>> import time - >>> from datetime import date - >>> today = date.today() + >>> import datetime as dt + >>> today = dt.date.today() >>> today datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) + >>> today == dt.date.fromtimestamp(time.time()) True - >>> my_birthday = date(today.year, 6, 24) + >>> my_birthday = dt.date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) ... @@ -861,8 +874,8 @@ More examples of working with :class:`date`: .. doctest:: - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 + >>> import datetime as dt + >>> d = dt.date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) @@ -878,7 +891,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -905,7 +918,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -913,7 +926,7 @@ from a :class:`date` object and a :class:`.time` object. Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. +:class:`!datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: @@ -937,6 +950,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -952,6 +966,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -972,6 +987,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1063,13 +1079,13 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. - .. versionchanged:: 3.15 - Accepts any real number as *timestamp*, not only integer or float. - .. classmethod:: datetime.fromordinal(ordinal) @@ -1086,7 +1102,7 @@ Other constructors, all class methods: are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. If the *date* argument is a :class:`.datetime` object, its time components + is used. If the *date* argument is a :class:`!datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. For any :class:`.datetime` object ``d``, @@ -1112,24 +1128,24 @@ Other constructors, all class methods: Examples:: - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') + >>> import datetime as dt + >>> dt.datetime.fromisoformat('2011-11-04') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') + >>> dt.datetime.fromisoformat('20111104') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23Z') datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') + >>> dt.datetime.fromisoformat('20111104T000523') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') + >>> dt.datetime.fromisoformat('2011-W01-2T00:05:23.283') datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1142,12 +1158,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1162,22 +1179,21 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. - .. versionchanged:: 3.13 + .. versionchanged:: 3.15 - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> date_string = "02/29" - >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -1255,6 +1271,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1290,7 +1307,7 @@ Supported operations: datetime, and no time zone adjustments are done even if the input is aware. (3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if + Subtraction of a :class:`.datetime` from a :class:`!datetime` is defined only if both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. @@ -1308,7 +1325,7 @@ Supported operations: :class:`.datetime` objects are equal if they represent the same date and time, taking into account the time zone. - Naive and aware :class:`!datetime` objects are never equal. + Naive and aware :class:`.datetime` objects are never equal. If both comparands are aware, and have the same :attr:`!tzinfo` attribute, the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and @@ -1316,7 +1333,7 @@ Supported operations: If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. - :class:`!datetime` instances in a repeated interval are never equal to + :class:`.datetime` instances in a repeated interval are never equal to :class:`!datetime` instances in other time zone. (5) @@ -1345,6 +1362,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1500,11 +1518,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1513,7 +1533,7 @@ Instance methods: Naive :class:`.datetime` instances are assumed to represent local time and this method relies on platform C functions to perform - the conversion. Since :class:`.datetime` supports a wider range of + the conversion. Since :class:`!datetime` supports a wider range of values than the platform C functions on many platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` for times far in the past or far in the future. @@ -1523,16 +1543,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. versionchanged:: 3.6 - This method no longer relies on the platform C :c:func:`mktime` - function to perform conversions. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1547,6 +1557,17 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1582,24 +1603,24 @@ Instance methods: Examples:: - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + >>> import datetime as dt + >>> dt.datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + >>> dt.datetime(2019, 5, 18, 15, 17, tzinfo=dt.timezone.utc).isoformat() '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example:: - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): + >>> import datetime as dt + >>> class TZ(dt.tzinfo): ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) + ... def utcoffset(self, when): + ... return dt.timedelta(hours=-6, minutes=-39) ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') + >>> dt.datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + >>> dt.datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional @@ -1623,11 +1644,11 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument:: - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP + >>> import datetime as dt + >>> dt.datetime.now().isoformat(timespec='minutes') # doctest: +SKIP '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') + >>> my_datetime = dt.datetime(2015, 1, 1, 12, 30, 59, 0) + >>> my_datetime.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' .. versionchanged:: 3.6 @@ -1644,8 +1665,8 @@ Instance methods: Return a string representing the date and time:: - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + >>> import datetime as dt + >>> dt.datetime(2002, 12, 4, 20, 30, 40).ctime() 'Wed Dec 4 20:30:40 2002' The output string will *not* include time zone information, regardless @@ -1675,34 +1696,34 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time, timezone + >>> import datetime as dt >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) + >>> d = dt.date(2005, 7, 14) + >>> t = dt.time(12, 30) + >>> dt.datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP + >>> dt.datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP + >>> dt.datetime.now(dt.timezone.utc) # doctest: +SKIP datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt + >>> my_datetime = dt.datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") + >>> my_datetime datetime.datetime(2006, 11, 21, 16, 30) >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() + >>> tt = my_datetime.timetuple() >>> for it in tt: # doctest: +SKIP ... print(it) ... @@ -1717,7 +1738,7 @@ Examples of working with :class:`.datetime` objects: -1 # dst - method tzinfo.dst() returned None >>> # Date in ISO format - >>> ic = dt.isocalendar() + >>> ic = my_datetime.isocalendar() >>> for it in ic: # doctest: +SKIP ... print(it) ... @@ -1726,55 +1747,55 @@ Examples of working with :class:`.datetime` objects: 2 # ISO weekday >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") + >>> my_datetime.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(my_datetime, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' The example below defines a :class:`tzinfo` subclass capturing time zone information for Kabul, Afghanistan, which used +4 UTC until 1945 and then +4:30 UTC thereafter:: - from datetime import timedelta, datetime, tzinfo, timezone + import datetime as dt - class KabulTz(tzinfo): + class KabulTz(dt.tzinfo): # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + UTC_MOVE_DATE = dt.datetime(1944, 12, 31, 20, tzinfo=dt.timezone.utc) - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + def utcoffset(self, when): + if when.year < 1945: + return dt.timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= when.timetuple()[:5] < (1945, 1, 1, 0, 30): # An ambiguous ("imaginary") half-hour range representing # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + # If when falls in the imaginary range, use fold to decide how + # to resolve. See PEP 495. + return dt.timedelta(hours=4, minutes=(30 if when.fold else 0)) else: - return timedelta(hours=4, minutes=30) + return dt.timedelta(hours=4, minutes=30) - def fromutc(self, dt): + def fromutc(self, when): # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): + if not isinstance(when, dt.datetime): raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") + if when.tzinfo is not self: + raise ValueError("when.tzinfo is not self") # A custom implementation is required for fromutc as # the input to this function is a datetime with utc values # but with a tzinfo set to self. # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) + if when.replace(tzinfo=dt.timezone.utc) >= self.UTC_MOVE_DATE: + return when + dt.timedelta(hours=4, minutes=30) else: - return dt + timedelta(hours=4) + return when + dt.timedelta(hours=4) - def dst(self, dt): + def dst(self, when): # Kabul does not observe daylight saving time. - return timedelta(0) + return dt.timedelta(0) - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: + def tzname(self, when): + if when >= self.UTC_MOVE_DATE: return "+04:30" return "+04" @@ -1783,17 +1804,17 @@ Usage of ``KabulTz`` from above:: >>> tz1 = KabulTz() >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> dt1 = dt.datetime(1900, 11, 21, 16, 30, tzinfo=tz1) >>> print(dt1.utcoffset()) 4:00:00 >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> dt2 = dt.datetime(2006, 6, 14, 13, 0, tzinfo=tz1) >>> print(dt2.utcoffset()) 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 = dt2.astimezone(dt.timezone.utc) >>> dt3 datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) >>> dt2 @@ -1801,9 +1822,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1824,6 +1846,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1882,6 +1905,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1904,8 +1928,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1926,22 +1950,22 @@ Other constructors: .. doctest:: - >>> from datetime import time - >>> time.fromisoformat('04:23:01') + >>> import datetime as dt + >>> dt.time.fromisoformat('04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') + >>> dt.time.fromisoformat('T04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') + >>> dt.time.fromisoformat('T042301') datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') + >>> dt.time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000384') + >>> dt.time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') + >>> dt.time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') + >>> dt.time.fromisoformat('04:23:01Z') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') + >>> dt.time.fromisoformat('04:23:01+00:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) @@ -1950,6 +1974,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -1974,7 +1999,7 @@ Instance methods: Return a new :class:`.time` with the same values, but with specified parameters updated. Note that ``tzinfo=None`` can be specified to create a - naive :class:`.time` from an aware :class:`.time`, without conversion of the + naive :class:`!time` from an aware :class:`!time`, without conversion of the time data. :class:`.time` objects are also supported by generic function @@ -2015,13 +2040,13 @@ Instance methods: Example:: - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + >>> import datetime as dt + >>> dt.time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') + >>> my_time = dt.time(hour=12, minute=34, second=56, microsecond=0) + >>> my_time.isoformat(timespec='microseconds') '12:34:56.000000' - >>> dt.isoformat(timespec='auto') + >>> my_time.isoformat(timespec='auto') '12:34:56' .. versionchanged:: 3.6 @@ -2066,29 +2091,31 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): + >>> import datetime as dt + >>> class TZ1(dt.tzinfo): + ... def utcoffset(self, when): + ... return dt.timedelta(hours=1) + ... def dst(self, when): + ... return dt.timedelta(0) + ... def tzname(self, when): ... return "+01:00" ... def __repr__(self): ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t = dt.time(12, 10, 30, tzinfo=TZ1()) >>> t datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() @@ -2105,25 +2132,25 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object + view their attributes as being in local time, and the :class:`!tzinfo` object supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + :class:`timezone`, a simple concrete subclass of :class:`!tzinfo` which can represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -2186,31 +2213,35 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz``. For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`!tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`!tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # a fixed-offset class: doesn't account for DST - return timedelta(0) + return dt.timedelta(0) or:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed + # transition times based on the input when.year, and expressed # in standard local time. - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) + if dston <= when.replace(tzinfo=None) < dstoff: + return dt.timedelta(hours=1) else: - return timedelta(0) + return dt.timedelta(0) The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. @@ -2227,17 +2258,17 @@ Examples of working with a :class:`.time` object:: valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for + of *dt* passed, especially if the :class:`!tzinfo` class is accounting for daylight time. The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the +response to their methods of the same names. A :class:`!datetime` object passes +itself as the argument, and a :class:`!time` object passes ``None`` as the argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. +accept a *dt* argument of ``None``, or of class :class:`!datetime`. When ``None`` is passed, it's up to the class designer to decide the best response. For example, returning ``None`` is appropriate if the class wishes to @@ -2245,10 +2276,10 @@ say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. -When a :class:`.datetime` object is passed in response to a :class:`.datetime` +When a :class:`.datetime` object is passed in response to a :class:`!datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local +rely on this, unless user code calls :class:`!tzinfo` methods directly. The +intent is that the :class:`!tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2276,20 +2307,22 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: Skipping code for error cases, the default :meth:`fromutc` implementation acts like:: - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() + import datetime as dt + + def fromutc(self, when): + # raise ValueError error if when.tzinfo is not self + dtoff = when.utcoffset() + dtdst = when.dst() # raise ValueError if dtoff is None or dtdst is None delta = dtoff - dtdst # this is self's standard offset if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() + when += delta # convert to standard local time + dtdst = when.dst() # raise ValueError if dtdst is None if dtdst: - return dt + dtdst + return when + dtdst else: - return dt + return when In the following :download:`tzinfo_examples.py <../includes/tzinfo_examples.py>` file there are some examples of @@ -2316,9 +2349,9 @@ When DST starts (the "start" line), the local wall clock leaps from 1:59 to ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. For example, at the Spring forward transition of 2016, we get:: - >>> from datetime import datetime, timezone + >>> import datetime as dt >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) + >>> u0 = dt.datetime(2016, 3, 13, 5, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2341,7 +2374,9 @@ form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) + >>> import datetime as dt + >>> from tzinfo_examples import HOUR, Eastern + >>> u0 = dt.datetime(2016, 11, 6, 4, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2358,7 +2393,7 @@ Note that the :class:`.datetime` instances that differ only by the value of the Applications that can't bear wall-time ambiguities should explicitly check the value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing +or any other fixed-offset :class:`!tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: @@ -2381,8 +2416,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2420,6 +2455,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2440,11 +2476,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2457,8 +2495,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2484,90 +2522,120 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') + >>> import datetime as dt + >>> dt.datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) >>> _.strftime('%a %d %b %Y, %I:%M%p') 'Mon 31 Jan 2022, 11:59PM' -The following is a list of all the format codes that the 1989 C standard -requires, and these work on all platforms with a standard C implementation. +The following is a list of all the format codes that the 2011 C standard +requires, and these work on all supported platforms. +-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | +| Directive | Meaning | Example | Notes | +| | | | | +===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | +| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | | | abbreviated name. | (en_US); | | | | || So, Mo, ..., Sa | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | +| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | | | | Saturday (en_US); | | | | || Sonntag, Montag, ..., | | | | | Samstag (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | +| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | | | name. | (en_US); | | | | || Jan, Feb, ..., Dez | | | | | (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | +| ``%B`` | Month as locale's full name. || January, February, | \(1) | | | | ..., December (en_US);| | | | || Januar, Februar, ..., | | | | | Dezember (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | -| | decimal number. | | | +| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | +| | time representation. | 1988 (en_US); | | +| | || Di 16 Aug 21:30:00 | | +| | | 1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | +| ``%C`` | The year divided by 100 and | 01, 02, ..., 99 | \(0) | +| | truncated to an integer as a | | | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9), | +| | zero-padded decimal number. | | \(10) | ++-----------+--------------------------------+------------------------+-------+ +| ``%D`` | Equivalent to ``%m/%d/%y``. | 11/28/25 | \(9) | +| | | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%e`` | The day of the month as a | ␣1, ␣2, ..., 31 | \(10) | +| | space-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%F`` | Equivalent to ``%Y-%m-%d``, | 2025-10-11, | | +| | the ISO 8601 format. | 1001-12-30 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%g`` | Last 2 digits of ISO 8601 year | 00, 01, ..., 99 | \(0) | +| | representing the year that | | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | +| | representing the year that | 2014, ..., 9998, 9999 | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%h`` | Equivalent to ``%b``. | See ``%b``. | \(0) | +-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE) | \(3) | +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | +| | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | | | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | -| | decimal number. | | \(9) | +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | +| | decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded to 6 | 999999 | | -| | digits. | | | +| ``%n`` | The newline character | ``\n`` | | +| | (``'\n'``). For | | | +| | :meth:`!strptime`, zero or | | | +| | more whitespace. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | -| | string if the object is | +063415, | | -| | naive). | -030712.345216 | | +| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | +| | AM or PM. || am, pm (de_DE) | \(3) | +-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | -| | if the object is naive). | | | +| ``%r`` | Locale's 12-hour clock time. | 12:00:00 AM | \(1), | +| | | | \(0) | +-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | -| | zero-padded decimal number. | | | +| ``%R`` | Equivalent to ``%H:%M``. | 10:01 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | +| | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%t`` | The tab character (``'\t'``). | ``\t`` | | +| | For :meth:`!strptime`, | | | +| | zero or more whitespace. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%T`` | ISO 8601 time format, | 10:01:59 | | +| | equivalent to ``%H:%M:%S``. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | +| | number where 1 is Monday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Sunday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2575,7 +2643,17 @@ requires, and these work on all platforms with a standard C implementation. | | Sunday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | +| | number with Monday as | | \(9) | +| | the first day of the week. | | | +| | Week 01 is the week containing | | | +| | Jan 4. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | +| | where 0 is Sunday and 6 is | | | +| | Saturday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | | | (Monday as the first day of | | \(9) | | | the week) as a zero-padded | | | | | decimal number. All days in a | | | @@ -2583,40 +2661,43 @@ requires, and these work on all platforms with a standard C implementation. | | Monday are considered to be in | | | | | week 0. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | +| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | | | representation. || 08/16/1988 (en_US); | | | | || 16.08.1988 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | | | representation. || 21:30:00 (de_DE) | | +-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | +| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | +| | number. | 2014, ..., 9998, 9999 | | +-----------+--------------------------------+------------------------+-------+ +| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | +| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | +| | string if the object is | +063415, | | +| | naive). | -030712.345216 | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | +| | if the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | % | | ++-----------+--------------------------------+------------------------+-------+ + +The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`~.datetime.strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. +Several additional directives not required by the C11 standard are included for +convenience. +-----------+--------------------------------+------------------------+-------+ | Directive | Meaning | Example | Notes | +===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | -| | number with Monday as | | \(9) | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | +| | number, zero-padded to 6 | 999999 | | +| | digits. | | | +-----------+--------------------------------+------------------------+-------+ | ``%:z`` | UTC offset in the form | (empty), +00:00, | \(6) | | | ``±HH:MM[:SS[.ffffff]]`` | -04:00, +10:30, | | @@ -2624,11 +2705,6 @@ convenience. These parameters all correspond to ISO 8601 date values. | | naive). | -03:07:12.345216 | | +-----------+--------------------------------+------------------------+-------+ -These may not be available on all platforms when used with the :meth:`~.datetime.strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`~.datetime.strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - The full set of format codes supported varies across platforms, because Python calls the platform C library's :c:func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your @@ -2639,21 +2715,61 @@ differences between platforms in handling of unsupported format specifiers. ``%G``, ``%u`` and ``%V`` were added. .. versionadded:: 3.12 - ``%:z`` was added for :meth:`~.datetime.strftime` + ``%:z`` was added for :meth:`~.datetime.strftime`. .. versionadded:: 3.15 - ``%:z`` was added for :meth:`~.datetime.strptime` + ``%D``, ``%F``, ``%n``, ``%t``, and ``%:z`` were added for + :meth:`~.datetime.strptime`. -Technical Detail + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a :meth:`~date.timetuple` method. -For the :meth:`.datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ +For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, +the default value is ``1900-01-01T00:00:00.000``: any components not specified +in the format string will be pulled from the default value. + +.. note:: + Format strings without separators can be ambiguous for parsing. For + example, with ``%Y%m%d``, the string ``2026111`` may be parsed either as + ``2026-11-01`` or as ``2026-01-11``. + Use separators to ensure the input is parsed as intended. + +.. note:: + When used to parse partial dates lacking a year, :meth:`.datetime.strptime` + and :meth:`.date.strptime` will raise when encountering February 29 because + the default year of 1900 is *not* a leap year. Always add a default leap + year to partial date strings before parsing. + +.. testsetup:: + + # doctest seems to turn the warning into an error which makes it + # show up and require matching and prevents the actual interesting + # exception from being raised. + # Manually apply the catch_warnings context manager + import warnings + catch_warnings = warnings.catch_warnings() + catch_warnings.__enter__() + warnings.simplefilter("ignore") + +.. testcleanup:: + + catch_warnings.__exit__() + +.. doctest:: + + >>> import datetime as dt + >>> value = "2/29" + >>> dt.datetime.strptime(value, "%m/%d") + Traceback (most recent call last): + ... + ValueError: day 29 must be in range 1..28 for month 2 in year 1900 + >>> dt.datetime.strptime(f"1904 {value}", "%Y %m/%d") + datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2679,6 +2795,9 @@ an empty string instead. Notes: +(0) + This format code is currently unsupported by :meth:`~.datetime.strptime`. + (1) Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for @@ -2790,23 +2909,24 @@ Notes: include a year in the format. If the value you need to parse lacks a year, append an explicit dummy leap year. Otherwise your code will raise an exception when it encounters leap day because the default year used by the - parser is not a leap year. Users run into this bug every four years... + parser (1900) is not a leap year. Users run into that bug every leap year. .. doctest:: >>> month_day = "02/29" - >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + Using ``%d`` without a year now raises :exc:`ValueError`. + + .. deprecated-removed:: 3.15 3.17 :meth:`~.datetime.strptime` calls using a format string containing - a day of month without a year now emit a - :exc:`DeprecationWarning`. In 3.15 or later we may change this into - an error or change the default year to a leap year. See :gh:`70647`. + ``%e`` without a year now emit a :exc:`DeprecationWarning`. .. rubric:: Footnotes -.. [#] If, that is, we ignore the effects of Relativity +.. [#] If, that is, we ignore the effects of relativity. .. [#] This matches the definition of the "proleptic Gregorian" calendar in Dershowitz and Reingold's book *Calendrical Calculations*, @@ -2817,5 +2937,3 @@ Notes: .. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar `_ for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 02eb68d7b49b048..646981e8692cc53 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,7 +8,7 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database: +:mod:`!dbm` is a generic interface to variants of the DBM database: * :mod:`dbm.sqlite3` * :mod:`dbm.gnu` @@ -107,7 +107,7 @@ will automatically close them when done. .. versionchanged:: 3.2 :meth:`!get` and :meth:`!setdefault` methods are now available for all - :mod:`dbm` backends. + :mod:`!dbm` backends. .. versionchanged:: 3.4 Added native support for the context management protocol to the objects @@ -118,7 +118,7 @@ will automatically close them when done. instead of :exc:`KeyError`. .. versionchanged:: 3.13 - :meth:`!clear` methods are now available for all :mod:`dbm` backends. + :meth:`!clear` methods are now available for all :mod:`!dbm` backends. The following example records some hostnames and a corresponding title, and @@ -157,11 +157,10 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- +:mod:`!dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- .. module:: dbm.sqlite3 - :platform: All :synopsis: SQLite backend for dbm .. versionadded:: 3.13 @@ -171,8 +170,8 @@ The individual submodules are described in the following sections. -------------- This module uses the standard library :mod:`sqlite3` module to provide an -SQLite backend for the :mod:`dbm` module. -The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +SQLite backend for the :mod:`!dbm` module. +The files created by :mod:`!dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. .. include:: ../includes/wasm-notavail.rst @@ -215,36 +214,37 @@ or any other SQLite browser, including the SQLite CLI. .. note:: While reorganizing, as much as two times the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. .. versionadded:: 3.15 -:mod:`dbm.gnu` --- GNU database manager ---------------------------------------- +:mod:`!dbm.gnu` --- GNU database manager +---------------------------------------- .. module:: dbm.gnu - :platform: Unix :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +The :mod:`!dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`!dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -335,7 +335,7 @@ functionality like crash tolerance. .. note:: While reorganizing, as much as one time the size of the original database is required - in free disk space. However, be aware that this factor changes for each :mod:`dbm` submodule. + in free disk space. However, be aware that this factor changes for each :mod:`!dbm` submodule. .. method:: gdbm.sync() @@ -343,25 +343,24 @@ functionality like crash tolerance. unwritten data to be written to the disk. -:mod:`dbm.ndbm` --- New Database Manager ----------------------------------------- +:mod:`!dbm.ndbm` --- New Database Manager +----------------------------------------- .. module:: dbm.ndbm - :platform: Unix :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the +The :mod:`!dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`dbm.gnu` and :mod:`!dbm.ndbm` are incompatible and can not be used interchangeably. .. warning:: @@ -373,9 +372,11 @@ This module can be used with the "classic" NDBM interface or the .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised + Raised on :mod:`!dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -425,8 +426,8 @@ This module can be used with the "classic" NDBM interface or the Close the NDBM database. -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ +:mod:`!dbm.dumb` --- Portable DBM implementation +------------------------------------------------ .. module:: dbm.dumb :synopsis: Portable implementation of the simple DBM interface. @@ -437,23 +438,23 @@ This module can be used with the "classic" NDBM interface or the .. note:: - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` + The :mod:`!dbm.dumb` module is intended as a last resort fallback for the + :mod:`!dbm` module when a more robust module is not available. The :mod:`!dbm.dumb` module is not written for speed and is not nearly as heavily used as the other database modules. -------------- -The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +The :mod:`!dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. -Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +Unlike other :mod:`!dbm` backends, such as :mod:`dbm.gnu`, no external library is required. The :mod:`!dbm.dumb` module defines the following: .. exception:: error - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -484,7 +485,7 @@ The :mod:`!dbm.dumb` module defines the following: Python's AST compiler. .. warning:: - :mod:`dbm.dumb` does not support concurrent read/write access. (Multiple + :mod:`!dbm.dumb` does not support concurrent read/write access. (Multiple simultaneous read accesses are safe.) When a program has the database open for writing, no other program should have it open for reading or writing. @@ -517,7 +518,7 @@ The :mod:`!dbm.dumb` module defines the following: .. note:: While reorganizing, no additional free disk space is required. However, be aware - that this factor changes for each :mod:`dbm` submodule. + that this factor changes for each :mod:`!dbm` submodule. .. versionadded:: 3.15 diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 60223657a440434..f87c2481fb89cdb 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -1,5 +1,5 @@ *********************** -Debugging and Profiling +Debugging and profiling *********************** These libraries help you with Python development: the debugger enables you to @@ -15,7 +15,8 @@ intrusive debugging or patching. bdb.rst faulthandler.rst pdb.rst - profile.rst + profiling.rst + pstats.rst timeit.rst trace.rst tracemalloc.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 985153b5443f5ca..2af5dfce9612b37 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -4,14 +4,6 @@ .. module:: decimal :synopsis: Implementation of the General Decimal Arithmetic Specification. -.. moduleauthor:: Eric Price -.. moduleauthor:: Facundo Batista -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Aahz -.. moduleauthor:: Tim Peters -.. moduleauthor:: Stefan Krah -.. sectionauthor:: Raymond D. Hettinger - **Source code:** :source:`Lib/decimal.py` .. import modules for testing inline doctests with the Sphinx doctest builder @@ -30,14 +22,16 @@ -------------- -The :mod:`decimal` module provides support for fast correctly rounded +The :mod:`!decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. +* Decimal "is based on a `floating-point model + `__ which was designed + with people in mind, and necessarily has a paramount guiding principle -- + computers must provide an arithmetic that works in the same way as the + arithmetic that people learn at school." -- excerpt from the decimal + arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like ``1.1`` and ``2.2`` do not have exact representations in binary @@ -238,6 +232,26 @@ floating-point flying circus: >>> c % a Decimal('0.77') +Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in +fixed-point or scientific notation, using the same formatting syntax (see +:ref:`formatspec`) as builtin :class:`float` type: + +.. doctest:: + + >>> format(Decimal('2.675'), "f") + '2.675' + >>> format(Decimal('2.675'), ".2f") + '2.68' + >>> f"{Decimal('2.675'):.2f}" + '2.68' + >>> format(Decimal('2.675'), ".2e") + '2.68e+0' + >>> with localcontext() as ctx: + ... ctx.rounding = ROUND_DOWN + ... print(format(Decimal('2.675'), ".2f")) + ... + 2.67 + And some mathematical functions are also available to Decimal: >>> getcontext().prec = 28 @@ -264,10 +278,10 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`!decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -1575,7 +1589,7 @@ Constants Specification that this implementation complies with. See https://speleotrove.com/decimal/decarith.html for the specification. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are only relevant for the C module. They @@ -1825,7 +1839,7 @@ properties of addition: >>> u * (v+w) Decimal('0.0060000') -The :mod:`decimal` module makes it possible to restore the identities by +The :mod:`!decimal` module makes it possible to restore the identities by expanding the precision sufficiently to avoid loss of significance: .. doctest:: newcontext @@ -1847,7 +1861,7 @@ expanding the precision sufficiently to avoid loss of significance: Special values ^^^^^^^^^^^^^^ -The number system for the :mod:`decimal` module provides special values +The number system for the :mod:`!decimal` module provides special values including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, and two zeros, ``+0`` and ``-0``. @@ -2109,20 +2123,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2140,10 +2154,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2175,21 +2189,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2207,10 +2221,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2225,9 +2239,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2236,19 +2250,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2276,9 +2290,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index e0693e8eb6ed226..06f2d6ddbf4f48b 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,31 +1,55 @@ -Tkinter Dialogs +Tkinter dialogs =============== -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.simpledialog - :platform: Tk :synopsis: Simple dialog windows **Source code:** :source:`Lib/tkinter/simpledialog.py` -------------- -The :mod:`tkinter.simpledialog` module contains convenience classes and +The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) +.. function:: askfloat(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askinteger(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askstring(title, prompt, *, initialvalue=None, show=None, parent=None, use_ttk=True) - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. + Prompt the user to enter a value of the desired type and return it, or + ``None`` if the dialog is cancelled. -.. class:: Dialog(parent, title=None) + *title* is the dialog title and *prompt* the message shown above the entry. + *initialvalue* is the value initially placed in the entry. + *parent* is the window over which the dialog is shown. + :func:`askinteger` and :func:`askfloat` also accept *minvalue* and + *maxvalue*, which bound the accepted value. + :func:`askstring` also accepts *show*, a character used to mask the entered + text, for example ``'*'`` to hide a password. + They use the themed :mod:`tkinter.ttk` widgets; pass ``use_ttk=False`` for + the classic widgets. + +.. class:: Dialog(parent, title=None, *, use_ttk=False) The base class for custom dialogs. + Instantiating it shows the dialog modally and returns once the user closes + it; the entered value is then available in the :attr:`!result` attribute. + When *use_ttk* is false (the default), the dialog is built from the classic + :mod:`tkinter` widgets, modelled on the classic ``tk_dialog``; when true, + from the themed :mod:`tkinter.ttk` widgets, modelled on the Tk message box. + The default is classic for compatibility, since the themed widgets set a + themed background that classic widgets added in :meth:`body` would not match. + + .. versionchanged:: next + Added the *use_ttk* parameter. + + .. attribute:: result + + The value produced by :meth:`apply`, or ``None`` if the dialog was + cancelled. .. method:: body(master) @@ -37,23 +61,77 @@ functions for creating simple modal dialogs to get a value from the user. Default behaviour adds OK and Cancel buttons. Override for custom button layouts. + .. method:: validate() + + Validate the data entered by the user. + Return true if it is valid, in which case the dialog proceeds to + :meth:`apply`; return false to keep the dialog open. + The default implementation always returns true; override it to check the + input. + + .. method:: apply() + + Process the data entered by the user, for example by storing it in the + :attr:`!result` attribute. + Called after :meth:`validate` succeeds and just before the dialog is + destroyed. + The default implementation does nothing; override it to act on or store + the result. + + .. method:: destroy() + + Destroy the dialog window, clearing the reference to the widget that had + the initial focus. -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: SimpleDialog(master, text='', buttons=[], default=None, cancel=None, title=None, class_=None, *, bitmap=None, detail='', use_ttk=True) + + A simple modal dialog that displays the message *text* above a row of push + buttons given by *buttons*, and returns the index of the button the user + presses. + Each entry of *buttons* is either a button label, or a mapping of button + options such as ``{'text': 'OK', 'underline': 0}``; an ``underline`` option + makes :kbd:`Alt` plus the underlined character invoke the button. + *default* is the index of the default button, activated by the Return key + when no button has the focus, *cancel* the index returned when the window is + closed through the window manager, *title* the window title, and *class_* + the Tk class name of the window. + *bitmap* is the name of a bitmap displayed beside the message + (for example ``'warning'`` or ``'question'``); the standard names + ``'error'``, ``'info'``, ``'question'`` and ``'warning'`` are shown as + themed icons when *use_ttk* is true. + *detail* is a secondary message displayed below *text*. + When *use_ttk* is true (the default), the dialog is built from the themed + :mod:`tkinter.ttk` widgets, modelled on the Tk message box; when false, from + the classic :mod:`tkinter` widgets, modelled on ``tk_dialog``. + + .. versionchanged:: next + The dialog is now built from the themed :mod:`tkinter.ttk` widgets by + default, instead of the classic :mod:`tkinter` widgets. + Added the *bitmap*, *detail* and *use_ttk* parameters. + Entries of *buttons* may be mappings of button options. + + .. method:: go() + + Display the dialog, wait until the user presses a button or closes the + window, and return the index of the chosen button. + + + +:mod:`!tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.filedialog - :platform: Tk :synopsis: Dialog classes for file selection **Source code:** :source:`Lib/tkinter/filedialog.py` -------------- -The :mod:`tkinter.filedialog` module provides classes and factory functions for +The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. -Native Load/Save Dialogs +Native load/save dialogs ------------------------ The following classes and functions provide file dialog windows that combine a @@ -79,34 +157,47 @@ listed below: **Static factory functions** The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. +wait for the user's selection, and return it. +The exact return value depends on the function (see below); when the dialog is +cancelled it is the empty value documented for that function -- an empty +string, an empty tuple, an empty list or ``None``. .. function:: askopenfile(mode="r", **options) askopenfiles(mode="r", **options) - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. + Create an :class:`Open` dialog. + :func:`askopenfile` returns the opened file object, or ``None`` if the + dialog is cancelled. + :func:`askopenfiles` returns a list of the opened file objects, or an empty + list if cancelled. + The files are opened in mode *mode* (read-only ``'r'`` by default). .. function:: asksaveasfile(mode="w", **options) - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + Create a :class:`SaveAs` dialog and return the opened file object, or + ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (``'w'`` by default). .. function:: askopenfilename(**options) askopenfilenames(**options) - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). + Create an :class:`Open` dialog. + :func:`askopenfilename` returns the selected filename as a string, or an + empty string if the dialog is cancelled. + :func:`askopenfilenames` returns a tuple of the selected filenames, or an + empty tuple if cancelled. .. function:: asksaveasfilename(**options) - Create a :class:`SaveAs` dialog and return the selected filename. + Create a :class:`SaveAs` dialog and return the selected filename as a + string, or an empty string if the dialog is cancelled. .. function:: askdirectory(**options) - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. + Prompt the user to select a directory, and return its path as a string, or + an empty string if the dialog is cancelled. + Additional keyword option: *mustexist* - if true, the user may only select + an existing directory (false by default). .. class:: Open(master=None, **options) SaveAs(master=None, **options) @@ -126,9 +217,19 @@ These do not emulate the native look-and-feel of the platform. .. note:: The *FileDialog* class should be subclassed for custom event handling and behaviour. -.. class:: FileDialog(master, title=None) +.. class:: FileDialog(master, title=None, *, use_ttk=True) Create a basic file selection dialog. + Its layout -- a filter entry, side-by-side directory and file lists, and a + selection entry -- follows the classic Motif file selection dialog. + When *use_ttk* is true (the default), the dialog is built from the themed + :mod:`tkinter.ttk` widgets; when false, from the classic :mod:`tkinter` + widgets. + + .. versionchanged:: next + The dialog is now built from the themed :mod:`tkinter.ttk` widgets by + default, instead of the classic :mod:`tkinter` widgets. + Added the *use_ttk* parameter. .. method:: cancel_command(event=None) @@ -170,6 +271,13 @@ These do not emulate the native look-and-feel of the platform. Exit dialog returning current selection. + .. method:: ok_command() + + Called when the user confirms the current selection. + The base implementation accepts the selection and closes the dialog; + :class:`LoadFileDialog` and :class:`SaveFileDialog` override it to check + the selection first. + .. method:: quit(how=None) Exit dialog returning filename, if any. @@ -183,39 +291,44 @@ These do not emulate the native look-and-feel of the platform. Update the current file selection to *file*. -.. class:: LoadFileDialog(master, title=None) +.. class:: LoadFileDialog(master, title=None, *, use_ttk=True) A subclass of FileDialog that creates a dialog window for selecting an existing file. + .. versionchanged:: next + Added the *use_ttk* parameter. + .. method:: ok_command() Test that a file is provided and that the selection indicates an already existing file. -.. class:: SaveFileDialog(master, title=None) +.. class:: SaveFileDialog(master, title=None, *, use_ttk=True) A subclass of FileDialog that creates a dialog window for selecting a destination file. + .. versionchanged:: next + Added the *use_ttk* parameter. + .. method:: ok_command() Test whether or not the selection points to a valid file that is not a directory. Confirmation is required if an already existing file is selected. -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.commondialog - :platform: Tk :synopsis: Tkinter base class for dialogs **Source code:** :source:`Lib/tkinter/commondialog.py` -------------- -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +The :mod:`!tkinter.commondialog` module provides the :class:`Dialog` class that is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) @@ -225,6 +338,40 @@ is the base class for dialogs defined in other supporting modules. Render the Dialog window. +:mod:`!tkinter.dialog` --- Classic Tk dialog boxes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.dialog + :synopsis: A simple dialog box built on the classic Tk widgets. + +**Source code:** :source:`Lib/tkinter/dialog.py` + +-------------- + +The :mod:`!tkinter.dialog` module provides a simple modal dialog box built on +the classic (non-themed) Tk widgets. + +.. data:: DIALOG_ICON + + The name of the default bitmap (``'questhead'``) displayed by a + :class:`Dialog`. + +.. class:: Dialog(master=None, cnf={}, **kw) + + Display a modal dialog box built from the classic (non-themed) Tk widgets + and wait for the user to press one of its buttons. + The options, given through *cnf* or as keyword arguments, include *title* + (the window title), *text* (the message), *bitmap* (an icon, + :data:`DIALOG_ICON` by default), *default* (the index of the default button) + and *strings* (the sequence of button labels). + After construction, the :attr:`!num` attribute holds the index of the button + the user pressed. + + .. method:: destroy() + + Destroy the dialog window. + + .. seealso:: Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 9e5a62d8fe52609..25edb40e35a630a 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -4,10 +4,6 @@ .. module:: difflib :synopsis: Helpers for computing differences between objects. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. Markup by Fred L. Drake, Jr. - **Source code:** :source:`Lib/difflib.py` .. testsetup:: @@ -366,7 +362,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -594,7 +590,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -645,7 +641,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -694,7 +690,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of @@ -728,7 +724,7 @@ Finally, we compare the two: >>> result = list(d.compare(text1, text2)) -``result`` is a list of strings, so let's pretty-print it: +``result`` is a list of strings, so let's pretty-print it:: >>> from pprint import pprint >>> pprint(result) @@ -743,7 +739,7 @@ Finally, we compare the two: '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] -As a single multi-line string it looks like this: +As a single multi-line string it looks like this:: >>> import sys >>> sys.stdout.writelines(result) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index a24589fd0a5af38..5cc0d68a24acee9 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by +The :mod:`!dis` module supports the analysis of CPython :term:`bytecode` by disassembling it. The CPython bytecode which this module takes as an input is defined in the file :file:`Include/opcode.h` and used by the compiler and the interpreter. @@ -38,7 +38,7 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the + any :mod:`!dis` utility. Furthermore, the interpreter now adapts the bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. @@ -87,7 +87,7 @@ the following command can be used to display the disassembly of Command-line interface ---------------------- -The :mod:`dis` module can be invoked as a script from the command line: +The :mod:`!dis` module can be invoked as a script from the command line: .. code-block:: sh @@ -139,7 +139,7 @@ code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ show_caches=False, adaptive=False, show_offsets=False,\ - show_positions=False) + show_positions=False, show_jit=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -170,6 +170,9 @@ code. If *show_positions* is ``True``, :meth:`.dis` will include instruction source code positions in the output. + If *show_jit* is ``True``, :meth:`.dis` will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -205,6 +208,9 @@ code. .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + Example: .. doctest:: @@ -223,7 +229,7 @@ Example: Analysis functions ------------------ -The :mod:`dis` module also defines the following analysis functions that convert +The :mod:`!dis` module also defines the following analysis functions that convert the input directly to the desired output. They can be useful if only a single operation is being performed, so the intermediate analysis object isn't useful: @@ -259,7 +265,8 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -286,6 +293,9 @@ operation is being performed, so the intermediate analysis object isn't useful: If *adaptive* is ``True``, this function will display specialized bytecode that may be different from the original bytecode. + If *show_jit* is ``True``, this function will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. versionchanged:: 3.4 Added *file* parameter. @@ -304,8 +314,11 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ - show_offset=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -326,10 +339,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ - show_offsets=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -362,7 +379,10 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. -.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) + .. versionchanged:: next + Added the *show_jit* parameter. + +.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False, show_jit=False) Return an iterator over the instructions in the supplied function, method, source code string or code object. @@ -377,6 +397,8 @@ operation is being performed, so the intermediate analysis object isn't useful: The *adaptive* parameter works as it does in :func:`dis`. + The *show_jit* parameter works as it does in :func:`dis`. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -388,6 +410,9 @@ operation is being performed, so the intermediate analysis object isn't useful: field populated (regardless of the value of *show_caches*) and it no longer generates separate items for the cache entries. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: findlinestarts(code) This generator function uses the :meth:`~codeobject.co_lines` method @@ -400,7 +425,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.10 The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the - :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + :attr:`~codeobject.co_firstlineno` and :attr:`!codeobject.co_lnotab` attributes of the :ref:`code object `. .. versionchanged:: 3.13 @@ -768,7 +793,7 @@ not have to be) the original ``STACK[-2]``. end = STACK.pop() start = STACK.pop() container = STACK.pop() - values = STACK.pop() + value = STACK.pop() container[start:end] = value .. versionadded:: 3.12 @@ -1031,12 +1056,6 @@ iterations of the loop. The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. -.. opcode:: DELETE_NAME (namei) - - Implements ``del name``, where *namei* is the index into :attr:`~codeobject.co_names` - attribute of the :ref:`code object `. - - .. opcode:: UNPACK_SEQUENCE (count) Unpacks ``STACK[-1]`` into *count* individual values, which are put onto the stack @@ -1076,27 +1095,11 @@ iterations of the loop. where *namei* is the index of name in :attr:`~codeobject.co_names` of the :ref:`code object `. -.. opcode:: DELETE_ATTR (namei) - - Implements:: - - obj = STACK.pop() - del obj.name - - where *namei* is the index of name into :attr:`~codeobject.co_names` of the - :ref:`code object `. - - .. opcode:: STORE_GLOBAL (namei) Works as :opcode:`STORE_NAME`, but stores the name as a global. -.. opcode:: DELETE_GLOBAL (namei) - - Works as :opcode:`DELETE_NAME`, but deletes a global name. - - .. opcode:: LOAD_CONST (consti) Pushes ``co_consts[consti]`` onto the stack. @@ -1642,7 +1645,7 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`!LOAD_METHOD` for non-method calls. + :opcode:`LOAD_ATTR` for non-method calls. .. versionadded:: 3.11 @@ -1827,7 +1830,7 @@ iterations of the loop. ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Pseudo instructions were added to the :mod:`dis` module, and for them + Pseudo instructions were added to the :mod:`!dis` module, and for them it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index df3de8f622a0911..3298697af8511b6 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -4,16 +4,11 @@ .. module:: doctest :synopsis: Test pieces of code within docstrings. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Edward Loper - **Source code:** :source:`Lib/doctest.py` -------------- -The :mod:`doctest` module searches for pieces of text that look like interactive +The :mod:`!doctest` module searches for pieces of text that look like interactive Python sessions, and then executes those sessions to verify that they work exactly as shown. There are several common ways to use doctest: @@ -85,7 +80,7 @@ Here's a complete but small example module:: import doctest doctest.testmod() -If you run :file:`example.py` directly from the command line, :mod:`doctest` +If you run :file:`example.py` directly from the command line, :mod:`!doctest` works its magic: .. code-block:: shell-session @@ -94,7 +89,7 @@ works its magic: $ There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what +``-v`` to the script, and :mod:`!doctest` prints a detailed log of what it's trying, and prints a summary at the end: .. code-block:: shell-session @@ -130,7 +125,7 @@ And so on, eventually ending with: Test passed. $ -That's all you need to know to start making productive use of :mod:`doctest`! +That's all you need to know to start making productive use of :mod:`!doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file @@ -252,7 +247,7 @@ For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. Command-line Usage ------------------ -The :mod:`doctest` module can be invoked as a script from the command line: +The :mod:`!doctest` module can be invoked as a script from the command line: .. code-block:: bash @@ -450,7 +445,7 @@ The fine print: What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, each time :mod:`doctest` finds a docstring to test, it uses a +By default, each time :mod:`!doctest` finds a docstring to test, it uses a *shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can @@ -730,7 +725,7 @@ The second group of options controls how test failures are reported: There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: +useful unless you intend to extend :mod:`!doctest` internals via subclassing: .. function:: register_optionflag(name) @@ -833,7 +828,7 @@ disabling an option via ``-`` in a directive can be useful. Warnings ^^^^^^^^ -:mod:`doctest` is serious about requiring exact matches in expected output. If +:mod:`!doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't guarantee about output. For example, when printing a set, Python doesn't @@ -1035,7 +1030,7 @@ Unittest API ------------ As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can +their doctests systematically. :mod:`!doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include a :ref:`load_tests ` function in your test module:: @@ -1179,7 +1174,7 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +:mod:`!doctest` functions yourself, you can control the :mod:`!doctest` options in use directly, by passing option flags to :mod:`!doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control @@ -1187,13 +1182,13 @@ when and how tests get run. The framework author typically wants to control options), but there's no way to pass options through :mod:`!unittest` to :mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` +For this reason, :mod:`!doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: .. function:: set_unittest_reportflags(flags) - Set the :mod:`doctest` reporting flags to use. + Set the :mod:`!doctest` reporting flags to use. Argument *flags* takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. @@ -1923,7 +1918,7 @@ There are two exceptions that may be raised by :class:`DebugRunner` instances: Soapbox ------- -As mentioned in the introduction, :mod:`doctest` has grown to have three primary +As mentioned in the introduction, :mod:`!doctest` has grown to have three primary uses: #. Checking examples in docstrings. @@ -1941,7 +1936,7 @@ this that needs to be learned---it may not be natural at first. Examples should add genuine value to the documentation. A good example can often be worth many words. If done with care, the examples will be invaluable for your users, and will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` +by and things change. I'm still amazed at how often one of my :mod:`!doctest` examples stops working after a "harmless" change. Doctest also makes an excellent tool for regression testing, especially if you diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 6875af2be49d7a9..76a57031862c85b 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -19,7 +19,7 @@ registry and several convenience methods for manipulating this registry. Instances of :class:`Charset` are used in several other modules within the :mod:`email` package. -Import this class from the :mod:`email.charset` module. +Import this class from the :mod:`!email.charset` module. .. class:: Charset(input_charset=DEFAULT_CHARSET) @@ -164,7 +164,7 @@ Import this class from the :mod:`email.charset` module. This method allows you to compare two :class:`Charset` instances for inequality. -The :mod:`email.charset` module also provides the following functions for adding +The :mod:`!email.charset` module also provides the following functions for adding new entries to the global character set, alias, and codec registries: diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index b33fe82a6e4c9fc..04a41667f7dc2cd 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -4,9 +4,6 @@ .. module:: email.contentmanager :synopsis: Storing and Retrieving Content from MIME Parts -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/contentmanager.py` ------------ diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 9c8c8c9234ed7a7..1a9a1cad3a619e8 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the +:mod:`!encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 689e7397cbcf1f2..2f7c9140cfcbe55 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -8,7 +8,7 @@ -------------- -The following exception classes are defined in the :mod:`email.errors` module: +The following exception classes are defined in the :mod:`!email.errors` module: .. exception:: MessageError() diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index a3132d02687bc9e..6f4f813a0f84d8c 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -232,7 +232,7 @@ a formatted string representation of a message object. For more detail, see :mod:`email.message`. -The :mod:`email.generator` module also provides a derived class, +The :mod:`!email.generator` module also provides a derived class, :class:`DecodedGenerator`, which is like the :class:`Generator` base class, except that non-\ :mimetype:`text` parts are not serialized, but are instead represented in the output stream by a string derived from a template filled diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index f49885b87852357..e7e21d036e07deb 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -28,13 +28,13 @@ transferred using only 7-bit ASCII characters, so a slew of RFCs have been written describing how to encode email containing non-ASCII characters into :rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. +in its :mod:`!email.header` and :mod:`email.charset` modules. If you want to include non-ASCII characters in your email headers, say in the :mailheader:`Subject` or :mailheader:`To` fields, you should use the :class:`Header` class and assign the field in the :class:`~email.message.Message` object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. +value. Import the :class:`Header` class from the :mod:`!email.header` module. For example:: >>> from email.message import Message @@ -170,7 +170,7 @@ Here is the :class:`Header` class description: This method allows you to compare two :class:`Header` instances for inequality. -The :mod:`email.header` module also provides the following convenient functions. +The :mod:`!email.header` module also provides the following convenient functions. .. function:: decode_header(header) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index ff8b601fe3d1bb1..619c17c98e8d89c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -4,9 +4,6 @@ .. module:: email.headerregistry :synopsis: Automatic Parsing of headers based on the field name -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - **Source code:** :source:`Lib/email/headerregistry.py` -------------- @@ -96,9 +93,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. @@ -269,6 +267,10 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A dictionary mapping parameter names to parameter values. + .. versionchanged:: 3.15 + It is now a :class:`frozendict` instead of a + :class:`types.MappingProxyType`. + .. class:: ContentTypeHeader diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index 090981d84b4de3d..ed300cdb30fdd69 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -10,7 +10,7 @@ Iterating over a message object tree is fairly easy with the :meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over +:mod:`!email.iterators` module provides some useful higher level iterations over message object trees. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 71d6e321f387bcb..b70df130e06dfa5 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -3,9 +3,6 @@ .. module:: email.message :synopsis: The base class representing email messages. -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray , - Barry A. Warsaw **Source code:** :source:`Lib/email/message.py` @@ -14,7 +11,7 @@ .. versionadded:: 3.6 [1]_ The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for +class, imported from the :mod:`!email.message` module. It is the base class for the :mod:`email` object model. :class:`EmailMessage` provides the core functionality for setting and querying header fields, for accessing message bodies, and for creating or modifying structured messages. @@ -57,7 +54,7 @@ message objects. :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. + :mod:`~email.policy` documentation. [2]_ .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) @@ -749,3 +746,9 @@ message objects. .. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. + +.. [2] The :class:`EmailMessage` class requires a policy that provides a + ``content_manager`` attribute for content management methods like + ``set_content()`` and ``get_content()`` to work. The legacy + :const:`~email.policy.compat32` policy does not support these methods + and should not be used with :class:`EmailMessage`. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 6a70714dc3ee42b..6a67bf7c8e555dd 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -125,10 +125,10 @@ Here is the API for the :class:`BytesFeedParser`: Parser API ^^^^^^^^^^ -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, +The :class:`BytesParser` class, imported from the :mod:`!email.parser` module, provides an API that can be used to parse a message when the complete contents of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, +:mod:`!email.parser` module also provides :class:`Parser` for parsing strings, and header-only parsers, :class:`BytesHeaderParser` and :class:`HeaderParser`, which can be used if you're only interested in the headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` @@ -155,7 +155,7 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.BufferedIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 6b997ee784f6e44..816d02d86f4fc4b 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -4,9 +4,6 @@ .. module:: email.policy :synopsis: Controlling the parsing and generating of messages -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - .. versionadded:: 3.3 **Source code:** :source:`Lib/email/policy.py` @@ -406,11 +403,26 @@ added matters. To illustrate:: .. attribute:: utf8 If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages formatted in this way may be passed to SMTP servers that support the ``SMTPUTF8`` extension (:rfc:`6531`). + When ``False``, the generator will raise + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII + characters in a context where :rfc:`2047` does not permit encoded words. + This particularly applies to mailboxes ("addr-spec") with non-ASCII + characters, which can be created via + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII + domain name with ``utf8=False``, first encode the domain using the + third-party :pypi:`idna` or :pypi:`uts46` module or with + :mod:`encodings.idna`. It is not possible to use a non-ASCII username + ("local-part") in a mailbox when ``utf8=False``. + + .. versionchanged:: 3.15 + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, + mostly notably in addr-specs.) .. attribute:: refold_source @@ -602,7 +614,7 @@ The header objects and their attributes are described in This concrete :class:`Policy` is the backward compatibility policy. It replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, + :mod:`!policy` module also defines an instance of this class, :const:`compat32`, that is used as the default policy. Thus the default behavior of the email package is to maintain compatibility with Python 3.2. @@ -662,6 +674,13 @@ The header objects and their attributes are described in An instance of :class:`Compat32`, providing backward compatibility with the behavior of the email package in Python 3.2. + .. note:: + + The :const:`compat32` policy should not be used as a policy for + :class:`~email.message.EmailMessage` objects, and should only be used + to serialize messages that were created using the :const:`compat32` + policy. + .. rubric:: Footnotes diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 66c42e4a5008eef..98b47ffd74096c0 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -4,18 +4,15 @@ .. module:: email :synopsis: Package supporting the parsing, manipulating, and generating email messages. -.. moduleauthor:: Barry A. Warsaw , - R. David Murray -.. sectionauthor:: R. David Murray **Source code:** :source:`Lib/email/__init__.py` -------------- -The :mod:`email` package is a library for managing email messages. It is +The :mod:`!email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`!email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -68,7 +65,7 @@ high level structure in question, and not the details of how those structures are represented. Since MIME content types are used widely in modern internet software (not just email), this will be a familiar concept to many programmers. -The following sections describe the functionality of the :mod:`email` package. +The following sections describe the functionality of the :mod:`!email` package. We start with the :mod:`~email.message` object model, which is the primary interface an application will use, and follow that with the :mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the @@ -102,7 +99,7 @@ compatibility reasons. :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` API. -Contents of the :mod:`email` package documentation: +Contents of the :mod:`!email` package documentation: .. toctree:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 611549604fda15e..0a96dbfd196a051 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -8,7 +8,7 @@ -------------- -There are a couple of useful utilities provided in the :mod:`email.utils` +There are a couple of useful utilities provided in the :mod:`!email.utils` module: .. function:: localtime(dt=None) @@ -70,7 +70,7 @@ of the new API. Add *strict* optional parameter and reject malformed inputs by default. -.. function:: formataddr(pair, charset='utf-8') +.. function:: formataddr(pair, charset='utf-8', *, strict=True) The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname, email_address)`` and returns the string value suitable for a :mailheader:`To` or @@ -82,9 +82,16 @@ of the new API. characters. Can be an instance of :class:`str` or a :class:`~email.charset.Charset`. Defaults to ``utf-8``. + If *strict* is true (the default), raise :exc:`ValueError` for inputs that + contain CR or LF, which are not allowed in an email address. Set *strict* + to ``False`` to allow non-strict inputs. + .. versionchanged:: 3.3 Added the *charset* option. + .. versionchanged:: next + Added the *strict* parameter. + .. function:: getaddresses(fieldvalues, *, strict=True) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 165b9a9f823154c..e0d77229b11802f 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +The :mod:`!ensurepip` package provides support for bootstrapping the ``pip`` installer into an existing Python installation or virtual environment. This bootstrapping approach reflects the fact that ``pip`` is an independent project with its own release cycle, and the latest available stable version @@ -42,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip @@ -97,7 +99,7 @@ Providing both of the script selection options will trigger an exception. Module API ---------- -:mod:`ensurepip` exposes two functions for programmatic use: +:mod:`!ensurepip` exposes two functions for programmatic use: .. function:: version() diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index a8a7e671aadca24..559a76bb3963e09 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -4,11 +4,6 @@ .. module:: enum :synopsis: Implementation of an enumeration class. -.. moduleauthor:: Ethan Furman -.. sectionauthor:: Barry Warsaw -.. sectionauthor:: Eli Bendersky -.. sectionauthor:: Ethan Furman - .. versionadded:: 3.4 **Source code:** :source:`Lib/enum.py` @@ -61,7 +56,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -120,30 +115,30 @@ Module Contents :class:`StrEnum` defaults to the lower-cased version of the member name, while other Enums default to 1 and increase from there. - :func:`~enum.property` + :deco:`~enum.property` Allows :class:`Enum` members to have attributes without conflicting with member names. The ``value`` and ``name`` attributes are implemented this way. - :func:`unique` + :deco:`unique` Enum class decorator that ensures only one name is bound to any one value. - :func:`verify` + :deco:`verify` Enum class decorator that checks user-selectable constraints on an enumeration. - :func:`member` + :deco:`member` Make ``obj`` a member. Can be used as a decorator. - :func:`nonmember` + :deco:`nonmember` Do not make ``obj`` a member. Can be used as a decorator. - :func:`global_enum` + :deco:`global_enum` Modify the :class:`str() ` and :func:`repr` of an enum to show its members as belonging to the module instead of its class, @@ -153,6 +148,12 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :func:`enum.bin` + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` @@ -160,7 +161,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -221,7 +222,7 @@ Data Types names of the members in *cls*:: >>> dir(Color) - ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__'] + ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', '_generate_next_value_', '_missing_'] .. method:: EnumType.__getitem__(cls, name) @@ -239,7 +240,7 @@ Data Types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 @@ -301,6 +302,28 @@ Data Types No longer used, kept for backward compatibility. (class attribute, removed during class creation). + The :attr:`~Enum._order_` attribute can be provided to help keep Python 2 / Python 3 code in sync. + It will be checked against the actual order of the enumeration and raise an error if the two do not match:: + + >>> class Color(Enum): + ... _order_ = 'RED GREEN BLUE' + ... RED = 1 + ... BLUE = 3 + ... GREEN = 2 + ... + Traceback (most recent call last): + ... + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] + + .. note:: + + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition + order is lost before it can be recorded. + + .. versionadded:: 3.6 + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -310,13 +333,15 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. + .. versionadded:: 3.7 + .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and any public methods defined on *self.__class__*:: >>> from enum import Enum - >>> from datetime import date + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -327,10 +352,10 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) - ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] + ['__class__', '__doc__', '__eq__', '__hash__', '__module__', '_add_alias_', '_add_value_alias_', '_generate_next_value_', '_missing_', 'name', 'today', 'value'] .. method:: Enum._generate_next_value_(name, start, count, last_values) @@ -340,7 +365,16 @@ Data Types :last_values: A list of the previous values. A *staticmethod* that is used to determine the next value returned by - :class:`auto`:: + :class:`auto`. + + .. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. + + This method may be overridden, e.g.:: >>> from enum import auto, Enum >>> class PowersOfThree(Enum): @@ -353,6 +387,10 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. versionadded:: 3.6 + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member @@ -391,6 +429,8 @@ Data Types >>> Build('deBUG') + .. versionadded:: 3.6 + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class @@ -462,7 +502,7 @@ Data Types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. - .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support`. .. method:: Enum._add_alias_ @@ -484,7 +524,8 @@ Data Types >>> Color(42) - Raises a :exc:`ValueError` if the value is already linked with a different member. + | Raises a :exc:`ValueError` if the value is already linked with a different member. + | See :ref:`multi-value-enum` for an example. .. versionadded:: 3.13 @@ -588,7 +629,7 @@ Data Types >>> white in purple False - .. method:: __iter__(self): + .. method:: __iter__(self) Returns all contained non-alias members:: @@ -599,7 +640,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __len__(self): + .. method:: __len__(self) Returns number of members in flag:: @@ -610,7 +651,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __bool__(self): + .. method:: __bool__(self) Returns *True* if any members in flag, *False* otherwise:: @@ -647,7 +688,7 @@ Data Types >>> purple ^ Color.GREEN - .. method:: __invert__(self): + .. method:: __invert__(self) Returns all the flags in *type(self)* that are not in *self*:: @@ -883,6 +924,8 @@ Data Types --------------- +.. _enum-dunder-sunder: + Supported ``__dunder__`` names """""""""""""""""""""""""""""" @@ -890,7 +933,7 @@ Supported ``__dunder__`` names items. It is only available on the class. :meth:`~Enum.__new__`, if specified, must create and return the enum members; -it is also a very good idea to set the member's :attr:`!_value_` appropriately. +it is also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. @@ -906,17 +949,10 @@ Supported ``_sunder_`` names from the final class - :attr:`~Enum._order_` -- no longer used, kept for backward compatibility (class attribute, removed during class creation) + - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing member. - :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an @@ -934,26 +970,27 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto *auto* can be used in place of a value. If used, the *Enum* machinery will call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus - one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater - than the highest value; for :class:`StrEnum` it will be the lower-cased version of + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the highest value seen + plus one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value seen; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. - *auto* instances are only resolved when at the top level of an assignment: + *auto* instances are only resolved when at the top level of an assignment, either by + itself or as part of a tuple: * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to - create the ``THREE`` enum member) + * ``THIRD = [auto(), -3]`` will *not* work (``[, -3]`` is used to + create the ``THIRD`` enum member) .. versionchanged:: 3.11.1 @@ -963,17 +1000,17 @@ Utilities and Decorators ``_generate_next_value_`` can be overridden to customize the values used by *auto*. - .. note:: in 3.13 the default ``_generate_next_value_`` will always return + .. note:: In version 3.13 the default ``_generate_next_value_`` will always return the highest member value incremented by 1, and will fail if any member is an incompatible type. .. decorator:: property - A decorator similar to the built-in *property*, but specifically for + A decorator similar to the built-in :deco:`property`, but specifically for enumerations. It allows member attributes to have the same names as members themselves. - .. note:: the *property* and the member must be defined in separate classes; + .. note:: The *property* and the member must be defined in separate classes; for example, the *value* and *name* attributes are defined in the *Enum* class, and *Enum* subclasses can define members with the names ``value`` and ``name``. @@ -1034,6 +1071,20 @@ Utilities and Decorators .. versionadded:: 3.11 +.. function:: bin(num, max_bits=None) + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + + >>> import enum + >>> enum.bin(10) + '0b0 1010' + >>> enum.bin(~10) # ~10 is -11 + '0b1 0101' + + .. versionadded:: 3.11 + --------------- Notes diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 16d42c010f6df08..3775d5ac81a2736 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -221,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (Note: the :meth:`!io.IOBase.read` and + without reading any data. (Note: the :meth:`io.TextIOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -266,6 +266,12 @@ The following exceptions are the exceptions that are usually raised. .. versionadded:: 3.6 +.. exception:: ImportCycleError + + A subclass of :exc:`ImportError` which is raised when a lazy import fails + because it (directly or indirectly) tries to import itself. + + .. versionadded:: 3.15 .. exception:: IndexError @@ -742,8 +748,8 @@ depending on the system error code. .. attribute:: characters_written - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the + An integer containing the number of **bytes** written to the stream + before it blocked. This attribute is available when using the buffered I/O classes from the :mod:`io` module. .. exception:: ChildProcessError @@ -978,6 +984,15 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 677966a8b2eaabb..905e1acd433617d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -31,7 +31,8 @@ tracebacks: * Each string is limited to 500 characters. * Only the filename, the function name and the line number are displayed. (no source code) -* It is limited to 100 frames and 100 threads. +* It is limited to 100 frames per thread, and 100 threads + (configurable via *max_threads*). * The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see @@ -55,16 +56,20 @@ at Python startup. Dumping the traceback --------------------- -.. function:: dump_traceback(file=sys.stderr, all_threads=True) +.. function:: dump_traceback(file=sys.stderr, all_threads=True, *, max_threads=100) Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. + ``False``, dump only the current thread. *max_threads* caps the number + of threads dumped. .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + Dumping the C stack ------------------- @@ -100,7 +105,7 @@ instead of the stack, even if the operating system supports dumping stacks. Fault handler state ------------------- -.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True) +.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True, *, max_threads=100) Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` @@ -116,6 +121,8 @@ Fault handler state traceback, unless the system does not support it. See :func:`dump_c_stack` for more information on compatibility. + *max_threads* caps the number of threads dumped when a fatal signal fires. + .. versionchanged:: 3.5 Added support for passing file descriptor to this function. @@ -133,6 +140,9 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -146,7 +156,7 @@ Fault handler state Dumping the tracebacks after a timeout -------------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False, *, max_threads=100) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -154,7 +164,7 @@ Dumping the tracebacks after a timeout :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. + sub-second resolution. *max_threads* caps the number of threads dumped. The *file* must be kept open until the traceback is dumped or :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file @@ -168,6 +178,9 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. @@ -176,11 +189,12 @@ Dumping the tracebacks after a timeout Dumping the traceback on a user signal -------------------------------------- -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) +.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False, *, max_threads=100) Register a user signal: install a handler for the *signum* signal to dump the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. + *max_threads* caps the number of threads dumped. The *file* must be kept open until the signal is unregistered by :func:`unregister`: see :ref:`issue with file descriptors `. @@ -190,6 +204,9 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index f57fcdf0bcf26df..c28e4d6c0cc80cc 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -2,11 +2,8 @@ ========================================================== .. module:: fcntl - :platform: Unix :synopsis: The fcntl() and ioctl() system calls. -.. sectionauthor:: Jaap Vermeulen - .. index:: pair: UNIX; file control pair: UNIX; I/O control @@ -53,7 +50,7 @@ descriptor. the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 - On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and + On Linux >= 4.5, the :mod:`!fcntl` module exposes the ``FICLONE`` and ``FICLONERANGE`` constants, which allow to share some data of one file with another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". @@ -91,7 +88,7 @@ The module defines the following functions: Perform the operation *cmd* on file descriptor *fd* (file objects providing a :meth:`~io.IOBase.fileno` method are accepted as well). The values used for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C + in the :mod:`!fcntl` module, using the same names as used in the relevant C header files. The argument *arg* can either be an integer value, a :term:`bytes-like object`, or a string. The type and size of *arg* must match the type and size of diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index abd1b8c826d1705..f8365b44c5a502d 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -4,17 +4,15 @@ .. module:: filecmp :synopsis: Compare files efficiently. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/filecmp.py` -------------- -The :mod:`filecmp` module defines functions to compare files and directories, +The :mod:`!filecmp` module defines functions to compare files and directories, with various optional time/correctness trade-offs. For comparing files, see also the :mod:`difflib` module. -The :mod:`filecmp` module defines the following functions: +The :mod:`!filecmp` module defines the following functions: .. function:: cmp(f1, f2, shallow=True) diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 8f32b11e5653652..5be16797be908cb 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -4,9 +4,6 @@ .. module:: fileinput :synopsis: Loop over standard input or a list of files. -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/fileinput.py` -------------- diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ee654b7a83e203b..fc99b7fd1e7523a 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -51,7 +51,7 @@ Unless stated otherwise, "filename string" and "pattern string" either refer to functions documented below do not allow to mix a :class:`!bytes` pattern with a :class:`!str` filename, and vice-versa. -Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +Finally, note that :deco:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. @@ -103,7 +103,8 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. The pattern is expected to be a :class:`str`. + using with :func:`re.prefixmatch`. The pattern is expected to be a + :class:`str`. Example: @@ -113,7 +114,7 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter >>> regex '(?s:.*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') + >>> reobj.prefixmatch('foobar.txt') diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index d6d1c7a461c51cd..b02e7b5b641136d 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -4,14 +4,11 @@ .. module:: fractions :synopsis: Rational numbers. -.. moduleauthor:: Jeffrey Yasskin -.. sectionauthor:: Jeffrey Yasskin - **Source code:** :source:`Lib/fractions.py` -------------- -The :mod:`fractions` module provides support for rational number arithmetic. +The :mod:`!fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of rational numbers, from diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index bb15322067245e8..e1baeff3f373bf1 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,7 +23,7 @@ The default encoding is UTF-8, following :rfc:`2640`. .. include:: ../includes/wasm-notavail.rst -Here's a sample session using the :mod:`ftplib` module:: +Here's a sample session using the :mod:`!ftplib` module:: >>> from ftplib import FTP >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port @@ -524,14 +524,9 @@ FTP_TLS objects :class:`!FTP_TLS` class inherits from :class:`FTP`, defining these additional methods and attributes: - .. attribute:: FTP_TLS.ssl_version - - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - .. method:: FTP_TLS.auth() - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. + Set up a secure control connection by using TLS. .. versionchanged:: 3.4 The method now supports hostname check with @@ -548,7 +543,7 @@ FTP_TLS objects .. method:: FTP_TLS.prot_p() - Set up secure data connection. + Set up secure data connection by using TLS. .. method:: FTP_TLS.prot_c() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3257daf89d327b3..a4d37beb436899a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,24 +19,25 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | :func:`bin` | | |func-frozendict|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | |func-frozenset|_ | | | | :func:`slice` | +| | :func:`breakpoint` | | | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | **G** | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`getattr` | | | | |func-str|_ | +| | | | :func:`globals` | | **O** | | :func:`sum` | +| | **C** | | | | :func:`object` | | :func:`super` | +| | :func:`callable` | | **H** | | :func:`oct` | | | +| | :func:`chr` | | :func:`hasattr` | | :func:`open` | | **T** | +| | :func:`classmethod` | | :func:`hash` | | :func:`ord` | | |func-tuple|_ | +| | :func:`compile` | | :func:`help` | | | | :func:`type` | +| | :func:`complex` | | :func:`hex` | | **P** | | | +| | | | | | :func:`pow` | | **V** | +| | **D** | | **I** | | :func:`print` | | :func:`vars` | +| | :func:`delattr` | | :func:`id` | | :func:`property` | | | +| | |func-dict|_ | | :func:`input` | | | | **Z** | +| | :func:`dir` | | :func:`int` | | | | :func:`zip` | +| | :func:`divmod` | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -44,6 +45,7 @@ are always available. They are listed here in alphabetical order. used, with replacement texts to make the output in the table consistent .. |func-dict| replace:: ``dict()`` +.. |func-frozendict| replace:: ``frozendict()`` .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` @@ -138,6 +140,8 @@ are always available. They are listed here in alphabetical order. >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') + See also :func:`enum.bin` to represent negative values as twos-complement. + See also :func:`format` for more information. @@ -340,8 +344,8 @@ are always available. They are listed here in alphabetical order. It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. + This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled + source is invalid. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -377,7 +381,7 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. @@ -481,10 +485,10 @@ are always available. They are listed here in alphabetical order. :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + See also :ref:`typesmapping` for documentation about this class. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built-in :class:`frozendict`, :class:`list`, + :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() @@ -526,7 +530,7 @@ are always available. They are listed here in alphabetical order. >>> dir() # show the names in the module namespace # doctest: +SKIP ['__builtins__', '__name__', 'struct'] >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', + ['Struct', '__all__', '__builtins__', '__doc__', '__file__', '__initializing__', '__loader__', '__name__', '__package__', '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', 'unpack', 'unpack_from'] @@ -592,7 +596,7 @@ are always available. They are listed here in alphabetical order. :param globals: The global namespace (default: ``None``). - :type globals: :class:`dict` | ``None`` + :type globals: :class:`dict` | :class:`frozendict` | ``None`` :param locals: The local namespace (default: ``None``). @@ -604,18 +608,19 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. - The *expression* argument is parsed and evaluated as a Python expression + The *source* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* mapping is omitted it defaults to the - *globals* dictionary. If both mappings are omitted, the expression is + inserted under that key before *source* is parsed. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. + If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the source is executed with the *globals* and *locals* in the environment where :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing @@ -641,7 +646,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval @@ -658,6 +663,10 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + .. index:: pair: built-in function; exec .. function:: exec(source, /, globals=None, locals=None, *, closure=None) @@ -665,7 +674,7 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as @@ -696,9 +705,10 @@ are always available. They are listed here in alphabetical order. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + :mod:`builtins` is inserted under that key. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. The *closure* argument specifies a closure--a tuple of cellvars. It's only valid when the *object* is a code object containing @@ -735,6 +745,10 @@ are always available. They are listed here in alphabetical order. The semantics of the default *locals* namespace have been adjusted as described for the :func:`locals` builtin. + .. versionchanged:: 3.15 + + *globals* can now be a :class:`frozendict`. + .. function:: filter(function, iterable, /) @@ -852,12 +866,27 @@ are always available. They are listed here in alphabetical order. if *format_spec* is not an empty string. +.. _func-frozendict: +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + :noindex: + + Create a new frozen dictionary. The :class:`frozendict` object is a built-in class. + See also :ref:`typesmapping` for documentation about this class. + + For other containers see the built-in :class:`dict`, :class:`list`, :class:`set`, + and :class:`tuple` classes, as well as the :mod:`collections` module. + + .. versionadded:: 3.15 + + .. _func-frozenset: .. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and + *iterable*. :class:`frozenset` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`set`, :class:`list`, @@ -1095,13 +1124,13 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: issubclass(class, classinfo, /) +.. function:: issubclass(cls, classinfo, /) - Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual + Return ``True`` if *cls* is a subclass (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a + or a :ref:`types-union`, in which case return ``True`` if *cls* is a subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` exception is raised. @@ -1651,7 +1680,7 @@ are always available. They are listed here in alphabetical order. If given, *doc* will be the docstring of the property attribute. Otherwise, the property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: + create read-only properties easily using :deco:`property` as a :term:`decorator`:: class Parrot: def __init__(self): @@ -1742,7 +1771,7 @@ are always available. They are listed here in alphabetical order. self.age = age def __repr__(self): - return f"Person('{self.name}', {self.age})" + return f"Person({self.name!r}, {self.age!r})" .. function:: reversed(object, /) @@ -1785,7 +1814,7 @@ are always available. They are listed here in alphabetical order. :noindex: Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and + *iterable*. :class:`set` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`frozenset`, :class:`list`, @@ -1815,6 +1844,69 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. +.. class:: sentinel(name, /, *, repr=None) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used by default as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + The optional *repr* argument can be used to specify a different representation:: + + >>> MISSING = sentinel("MISSING", repr="") + >>> MISSING + + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + ``sentinel`` does not support subclassing. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. This attribute is writable. + + .. versionadded:: 3.15 + + .. class:: slice(stop, /) slice(start, stop, step=None, /) @@ -1822,19 +1914,19 @@ are always available. They are listed here in alphabetical order. ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + Slice objects are also generated when :ref:`slicing syntax ` + is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. + + See :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. attribute:: slice.start - .. attribute:: slice.stop - .. attribute:: slice.step + slice.stop + slice.step - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an - :term:`iterator`. + These read-only attributes are set to the argument values + (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, @@ -1892,7 +1984,7 @@ are always available. They are listed here in alphabetical order. be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see - :func:`classmethod` for a variant that is useful for creating alternate class + :deco:`classmethod` for a variant that is useful for creating alternate class constructors. Like all decorators, it is also possible to call ``staticmethod`` as @@ -2089,6 +2181,10 @@ are always available. They are listed here in alphabetical order. Subclasses of :class:`!type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. + .. function:: vars() vars(object, /) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index b2e2e11c0dc414b..2b46978f058102b 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -4,13 +4,6 @@ .. module:: functools :synopsis: Higher-order functions and operations on callable objects. -.. moduleauthor:: Peter Harris -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa -.. moduleauthor:: Pablo Galindo -.. sectionauthor:: Peter Harris - **Source code:** :source:`Lib/functools.py` .. testsetup:: default @@ -20,11 +13,11 @@ -------------- -The :mod:`functools` module is for higher-order functions: functions that act on +The :mod:`!functools` module is for higher-order functions: functions that act on or return other functions. In general, any callable object can be treated as a function for the purposes of this module. -The :mod:`functools` module defines the following functions: +The :mod:`!functools` module defines the following functions: .. decorator:: cache(user_function) @@ -34,7 +27,7 @@ The :mod:`functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache` with a size limit. + :deco:`lru_cache` with a size limit. For example:: @@ -57,6 +50,10 @@ The :mod:`functools` module defines the following functions: another thread makes an additional call before the initial call has been completed and cached. + Call-once behavior is not guaranteed because locks are not held during the + function call. Potentially another call with the same arguments could + occur while the first call is still running. + .. versionadded:: 3.9 @@ -64,7 +61,7 @@ The :mod:`functools` module defines the following functions: Transform a method of a class into a property whose value is computed once and then cached as a normal attribute for the life of the instance. Similar - to :func:`property`, with the addition of caching. Useful for expensive + to :deco:`property`, with the addition of caching. Useful for expensive computed properties of instances that are otherwise effectively immutable. Example:: @@ -78,8 +75,8 @@ The :mod:`functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) - The mechanics of :func:`cached_property` are somewhat different from - :func:`property`. A regular property blocks attribute writes unless a + The mechanics of :deco:`cached_property` are somewhat different from + :deco:`property`. A regular property blocks attribute writes unless a setter is defined. In contrast, a *cached_property* allows writes. The *cached_property* decorator only runs on lookups and only when an @@ -111,14 +108,14 @@ The :mod:`functools` module defines the following functions: (as such classes don't provide a ``__dict__`` attribute at all). If a mutable mapping is not available or if space-efficient key sharing is - desired, an effect similar to :func:`cached_property` can also be achieved by - stacking :func:`property` on top of :func:`lru_cache`. See - :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. + desired, an effect similar to :deco:`cached_property` can also be achieved by + stacking :deco:`property` on top of :deco:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :deco:`cached_property`. .. versionadded:: 3.8 .. versionchanged:: 3.12 - Prior to Python 3.12, ``cached_property`` included an undocumented lock to + Prior to Python 3.12, :deco:`!cached_property` included an undocumented lock to ensure that in multi-threaded usage the getter function was guaranteed to run only once per instance. However, the lock was per-property, not per-instance, which could result in unacceptably high lock contention. In @@ -176,8 +173,8 @@ The :mod:`functools` module defines the following functions: the *maxsize* at its default value of 128:: @lru_cache - def count_vowels(sentence): - return sum(sentence.count(vowel) for vowel in 'AEIOUaeiou') + def count_vowels(word): + return sum(word.count(vowel) for vowel in 'AEIOUaeiou') If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can grow without bound. @@ -471,7 +468,7 @@ The :mod:`functools` module defines the following functions: Roughly equivalent to:: - initial_missing = object() + initial_missing = sentinel('initial_missing') def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) @@ -716,7 +713,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 - .. versionchanged:: next + .. versionchanged:: 3.15 Added support of non-:term:`descriptor` callables. @@ -735,7 +732,7 @@ The :mod:`functools` module defines the following functions: function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function + (e.g. bypassing a caching decorator such as :deco:`lru_cache`), this function automatically adds a ``__wrapped__`` attribute to the wrapper that refers to the function being wrapped. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 2ef5c4b35a25cc0..65533e7e57adc33 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -4,9 +4,6 @@ .. module:: gc :synopsis: Interface to the cycle-detecting garbage collector. -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - -------------- This module provides an interface to the optional garbage collector. It @@ -20,7 +17,7 @@ can be disabled by calling ``gc.disable()``. To debug a leaking program call ``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in gc.garbage for inspection. -The :mod:`gc` module provides the following functions: +The :mod:`!gc` module provides the following functions: .. function:: enable() @@ -40,18 +37,11 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -63,6 +53,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -78,13 +71,9 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.14) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. @@ -92,6 +81,9 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -108,43 +100,47 @@ The :mod:`gc` module provides the following functions: * ``uncollectable`` is the total number of objects which were found to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. + list) inside this generation; + + * ``candidates`` is the total number of objects in this generation which were + considered for collection and traversed; + + * ``duration`` is the total time in seconds spent in collections for this + generation. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Add ``duration`` and ``candidates``. + .. function:: set_threshold(threshold0, [threshold1, [threshold2]]) Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. - - In the free-threaded build, the increase in process memory usage is also - checked before running the collector. If the memory usage has not increased - by 10% since the last collection and the net number of object allocations - has not exceeded 40 times *threshold0*, the collection is not run. - - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. - See `Garbage collector design `_ for more information. + See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() @@ -313,6 +309,12 @@ values but should not rebind them): "uncollectable": When *phase* is "stop", the number of objects that could not be collected and were put in :data:`garbage`. + "candidates": When *phase* is "stop", the total number of objects in this + generation which were considered for collection and traversed. + + "duration": When *phase* is "stop", the time in seconds spent in the + collection. + Applications can add their own callbacks to this list. The primary use cases are: @@ -325,6 +327,9 @@ values but should not rebind them): .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Add "duration" and "candidates". + The following constants are provided for use with :func:`set_debug`: diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index a0c0c6dee2d5131..f60cc63414f764d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -4,17 +4,13 @@ .. module:: getpass :synopsis: Portable reading of passwords and retrieval of the userid. -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Fred L. Drake, Jr. -.. Windows (& Mac?) support by Guido van Rossum. - **Source code:** :source:`Lib/getpass.py` -------------- .. include:: ../includes/wasm-notavail.rst -The :mod:`getpass` module provides two functions: +The :mod:`!getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None, *, echo_char=None) @@ -22,7 +18,7 @@ The :mod:`getpass` module provides two functions: the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt is written to the file-like object *stream* using the replace error handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this + (:file:`/dev/tty`) or if that is unavailable to :data:`sys.stderr` (this argument is ignored on Windows). The *echo_char* argument controls how user input is displayed while typing. @@ -31,25 +27,39 @@ The :mod:`getpass` module provides two functions: typed character is replaced by it. For example, ``echo_char='*'`` will display asterisks instead of the actual input. - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and + If echo-free input is unavailable, :func:`getpass` falls back to printing + a warning message to *stream* and reading from :data:`sys.stdin` and issuing a :exc:`GetPassWarning`. .. note:: - If you call getpass from within IDLE, the input may be done in the + If you call :func:`getpass` from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. .. note:: On Unix systems, when *echo_char* is set, the terminal will be configured to operate in :manpage:`noncanonical mode `. - In particular, this means that line editing shortcuts such as - :kbd:`Ctrl+U` will not work and may insert unexpected characters into - the input. + Common terminal control characters are supported: + + * :kbd:`Ctrl+A` - Move cursor to beginning of line + * :kbd:`Ctrl+E` - Move cursor to end of line + * :kbd:`Ctrl+K` - Kill (delete) from cursor to end of line + * :kbd:`Ctrl+U` - Kill (delete) entire line + * :kbd:`Ctrl+W` - Erase previous word + * :kbd:`Ctrl+V` - Insert next character literally (quote) + * :kbd:`Backspace`/:kbd:`DEL` - Delete character before cursor + + These shortcuts work by reading the terminal's configured control + character mappings from termios settings. .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. + .. versionchanged:: 3.15 + When using non-empty *echo_char* on Unix, keyboard shortcuts (including + cursor movement and line editing) are now properly handled using the + terminal's control character configuration. + .. exception:: GetPassWarning A :exc:`UserWarning` subclass issued when password input may be echoed. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb2979..2ab7ba7df19cf14 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -4,14 +4,11 @@ .. module:: gettext :synopsis: Multilingual internationalization services. -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw - **Source code:** :source:`Lib/gettext.py` -------------- -The :mod:`gettext` module provides internationalization (I18N) and localization +The :mod:`!gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you @@ -25,7 +22,7 @@ Some hints on localizing your Python modules and applications are also given. GNU :program:`gettext` API -------------------------- -The :mod:`gettext` module defines the following API, which is very similar to +The :mod:`!gettext` module defines the following API, which is very similar to the GNU :program:`gettext` API. If you use this API you will affect the translation of your entire application globally. Often this is what you want if your application is monolingual, with the choice of language dependent on the @@ -37,7 +34,7 @@ class-based API instead. .. function:: bindtextdomain(domain, localedir=None) Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using + :mod:`!gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -54,19 +51,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -81,15 +78,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -114,7 +111,7 @@ Here's an example of typical usage for this API:: Class-based API --------------- -The class-based API of the :mod:`gettext` module gives you more flexibility and +The class-based API of the :mod:`!gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format @@ -226,20 +223,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -247,7 +244,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -325,7 +322,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -334,7 +331,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -355,7 +352,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -366,7 +363,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to @@ -393,7 +390,7 @@ The Catalog constructor .. index:: single: GNOME -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this +GNOME uses a version of the :mod:`!gettext` module by James Henstridge, but this version has a slightly different API. Its documented usage was:: import gettext @@ -425,7 +422,7 @@ take the following steps: #. create language-specific translations of the message catalogs -#. use the :mod:`gettext` module so that message strings are properly translated +#. use the :mod:`!gettext` module so that message strings are properly translated In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping @@ -473,10 +470,10 @@ supported natural language. They send back the completed language-specific versions as a :file:`.po` file that's compiled into a machine-readable :file:`.mo` binary catalog file using the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at +:mod:`!gettext` module for the actual translation processing at run-time. -How you use the :mod:`gettext` module in your code depends on whether you are +How you use the :mod:`!gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two sections will discuss each case. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 52c449281533372..bbeb808159d446b 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -29,7 +29,7 @@ not by actually invoking a subshell. The pathnames are returned in no particular order. If you need a specific order, sort the results. -Files beginning with a dot (``.``) can only be matched by +By default, files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. For tilde and shell variable expansion, use :func:`os.path.expanduser` and @@ -70,7 +70,8 @@ The :mod:`!glob` module defines the following functions: pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not match. - If *include_hidden* is true, "``**``" pattern will match hidden directories. + If *include_hidden* is true, wildcards can match path segments that + begin with a dot (``.``). .. audit-event:: glob.glob pathname,recursive glob.glob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.glob @@ -83,6 +84,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -106,6 +112,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -121,7 +132,7 @@ The :mod:`!glob` module defines the following functions: Escape all special characters (``'?'``, ``'*'`` and ``'['``). This is useful if you want to match an arbitrary literal string that may have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows + sharepoints are not escaped, for example on Windows ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. .. versionadded:: 3.4 @@ -130,7 +141,8 @@ The :mod:`!glob` module defines the following functions: .. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) Convert the given path specification to a regular expression for use with - :func:`re.match`. The path specification can contain shell-style wildcards. + :func:`re.prefixmatch`. The path specification can contain shell-style + wildcards. For example: @@ -140,7 +152,7 @@ The :mod:`!glob` module defines the following functions: >>> regex '(?s:(?:.+/)?[^/]*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foo/bar/baz.txt') + >>> reobj.prefixmatch('foo/bar/baz.txt') Path separators and segments are meaningful to this function, unlike diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 053d5f8231ba0e3..21f4d1fb938038e 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -204,7 +204,7 @@ Exceptions ---------- -The :mod:`graphlib` module defines the following exception classes: +The :mod:`!graphlib` module defines the following exception classes: .. exception:: CycleError diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index d1c7f22a2097803..f436970e791ace1 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -2,7 +2,6 @@ ================================== .. module:: grp - :platform: Unix :synopsis: The group database (getgrnam() and friends). -------------- diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index cb36be42a83f12a..9211e5f18c6b6e6 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -15,7 +15,7 @@ like the GNU programs :program:`gzip` and :program:`gunzip` would. The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the +The :mod:`!gzip` module provides the :class:`GzipFile` class, as well as the :func:`.open`, :func:`compress` and :func:`decompress` convenience functions. The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, automatically compressing or decompressing the data so that it looks like an @@ -28,7 +28,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None, *, mtime=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -43,9 +43,12 @@ The module defines the following items: The *compresslevel* argument is an integer from 0 to 9, as for the :class:`GzipFile` constructor. + The keyword-only argument *mtime* represents a Unix timestamp. + For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. + constructor: ``GzipFile(filename, mode, compresslevel, mtime=mtime)``. + In this case, the *encoding*, *errors* and *newline* arguments must not + be provided. For text mode, a :class:`GzipFile` object is created, and wrapped in an :class:`io.TextIOWrapper` instance with the specified encoding, error @@ -66,6 +69,10 @@ The module defines the following items: It is the default level used by most compression tools and a better tradeoff between speed and performance. + .. versionchanged:: next + Added keyword-only argument *mtime* which is passed to the class + constructor of :class:`~gzip.GzipFile`. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -108,9 +115,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. @@ -283,20 +294,20 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`gzip` module provides a simple command line interface to compress or +The :mod:`!gzip` module provides a simple command line interface to compress or decompress files. -Once executed the :mod:`gzip` module keeps the input file(s). +Once executed the :mod:`!gzip` module keeps the input file(s). .. versionchanged:: 3.8 Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index b21ecdaede622a7..5b666c5adac6456 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -4,9 +4,6 @@ .. module:: hashlib :synopsis: Secure hash and message digest algorithms. -.. moduleauthor:: Gregory P. Smith -.. sectionauthor:: Gregory P. Smith - **Source code:** :source:`Lib/hashlib.py` .. index:: @@ -53,15 +50,18 @@ hash supplied more than 2047 bytes of data at once in its constructor or .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, -:func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it may be missing or blocked -if you are using a rare "FIPS compliant" build of Python. -These correspond to :data:`algorithms_guaranteed`. +:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, +:func:`sha512`, :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, +:func:`sha3_512`, :func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and +:func:`blake2s`. These correspond to :data:`algorithms_guaranteed`. + +Any of these may nonetheless be missing or blocked in unusual environments, +such as a rare "FIPS compliant" build of Python or when OpenSSL's "FIPS mode" +is configured to exclude some algorithms from its default provider. Calling +the constructor of an algorithm that is unavailable raises :exc:`ValueError`. Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. +:mod:`!hashlib` was linked against a build of OpenSSL that provides others. Others *are not guaranteed available* on all installations and will only be accessible by name via :func:`new`. See :data:`algorithms_available`. @@ -379,8 +379,6 @@ include a `salt `_. BLAKE2 ------ -.. sectionauthor:: Dmitry Chestnykh - .. index:: single: blake2b, blake2s @@ -397,7 +395,7 @@ BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), **salted hashing**, **personalization**, and **tree hashing**. Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. +:mod:`!hashlib` objects. Creating hash objects diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 5049262306a228d..26cffa7c643028a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -4,11 +4,6 @@ .. module:: heapq :synopsis: Heap queue algorithm (a.k.a. priority queue). -.. moduleauthor:: Kevin O'Connor -.. sectionauthor:: Guido van Rossum -.. sectionauthor:: François Pinard -.. sectionauthor:: Raymond Hettinger - **Source code:** :source:`Lib/heapq.py` -------------- diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d5608bd7543eb18..2ee0c0bd9128b89 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -4,9 +4,6 @@ .. module:: hmac :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation -.. moduleauthor:: Gerhard Häring -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/hmac.py` -------------- diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index add18e4c87d2204..15d2dc2e9aa6bc9 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -4,8 +4,6 @@ .. module:: html.entities :synopsis: Definitions of HTML general entities. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/html/entities.py` -------------- diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 341a8337ba2ceb8..11f851d4f6c4b74 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -141,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -317,6 +317,18 @@ without further parsing: Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 7c258b324d98674..98ea09d8f72d8b6 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -319,6 +319,12 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. + .. note:: + + Note that you must have read the whole response or call :meth:`close` + if :meth:`getresponse` raised an non-:exc:`ConnectionError` exception + before you can send a new request to the server. + .. versionchanged:: 3.2 *body* can now be an iterable. @@ -334,16 +340,15 @@ HTTPConnection Objects Should be called after a request is sent to get the response from the server. Returns an :class:`HTTPResponse` instance. - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - .. versionchanged:: 3.5 If a :exc:`ConnectionError` or subclass is raised, the :class:`HTTPConnection` object will be ready to reconnect when a new request is sent. + Note that this does not apply to :exc:`OSError`\s raised by the underlying + socket. Instead the caller is responsible to call :meth:`close` on the + existing connection. + .. method:: HTTPConnection.set_debuglevel(level) @@ -428,6 +433,7 @@ HTTPConnection Objects The maximum number of allowed response headers to help prevent denial-of-service attacks. By default, the maximum number of allowed headers is set to 100. + The same limit applies to the trailer section of a chunked response. .. versionadded:: 3.15 diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index fcb0069b760e59b..ce65bc961a98a68 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -4,14 +4,11 @@ .. module:: http.cookiejar :synopsis: Classes for automatic handling of HTTP cookies. -.. moduleauthor:: John J. Lee -.. sectionauthor:: John J. Lee - **Source code:** :source:`Lib/http/cookiejar.py` -------------- -The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP +The :mod:`!http.cookiejar` module defines classes for automatic handling of HTTP cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -21,7 +18,7 @@ Both the regular Netscape cookie protocol and the protocol defined by :rfc:`2109` cookies are parsed as Netscape cookies and subsequently treated either as Netscape or RFC 2965 cookies according to the 'policy' in effect. Note that the great majority of cookies on the internet are Netscape cookies. -:mod:`http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which +:mod:`!http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which differs substantially from that set out in the original Netscape specification), including taking note of the ``max-age`` and ``port`` cookie-attributes introduced with RFC 2965. @@ -29,7 +26,7 @@ introduced with RFC 2965. .. note:: The various named parameters found in :mailheader:`Set-Cookie` and - :mailheader:`Set-Cookie2` headers (eg. ``domain`` and ``expires``) are + :mailheader:`Set-Cookie2` headers (for example, ``domain`` and ``expires``) are conventionally referred to as :dfn:`attributes`. To distinguish them from Python attributes, the documentation for this module uses the term :dfn:`cookie-attribute` instead. @@ -88,7 +85,7 @@ The following classes are provided: Constructor arguments should be passed as keyword arguments only. *blocked_domains* is a sequence of domain names that we never accept cookies - from, nor return cookies to. *allowed_domains* if not :const:`None`, this is a + from, nor return cookies to. *allowed_domains* if not ``None``, this is a sequence of the only domains for which we accept and return cookies. *secure_protocols* is a sequence of protocols for which secure cookies can be added to. By default *https* and *wss* (secure websocket) are considered @@ -96,12 +93,12 @@ The following classes are provided: :class:`CookiePolicy` and :class:`DefaultCookiePolicy` objects. :class:`DefaultCookiePolicy` implements the standard accept / reject rules for - Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (ie. cookies + Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) are treated according to the RFC 2965 rules. However, if RFC 2965 handling is turned off or :attr:`rfc2109_as_netscape` is ``True``, RFC 2109 cookies are 'downgraded' by the :class:`CookieJar` instance to Netscape cookies, by - setting the :attr:`version` attribute of the :class:`Cookie` instance to 0. + setting the :attr:`~Cookie.version` attribute of the :class:`Cookie` instance to 0. :class:`DefaultCookiePolicy` also provides some parameters to allow some fine-tuning of policy. @@ -109,8 +106,8 @@ The following classes are provided: .. class:: Cookie() This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not - expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie` - instances. Instead, if necessary, call :meth:`make_cookies` on a + expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` + instances. Instead, if necessary, call :meth:`~CookieJar.make_cookies` on a :class:`CookieJar` instance. @@ -121,13 +118,13 @@ The following classes are provided: Module :mod:`http.cookies` HTTP cookie classes, principally useful for server-side code. The - :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each + :mod:`!http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all - the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to + the major browsers (and :mod:`!http.cookiejar`) only bears a passing resemblance to the one sketched out in ``cookie_spec.html``. :rfc:`2109` - HTTP State Management Mechanism @@ -144,7 +141,7 @@ The following classes are provided: .. _cookie-jar-objects: -CookieJar and FileCookieJar Objects +CookieJar and FileCookieJar objects ----------------------------------- :class:`CookieJar` objects support the :term:`iterator` protocol for iterating over @@ -157,20 +154,27 @@ contained :class:`Cookie` objects. Add correct :mailheader:`Cookie` header to *request*. - If policy allows (ie. the :attr:`rfc2965` and :attr:`hide_cookie2` attributes of + If policy allows (that is, the :attr:`~CookiePolicy.rfc2965` and + :attr:`~CookiePolicy.hide_cookie2` attributes of the :class:`CookieJar`'s :class:`CookiePolicy` instance are true and false respectively), the :mailheader:`Cookie2` header is also added when appropriate. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the methods :meth:`get_full_url`, :meth:`has_header`, - :meth:`get_header`, :meth:`header_items`, :meth:`add_unredirected_header` - and the attributes :attr:`host`, :attr:`!type`, :attr:`unverifiable` - and :attr:`origin_req_host` as documented by :mod:`urllib.request`. + must support the methods :meth:`~urllib.request.Request.get_full_url`, + :meth:`~urllib.request.Request.has_header`, + :meth:`~urllib.request.Request.get_header`, + :meth:`~urllib.request.Request.header_items`, + :meth:`~urllib.request.Request.add_unredirected_header` + and the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.type`, :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host` as documented by + :mod:`urllib.request`. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.extract_cookies(response, request) @@ -183,20 +187,24 @@ contained :class:`Cookie` objects. as appropriate (subject to the :meth:`CookiePolicy.set_ok` method's approval). The *response* object (usually the result of a call to - :meth:`urllib.request.urlopen`, or similar) should support an :meth:`info` - method, which returns an :class:`email.message.Message` instance. + :meth:`urllib.request.urlopen`, or similar) should support an + :meth:`~http.client.HTTPResponse.info` method, which returns an + :class:`email.message.Message` instance. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the method :meth:`get_full_url` and the attributes - :attr:`host`, :attr:`unverifiable` and :attr:`origin_req_host`, + must support the method :meth:`~urllib.request.Request.get_full_url` and + the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host`, as documented by :mod:`urllib.request`. The request is used to set default values for cookie-attributes as well as for checking that the cookie is allowed to be set. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.set_policy(policy) @@ -239,13 +247,13 @@ contained :class:`Cookie` objects. Discard all session cookies. - Discards all contained cookies that have a true :attr:`discard` attribute + Discards all contained cookies that have a true :attr:`~Cookie.discard` attribute (usually because they had either no ``max-age`` or ``expires`` cookie-attribute, or an explicit ``discard`` cookie-attribute). For interactive browsers, the end of a session usually corresponds to closing the browser window. - Note that the :meth:`save` method won't save session cookies anyway, unless you - ask otherwise by passing a true *ignore_discard* argument. + Note that the :meth:`~FileCookieJar.save` method won't save session cookies + anyway, unless you ask otherwise by passing a true *ignore_discard* argument. :class:`FileCookieJar` implements the following additional methods: @@ -258,8 +266,9 @@ contained :class:`Cookie` objects. method unimplemented. *filename* is the name of file in which to save cookies. If *filename* is not - specified, :attr:`self.filename` is used (whose default is the value passed to - the constructor, if any); if :attr:`self.filename` is :const:`None`, + specified, :attr:`self.filename ` is used (whose + default is the value passed to the constructor, if any); if + :attr:`self.filename ` is ``None``, :exc:`ValueError` is raised. *ignore_discard*: save even cookies set to be discarded. *ignore_expires*: save @@ -352,7 +361,7 @@ writing. .. _cookie-policy-objects: -CookiePolicy Objects +CookiePolicy objects -------------------- Objects implementing the :class:`CookiePolicy` interface have the following @@ -436,7 +445,7 @@ setting and receiving any and all cookies (this is unlikely to be useful). .. _default-cookie-policy-objects: -DefaultCookiePolicy Objects +DefaultCookiePolicy objects --------------------------- Implements the standard rules for accepting and returning cookies. @@ -466,10 +475,10 @@ blocking some benign cookies). A domain blocklist and allowlist is provided (both off by default). Only domains not in the blocklist and present in the allowlist (if the allowlist is active) participate in cookie setting and returning. Use the *blocked_domains* -constructor argument, and :meth:`blocked_domains` and -:meth:`set_blocked_domains` methods (and the corresponding argument and methods -for *allowed_domains*). If you set an allowlist, you can turn it off again by -setting it to :const:`None`. +constructor argument, and :meth:`~DefaultCookiePolicy.blocked_domains` and +:meth:`~DefaultCookiePolicy.set_blocked_domains` methods (and the corresponding +argument and methods for *allowed_domains*). If you set an allowlist, you can +turn it off again by setting it to ``None``. Domains in block or allow lists that do not start with a dot must equal the cookie domain to be matched. For example, ``"example.com"`` matches a blocklist @@ -501,12 +510,12 @@ and ``".168.1.2"``, 192.168.1.2 is blocked, but 193.168.1.2 is not. .. method:: DefaultCookiePolicy.allowed_domains() - Return :const:`None`, or the sequence of allowed domains (as a tuple). + Return ``None``, or the sequence of allowed domains (as a tuple). .. method:: DefaultCookiePolicy.set_allowed_domains(allowed_domains) - Set the sequence of allowed domains, or :const:`None`. + Set the sequence of allowed domains, or ``None``. .. method:: DefaultCookiePolicy.is_not_allowed(domain) @@ -522,9 +531,9 @@ all be assigned to. .. attribute:: DefaultCookiePolicy.rfc2109_as_netscape If true, request that the :class:`CookieJar` instance downgrade :rfc:`2109` cookies - (ie. cookies received in a :mailheader:`Set-Cookie` header with a version + (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) to Netscape cookies by setting the version attribute of - the :class:`Cookie` instance to 0. The default value is :const:`None`, in which + the :class:`Cookie` instance to 0. The default value is ``None``, in which case RFC 2109 cookies are downgraded if and only if :rfc:`2965` handling is turned off. Therefore, RFC 2109 cookies are downgraded by default. @@ -577,7 +586,7 @@ both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNoDots - When setting cookies, the 'host prefix' must not contain a dot (eg. + When setting cookies, the 'host prefix' must not contain a dot (for example, ``www.foo.bar.com`` can't set a cookie for ``.bar.com``, because ``www.foo`` contains a dot). @@ -585,7 +594,7 @@ both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNonDomain Cookies that did not explicitly specify a ``domain`` cookie-attribute can only - be returned to a domain equal to the domain that set the cookie (eg. + be returned to a domain equal to the domain that set the cookie (for example, ``spam.example.com`` won't be returned cookies from ``example.com`` that had no ``domain`` cookie-attribute). @@ -600,7 +609,7 @@ combinations of the above flags: .. attribute:: DefaultCookiePolicy.DomainLiberal - Equivalent to 0 (ie. all of the above Netscape domain strictness flags switched + Equivalent to 0 (that is, all of the above Netscape domain strictness flags switched off). @@ -609,7 +618,7 @@ combinations of the above flags: Equivalent to ``DomainStrictNoDots|DomainStrictNonDomain``. -Cookie Objects +Cookie objects -------------- :class:`Cookie` instances have Python attributes roughly corresponding to the @@ -617,7 +626,7 @@ standard cookie-attributes specified in the various cookie standards. The correspondence is not one-to-one, because there are complicated rules for assigning default values, because the ``max-age`` and ``expires`` cookie-attributes contain equivalent information, and because :rfc:`2109` cookies -may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape) +may be 'downgraded' by :mod:`!http.cookiejar` from version 1 to version 0 (Netscape) cookies. Assignment to these attributes should not be necessary other than in rare @@ -627,9 +636,9 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.version - Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and + Integer or ``None``. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -640,13 +649,13 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.value - Cookie value (a string), or :const:`None`. + Cookie value (a string), or ``None``. .. attribute:: Cookie.port - String representing a port or a set of ports (eg. '80', or '80,8080'), or - :const:`None`. + String representing a port or a set of ports (for example, '80', or '80,8080'), or + ``None``. .. attribute:: Cookie.domain @@ -656,7 +665,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.path - Cookie path (a string, eg. ``'/acme/rocket_launchers'``). + Cookie path (a string, for example, ``'/acme/rocket_launchers'``). .. attribute:: Cookie.secure @@ -666,7 +675,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.expires - Integer expiry date in seconds since epoch, or :const:`None`. See also the + Integer expiry date in seconds since epoch, or ``None``. See also the :meth:`is_expired` method. @@ -678,21 +687,21 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.comment String comment from the server explaining the function of this cookie, or - :const:`None`. + ``None``. .. attribute:: Cookie.comment_url URL linking to a comment from the server explaining the function of this cookie, - or :const:`None`. + or ``None``. .. attribute:: Cookie.rfc2109 - ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie + ``True`` if this cookie was received as an :rfc:`2109` cookie (that is, the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -744,7 +753,7 @@ The :class:`Cookie` class also defines the following method: Examples -------- -The first example shows the most common usage of :mod:`http.cookiejar`:: +The first example shows the most common usage of :mod:`!http.cookiejar`:: import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 88e978d7f5eafbc..4965c5fc3ba1d86 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -4,14 +4,11 @@ .. module:: http.cookies :synopsis: Support for HTTP state management (cookies). -.. moduleauthor:: Timothy O'Malley -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/http/cookies.py` -------------- -The :mod:`http.cookies` module defines classes for abstracting the concept of +The :mod:`!http.cookies` module defines classes for abstracting the concept of cookies, an HTTP state management mechanism. It supports both simple string-only cookies, and provides an abstraction for having any serializable data-type as cookie value. @@ -28,10 +25,8 @@ The character set, :data:`string.ascii_letters`, :data:`string.digits` and in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed '``:``' as a valid cookie name character. + Allowed ':' as a valid cookie name character. -.. versionchanged:: 3.15 - Allowed '``"``' as a valid cookie value character. .. note:: @@ -67,7 +62,7 @@ in a cookie name (as :attr:`~Morsel.key`). Module :mod:`http.cookiejar` HTTP cookie handling for web *clients*. The :mod:`http.cookiejar` and - :mod:`http.cookies` modules do not depend on each other. + :mod:`!http.cookies` modules do not depend on each other. :rfc:`2109` - HTTP State Management Mechanism This is the state management specification implemented by this module. @@ -112,6 +107,12 @@ Cookie Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.BaseCookie.output` instead to generate HTTP + headers. + .. method:: BaseCookie.load(rawdata) @@ -228,6 +229,12 @@ Morsel Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.Morsel.output` instead to generate HTTP + headers. + .. method:: Morsel.OutputString(attrs=None) @@ -266,7 +273,7 @@ Morsel Objects Example ------- -The following example demonstrates how to use the :mod:`http.cookies` module. +The following example demonstrates how to use the :mod:`!http.cookies` module. .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -294,9 +301,9 @@ The following example demonstrates how to use the :mod:`http.cookies` module. Set-Cookie: chips=ahoy Set-Cookie: vienna=finger >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" @@ -316,10 +323,3 @@ The following example demonstrates how to use the :mod:`http.cookies` module. >>> print(C) Set-Cookie: number=7 Set-Cookie: string=seven - >>> import json - >>> C = cookies.SimpleCookie() - >>> C.load(f'cookies=7; mixins="{json.dumps({"chips": "dark chocolate"})}"; state=gooey') - >>> print(C) - Set-Cookie: cookies=7 - Set-Cookie: mixins="{"chips": "dark chocolate"}" - Set-Cookie: state=gooey diff --git a/Doc/library/http.rst b/Doc/library/http.rst index b0bdfc65e4508dd..43a801416e24f99 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -12,7 +12,7 @@ -------------- -:mod:`http` is a package that collects several modules for working with the +:mod:`!http` is a package that collects several modules for working with the HyperText Transfer Protocol: * :mod:`http.client` is a low-level HTTP protocol client; for high-level URL @@ -22,7 +22,7 @@ HyperText Transfer Protocol: * :mod:`http.cookiejar` provides persistence of cookies -The :mod:`http` module also defines the following enums that help you work with http related code: +The :mod:`!http` module also defines the following enums that help you work with http related code: .. class:: HTTPStatus diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 58f09634f95e0f0..0d427db19c0fcdb 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -19,7 +19,7 @@ This module defines classes for implementing HTTP servers. .. warning:: - :mod:`http.server` is not recommended for production. It only implements + :mod:`!http.server` is not recommended for production. It only implements :ref:`basic security checks `. .. include:: ../includes/wasm-notavail.rst @@ -39,7 +39,17 @@ handler. Code to create and run the server looks like this:: This class builds on the :class:`~socketserver.TCPServer` class by storing the server address as instance variables named :attr:`server_name` and :attr:`server_port`. The server is accessible by the handler, typically - through the handler's :attr:`server` instance variable. + through the handler's :attr:`~socketserver.BaseRequestHandler.server` + instance variable. + + .. attribute:: server_name + + The HTTP server's fully qualified domain name. + + .. attribute:: server_port + + The HTTP server's port number obtained from *server_address*. + .. class:: ThreadingHTTPServer(server_address, RequestHandlerClass) @@ -60,7 +70,7 @@ handler. Code to create and run the server looks like this:: object fails with a :exc:`RuntimeError`. The *certfile* argument is the path to the SSL certificate chain file, - and the *keyfile* is the path to file containing the private key. + and the *keyfile* is the path to the file containing the private key. A *password* can be specified for files protected and wrapped with PKCS#8, but beware that this could possibly expose hardcoded passwords in clear. @@ -99,7 +109,7 @@ instantiation, of which this module provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -140,7 +150,7 @@ instantiation, of which this module provides three different variants: .. attribute:: path - Contains the request path. If query component of the URL is present, + Contains the request path. If the query component of the URL is present, then ``path`` includes the query. Using the terminology of :rfc:`3986`, ``path`` here includes ``hier-part`` and the ``query``. @@ -190,7 +200,7 @@ instantiation, of which this module provides three different variants: Specifies a format string that should be used by :meth:`send_error` method for building an error response to the client. The string is filled by default with variables from :attr:`responses` based on the status code - that passed to :meth:`send_error`. + passed to :meth:`send_error`. .. attribute:: error_content_type @@ -238,10 +248,10 @@ instantiation, of which this module provides three different variants: .. method:: handle_expect_100() When an HTTP/1.1 conformant server receives an ``Expect: 100-continue`` - request header it responds back with a ``100 Continue`` followed by ``200 - OK`` headers. + request header it responds with a ``100 Continue`` followed by ``200 OK`` + headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -287,16 +297,20 @@ instantiation, of which this module provides three different variants: specifying its value. Note that, after the send_header calls are done, :meth:`end_headers` MUST BE called in order to complete the operation. + This method does not reject input containing CRLF sequences. + .. versionchanged:: 3.2 Headers are stored in an internal buffer. .. method:: send_response_only(code, message=None) Sends the response header only, used for the purposes when ``100 - Continue`` response is sent by the server to the client. The headers not - buffered and sent directly the output stream.If the *message* is not + Continue`` response is sent by the server to the client. The headers are + not buffered and sent directly the output stream. If the *message* is not specified, the HTTP message corresponding the response *code* is sent. + This method does not reject *message* containing CRLF sequences. + .. versionadded:: 3.2 .. method:: end_headers() @@ -334,7 +348,7 @@ instantiation, of which this module provides three different variants: to create custom error logging mechanisms. The *format* argument is a standard printf-style format string, where the additional arguments to :meth:`log_message` are applied as inputs to the formatting. The client - ip address and current date and time are prefixed to every message logged. + IP address and current date and time are prefixed to every message logged. .. method:: version_string() @@ -362,7 +376,8 @@ instantiation, of which this module provides three different variants: delays, it now always returns the IP address. -.. class:: SimpleHTTPRequestHandler(request, client_address, server, directory=None) +.. class:: SimpleHTTPRequestHandler(request, client_address, server, \ + *, directory=None, extra_response_headers=None) This class serves files from the directory *directory* and below, or the current directory if *directory* is not provided, directly @@ -374,6 +389,9 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Added *extra_response_headers* parameter. + A lot of the work, such as parsing the request, is done by the base class :class:`BaseHTTPRequestHandler`. This class implements the :func:`do_GET` and :func:`do_HEAD` functions. @@ -386,6 +404,22 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: default_content_type + + Specifies the Content-Type header value sent when the MIME type + cannot be guessed from the file extension of the requested URL. + By default, it is set to ``'application/octet-stream'``. + + .. versionadded:: 3.15 + + .. attribute:: index_pages + + Specifies the filenames that are treated as directory index pages. + + Defaults to ``("index.html", "index.htm")``. + + .. versionadded:: 3.12 + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -396,6 +430,15 @@ instantiation, of which this module provides three different variants: This dictionary is no longer filled with the default system mappings, but only contains overrides. + .. attribute:: extra_response_headers + + A sequence of ``(name, value)`` pairs containing user-defined extra HTTP + response headers to add to each successful HTTP status 200 response. These + headers are not included in other status code responses. + + Headers that the server sends automatically such as ``Content-Type`` + will not be overwritten by :attr:`!extra_response_headers`. + The :class:`SimpleHTTPRequestHandler` class defines the following methods: .. method:: do_HEAD() @@ -409,8 +452,8 @@ instantiation, of which this module provides three different variants: The request is mapped to a local file by interpreting the request as a path relative to the current working directory. - If the request was mapped to a directory, the directory is checked for a - file named ``index.html`` or ``index.htm`` (in that order). If found, the + If the request was mapped to a directory, the directory is checked for + an index page as specified by :attr:`index_pages`. If found, the file's contents are returned; otherwise a directory listing is generated by calling the :meth:`list_directory` method. This method uses :func:`os.listdir` to scan the directory, and returns a ``404`` error @@ -428,6 +471,9 @@ instantiation, of which this module provides three different variants: followed by a ``'Content-Length:'`` header with the file's size and a ``'Last-Modified:'`` header with the file's modification time. + The instance attribute :attr:`extra_response_headers` is a sequence of + ``(name, value)`` pairs containing user-defined extra response headers. + Then follows a blank line signifying the end of the headers, and then the contents of the file are output. @@ -437,9 +483,32 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.7 Support of the ``'If-Modified-Since'`` header. -The :class:`SimpleHTTPRequestHandler` class can be used in the following -manner in order to create a very basic webserver serving files relative to -the current directory:: + .. method:: list_directory(path) + + Helper to list the contents of *path* when no index page is present. + + This returns either a :term:`file-like object` (which must be closed + by the caller) or ``None`` to indicate an error, in which case the + caller has nothing further to do. In either case, the headers are sent. + + .. method:: guess_type(path) + + Guess the type of the file at the given *path*. + + This returns a string of the form ``type/subtype``, usable for + a MIME Content-type header. + + The default implementation looks the file's extension up in + :attr:`extensions_map`, falling back to + :func:`mimetypes.guess_file_type` and then to + :attr:`default_content_type`. + + .. versionchanged:: 3.13 + Add :func:`mimetypes.guess_file_type` as a fallback. + + +The :class:`SimpleHTTPRequestHandler` class can be used to create a very basic +webserver serving files relative to the current directory as follows:: import http.server import socketserver @@ -455,7 +524,7 @@ the current directory:: :class:`SimpleHTTPRequestHandler` can also be subclassed to enhance behavior, such as using different index file names by overriding the class attribute -:attr:`index_pages`. +:attr:`~SimpleHTTPRequestHandler.index_pages`. .. _http-server-cli: @@ -463,9 +532,11 @@ such as using different index file names by overriding the class attribute Command-line interface ---------------------- -:mod:`http.server` can also be invoked directly using the :option:`-m` +:mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -476,7 +547,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -485,7 +558,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -498,7 +573,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -508,15 +585,31 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 .. versionadded:: 3.11 +.. option:: --content-type + + Specifies the default Content-Type HTTP header used when the MIME type + cannot be guessed from the URL's file extension. By default, the server + uses ``'application/octet-stream'``: + + .. code-block:: bash + + python -m http.server --content-type text/html + + .. versionadded:: 3.15 + .. option:: --tls-cert - Specifies a TLS certificate chain for HTTPS connections:: + Specifies a TLS certificate chain for HTTPS connections: + + .. code-block:: bash python -m http.server --tls-cert fullchain.pem @@ -532,17 +625,28 @@ The following options are accepted: .. option:: --tls-password-file - Specifies the password file for password-protected private keys:: + Specifies the password file for password-protected private keys: + + .. code-block:: bash python -m http.server \ --tls-cert cert.pem \ --tls-key key.pem \ --tls-password-file password.txt - This option requires `--tls-cert`` to be specified. + This option requires ``--tls-cert`` to be specified. .. versionadded:: 3.14 +.. option:: -H, --header
+ + Specify an additional HTTP Response Header to send on successful HTTP 200 + responses. Can be used multiple times to send additional custom response + headers. Headers that are sent automatically by the server (for instance + Content-Type) will not be overwritten by the server. + + .. versionadded:: 3.15 + .. _http.server-security: @@ -552,9 +656,14 @@ Security considerations .. index:: pair: http.server; security :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling -requests, this makes it possible for files outside of the specified directory +requests which makes it possible for files outside of the specified directory to be served. +Methods :meth:`BaseHTTPRequestHandler.send_header` and +:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input +and do not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP header injection attacks. + Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 52e3726a0f5af5b..c7c30e5300c2a4a 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -3,8 +3,6 @@ IDLE --- Python editor and shell ================================ -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/idlelib/` .. index:: @@ -158,7 +156,7 @@ Go to Line Show Completions Open a scrollable list allowing selection of existing names. See - :ref:`Completions ` in the Editing and navigation section below. + :ref:`Completions ` in the Editing and Navigation section below. Expand Word Expand a prefix you have typed to match a full word in the same window; @@ -167,7 +165,7 @@ Expand Word Show Call Tip After an unclosed parenthesis for a function, open a small window with function parameter hints. See :ref:`Calltips ` in the - Editing and navigation section below. + Editing and Navigation section below. Show Surrounding Parens Highlight the surrounding parenthesis. @@ -178,9 +176,9 @@ Format menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Format Paragraph - Reformat the current blank-line-delimited paragraph in comment block or - multiline string or selected line in a string. All lines in the - paragraph will be formatted to less than N columns, where N defaults to 72. + Rewrap the text block containing the text insert cursor. + Avoid code lines. See :ref:`Format block` in the + Editing and Navigation section below. Indent Region Shift selected lines right by the indent width (default 4 spaces). @@ -566,6 +564,20 @@ In an editor, import statements have no effect until one runs the file. One might want to run a file after writing import statements, after adding function definitions, or after opening an existing file. +.. _format-block: + +Format block +^^^^^^^^^^^^ + +Reformat Paragraph rewraps a block ('paragraph') of contiguous equally +indented non-blank comments, a similar block of text within a multiline +string, or a selected subset of either. +If needed, add a blank line to separate string from code. +Partial lines in a selection expand to complete lines. +The resulting lines have the same indent as before +but have maximum total length of N columns (characters). +Change the default N of 72 on the Window tab of IDLE Settings. + .. _code-context: Code Context @@ -661,7 +673,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0b0537d3bbd1047..53934dc698f19e0 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -4,14 +4,6 @@ .. module:: imaplib :synopsis: IMAP4 protocol client (requires sockets). -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Piers Lauder -.. revised by ESR, January 2000 -.. changes for IMAP4_SSL by Tino Lange , March 2002 -.. changes for IMAP4_stream by Piers Lauder , - November 2002 -.. changes for IMAP4 IDLE by Forest , August 2024 - **Source code:** :source:`Lib/imaplib.py` .. index:: @@ -24,12 +16,12 @@ This module defines three classes, :class:`IMAP4`, :class:`IMAP4_SSL` and :class:`IMAP4_stream`, which encapsulate a connection to an IMAP4 server and implement a large subset of the IMAP4rev1 client protocol as defined in -:rfc:`2060`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but +:rfc:`3501`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst -Three classes are provided by the :mod:`imaplib` module, :class:`IMAP4` is the +Three classes are provided by the :mod:`!imaplib` module, :class:`IMAP4` is the base class: @@ -97,6 +89,13 @@ There's also a subclass for secure connections: (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. @@ -128,11 +127,11 @@ The second subclass allows for connections created by a child process: The following utility functions are defined: -.. function:: Internaldate2tuple(datestr) +.. function:: Internaldate2tuple(resp) - Parse an IMAP4 ``INTERNALDATE`` string and return corresponding local - time. The return value is a :class:`time.struct_time` tuple or - ``None`` if the string has wrong format. + Parse a :term:`bytes-like object` containing an IMAP4 ``INTERNALDATE`` + response and return the corresponding local time. The return value is a + :class:`time.struct_time` tuple or ``None`` if the input has wrong format. .. function:: Int2AP(num) @@ -140,9 +139,11 @@ The following utility functions are defined: [``A`` .. ``P``]. -.. function:: ParseFlags(flagstr) +.. function:: ParseFlags(resp) - Converts an IMAP4 ``FLAGS`` response to a tuple of individual flags. + Converts a :term:`bytes-like object` containing an IMAP4 ``FLAGS`` response + to a tuple of individual flags as :class:`bytes`. The return value is an + empty tuple if the input has wrong format. .. function:: Time2Internaldate(date_time) @@ -187,6 +188,9 @@ enclosed with either parentheses or double quotes) each string is quoted. However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). +In general, pass arguments unquoted and let the module quote them as needed. +An argument that is already enclosed in double quotes is left unchanged, +so that code which quotes arguments itself keeps working. Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually ``'OK'`` or ``'NO'``, and *data* is either the text from the command response, @@ -203,10 +207,26 @@ upper bound (``'3:*'``). An :class:`IMAP4` instance has the following methods: -.. method:: IMAP4.append(mailbox, flags, date_time, message) +.. method:: IMAP4.append(mailbox, flags, date_time, message, *, translate_line_endings=True) Append *message* to named mailbox. + *flags* may be ``None`` or a string of IMAP flag tokens. Multiple + flags are separated by spaces, for example ``r'\Seen \Answered'``. + If *flags* is not already enclosed in parentheses, parentheses are + added automatically. + + If *translate_line_endings* is true (the default), + line endings in *message* are translated to CRLF. + Pass ``False`` to send the message literal exactly as given, + which is required to preserve messages that contain bare CR or LF. + In that case *message* must already use CRLF line endings as required + by :rfc:`3501`; for example, serialize :mod:`email` messages using + :class:`email.policy.SMTP`. + + .. versionchanged:: next + Added the *translate_line_endings* parameter. + .. method:: IMAP4.authenticate(mechanism, authobject) @@ -308,6 +328,19 @@ An :class:`IMAP4` instance has the following methods: of the IMAP4 QUOTA extension defined in rfc2087. +.. method:: IMAP4.id(fields=None) + + Send client identification information to the server + and return the identification information sent back by the server + (the ``ID`` command, defined in :rfc:`2971`). + *fields* is a mapping of field names to values + (for example, ``{'name': 'myclient', 'version': '1.0'}``); + a value can be ``None``. + The server must support the ``ID`` capability. + + .. versionadded:: next + + .. method:: IMAP4.idle(duration=None) Return an :class:`!Idler`: an iterable context manager implementing the @@ -395,7 +428,7 @@ An :class:`IMAP4` instance has the following methods: .. versionadded:: 3.14 -.. method:: IMAP4.list([directory[, pattern]]) +.. method:: IMAP4.list(directory='', pattern='*') List mailbox names in *directory* matching *pattern*. *directory* defaults to the top-level mail folder, and *pattern* defaults to match anything. Returned @@ -410,13 +443,30 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.login_cram_md5(user, password) Force use of ``CRAM-MD5`` authentication when identifying the client to protect - the password. Will only work if the server ``CAPABILITY`` response includes the - phrase ``AUTH=CRAM-MD5``. + the password. It will only work if the server ``CAPABILITY`` response includes + the phrase ``AUTH=CRAM-MD5``. .. versionchanged:: 3.15 An :exc:`IMAP4.error` is raised if MD5 support is not available. +.. method:: IMAP4.login_plain(user, password) + + Authenticate using the ``PLAIN`` SASL mechanism (:rfc:`4616`). + + This is a plaintext authentication mechanism that can be used instead + of :meth:`login` when UTF-8 support is required (see :rfc:`6855`). + Since the credentials are only base64-encoded, not encrypted, this + method should only be used over a TLS-protected connection, such as + :class:`IMAP4_SSL` or after :meth:`starttls`. + + It will only work if the server supports the ``PLAIN`` mechanism, + which it need not advertise as ``AUTH=PLAIN`` in its ``CAPABILITY`` + response. + + .. versionadded:: next + + .. method:: IMAP4.logout() Shutdown connection to server. Returns server ``BYE`` response. @@ -425,13 +475,22 @@ An :class:`IMAP4` instance has the following methods: The method no longer ignores silently arbitrary exceptions. -.. method:: IMAP4.lsub(directory='""', pattern='*') +.. method:: IMAP4.lsub(directory='', pattern='*') List subscribed mailbox names in directory matching pattern. *directory* defaults to the top level directory and *pattern* defaults to match any mailbox. Returned data are tuples of message part envelope and data. +.. method:: IMAP4.move(message_set, new_mailbox) + + Move *message_set* messages onto end of *new_mailbox*. + + The server must support the ``MOVE`` capability (:rfc:`6851`). + + .. versionadded:: next + + .. method:: IMAP4.myrights(mailbox) Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). @@ -589,6 +648,13 @@ An :class:`IMAP4` instance has the following methods: encryption on the IMAP connection. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + .. versionadded:: 3.2 .. versionchanged:: 3.4 @@ -605,7 +671,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.store(message_set, command, flag_list) Alters flag dispositions for messages in mailbox. *command* is specified by - section 6.4.6 of :rfc:`2060` as being one of "FLAGS", "+FLAGS", or "-FLAGS", + section 6.4.6 of :rfc:`3501` as being one of "FLAGS", "+FLAGS", or "-FLAGS", optionally with a suffix of ".SILENT". For example, to set the delete flag on all messages:: @@ -619,11 +685,11 @@ An :class:`IMAP4` instance has the following methods: Creating flags containing ']' (for example: "[test]") violates :rfc:`3501` (the IMAP protocol). However, imaplib has historically - allowed creation of such tags, and popular IMAP servers, such as Gmail, + allowed creation of such flags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also - create such tags. Although it is an RFC violation and IMAP clients and + create such flags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib still continues to allow - such tags to be created for backward compatibility reasons, and as of + such flags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. @@ -703,6 +769,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: 3.15 3.19 + + .. _imap4-example: IMAP4 Example @@ -723,3 +799,10 @@ retrieves and prints all messages:: M.close() M.logout() +.. note:: + + A ``FETCH`` response may contain additional or unsolicited data + (see :rfc:`3501`, section 7.4.2), + so production code should inspect the whole response + rather than rely on ``data[0][1]``. + diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 12014309e26ec93..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -18,11 +18,9 @@ the metadata of an installed `Distribution Package `_\s, modules, if any). Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources`, -this package can eliminate the need to use the older and less efficient -``pkg_resources`` package. +provides the entry point and metadata APIs that were previously +exposed by the now-removed ``pkg_resources`` package. Along with +:mod:`importlib.resources`, it supersedes ``pkg_resources``. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as @@ -107,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -125,8 +130,8 @@ Entry points :meth:`!select` method for comparison to the attributes of the individual entry point definitions. - Note: it is not currently possible to query for entry points based on - their :attr:`!EntryPoint.dist` attribute (as different :class:`!Distribution` + Note: to query for entry points based on :attr:`!EntryPoint.dist` attribute, + use :meth:`Distribution.entry_points` instead (as different :class:`Distribution` instances do not currently compare equal, even if they have the same attributes) .. class:: EntryPoints @@ -226,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -254,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -291,7 +305,7 @@ Distribution files .. function:: files(distribution_name) Return the full set of files contained within the named - distribution package. + distribution package as :class:`PackagePath` instances. Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. @@ -304,12 +318,22 @@ Distribution files A :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties corresponding to the distribution - package's installation metadata for that file. + package's installation metadata for that file, also: + + .. method:: locate() + + If possible, return the concrete :class:`SimplePath` allowing to access data, + or raise a :exc:`NotImplementedError` otherwise. + +.. class:: SimplePath + + A protocol representing a minimal subset of :class:`pathlib.Path` that allows to + check if it ``exists()``, to traverse using ``joinpath()`` and ``parent``, + and to retrieve data using ``read_text()`` and ``read_bytes()``. The :func:`!files` function takes a `Distribution Package `_ -name and returns all of the files installed by this distribution. Each file is reported -as a :class:`PackagePath` instance. For example:: +name and returns all of the files installed by this distribution. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP >>> util # doctest: +SKIP @@ -402,6 +426,18 @@ function is not reliable with such installs. Distributions ============= +While the module level API described above is the most common and convenient usage, +all that information is accessible from the :class:`Distribution` class. +:class:`!Distribution` is an abstract object that represents the metadata for +a Python `Distribution Package `_. +Get the concrete :class:`!Distribution` subclass instance for an installed +distribution package by calling the :func:`distribution` function:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + >>> type(dist) # doctest: +SKIP + + .. function:: distribution(distribution_name) Return a :class:`Distribution` instance describing the named @@ -410,6 +446,14 @@ Distributions Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. +Thus, an alternative way to get e.g. the version number is through the +:attr:`Distribution.version` attribute:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +The same applies for :func:`entry_points` and :func:`files`. + .. class:: Distribution Details of an installed distribution package. @@ -418,43 +462,88 @@ Distributions equal, even if they relate to the same installed distribution and accordingly have the same attributes. -While the module level API described above is the most common and convenient usage, -you can get all of that information from the :class:`!Distribution` class. -:class:`!Distribution` is an abstract object that represents the metadata for -a Python `Distribution Package `_. -You can get the concrete :class:`!Distribution` subclass instance for an installed -distribution package by calling the :func:`distribution` function:: + .. staticmethod:: at(path) + .. classmethod:: from_name(name) - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - >>> type(dist) # doctest: +SKIP - + Return a :class:`!Distribution` instance at the given path or + with the given name. -Thus, an alternative way to get the version number is through the -:class:`!Distribution` instance:: + .. classmethod:: discover(*, context=None, **kwargs) - >>> dist.version # doctest: +SKIP - '0.32.3' + Returns an iterable of :class:`!Distribution` instances for all packages + (see distribution-discovery_). -There are all kinds of additional metadata available on :class:`!Distribution` -instances:: + The optional argument *context* is a :class:`DistributionFinder.Context` + instance, used to modify the search for distributions. Alternatively, + *kwargs* may contain keyword arguments for constructing a new + :class:`!DistributionFinder.Context`. - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' + .. attribute:: metadata + :type: PackageMetadata -For editable packages, an ``origin`` property may present :pep:`610` -metadata:: + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. - >>> dist.origin.url - 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + There are all kinds of additional metadata available on :class:`!Distribution` + instances as a :class:`PackageMetadata` instance:: -The full set of available metadata is not described here. -See the PyPA `Core metadata specification `_ for additional details. + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' + + The full set of available metadata is not described here. + See the PyPA `Core metadata specification `_ for additional details. -.. versionadded:: 3.13 - The ``.origin`` property was added. + .. attribute:: name + :type: str + .. attribute:: requires + :type: list[str] + .. attribute:: version + :type: str + + A few metadata fields are also available as shortcut properties. + + .. versionadded:: 3.10 + + The ``name`` shortcut was added. + + .. attribute:: origin + + For editable packages, an ``origin`` property may present :pep:`610` + metadata (for non-editable packages, ``origin`` is :const:`None`):: + + >>> dist.origin.url + 'file:///path/to/wheel-0.32.3.editable-py3-none-any.whl' + + The ``origin`` object follows the `Direct URL Data Structure + `_. + + .. versionadded:: 3.13 + + .. attribute:: entry_points + :type: EntryPoints + + The entry points provided by this distribution package. + + .. attribute:: files + :type: list[PackagePath] | None + + All files contained in this distribution package. + Like :func:`files`, this returns :const:`None` if there are no records. + + The following two abstract methods need to be implemented when implementing-custom-providers_: + + .. method:: locate_file(path) + + Like :meth:`!PackagePath.locate`, return a :class:`SimplePath` for the given path. + Takes a :class:`os.PathLike` or a :class:`str`. + + .. method:: read_text(filename) + + A shortcut for ``distribution.locate_file(filename).read_text()``. + +.. _distribution-discovery: Distribution Discovery ====================== @@ -466,6 +555,61 @@ This metadata finder search defaults to ``sys.path``, but varies slightly in how - ``importlib.metadata`` does not honor :class:`bytes` objects on ``sys.path``. - ``importlib.metadata`` will incidentally honor :py:class:`pathlib.Path` objects on ``sys.path`` even though such values will be ignored for imports. +.. class:: DistributionFinder + + A :class:`~importlib.abc.MetaPathFinder` subclass capable of discovering + installed distributions. + + Custom providers should implement this interface in order to + supply metadata. + + .. class:: Context(**kwargs) + + A :class:`!Context` gives a custom provider a means to + solicit additional details from the callers of distribution discovery + functions like :func:`distributions` or :meth:`Distribution.discover` + beyond :attr:`!.name` and :attr:`!.path` when searching + for distributions. + + For example, a provider could expose suites of packages in either a + "public" or "private" ``realm``. A caller of distribution discovery + functions may wish to query only for distributions in a particular realm + and could call ``distributions(realm="private")`` to signal to the + custom provider to only include distributions from that + realm. + + Each :class:`!DistributionFinder` must expect any parameters and should + attempt to honor the canonical parameters defined below when + appropriate. + + See the section on :ref:`implementing-custom-providers` for more details. + + .. attribute:: name + + Specific name for which a distribution finder should match. + + A :attr:`!.name` of ``None`` matches all distributions. + + .. attribute:: path + + A property providing the sequence of directory paths that a + distribution finder should search. + + Typically refers to Python installed package paths such as + "site-packages" directories and defaults to :attr:`sys.path`. + + +.. function:: distributions(**kwargs) + + Returns an iterable of :class:`Distribution` instances for all packages. + + The *kwargs* argument may contain either a keyword argument ``context``, a + :class:`DistributionFinder.Context` instance, or pass keyword arguments for + constructing a new :class:`!DistributionFinder.Context`. The + :class:`!DistributionFinder.Context` is used to modify the search for + distributions. + +.. _implementing-custom-providers: Implementing Custom Providers ============================= @@ -493,7 +637,7 @@ interface expected of finders by Python's import system. ``importlib.metadata`` extends this protocol by looking for an optional ``find_distributions`` callable on the finders from :data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract +:class:`DistributionFinder` abstract base class, which defines this abstract method:: @abc.abstractmethod @@ -502,14 +646,16 @@ method:: loading the metadata for packages for the indicated ``context``. """ -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context sought by the consumer. +The :class:`DistributionFinder.Context` object provides +:attr:`~DistributionFinder.Context.path` and +:attr:`~DistributionFinder.Context.name` properties indicating the path to +search and name to match and may supply other relevant context sought by the +consumer. In practice, to support finding distribution package metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the +:class:`!Distribution` and implement the abstract methods. Then from +a custom finder, return instances of this derived :class:`!Distribution` in the ``find_distributions()`` method. Example @@ -529,7 +675,7 @@ Imagine a custom finder that loads Python modules from a database:: That importer now presumably provides importable modules from a database, but it provides no metadata or entry points. For this custom importer to provide metadata, it would also need to implement -``DistributionFinder``:: +:class:`DistributionFinder`:: from importlib.metadata import DistributionFinder @@ -586,9 +732,5 @@ packages served by the ``DatabaseImporter``, assuming that the ``.entry_points`` attributes. The ``DatabaseDistribution`` may also provide other metadata files, like -``RECORD`` (required for ``Distribution.files``) or override the -implementation of ``Distribution.files``. See the source for more inspiration. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +``RECORD`` (required for :attr:`!Distribution.files`) or override the +implementation of :attr:`!Distribution.files`. See the source for more inspiration. diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 8253a33f591a0bc..45979c5691270aa 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -63,11 +63,14 @@ If the resource does not concretely exist on the file system, raise :exc:`FileNotFoundError`. - .. method:: is_resource(name) + .. method:: is_resource(path) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *path* is considered a resource. + :exc:`FileNotFoundError` is raised if *path* does not exist. + + .. versionchanged:: 3.10 + The argument *name* was renamed to *path*. .. method:: contents() :abstractmethod: diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 20297f9fe307b54..72db66f9f06f890 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -31,15 +31,13 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. -.. note:: +.. warning:: + + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. - This module provides functionality similar to `pkg_resources - `_ `Basic - Resource Access - `_ - without the performance overhead of that package. This makes reading - resources included in packages easier, with more stable and consistent - semantics. +.. note:: The standalone backport of this module provides more information on `using importlib.resources @@ -242,7 +240,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 03ba23b6216cbfb..0b76020eacc1da2 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -4,9 +4,6 @@ .. module:: importlib :synopsis: The implementation of the import machinery. -.. moduleauthor:: Brett Cannon -.. sectionauthor:: Brett Cannon - .. versionadded:: 3.1 **Source code:** :source:`Lib/importlib/__init__.py` @@ -17,7 +14,7 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is three-fold. +The purpose of the :mod:`!importlib` package is three-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the @@ -215,8 +212,8 @@ Functions in unexpected behavior. It's recommended to use the :class:`threading.Lock` or other synchronization primitives for thread-safe module reloading. -:mod:`importlib.abc` -- Abstract base classes related to import ---------------------------------------------------------------- +:mod:`!importlib.abc` -- Abstract base classes related to import +---------------------------------------------------------------- .. module:: importlib.abc :synopsis: Abstract base classes related to import @@ -226,7 +223,7 @@ Functions -------------- -The :mod:`importlib.abc` module contains all of the core abstract base classes +The :mod:`!importlib.abc` module contains all of the core abstract base classes used by :keyword:`import`. Some subclasses of the core abstract base classes are also provided to help in implementing the core ABCs. @@ -275,6 +272,28 @@ ABC hierarchy:: .. versionchanged:: 3.4 Returns ``None`` when called instead of :data:`NotImplemented`. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: PathEntryFinder @@ -307,6 +326,28 @@ ABC hierarchy:: :meth:`importlib.machinery.PathFinder.invalidate_caches` when invalidating the caches of all cached finders. + .. method:: discover(parent=None) + + An optional method which searches for possible specs with given *parent* + module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will + search for top-level modules. + + Returns an iterable of possible specs. + + Raises :exc:`ValueError` if *parent* is not a package module. + + .. warning:: + This method can potentially yield a very large number of objects, and + it may carry out IO operations when computing these values. + + Because of this, it will generally be desirable to compute the result + values on-the-fly, as they are needed. As such, the returned object is + only guaranteed to be an :class:`iterable `, + instead of a :class:`list` or other + :class:`collection ` type. + + .. versionadded:: 3.15 + .. class:: Loader @@ -320,6 +361,9 @@ ABC hierarchy:: .. versionchanged:: 3.7 Introduced the optional :meth:`get_resource_reader` method. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: create_module(spec) A method that returns the module object to use when @@ -344,47 +388,6 @@ ABC hierarchy:: .. versionchanged:: 3.6 :meth:`create_module` must also be defined. - .. method:: load_module(fullname) - - A legacy method for loading a module. If the module cannot be - loaded, :exc:`ImportError` is raised, otherwise the loaded module is - returned. - - If the requested module already exists in :data:`sys.modules`, that - module should be used and reloaded. - Otherwise the loader should create a new module and insert it into - :data:`sys.modules` before any loading begins, to prevent recursion - from the import. If the loader inserted a module and the load fails, it - must be removed by the loader from :data:`sys.modules`; modules already - in :data:`sys.modules` before the loader began execution should be left - alone. - - The loader should set several attributes on the module - (note that some of these attributes can change when a module is - reloaded): - - - :attr:`module.__name__` - - :attr:`module.__file__` - - :attr:`module.__cached__` *(deprecated)* - - :attr:`module.__path__` - - :attr:`module.__package__` *(deprecated)* - - :attr:`module.__loader__` *(deprecated)* - - When :meth:`exec_module` is available then backwards-compatible - functionality is provided. - - .. versionchanged:: 3.4 - Raise :exc:`ImportError` when called instead of - :exc:`NotImplementedError`. Functionality provided when - :meth:`exec_module` is available. - - .. deprecated-removed:: 3.4 3.15 - The recommended API for loading a module is :meth:`exec_module` - (and :meth:`create_module`). Loaders should implement it instead of - :meth:`load_module`. The import machinery takes care of all the - other responsibilities of :meth:`load_module` when - :meth:`exec_module` is implemented. - .. class:: ResourceLoader @@ -480,7 +483,7 @@ ABC hierarchy:: .. versionchanged:: 3.5 Made the method static. - .. versionadded:: next + .. versionadded:: 3.15 Added the *fullname* parameter. @@ -490,13 +493,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - use :meth:`exec_module` instead. - .. class:: ExecutionLoader @@ -530,6 +526,9 @@ ABC hierarchy:: .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader can handle. @@ -538,13 +537,6 @@ ABC hierarchy:: Path to the file of the module. - .. method:: load_module(fullname) - - Calls super's ``load_module()``. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`Loader.exec_module` instead. - .. method:: get_filename(fullname) :abstractmethod: @@ -576,6 +568,9 @@ ABC hierarchy:: optimization to speed up loading by removing the parsing step of Python's compiler, and so no bytecode-specific API is exposed. + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. method:: path_stats(path) Optional abstract method which returns a :class:`dict` containing @@ -629,13 +624,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: load_module(fullname) - - Concrete implementation of :meth:`Loader.load_module`. - - .. deprecated-removed:: 3.4 3.15 - Use :meth:`exec_module` instead. - .. method:: get_source(fullname) Concrete implementation of :meth:`InspectLoader.get_source`. @@ -649,174 +637,8 @@ ABC hierarchy:: itself does not end in ``__init__``. -.. class:: ResourceReader - - *Superseded by TraversableResources* - - An :term:`abstract base class` to provide the ability to read - *resources*. - - From the perspective of this ABC, a *resource* is a binary - artifact that is shipped within a package. Typically this is - something like a data file that lives next to the ``__init__.py`` - file of the package. The purpose of this class is to help abstract - out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored e.g. in a zip file - versus on the file system. - - For any of methods of this class, a *resource* argument is - expected to be a :term:`path-like object` which represents - conceptually just a file name. This means that no subdirectory - paths should be included in the *resource* argument. This is - because the location of the package the reader is for, acts as the - "directory". Hence the metaphor for directories and file - names is packages and resources, respectively. This is also why - instances of this class are expected to directly correlate to - a specific package (instead of potentially representing multiple - packages or a module). - - Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_reader(fullname)`` which - returns an object implementing this ABC's interface. If the module - specified by fullname is not a package, this method should return - :const:`None`. An object compatible with this ABC should only be - returned when the specified module is a package. - - .. versionadded:: 3.7 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: open_resource(resource) - :abstractmethod: - - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. - - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. - - .. method:: resource_path(resource) - :abstractmethod: - - Returns the file system path to the *resource*. - - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. - - .. method:: is_resource(name) - :abstractmethod: - - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. - - .. method:: contents() - :abstractmethod: - - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. - - -.. class:: Traversable - - An object with a subset of :class:`pathlib.Path` methods suitable for - traversing directories and opening files. - - For a representation of the object on the file-system, use - :meth:`importlib.resources.as_file`. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.Traversable` instead. - - .. attribute:: name - - Abstract. The base name of this object without any parent references. - - .. method:: iterdir() - :abstractmethod: - - Yield ``Traversable`` objects in ``self``. - - .. method:: is_dir() - :abstractmethod: - - Return ``True`` if ``self`` is a directory. - - .. method:: is_file() - :abstractmethod: - - Return ``True`` if ``self`` is a file. - - .. method:: joinpath(child) - :abstractmethod: - - Return Traversable child in ``self``. - - .. method:: __truediv__(child) - :abstractmethod: - - Return ``Traversable`` child in ``self``. - - .. method:: open(mode='r', *args, **kwargs) - :abstractmethod: - - *mode* may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as :attr:`pathlib.Path.open`). - - When opening as text, accepts encoding parameters such as those - accepted by :class:`io.TextIOWrapper`. - - .. method:: read_bytes() - - Read contents of ``self`` as bytes. - - .. method:: read_text(encoding=None) - - Read contents of ``self`` as text. - - -.. class:: TraversableResources - - An abstract base class for resource readers capable of serving - the :meth:`importlib.resources.files` interface. Subclasses - :class:`importlib.resources.abc.ResourceReader` and provides - concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s - abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableResources` also supplies ResourceReader. - - Loaders that wish to support resource reading are expected to - implement this interface. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: files() - :abstractmethod: - - Returns a :class:`importlib.resources.abc.Traversable` object for the loaded - package. - - - -:mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------- +:mod:`!importlib.machinery` -- Importers and path hooks +------------------------------------------------------- .. module:: importlib.machinery :synopsis: Importers and path hooks @@ -1048,7 +870,7 @@ find and load modules. :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, forcing the path value to be recomputed next time it is accessed. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: SourceFileLoader(fullname, path) @@ -1059,6 +881,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module that this loader will handle. @@ -1079,15 +904,6 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.SourceLoader.set_data`. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: SourcelessFileLoader(fullname, path) @@ -1101,6 +917,9 @@ find and load modules. .. versionadded:: 3.3 + .. versionchanged:: 3.15 + Removed the ``load_module()`` method. + .. attribute:: name The name of the module the loader will handle. @@ -1122,15 +941,6 @@ find and load modules. Returns ``None`` as bytecode files have no source when this loader is used. - .. method:: load_module(name=None) - - Concrete implementation of :meth:`importlib.abc.Loader.load_module` where - specifying the name of the module to load is optional. - - .. deprecated-removed:: 3.6 3.15 - - Use :meth:`importlib.abc.Loader.exec_module` instead. - .. class:: ExtensionFileLoader(fullname, path) @@ -1262,8 +1072,7 @@ find and load modules. .. attribute:: cached - The filename of a compiled version of the module's code - (see :attr:`module.__cached__`). + The filename of a compiled version of the module's code. The :term:`finder` should always set this attribute but it may be ``None`` for modules that do not need compiled code stored. @@ -1344,8 +1153,8 @@ find and load modules. Path to the ``.fwork`` file for the extension module. -:mod:`importlib.util` -- Utility code for importers ---------------------------------------------------- +:mod:`!importlib.util` -- Utility code for importers +---------------------------------------------------- .. module:: importlib.util :synopsis: Utility code for importers @@ -1365,7 +1174,7 @@ an :term:`importer`. .. versionadded:: 3.4 -.. function:: cache_from_source(path, debug_override=None, *, optimization=None) +.. function:: cache_from_source(path, *, optimization=None) Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated with the source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return @@ -1384,12 +1193,6 @@ an :term:`importer`. ``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised. - The *debug_override* parameter is deprecated and can be used to override - the system's value for ``__debug__``. A ``True`` value is the equivalent of - setting *optimization* to the empty string. A ``False`` value is the same as - setting *optimization* to ``1``. If both *debug_override* an *optimization* - are not ``None`` then :exc:`TypeError` is raised. - .. versionadded:: 3.4 .. versionchanged:: 3.5 @@ -1399,6 +1202,9 @@ an :term:`importer`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + The *debug_override* parameter was removed. + .. function:: source_from_cache(path) diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 163e1679c65ef83..8fc77be520d4268 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -43,6 +43,7 @@ the `Python Package Index `_. constants.rst stdtypes.rst exceptions.rst + threadsafety.rst text.rst binary.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 13a352cbdb2cdca..a0f7379b12a8a62 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -9,14 +9,11 @@ .. module:: inspect :synopsis: Extract information and source code from live objects. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/inspect.py` -------------- -The :mod:`inspect` module provides several useful functions to help get +The :mod:`!inspect` module provides several useful functions to help get information about live objects such as modules, classes, methods, functions, tracebacks, frame objects, and code objects. For example, it can help you examine the contents of a class, retrieve the source code of a method, extract @@ -198,10 +195,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | read more :ref:`here | | | | `| +-----------------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------------+-------------------+---------------------------+ | | co_freevars | tuple of names of free | | | | variables (referenced via | | | | a function's closure) | @@ -262,6 +255,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | ``yield from``, or | | | | ``None`` | +-----------------+-------------------+---------------------------+ +| | gi_state | state of the generator, | +| | | one of ``GEN_CREATED``, | +| | | ``GEN_RUNNING``, | +| | | ``GEN_SUSPENDED``, or | +| | | ``GEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | async generator | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -273,8 +272,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | ag_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | ag_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | ag_code | code | +-----------------+-------------------+---------------------------+ +| | ag_state | state of the async | +| | | generator, one of | +| | | ``AGEN_CREATED``, | +| | | ``AGEN_RUNNING``, | +| | | ``AGEN_SUSPENDED``, or | +| | | ``AGEN_CLOSED`` | ++-----------------+-------------------+---------------------------+ | coroutine | __name__ | name | +-----------------+-------------------+---------------------------+ | | __qualname__ | qualified name | @@ -286,12 +295,21 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | cr_running | is the coroutine running? | +-----------------+-------------------+---------------------------+ +| | cr_suspended | is the coroutine | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | cr_code | code | +-----------------+-------------------+---------------------------+ | | cr_origin | where coroutine was | | | | created, or ``None``. See | | | | |coroutine-origin-link| | +-----------------+-------------------+---------------------------+ +| | cr_state | state of the coroutine, | +| | | one of ``CORO_CREATED``, | +| | | ``CORO_RUNNING``, | +| | | ``CORO_SUSPENDED``, or | +| | | ``CORO_CLOSED`` | ++-----------------+-------------------+---------------------------+ | builtin | __doc__ | documentation string | +-----------------+-------------------+---------------------------+ | | __name__ | original name of this | @@ -319,10 +337,27 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.11 + + Add ``gi_suspended`` attribute to generators. + +.. versionchanged:: 3.11 + + Add ``cr_suspended`` attribute to coroutines. + +.. versionchanged:: 3.12 + + Add ``ag_suspended`` attribute to async generators. + .. versionchanged:: 3.14 Add ``f_generator`` attribute to frames. +.. versionchanged:: 3.15 + + Add ``gi_state`` attribute to generators, ``cr_state`` attribute to + coroutines, and ``ag_state`` attribute to async generators. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` @@ -381,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -399,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -506,7 +583,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` - if the wrapped function is a :term:`coroutine function`. + if the wrapped function is a :term:`asynchronous generator` function. .. function:: isasyncgen(object) @@ -636,11 +713,11 @@ Retrieving source code .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. - .. versionchanged:: next + .. versionchanged:: 3.15 Added parameters *inherit_class_doc* and *fallback_to_class_doc*. Documentation strings on :class:`~functools.cached_property` - objects are now inherited if not overriden. + objects are now inherited if not overridden. .. function:: getcomments(object) @@ -1154,7 +1231,7 @@ Classes and functions times. -.. function:: getfullargspec(func) +.. function:: getfullargspec(func, *, annotation_format=Format.VALUE) Get the names and default values of a Python function's parameters. A :term:`named tuple` is returned: @@ -1184,6 +1261,14 @@ Classes and functions APIs. This function is retained primarily for use in code that needs to maintain compatibility with the Python 2 ``inspect`` module API. + A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Note that with the default ``VALUE`` format, creation of some argspecs + may raise an exception. + .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -1201,6 +1286,9 @@ Classes and functions order of keyword-only parameters as of version 3.7, although in practice this order had always been preserved in Python 3. + .. versionchanged:: 3.15 + The *annotation_format* parameter was added. + .. function:: getargvalues(frame) @@ -1526,10 +1614,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` @@ -1749,7 +1838,7 @@ which is a bitmap of the following flags: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation detail, and can be removed or deprecated in future Python releases. - It's recommended to use public APIs from the :mod:`inspect` module + It's recommended to use public APIs from the :mod:`!inspect` module for any introspection needs. @@ -1788,18 +1877,25 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`inspect` module also provides a basic introspection capability +The :mod:`!inspect` module also provides a basic introspection capability from the command line. .. program:: inspect By default, accepts the name of a module and prints the source of that module. A class or function within the module can be printed instead by -appended a colon and the qualified name of the target object. +appending a colon and the qualified name of the target object. .. option:: --details Print information about the specified object rather than the source code + +.. versionchanged:: 3.15 + + The ``--details`` option now supports basic introspection for modules + without available source code and indicates when modules are frozen. + It also indicates when the given target reference is not the canonical + name of the referenced object. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index dfebccb5a9cb91c..c0d7ee877536ad5 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -4,14 +4,6 @@ .. module:: io :synopsis: Core tools for working with streams. -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Mike Verdone -.. moduleauthor:: Mark Russell -.. moduleauthor:: Antoine Pitrou -.. moduleauthor:: Amaury Forgeot d'Arc -.. moduleauthor:: Benjamin Peterson -.. sectionauthor:: Benjamin Peterson - **Source code:** :source:`Lib/io.py` -------------- @@ -24,7 +16,7 @@ Overview .. index:: single: file object; io module -The :mod:`io` module provides Python's main facilities for dealing with various +The :mod:`!io` module provides Python's main facilities for dealing with various types of I/O. There are three main types of I/O: *text I/O*, *binary I/O* and *raw I/O*. These are generic categories, and various backing stores can be used for each of them. A concrete object belonging to any of these @@ -46,6 +38,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -73,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -111,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -292,7 +293,7 @@ interface to a buffered raw stream (:class:`BufferedIOBase`). Finally, Argument names are not part of the specification, and only the arguments of :func:`open` are intended to be used as keyword arguments. -The following table summarizes the ABCs provided by the :mod:`io` module: +The following table summarizes the ABCs provided by the :mod:`!io` module: .. tabularcolumns:: |l|l|L|L| @@ -486,8 +487,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -501,13 +505,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -521,6 +531,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -587,7 +604,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: read1(size=-1, /) @@ -600,7 +617,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: readinto(b, /) @@ -649,7 +666,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -669,10 +690,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -684,6 +701,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -720,7 +744,7 @@ than raw I/O does. contains initial data. Methods may be used from multiple threads without external locking in - :term:`free threading` builds. + :term:`free-threaded builds `. :class:`BytesIO` provides or overrides these methods in addition to those from :class:`BufferedIOBase` and :class:`IOBase`: @@ -747,6 +771,17 @@ than raw I/O does. Return :class:`bytes` containing the entire contents of the buffer. + .. method:: peek(size=0, /) + + Return a copy of the buffer from the current position onwards without + advancing the position. + + If *size* is less than one or omitted, at most + :data:`DEFAULT_BUFFER_SIZE` bytes are returned. + Otherwise, at most *size* bytes are returned. + Return an empty :class:`bytes` object at EOF. + + .. versionadded:: next .. method:: read1(size=-1, /) @@ -832,9 +867,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9e887d8e65741b2..9ccd8602bcb2c33 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -4,13 +4,11 @@ .. module:: ipaddress :synopsis: IPv4/IPv6 manipulation library. -.. moduleauthor:: Peter Moody - **Source code:** :source:`Lib/ipaddress.py` -------------- -:mod:`ipaddress` provides the capabilities to create, manipulate and +:mod:`!ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. The functions and classes in this module make it straightforward to handle @@ -34,7 +32,7 @@ This is the full module API reference—for an overview and introduction, see Convenience factory functions ----------------------------- -The :mod:`ipaddress` module provides factory functions to conveniently create +The :mod:`!ipaddress` module provides factory functions to conveniently create IP addresses, networks and interfaces: .. function:: ip_address(address) @@ -369,9 +367,9 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: ipv4_mapped - For addresses that appear to be IPv4 mapped addresses (starting with - ``::FFFF/96``), this property will report the embedded IPv4 address. - For any other address, this property will be ``None``. + For addresses that appear to be IPv4 mapped addresses in the range + ``::FFFF:0:0/96`` as defined by :RFC:`4291`, this property reports the + embedded IPv4 address. For any other address, this property will be ``None``. .. attribute:: scope_id @@ -1027,7 +1025,7 @@ The module also provides the following module level functions: IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make sense. There are some times however, where you may wish to - have :mod:`ipaddress` sort these anyway. If you need to do this, you can use + have :mod:`!ipaddress` sort these anyway. If you need to do this, you can use this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index aa46920d3526f0f..e1730608887b3dd 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -4,9 +4,6 @@ .. module:: itertools :synopsis: Functions creating iterators for efficient looping. -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - .. testsetup:: from itertools import * @@ -30,32 +27,24 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. - -**Infinite iterators:** - -================== ================= ================================================= ========================================= -Iterator Arguments Results Example -================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** +**General iterators:** ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) → AB CD EF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` :func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` @@ -181,7 +170,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 2) → AB CD EF G + # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) @@ -208,6 +197,10 @@ loops that truncate the stream. for iterable in iterables: yield from iterable + Note that :ref:`unpacking in comprehensions ` + provides similar functionality so that ``list(chain(p, q))`` could be + written as ``[*s for s in (p, q)]``. + .. classmethod:: chain.from_iterable(iterable) @@ -219,6 +212,10 @@ loops that truncate the stream. for iterable in iterables: yield from iterable + Note that :ref:`unpacking in comprehensions ` + provides similar functionality so that ``list(chain.from_iterable(iterables))`` + could be written as ``[*s for s in iterables]``. + .. function:: combinations(iterable, r) @@ -819,7 +816,7 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes. Currently, the ``sliding_window()``, ``derangements()``, and ``sieve()`` recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from @@ -838,11 +835,18 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: + from itertools import (accumulate, batched, chain, combinations, compress, + count, cycle, filterfalse, groupby, islice, permutations, product, + repeat, starmap, tee, zip_longest) from collections import Counter, deque from contextlib import suppress from functools import reduce - from math import comb, prod, sumprod, isqrt - from operator import itemgetter, getitem, mul, neg + from heapq import heappush, heappushpop, heappush_max, heappushpop_max + from math import comb, isqrt, prod, sumprod + from operator import getitem, is_not, itemgetter, mul, neg, truediv + + + # ==== Basic one liners ==== def take(n, iterable): "Return first n items of the iterable as a list." @@ -853,10 +857,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -899,8 +899,8 @@ and :term:`generators ` which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) → a or b or c or x - # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + # first_true([a, b, c], x) → a or b or c or x + # first_true([a, b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): @@ -908,6 +908,9 @@ and :term:`generators ` which incur interpreter overhead. # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + + # ==== Data pipelines ==== + def unique_justseen(iterable, key=None): "Yield unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') → A B C D A B @@ -933,14 +936,14 @@ and :term:`generators ` which incur interpreter overhead. yield element def unique(iterable, key=None, reverse=False): - "Yield unique elements in sorted order. Supports unhashable inputs." - # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - sequenced = sorted(iterable, key=key, reverse=reverse) - return unique_justseen(sequenced, key=key) + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + # sliding_window('ABCDEFG', 3) → ABC BCD CDE DEF EFG iterator = iter(iterable) window = deque(islice(iterator, n - 1), maxlen=n) for x in iterator: @@ -949,7 +952,7 @@ and :term:`generators ` which incur interpreter overhead. def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF iterators = [iter(iterable)] * n @@ -978,6 +981,16 @@ and :term:`generators ` which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(getitem, repeat(seq), slices) + def derangements(iterable, r=None): + "Produce r length permutations without fixed points." + # derangements('ABCD') → BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA + # Algorithm credited to Stefan Pochmann + seq = tuple(iterable) + pos = tuple(range(len(seq))) + have_moved = map(map, repeat(is_not), repeat(pos), permutations(pos, r=r)) + valid_derangements = map(all, have_moved) + return compress(permutations(seq, r=r), valid_derangements) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') → 0 1 4 7 @@ -1005,9 +1018,7 @@ and :term:`generators ` which incur interpreter overhead. yield function() -The following recipes have a more mathematical flavor: - -.. testcode:: + # ==== Mathematical operations ==== def multinomial(*counts): "Number of distinct arrangements of a multiset." @@ -1026,9 +1037,12 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) → 1400 return sumprod(*tee(iterable)) + + # ==== Matrix operations ==== + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2) (3, 4, 5) return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): @@ -1038,10 +1052,13 @@ The following recipes have a more mathematical flavor: def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80) (41, 60) n = len(m2[0]) return batched(starmap(sumprod, product(m1, transpose(m2))), n) + + # ==== Polynomial arithmetic ==== + def convolve(signal, kernel): """Discrete linear convolution of two iterables. Equivalent to polynomial multiplication. @@ -1096,6 +1113,9 @@ The following recipes have a more mathematical flavor: powers = reversed(range(1, n)) return list(map(mul, coefficients, powers)) + + # ==== Number theory ==== + def sieve(n): "Primes less than n." # sieve(30) → 2 3 5 7 11 13 17 19 23 29 @@ -1134,6 +1154,49 @@ The following recipes have a more mathematical flavor: return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1210,10 +1273,6 @@ The following recipes have a more mathematical flavor: [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] - - >>> for _ in loops(5): ... print('hi') ... @@ -1663,6 +1722,36 @@ The following recipes have a more mathematical flavor: ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> ' '.join(map(''.join, derangements('ABCD'))) + 'BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA' + >>> ' '.join(map(''.join, derangements('ABCD', 3))) + 'BAD BCA BCD BDA CAB CAD CDA CDB DAB DCA DCB' + >>> ' '.join(map(''.join, derangements('ABCD', 2))) + 'BA BC BD CA CD DA DC' + >>> ' '.join(map(''.join, derangements('ABCD', 1))) + 'B C D' + >>> ' '.join(map(''.join, derangements('ABCD', 0))) + '' + >>> # Compare number of derangements to https://oeis.org/A000166 + >>> [len(list(derangements(range(n)))) for n in range(10)] + [1, 0, 1, 2, 9, 44, 265, 1854, 14833, 133496] + >>> # Verify that identical objects are treated as unique by position + >>> identical = 'X' + >>> distinct = 'x' + >>> seq1 = ('A', identical, 'B', identical) + >>> result1 = ' '.join(map(''.join, derangements(seq1))) + >>> result1 + 'XAXB XBXA XXAB BAXX BXAX BXXA XAXB XBAX XBXA' + >>> seq2 = ('A', identical, 'B', distinct) + >>> result2 = ' '.join(map(''.join, derangements(seq2))) + >>> result2 + 'XAxB XBxA XxAB BAxX BxAX BxXA xAXB xBAX xBXA' + >>> result1 == result2 + False + >>> result1.casefold() == result2.casefold() + True + + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) @@ -1743,11 +1832,37 @@ The following recipes have a more mathematical flavor: True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: # Old recipes and their tests which are guaranteed to continue to work. + def tabulate(function, start=0): + "Return function(0), function(1), ..." + return map(function, count(start)) + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1827,6 +1942,10 @@ The following recipes have a more mathematical flavor: .. doctest:: :hide: + >>> list(islice(tabulate(lambda x: 2*x), 4)) + [0, 2, 4, 6] + + >>> dotproduct([1,2,3], [4,5,6]) 32 diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8b4217c210d5b3e..383ccad9df041b5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -4,9 +4,6 @@ .. module:: json :synopsis: Encode and decode the JSON format. -.. moduleauthor:: Bob Ippolito -.. sectionauthor:: Bob Ippolito - **Source code:** :source:`Lib/json/__init__.py` -------------- @@ -121,7 +118,7 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json` from the shell to validate and pretty-print: +Using :mod:`!json` from the shell to validate and pretty-print: .. code-block:: shell-session @@ -214,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: @@ -267,7 +264,7 @@ Basic Usage .. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, \ parse_int=None, parse_constant=None, \ - object_pairs_hook=None, **kw) + object_pairs_hook=None, array_hook=None, **kw) Deserialize *fp* to a Python object using the :ref:`JSON-to-Python conversion table `. @@ -304,6 +301,15 @@ Basic Usage Default ``None``. :type object_pairs_hook: :term:`callable` | None + :param array_hook: + If set, a function that is called with the result of + any JSON array literal decoded with as a Python list. + The return value of this function will be used + instead of the :class:`list`. + This feature can be used to implement custom decoders. + Default ``None``. + :type array_hook: :term:`callable` | None + :param parse_float: If set, a function that is called with the string of every JSON float to be decoded. @@ -352,7 +358,10 @@ Basic Usage conversion length limitation ` to help avoid denial of service attacks. -.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) + .. versionchanged:: 3.15 + Added the optional *array_hook* parameter. + +.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, array_hook=None, **kw) Identical to :func:`load`, but instead of a file-like object, deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` @@ -370,7 +379,7 @@ Basic Usage Encoders and Decoders --------------------- -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) +.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None, array_hook=None) Simple JSON decoder. @@ -415,6 +424,14 @@ Encoders and Decoders .. versionchanged:: 3.1 Added support for *object_pairs_hook*. + *array_hook* is an optional function that will be called with the + result of every JSON array decoded as a list. The return value of + *array_hook* will be used instead of the :class:`list`. This feature can be + used to implement custom decoders. + + .. versionchanged:: 3.15 + Added support for *array_hook*. + *parse_float* is an optional function that will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for @@ -747,8 +764,8 @@ Command-line interface -------------- -The :mod:`json` module can be invoked as a script via ``python -m json`` -to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +The :mod:`!json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`!json.tool` submodule implements this interface. If the optional ``infile`` and ``outfile`` arguments are not @@ -769,7 +786,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: alphabetically by key. .. versionchanged:: 3.14 - The :mod:`json` module may now be directly executed as + The :mod:`!json` module may now be directly executed as ``python -m json``. For backwards compatibility, invoking the CLI as ``python -m json.tool`` remains supported. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index e766a9280946d30..ff07499e58f0f6d 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -4,13 +4,11 @@ .. module:: linecache :synopsis: Provides random access to individual lines from text files. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/linecache.py` -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`!linecache` module allows one to get any line from a Python source file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. @@ -19,7 +17,7 @@ The :func:`tokenize.open` function is used to open files. This function uses :func:`tokenize.detect_encoding` to get the encoding of the file; in the absence of an encoding token, the file encoding defaults to UTF-8. -The :mod:`linecache` module defines the following functions: +The :mod:`!linecache` module defines the following functions: .. function:: getline(filename, lineno, module_globals=None) @@ -31,7 +29,7 @@ The :mod:`linecache` module defines the following functions: .. index:: triple: module; search; path If *filename* indicates a frozen module (starting with ``' -.. sectionauthor:: Martin von Löwis - **Source code:** :source:`Lib/locale.py` -------------- -The :mod:`locale` module opens access to the POSIX locale database and +The :mod:`!locale` module opens access to the POSIX locale database and functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, +The :mod:`!locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. -The :mod:`locale` module defines the following exception and functions: +The :mod:`!locale` module defines the following exception and functions: .. exception:: Error @@ -48,8 +45,19 @@ The :mod:`locale` module defines the following exception and functions: If *locale* is omitted or ``None``, the current setting for *category* is returned. + Example:: + + >>> import locale + >>> loc = locale.setlocale(locale.LC_ALL) # get current locale + # use German locale; name and availability varies with platform + >>> locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') + >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut + >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale + >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale + >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale + :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: + start with a call of:: import locale locale.setlocale(locale.LC_ALL, '') @@ -359,8 +367,6 @@ The :mod:`locale` module defines the following exception and functions: determined. The "C" locale is represented as ``(None, None)``. - .. deprecated-removed:: 3.11 3.15 - .. function:: getlocale(category=LC_CTYPE) @@ -531,7 +537,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_COLLATE Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. + :func:`strxfrm` of the :mod:`!locale` module are affected. .. data:: LC_TIME @@ -560,7 +566,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_NUMERIC Locale category for formatting numbers. The functions :func:`format_string`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are + :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`!locale` module are affected by that category. All other numeric formatting operations are not affected. @@ -580,18 +586,6 @@ The :mod:`locale` module defines the following exception and functions: :func:`localeconv`. -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - Background, details, hints, tips and caveats -------------------------------------------- @@ -696,7 +690,7 @@ the current locale is. But since the return value can only be used portably to restore it, that is not very useful (except perhaps to find out whether or not the locale is ``C``). -When Python code uses the :mod:`locale` module to change the locale, this also +When Python code uses the :mod:`!locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 96cca3073fec7e7..3842c1e9d477d29 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -4,9 +4,6 @@ .. module:: logging.config :synopsis: Configuration of the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/config.py` .. sidebar:: Important @@ -28,7 +25,7 @@ Configuration functions ^^^^^^^^^^^^^^^^^^^^^^^ The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the +:mod:`!logging.config` module. Their use is optional --- you can configure the logging module using these functions or by making calls to the main API (defined in :mod:`logging` itself) and defining handlers which are declared either in :mod:`logging` or :mod:`logging.handlers`. @@ -55,7 +52,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in Parsing is performed by the :class:`DictConfigurator` class, whose constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module + has a :meth:`configure` method. The :mod:`!logging.config` module has a callable attribute :attr:`dictConfigClass` which is initially set to :class:`DictConfigurator`. You can replace the value of :attr:`dictConfigClass` with a @@ -127,12 +124,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionchanged:: 3.10 - Added the *encoding* parameter. + .. versionchanged:: 3.10 + Added the *encoding* parameter. - .. versionchanged:: 3.12 - An exception will be thrown if the provided file - doesn't exist or is invalid or empty. + .. versionchanged:: 3.12 + An exception will be thrown if the provided file + doesn't exist or is invalid or empty. .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index c9cfbdb4126fdac..5152c7561fa1f26 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -4,9 +4,6 @@ .. module:: logging.handlers :synopsis: Handlers for the logging module. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/handlers.py` .. sidebar:: Important @@ -160,7 +157,7 @@ WatchedFileHandler .. currentmodule:: logging.handlers -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` +The :class:`WatchedFileHandler` class, located in the :mod:`!logging.handlers` module, is a :class:`FileHandler` which watches the file it is logging to. If the file changes, it is closed and reopened using the file name. @@ -213,7 +210,7 @@ for this value. BaseRotatingHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` +The :class:`BaseRotatingHandler` class, located in the :mod:`!logging.handlers` module, is the base class for the rotating file handlers, :class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should not need to instantiate this class, but it has attributes and methods you may @@ -307,7 +304,7 @@ For an example, see :ref:`cookbook-rotator-namer`. RotatingFileHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` +The :class:`RotatingFileHandler` class, located in the :mod:`!logging.handlers` module, supports rotation of disk log files. @@ -362,7 +359,7 @@ TimedRotatingFileHandler ^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain +:mod:`!logging.handlers` module, supports rotation of disk log files at certain timed intervals. @@ -405,7 +402,8 @@ timed intervals. rollover interval. When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else + is created), the creation time (if supported by the OS and file system) + or the last modification of an existing log file, or else the current time, is used to compute when the next rotation will occur. If the *utc* argument is true, times in UTC will be used; otherwise @@ -452,6 +450,10 @@ timed intervals. .. versionchanged:: 3.9 The *errors* parameter was added. + .. versionchanged:: next + Use the creation time instead of the last modification time, if supported by the OS and file system. + + .. method:: doRollover() Does a rollover, as described above. @@ -475,7 +477,7 @@ timed intervals. SocketHandler ^^^^^^^^^^^^^ -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SocketHandler` class, located in the :mod:`!logging.handlers` module, sends logging output to a network socket. The base class uses a TCP socket. @@ -571,7 +573,7 @@ sends logging output to a network socket. The base class uses a TCP socket. DatagramHandler ^^^^^^^^^^^^^^^ -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` +The :class:`DatagramHandler` class, located in the :mod:`!logging.handlers` module, inherits from :class:`SocketHandler` to support sending logging messages over UDP sockets. @@ -618,7 +620,7 @@ over UDP sockets. SysLogHandler ^^^^^^^^^^^^^ -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SysLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. @@ -797,7 +799,7 @@ supports sending logging messages to a remote or local Unix syslog. NTEventLogHandler ^^^^^^^^^^^^^^^^^ -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` +The :class:`NTEventLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a local Windows NT, Windows 2000 or Windows XP event log. Before you can use it, you need Mark Hammond's Win32 extensions for Python installed. @@ -864,7 +866,7 @@ extensions for Python installed. SMTPHandler ^^^^^^^^^^^ -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SMTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to an email address via SMTP. @@ -905,7 +907,7 @@ supports sending logging messages to an email address via SMTP. MemoryHandler ^^^^^^^^^^^^^ -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, +The :class:`MemoryHandler` class, located in the :mod:`!logging.handlers` module, supports buffering of logging records in memory, periodically flushing them to a :dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. @@ -985,7 +987,7 @@ should, then :meth:`flush` is expected to do the flushing. HTTPHandler ^^^^^^^^^^^ -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`HTTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a web server, using either ``GET`` or ``POST`` semantics. @@ -1037,7 +1039,7 @@ QueueHandler .. versionadded:: 3.2 -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, +The :class:`QueueHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. @@ -1130,7 +1132,7 @@ QueueListener .. versionadded:: 3.2 -The :class:`QueueListener` class, located in the :mod:`logging.handlers` +The :class:`QueueListener` class, located in the :mod:`!logging.handlers` module, supports receiving logging messages from a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The messages are received from a queue in an internal thread and passed, on diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 0cf5b1c0d9bc3ee..a3d117c10702414 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -4,9 +4,6 @@ .. module:: logging :synopsis: Flexible event logging system for applications. -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - **Source code:** :source:`Lib/logging/__init__.py` .. index:: pair: Errors; logging @@ -693,7 +690,7 @@ Formatter Objects :param defaults: A dictionary with default values to use in custom fields. For example, ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - :type defaults: dict[str, Any] + :type defaults: dict[str, typing.Any] .. versionchanged:: 3.2 Added the *style* parameter. @@ -1011,6 +1008,11 @@ the options available to you. | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | +----------------+-------------------------+-----------------------------------------------+ +| exc_text | You shouldn't need to | Exception information formatted as a string. | +| | format this yourself. | This is set when :meth:`Formatter.format` is | +| | | invoked, or ``None`` if no exception has | +| | | occurred. | ++----------------+-------------------------+-----------------------------------------------+ | filename | ``%(filename)s`` | Filename portion of ``pathname``. | +----------------+-------------------------+-----------------------------------------------+ | funcName | ``%(funcName)s`` | Name of function containing the logging call. | @@ -1546,7 +1548,7 @@ Module-Level Attributes Integration with the warnings module ------------------------------------ -The :func:`captureWarnings` function can be used to integrate :mod:`logging` +The :func:`captureWarnings` function can be used to integrate :mod:`!logging` with the :mod:`warnings` module. .. function:: captureWarnings(capture) @@ -1577,7 +1579,7 @@ with the :mod:`warnings` module. library. `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the + This is the original source for the :mod:`!logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard + and 2.2.x, which do not include the :mod:`!logging` package in the standard library. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 8a4f68f35025216..cd72174d54f6e62 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -4,9 +4,6 @@ .. module:: lzma :synopsis: A Python wrapper for the liblzma compression library. -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Nadeem Vawda - .. versionadded:: 3.3 **Source code:** :source:`Lib/lzma.py` @@ -359,12 +356,26 @@ options. Valid filter IDs are as follows: * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + * :const:`!FILTER_X86` + * :const:`!FILTER_IA64` + * :const:`!FILTER_ARM` + * :const:`!FILTER_ARMTHUMB` + * :const:`!FILTER_POWERPC` + * :const:`!FILTER_SPARC` + + The above work on all lzma runtime library versions. + + * :const:`!FILTER_ARM64` + + Only works if the lzma version is 5.4.0 or later. + + .. versionadded:: next + + * :const:`!FILTER_RISCV` + + Only works if the lzma version is 5.6.0 or later. + + .. versionadded:: next A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 62e289573c0c7e0..5b9741bdbcad19e 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -4,9 +4,6 @@ .. module:: mailbox :synopsis: Manipulate mailboxes in various formats -.. moduleauthor:: Gregory K. Johnson -.. sectionauthor:: Gregory K. Johnson - **Source code:** :source:`Lib/mailbox.py` -------------- @@ -78,6 +75,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. message. Failing to lock the mailbox runs the risk of losing messages or corrupting the entire mailbox. + The :class:`!Mailbox` class supports the :keyword:`with` statement. When used + as a context manager, :class:`!Mailbox` calls :meth:`lock` when the context is entered, + returns the mailbox object as the context object, and at context end calls :meth:`close`, + thereby releasing the lock. + + .. versionchanged:: 3.15 + Support for the :keyword:`with` statement was added. + :class:`!Mailbox` instances have the following methods: @@ -1025,7 +1030,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. If "info" contains experimental information rather than flags, the current "info" is not modified. @@ -1190,7 +1195,7 @@ When a :class:`!MaildirMessage` instance is created based upon a .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!mboxMessage` instance is created based upon a @@ -1562,7 +1567,7 @@ When a :class:`!BabylMessage` instance is created based upon an .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!MMDFMessage` instance is created based upon a @@ -1641,7 +1646,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() - The based class for all other module-specific exceptions. + The base class for all other module-specific exceptions. .. exception:: NoSuchMailboxError() @@ -1661,7 +1666,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: Raised when some mailbox-related condition beyond the control of the program causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name + another program already holds, or when a uniquely generated file name already exists. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index e8e9071a5c9ef49..4fe34f0a3a3f911 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -20,7 +20,7 @@ rarely does). [#]_ This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and +:mod:`shelve`. The :mod:`!marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format in backward incompatible ways should the need arise. @@ -34,7 +34,7 @@ supports a substantially wider range of objects than marshal. .. warning:: - The :mod:`marshal` module is not intended to be secure against erroneous or + The :mod:`!marshal` module is not intended to be secure against erroneous or maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. @@ -51,8 +51,9 @@ this module. The following types are supported: * Strings (:class:`str`) and :class:`bytes`. :term:`Bytes-like objects ` like :class:`bytearray` are marshalled as :class:`!bytes`. -* Containers: :class:`tuple`, :class:`list`, :class:`set`, :class:`frozenset`, - and (since :data:`version` 5), :class:`slice`. +* Containers: :class:`tuple`, :class:`list`, :class:`dict`, :class:`frozendict` + (since :data:`version` 6), :class:`set`, :class:`frozenset`, and + :class:`slice` (since :data:`version` 5). It should be understood that these are supported only if the values contained therein are themselves supported. Recursive containers are supported since :data:`version` 3. @@ -71,6 +72,10 @@ this module. The following types are supported: Added format version 5, which allows marshalling slices. +.. versionchanged:: 3.15 + + Added format version 6, which allows marshalling :class:`frozendict`. + The module defines these functions: @@ -173,6 +178,8 @@ In addition, the following constants are defined: 4 Python 3.4 Efficient representation of short strings ------- --------------- ---------------------------------------------------- 5 Python 3.14 Support for :class:`slice` objects + ------- --------------- ---------------------------------------------------- + 6 Python 3.15 Support for :class:`frozendict` objects ======= =============== ==================================================== diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst index 6a9fe74c5e861ba..c3f34cdfd85410c 100644 --- a/Doc/library/math.integer.rst +++ b/Doc/library/math.integer.rst @@ -1,10 +1,10 @@ -:mod:`math.integer` --- integer-specific mathematics functions -============================================================== +:mod:`!math.integer` --- integer-specific mathematics functions +=============================================================== .. module:: math.integer :synopsis: Integer-specific mathematics functions. -.. versionadded:: next +.. versionadded:: 3.15 -------------- diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 54c98346b2798bb..efe411e5a43f27d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -83,13 +83,20 @@ noted otherwise, all return values are floats. **Trigonometric functions** -------------------------------------------------------------------------------------------------- -:func:`acos(x) ` Arc cosine of *x* -:func:`asin(x) ` Arc sine of *x* -:func:`atan(x) ` Arc tangent of *x* -:func:`atan2(y, x) ` ``atan(y / x)`` -:func:`cos(x) ` Cosine of *x* -:func:`sin(x) ` Sine of *x* -:func:`tan(x) ` Tangent of *x* +:func:`acos(x) ` Arc cosine of *x*, in radians +:func:`acospi(x) ` Arc cosine of *x*, in half-turns +:func:`asin(x) ` Arc sine of *x*, in radians +:func:`asinpi(x) ` Arc sine of *x*, in half-turns +:func:`atan(x) ` Arc tangent of *x*, in radians +:func:`atanpi(x) ` Arc tangent of *x*, in half-turns +:func:`atan2(y, x) ` ``atan(y / x)``, in radians +:func:`atan2pi(y, x) ` ``atan(y / x)``, in half-turns +:func:`cos(x) ` Cosine of *x* radians +:func:`cospi(x) ` Cosine of *x⋅π* radians +:func:`sin(x) ` Sine of *x* radians +:func:`sinpi(x) ` Sine of *x⋅π* radians +:func:`tan(x) ` Tangent of *x* radians +:func:`tanpi(x) ` Tangent of *x⋅π* radians **Hyperbolic functions** -------------------------------------------------------------------------------------------------- @@ -255,10 +262,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of @@ -506,7 +515,7 @@ Summation and product functions Roughly equivalent to:: - sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q, strict=True))) .. versionadded:: 3.8 @@ -597,18 +606,42 @@ Trigonometric functions ``pi``. +.. function:: acospi(x) + + Return the arc cosine of *x*, in half-turns. The result is between ``0`` and + ``1``. + + .. versionadded:: next + + .. function:: asin(x) Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: asinpi(x) + + Return the arc sine of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan(x) Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: atanpi(x) + + Return the arc tangent of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan2(y, x) Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. @@ -619,21 +652,54 @@ Trigonometric functions -1)`` is ``-3*pi/4``. +.. function:: atan2pi(y, x) + + Return ``atanpi(y / x)``, in half-turns. The result is between ``-1`` and ``1``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2pi` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atanpi(1)`` and ``atan2pi(1, 1)`` are both ``0.25``, but + ``atan2pi(-1, -1)`` is ``-0.75``. + + .. versionadded:: next + + .. function:: cos(x) Return the cosine of *x* radians. +.. function:: cospi(x) + + Return the cosine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: sin(x) Return the sine of *x* radians. +.. function:: sinpi(x) + + Return the sine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: tan(x) Return the tangent of *x* radians. +.. function:: tanpi(x) + + Return the tangent of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + Hyperbolic functions -------------------- @@ -720,7 +786,7 @@ Special functions Number-theoretic functions -------------------------- -For backward compatibility, the :mod:`math` module provides also aliases of +For backward compatibility, the :mod:`!math` module provides also aliases of the following functions from the :mod:`math.integer` module: .. list-table:: @@ -781,9 +847,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: next - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants @@ -846,7 +911,7 @@ Constants .. impl-detail:: - The :mod:`math` module consists mostly of thin wrappers around the platform C + The :mod:`!math` module consists mostly of thin wrappers around the platform C math library functions. Behavior in exceptional cases follows Annex F of the C99 standard where appropriate. The current implementation will raise :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 13511b16a0ed8ce..5c29fff146eef00 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -4,15 +4,13 @@ .. module:: mimetypes :synopsis: Mapping of filename extensions to MIME types. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/mimetypes.py` .. index:: pair: MIME; content type -------------- -The :mod:`mimetypes` module converts between a filename or URL and the MIME type +The :mod:`!mimetypes` module converts between a filename or URL and the MIME type associated with the filename extension. Conversions are provided from filename to MIME type and from MIME type to filename extension; encodings are not supported for the latter conversion. @@ -41,8 +39,8 @@ the information :func:`init` sets up. (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use as a :mailheader:`Content-Encoding` header, **not** as a :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. + Encoding suffixes are case-sensitive. Suffix mappings and type suffixes are + first tried case-sensitively, then case-insensitively. The optional *strict* argument is a flag specifying whether the list of known MIME types is limited to only the official types `registered with IANA @@ -56,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. @@ -131,10 +129,20 @@ behavior of the module. Add a mapping from the MIME type *type* to the extension *ext*. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Valid extensions are empty or start with a ``'.'``. + + Registered lower-case extensions are matched case-insensitively. When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. + + .. data:: inited @@ -196,7 +204,7 @@ MimeTypes objects The :class:`MimeTypes` class may be useful for applications which may want more than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. +:mod:`!mimetypes` module. .. class:: MimeTypes(filenames=(), strict=True) @@ -306,12 +314,16 @@ than one MIME-type database; it provides an interface similar to the one of the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched case-insensitively. + When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. - .. deprecated-removed:: 3.14 3.16 - Invalid, undotted extensions will raise a - :exc:`ValueError` in Python 3.16. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. .. _mimetypes-cli: @@ -350,7 +362,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -377,7 +389,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -399,7 +411,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index f32aa322c40dbb3..774d05fdcaa3007 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -212,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush([offset[, size]], *, flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -221,6 +221,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length whole extent of the mapping is flushed. *offset* must be a multiple of the :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. + The *flags* parameter specifies the synchronization behavior. + *flags* must be one of the :ref:`MS_* constants ` available + on the system. + + On Windows, the *flags* parameter is ignored. + ``None`` is returned to indicate success. An exception is raised when the call failed. @@ -235,6 +241,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length specified alone, and the flush operation will extend from *offset* to the end of the mmap. + .. versionchanged:: 3.15 + Added *flags* parameter to control synchronization behavior. + .. method:: madvise(option[, start[, length]]) @@ -328,6 +337,17 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. versionadded:: 3.13 + .. method:: set_name(name, /) + + Annotate the memory mapping with the given *name* for easier identification + in ``/proc//maps`` if the kernel supports the feature and :option:`-X dev <-X>` is passed + to Python or if Python is built in :ref:`debug mode `. + The length of *name* must not exceed 67 bytes including the ``'\0'`` terminator. + + .. availability:: Linux >= 5.17 (kernel built with ``CONFIG_ANON_VMA_NAME`` option) + + .. versionadded:: 3.15 + .. method:: size() Return the length of the file, which can be larger than the size of the @@ -450,3 +470,22 @@ MAP_* Constants :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and :data:`MAP_UNIX03` constants. +.. _ms-constants: + +MS_* Constants +++++++++++++++ + +.. data:: MS_SYNC + MS_ASYNC + MS_INVALIDATE + + These flags control the synchronization behavior for :meth:`mmap.flush`: + + * :data:`MS_SYNC` - Synchronous flush: writes are scheduled and the call + blocks until they are physically written to storage. + * :data:`MS_ASYNC` - Asynchronous flush: writes are scheduled but the call + returns immediately without waiting for completion. + * :data:`MS_INVALIDATE` - Invalidate cached data: invalidates other mappings + of the same file so they can see the changes. + + .. versionadded:: 3.15 diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst index 823d853f1ed8ebe..d26dcbd5f687257 100644 --- a/Doc/library/modulefinder.rst +++ b/Doc/library/modulefinder.rst @@ -4,8 +4,6 @@ .. module:: modulefinder :synopsis: Find modules used by a script. -.. sectionauthor:: A.M. Kuchling - **Source code:** :source:`Lib/modulefinder.py` -------------- diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index a2c5e375d2cc4fd..6b49c1a9ccd6e11 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -2,10 +2,9 @@ =========================================================== .. module:: msvcrt - :platform: Windows :synopsis: Miscellaneous useful routines from the MS VC++ runtime. -.. sectionauthor:: Fred L. Drake, Jr. +**Source code:** :source:`PC/msvcrtmodule.c` -------------- diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 714207cb0aefcd5..2d13053915830b0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -13,16 +13,16 @@ Introduction ------------ -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package +:mod:`!multiprocessing` is a package that supports spawning processes using an +API similar to the :mod:`threading` module. The :mod:`!multiprocessing` package offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully +to this, the :mod:`!multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces the +The :mod:`!multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -43,7 +43,7 @@ will print to standard output :: [1, 4, 9] -The :mod:`multiprocessing` module also introduces APIs which do not have +The :mod:`!multiprocessing` module also introduces APIs which do not have analogs in the :mod:`threading` module, like the ability to :meth:`terminate `, :meth:`interrupt ` or :meth:`kill ` a running process. @@ -61,7 +61,7 @@ analogs in the :mod:`threading` module, like the ability to :meth:`terminate The :class:`Process` class ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` +In :mod:`!multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` follows the API of :class:`threading.Thread`. A trivial example of a multiprocess program is :: @@ -100,10 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -The arguments to :class:`Process` usually need to be unpickleable from within -the child process. If you tried typing the above example directly into a REPL it -could lead to an :exc:`AttributeError` in the child process trying to locate the -*f* function in the ``__main__`` module. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -111,7 +111,7 @@ could lead to an :exc:`AttributeError` in the child process trying to locate the Contexts and start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the platform, :mod:`multiprocessing` supports three ways +Depending on the platform, :mod:`!multiprocessing` supports three ways to start a process. These *start methods* are .. _multiprocessing-start-method-spawn: @@ -240,7 +240,7 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -Libraries using :mod:`multiprocessing` or +Libraries using :mod:`!multiprocessing` or :class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow their users to provide their own multiprocessing context. Using a specific context of your own within a library can lead to incompatibilities with the @@ -258,7 +258,7 @@ requires a specific start method. Exchanging objects between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` supports two types of communication channel between +:mod:`!multiprocessing` supports two types of communication channel between processes: **Queues** @@ -279,7 +279,7 @@ processes: p.join() Queues are thread and process safe. - Any object put into a :mod:`~multiprocessing` queue will be serialized. + Any object put into a :mod:`!multiprocessing` queue will be serialized. **Pipes** @@ -313,7 +313,7 @@ processes: Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` contains equivalents of all the synchronization +:mod:`!multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure that only one process prints to standard output at a time:: @@ -344,7 +344,7 @@ avoid using shared state as far as possible. This is particularly true when using multiple processes. However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. +:mod:`!multiprocessing` provides a couple of ways of doing so. **Shared memory** @@ -518,9 +518,24 @@ process which created it. Reference --------- -The :mod:`multiprocessing` package mostly replicates the API of the +The :mod:`!multiprocessing` package mostly replicates the API of the :mod:`threading` module. +.. _global-start-method: + +Global start method +^^^^^^^^^^^^^^^^^^^ + +Python supports several ways to create and initialize a process. +The global start method sets the default mechanism for creating a process. + +Several multiprocessing functions and methods that may also instantiate +certain objects will implicitly set the global start method to the system's default, +if it hasn’t been set already. The global start method can only be set once. +If you need to change the start method from the system default, you must +proactively set the global start method before calling functions or methods, +or creating these objects. + :class:`Process` and exceptions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -689,7 +704,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). - When :mod:`multiprocessing` is initialized the main process is assigned a + When :mod:`!multiprocessing` is initialized the main process is assigned a random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the @@ -790,7 +805,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: ProcessError - The base class of all :mod:`multiprocessing` exceptions. + The base class of all :mod:`!multiprocessing` exceptions. .. exception:: BufferTooShort @@ -830,19 +845,19 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. -One difference from other Python queue implementations, is that :mod:`multiprocessing` +One difference from other Python queue implementations, is that :mod:`!multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. .. note:: - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and + :mod:`!multiprocessing` uses the usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from + the :mod:`!multiprocessing` namespace so you need to import them from :mod:`queue`. .. note:: @@ -890,7 +905,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the @@ -910,11 +925,15 @@ For an example of the usage of queues for interprocess communication see locks/semaphores. When a process first puts an item on the queue a feeder thread is started which transfers objects from a buffer into the pipe. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -1025,6 +1044,9 @@ For an example of the usage of queues for interprocess communication see It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: close() Close the queue: release internal resources. @@ -1055,6 +1077,9 @@ For an example of the usage of queues for interprocess communication see :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which additionally has :meth:`task_done` and :meth:`join` methods. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: task_done() Indicate that a formerly enqueued task is complete. Used by queue @@ -1128,7 +1153,7 @@ Miscellaneous .. function:: freeze_support() - Add support for when a program which uses :mod:`multiprocessing` has been + Add support for when a program which uses :mod:`!multiprocessing` has been frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) @@ -1164,11 +1189,11 @@ Miscellaneous .. function:: get_context(method=None) Return a context object which has the same attributes as the - :mod:`multiprocessing` module. + :mod:`!multiprocessing` module. If *method* is ``None`` then the default context is returned. Note that if - the global start method has not been set, this will set it to the - default method. + the global start method has not been set, this will set it to the system default + See :ref:`global-start-method` for more details. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1179,10 +1204,9 @@ Miscellaneous Return the name of start method used for starting processes. - If the global start method has not been set and *allow_none* is - ``False``, then the start method is set to the default and the name - is returned. If the start method has not been set and *allow_none* is - ``True`` then ``None`` is returned. + If the global start method is not set and *allow_none* is ``False``, the global start + method is set to the default, and its name is returned. See + :ref:`global-start-method` for more details. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. @@ -1199,7 +1223,7 @@ Miscellaneous Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: + do something like :: set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) @@ -1211,22 +1235,32 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_forkserver_preload(module_names) +.. function:: set_forkserver_preload(module_names, *, on_error='ignore') Set a list of module names for the forkserver main process to attempt to import so that their already imported state is inherited by forked - processes. Any :exc:`ImportError` when doing so is silently ignored. - This can be used as a performance enhancement to avoid repeated work - in every process. + processes. This can be used as a performance enhancement to avoid repeated + work in every process. For this to work, it must be called before the forkserver process has been launched (before creating a :class:`Pool` or starting a :class:`Process`). + The *on_error* parameter controls how :exc:`ImportError` exceptions during + module preloading are handled: ``"ignore"`` (default) silently ignores + failures, ``"warn"`` causes the forkserver subprocess to emit an + :exc:`ImportWarning` to stderr, and ``"fail"`` causes the forkserver + subprocess to exit with the exception traceback on stderr, making + subsequent process creation fail with :exc:`EOFError` or + :exc:`ConnectionError`. + Only meaningful when using the ``'forkserver'`` start method. See :ref:`multiprocessing-start-methods`. .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *on_error* parameter. + .. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. @@ -1246,7 +1280,7 @@ Miscellaneous .. note:: - :mod:`multiprocessing` contains no analogues of + :mod:`!multiprocessing` contains no analogues of :func:`threading.active_count`, :func:`threading.enumerate`, :func:`threading.settrace`, :func:`threading.setprofile`, :class:`threading.Timer`, or :class:`threading.local`. @@ -1302,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1327,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1409,6 +1443,9 @@ object -- see :ref:`multiprocessing-managers`. A barrier object: a clone of :class:`threading.Barrier`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. versionadded:: 3.3 .. class:: BoundedSemaphore([value]) @@ -1416,6 +1453,9 @@ object -- see :ref:`multiprocessing-managers`. A bounded semaphore object: a close analog of :class:`threading.BoundedSemaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. @@ -1434,7 +1474,10 @@ object -- see :ref:`multiprocessing-managers`. A condition variable: an alias for :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. + object from :mod:`!multiprocessing`. + + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. versionchanged:: 3.3 The :meth:`~threading.Condition.wait_for` method was added. @@ -1443,6 +1486,8 @@ object -- see :ref:`multiprocessing-managers`. A clone of :class:`threading.Event`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. class:: Lock() @@ -1458,6 +1503,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.Lock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`Lock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1515,6 +1563,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.RLock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`RLock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1574,15 +1625,28 @@ object -- see :ref:`multiprocessing-managers`. A semaphore object: a close analog of :class:`threading.Semaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + + .. method:: get_value() + + Return the current value of semaphore. + + Note that this may raise :exc:`NotImplementedError` on platforms like + macOS where ``sem_getvalue()`` is not implemented. + + .. method:: locked() Return a boolean indicating whether this object is locked right now. .. versionadded:: 3.14 + .. note:: On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with @@ -1640,11 +1704,14 @@ inherited by child processes. value is actually a synchronized wrapper for the array. *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. + it is either a :ref:`ctypes type ` or a one + character typecode of the kind used by the :mod:`array` module with the + exception of ``'w'``, which is not supported. In addition, the ``'c'`` + typecode is an alias for :class:`ctypes.c_char`. If *size_or_initializer* + is an integer, then it determines the length of the array, and the array + will be initially zeroed. Otherwise, *size_or_initializer* is a sequence + which is used to initialize the array and whose length determines the length + of the array. If *lock* is ``True`` (the default) then a new lock object is created to synchronize access to the value. If *lock* is a :class:`Lock` or @@ -1656,16 +1723,19 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. -The :mod:`multiprocessing.sharedctypes` module -"""""""""""""""""""""""""""""""""""""""""""""" +The :mod:`!multiprocessing.sharedctypes` module +""""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating +The :mod:`!multiprocessing.sharedctypes` module provides functions for allocating :mod:`ctypes` objects from shared memory which can be inherited by child processes. @@ -1708,7 +1778,7 @@ processes. attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1722,9 +1792,13 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. -.. function:: Value(typecode_or_type, *args, lock=True) + Note that *lock* and *ctx* are keyword-only parameters. + +.. function:: Value(typecode_or_type, *args, lock=True, ctx=None) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1737,19 +1811,27 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + + Note that *lock* and *ctx* are keyword-only parameters. .. function:: copy(obj) Return a ctypes object allocated from shared memory which is a copy of the ctypes object *obj*. -.. function:: synchronized(obj[, lock]) +.. function:: synchronized(obj, lock=None, ctx=None) Return a process-safe wrapper object for a ctypes object which uses *lock* to synchronize access. If *lock* is ``None`` (the default) then a :class:`multiprocessing.RLock` object is created automatically. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + A synchronized wrapper will have two methods in addition to those of the object it wraps: :meth:`get_obj` returns the wrapped object and :meth:`get_lock` returns the lock object used for synchronization. @@ -1867,8 +1949,9 @@ their parent process exits. The manager classes are defined in the *serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or ``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization). - *ctx* is a context object, or ``None`` (use the current context). See the - :func:`get_context` function. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. *shutdown_timeout* is a timeout in seconds used to wait until the process used by the manager completes in the :meth:`shutdown` method. If the @@ -2252,7 +2335,7 @@ demonstrates a level of control over the synchronization. .. note:: - The proxy types in :mod:`multiprocessing` do nothing to support comparisons + The proxy types in :mod:`!multiprocessing` do nothing to support comparisons by value. So, for instance, we have: .. doctest:: @@ -2361,7 +2444,9 @@ with the :class:`Pool` class. the worker processes. Usually a pool is created using the function :func:`multiprocessing.Pool` or the :meth:`Pool` method of a context object. In both cases *context* is set - appropriately. + appropriately. If ``None``, calling this function will have the side effect + of setting the current global start method if it has not been set already. + See the :func:`get_context` function. Note that the methods of the pool object should only be called by the process which created the pool. @@ -2392,7 +2477,7 @@ with the :class:`Pool` class. duration of the Pool's work queue. A frequent pattern found in other systems (such as Apache, mod_wsgi, etc) to free resources held by workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new + amount of work before exiting, being cleaned up and a new process spawned to replace the old one. The *maxtasksperchild* argument to the :class:`Pool` exposes this ability to the end user. @@ -2577,7 +2662,7 @@ Usually message passing between processes is done using queues or by using :class:`~Connection` objects returned by :func:`~multiprocessing.Pipe`. -However, the :mod:`multiprocessing.connection` module allows some extra +However, the :mod:`!multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing with sockets or Windows named pipes. It also has support for *digest authentication* using the :mod:`hmac` module, and for polling @@ -2835,6 +2920,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ @@ -2846,7 +2941,7 @@ handler type) for messages from different processes to get mixed up. .. currentmodule:: multiprocessing .. function:: get_logger() - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one + Returns the logger used by :mod:`!multiprocessing`. If necessary, a new one will be created. When first created the logger has level :const:`logging.NOTSET` and no @@ -2884,18 +2979,18 @@ Below is an example session with logging turned on:: For a full table of logging levels, see the :mod:`logging` module. -The :mod:`multiprocessing.dummy` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`!multiprocessing.dummy` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is +:mod:`!multiprocessing.dummy` replicates the API of :mod:`!multiprocessing` but is no more than a wrapper around the :mod:`threading` module. .. currentmodule:: multiprocessing.pool -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +In particular, the ``Pool`` function provided by :mod:`!multiprocessing.dummy` returns an instance of :class:`ThreadPool`, which is a subclass of :class:`Pool` that supports all the same method calls but uses a pool of worker threads rather than worker processes. @@ -2940,7 +3035,7 @@ Programming guidelines ---------------------- There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. +:mod:`!multiprocessing`. All start methods @@ -2981,7 +3076,7 @@ Joining zombie processes Better to inherit than pickle/unpickle When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child + from :mod:`!multiprocessing` need to be picklable so that child processes can use them. However, one should generally avoid sending shared objects to other processes using pipes or queues. Instead you should arrange the program so that a process which @@ -3071,7 +3166,7 @@ Explicitly pass resources to child processes Beware of replacing :data:`sys.stdin` with a "file like object" - :mod:`multiprocessing` originally unconditionally called:: + :mod:`!multiprocessing` originally unconditionally called:: os.close(sys.stdin.fileno()) diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 74c97e8c9a97598..3fbd2b57426ebe7 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -4,9 +4,6 @@ .. module:: netrc :synopsis: Loading of .netrc files. -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/netrc.py` -------------- diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 57b35017072c97e..0ed24b4e6c5225d 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -90,20 +90,7 @@ Notes for type implementers Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different -extensions of the real numbers. For example, :class:`fractions.Fraction` -implements :func:`hash` as follows:: - - def __hash__(self): - if self.denominator == 1: - # Get integers right. - return hash(self.numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self.numerator, self.denominator)) +extensions of the real numbers. See also :ref:`numeric-hash`. Adding More Numeric ABCs diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e8e71068dd99ebc..3d1c8cda13be381 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -4,8 +4,6 @@ .. module:: operator :synopsis: Functions corresponding to the standard operators. -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/operator.py` .. testsetup:: @@ -15,7 +13,7 @@ -------------- -The :mod:`operator` module exports a set of efficient functions corresponding to +The :mod:`!operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is equivalent to the expression ``x+y``. Many function names are those used for special methods, without the double underscores. For backward compatibility, @@ -112,7 +110,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -136,13 +134,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -154,7 +152,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -174,25 +172,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -211,7 +209,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -275,7 +273,7 @@ The following operation works with callables: .. versionadded:: 3.11 -The :mod:`operator` module also defines tools for generalized attribute and item +The :mod:`!operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that expect a function argument. @@ -390,7 +388,7 @@ Mapping Operators to Functions ------------------------------ This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. +Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Operation | Syntax | Function | @@ -405,13 +403,18 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index ff327cf9162a8cc..905212965bd70fe 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -4,9 +4,6 @@ .. module:: optparse :synopsis: Command-line option parsing library. -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/optparse.py` -------------- @@ -20,7 +17,7 @@ The standard library includes three argument parsing libraries: * :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. Included in the standard library since before the initial Python 1.0 release. -* :mod:`optparse`: a declarative replacement for ``getopt`` that +* :mod:`!optparse`: a declarative replacement for ``getopt`` that provides equivalent functionality without requiring each application to implement its own procedural option parsing logic. Included in the standard library since the Python 2.3 release. @@ -37,10 +34,10 @@ the highest level of baseline functionality with the least application level cod However, it also serves a niche use case as a tool for prototyping and testing command line argument handling in ``getopt``-based C applications. -:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +:mod:`!optparse` should be considered as an alternative to :mod:`argparse` in the following cases: -* an application is already using :mod:`optparse` and doesn't want to risk the +* an application is already using :mod:`!optparse` and doesn't want to risk the subtle behavioural changes that may arise when migrating to :mod:`argparse` * the application requires additional control over the way options and positional parameters are interleaved on the command line (including @@ -55,7 +52,7 @@ following cases: behavior which ``argparse`` does not support, but which can be implemented in terms of the lower level interface offered by ``optparse`` -These considerations also mean that :mod:`optparse` is likely to provide a +These considerations also mean that :mod:`!optparse` is likely to provide a better foundation for library authors writing third party command line argument processing libraries. @@ -126,15 +123,15 @@ application use case. Introduction ------------ -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing +:mod:`!optparse` is a more convenient, flexible, and powerful library for parsing command-line options than the minimalist :mod:`getopt` module. -:mod:`optparse` uses a more declarative style of command-line parsing: +:mod:`!optparse` uses a more declarative style of command-line parsing: you create an instance of :class:`OptionParser`, populate it with options, and parse the command line. -:mod:`optparse` allows users to specify options in the conventional +:mod:`!optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. -Here's an example of using :mod:`optparse` in a simple script:: +Here's an example of using :mod:`!optparse` in a simple script:: from optparse import OptionParser ... @@ -152,11 +149,11 @@ on the command-line, for example:: --file=outfile -q -As it parses the command line, :mod:`optparse` sets attributes of the +As it parses the command line, :mod:`!optparse` sets attributes of the ``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short +``False``. :mod:`!optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their arguments in a variety of ways. Thus, the following command lines are all equivalent to the above example:: @@ -171,7 +168,7 @@ Additionally, users can run one of the following :: -h --help -and :mod:`optparse` will print out a brief summary of your script's options: +and :mod:`!optparse` will print out a brief summary of your script's options: .. code-block:: text @@ -191,7 +188,7 @@ where the value of *yourscript* is determined at runtime (normally from Background ---------- -:mod:`optparse` was explicitly designed to encourage the creation of programs +:mod:`!optparse` was explicitly designed to encourage the creation of programs with straightforward command-line interfaces that follow the conventions established by the :c:func:`!getopt` family of functions available to C developers. To that end, it supports only the most common command-line syntax and semantics @@ -223,7 +220,7 @@ option options to be merged into a single argument, e.g. ``-x -F`` is equivalent to ``-xF``. The GNU project introduced ``--`` followed by a series of hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. + only two option syntaxes provided by :mod:`!optparse`. Some other option syntaxes that the world has seen include: @@ -240,7 +237,7 @@ option * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, ``/file`` - These option syntaxes are not supported by :mod:`optparse`, and they never + These option syntaxes are not supported by :mod:`!optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting Windows or certain legacy platforms (e.g. VMS, MS-DOS). @@ -248,7 +245,7 @@ option option argument an argument that follows an option, is closely associated with that option, and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from + :mod:`!optparse`, option arguments may either be in a separate argument from their option: .. code-block:: text @@ -268,7 +265,7 @@ option argument will take an argument if they see it, and won't if they don't. This is somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not + interpret ``-ab``? Because of this ambiguity, :mod:`!optparse` does not support this feature. positional argument @@ -278,7 +275,7 @@ positional argument required option an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't + "required option" is self-contradictory in English. :mod:`!optparse` doesn't prevent you from implementing required options, but doesn't give you much help at it either. @@ -357,9 +354,9 @@ too many options can overwhelm users and make your code much harder to maintain. Tutorial -------- -While :mod:`optparse` is quite flexible and powerful, it's also straightforward +While :mod:`!optparse` is quite flexible and powerful, it's also straightforward to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. +any :mod:`!optparse`\ -based program. First, you need to import the OptionParser class; then, early in the main program, create an OptionParser instance:: @@ -374,7 +371,7 @@ Then you can start defining options. The basic syntax is:: attr=value, ...) Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what +and several option attributes that tell :mod:`!optparse` what to expect and what to do when it encounters that option on the command line. Typically, each option will have one short option string and one long option @@ -389,10 +386,10 @@ string overall. The option strings passed to :meth:`OptionParser.add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` +*encountering an option* on the command line; in reality, :mod:`!optparse` encounters *option strings* and looks up options from them. -Once all of your options are defined, instruct :mod:`optparse` to parse your +Once all of your options are defined, instruct :mod:`!optparse` to parse your program's command line:: (options, args) = parser.parse_args() @@ -420,14 +417,14 @@ most fundamental. Understanding option actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; +Actions tell :mod:`!optparse` what to do when it encounters an option on the +command line. There is a fixed set of actions hard-coded into :mod:`!optparse`; adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store +:ref:`optparse-extending-optparse`. Most actions tell :mod:`!optparse` to store a value in some variable---for example, take a string from the command line and store it in an attribute of ``options``. -If you don't specify an option action, :mod:`optparse` defaults to ``store``. +If you don't specify an option action, :mod:`!optparse` defaults to ``store``. .. _optparse-store-action: @@ -435,7 +432,7 @@ If you don't specify an option action, :mod:`optparse` defaults to ``store``. The store action ^^^^^^^^^^^^^^^^ -The most common option action is ``store``, which tells :mod:`optparse` to take +The most common option action is ``store``, which tells :mod:`!optparse` to take the next argument (or the remainder of the current argument), ensure that it is of the correct type, and store it to your chosen destination. @@ -444,16 +441,16 @@ For example:: parser.add_option("-f", "--file", action="store", type="string", dest="filename") -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: +Now let's make up a fake command line and ask :mod:`!optparse` to parse it:: args = ["-f", "foo.txt"] (options, args) = parser.parse_args(args) -When :mod:`optparse` sees the option string ``-f``, it consumes the next +When :mod:`!optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. +Some other option types supported by :mod:`!optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: parser.add_option("-n", type="int", dest="num") @@ -470,19 +467,19 @@ right up against the option: since ``-n42`` (one argument) is equivalent to will print ``42``. -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with +If you don't specify a type, :mod:`!optparse` assumes ``string``. Combined with the fact that the default action is ``store``, that means our first example can be a lot shorter:: parser.add_option("-f", "--file", dest="filename") -If you don't supply a destination, :mod:`optparse` figures out a sensible +If you don't supply a destination, :mod:`!optparse` figures out a sensible default from the option strings: if the first long option string is ``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the +long option strings, :mod:`!optparse` looks at the first short option string: the default destination for ``-f`` is ``f``. -:mod:`optparse` also includes the built-in ``complex`` type. Adding +:mod:`!optparse` also includes the built-in ``complex`` type. Adding types is covered in section :ref:`optparse-extending-optparse`. @@ -492,7 +489,7 @@ Handling boolean (flag) options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, +seen---are quite common. :mod:`!optparse` supports them with two separate actions, ``store_true`` and ``store_false``. For example, you might have a ``verbose`` flag that is turned on with ``-v`` and off with ``-q``:: @@ -503,7 +500,7 @@ Here we have two different options with the same destination, which is perfectly OK. (It just means you have to be a bit careful when setting default values---see below.) -When :mod:`optparse` encounters ``-v`` on the command line, it sets +When :mod:`!optparse` encounters ``-v`` on the command line, it sets ``options.verbose`` to ``True``; when it encounters ``-q``, ``options.verbose`` is set to ``False``. @@ -513,7 +510,7 @@ When :mod:`optparse` encounters ``-v`` on the command line, it sets Other actions ^^^^^^^^^^^^^ -Some other actions supported by :mod:`optparse` are: +Some other actions supported by :mod:`!optparse` are: ``"store_const"`` store a constant value, pre-set via :attr:`Option.const` @@ -539,11 +536,11 @@ Default values All of the above examples involve setting some variable (the "destination") when certain command-line options are seen. What happens if those options are never seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you +is usually fine, but sometimes you want more control. :mod:`!optparse` lets you supply a default value for each destination, which is assigned before the command line is parsed. -First, consider the verbose/quiet example. If we want :mod:`optparse` to set +First, consider the verbose/quiet example. If we want :mod:`!optparse` to set ``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: parser.add_option("-v", action="store_true", dest="verbose", default=True) @@ -582,7 +579,7 @@ values, not both. Generating help ^^^^^^^^^^^^^^^ -:mod:`optparse`'s ability to generate help and usage text automatically is +:mod:`!optparse`'s ability to generate help and usage text automatically is useful for creating user-friendly command-line interfaces. All you have to do is supply a :attr:`~Option.help` value for each option, and optionally a short usage message for your whole program. Here's an OptionParser populated with @@ -603,7 +600,7 @@ user-friendly (documented) options:: help="interaction mode: novice, intermediate, " "or expert [default: %default]") -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the +If :mod:`!optparse` encounters either ``-h`` or ``--help`` on the command-line, or if you just call :meth:`parser.print_help`, it prints the following to standard output: @@ -620,26 +617,26 @@ following to standard output: -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert [default: intermediate] -(If the help output is triggered by a help option, :mod:`optparse` exits after +(If the help output is triggered by a help option, :mod:`!optparse` exits after printing the help text.) -There's a lot going on here to help :mod:`optparse` generate the best possible +There's a lot going on here to help :mod:`!optparse` generate the best possible help message: * the script defines its own usage message:: usage = "usage: %prog [options] arg1 arg2" - :mod:`optparse` expands ``%prog`` in the usage string to the name of the + :mod:`!optparse` expands ``%prog`` in the usage string to the name of the current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string is then printed before the detailed option help. - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible + If you don't supply a usage string, :mod:`!optparse` uses a bland but sensible default: ``"Usage: %prog [options]"``, which is fine if your script doesn't take any positional arguments. * every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making + line-wrapping---\ :mod:`!optparse` takes care of wrapping lines and making the help output look good. * options that take a value indicate this fact in their automatically generated @@ -649,7 +646,7 @@ help message: Here, "MODE" is called the meta-variable: it stands for the argument that the user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses + :mod:`!optparse` converts the destination variable name to uppercase and uses that for the meta-variable. Sometimes, that's not what you want---for example, the ``--filename`` option explicitly sets ``metavar="FILE"``, resulting in this automatically generated option description:: @@ -663,7 +660,7 @@ help message: way to make your help text a lot clearer and more useful for end users. * options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's + string---\ :mod:`!optparse` will replace it with :func:`str` of the option's default value. If an option has no default value (or the default value is ``None``), ``%default`` expands to ``none``. @@ -779,14 +776,14 @@ option groups is: Printing a version string ^^^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the brief usage string, :mod:`optparse` can also print a version +Similar to the brief usage string, :mod:`!optparse` can also print a version string for your program. You have to supply the string as the ``version`` argument to OptionParser:: parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") ``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` +``version`` can contain anything you like. When you supply it, :mod:`!optparse` automatically adds a ``--version`` option to your parser. If it encounters this option on the command line, it expands your ``version`` string (by replacing ``%prog``), prints it to stdout, and exits. @@ -815,10 +812,10 @@ The following two methods can be used to print and get the ``version`` string: .. _optparse-how-optparse-handles-errors: -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How :mod:`!optparse` handles errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two broad classes of errors that :mod:`optparse` has to worry about: +There are two broad classes of errors that :mod:`!optparse` has to worry about: programmer errors and user errors. Programmer errors are usually erroneous calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown option attributes, missing option attributes, etc. These are dealt with in the @@ -826,7 +823,7 @@ usual way: raise an exception (either :exc:`optparse.OptionError` or :exc:`TypeError`) and let the program crash. Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect +no matter how stable your code is. :mod:`!optparse` can automatically detect some user errors, such as bad option arguments (passing ``-n 4x`` where ``-n`` takes an integer argument), missing arguments (``-n`` at the end of the command line, where ``-n`` takes an argument of any type). Also, @@ -838,7 +835,7 @@ condition:: if options.a and options.b: parser.error("options -a and -b are mutually exclusive") -In either case, :mod:`optparse` handles the error the same way: it prints the +In either case, :mod:`!optparse` handles the error the same way: it prints the program's usage message and an error message to standard error and exits with error status 2. @@ -861,11 +858,11 @@ Or, where the user fails to pass a value at all: foo: error: -n option requires an argument -:mod:`optparse`\ -generated error messages take care always to mention the +:mod:`!optparse`\ -generated error messages take care always to mention the option involved in the error; be sure to do the same when calling :func:`OptionParser.error` from your application code. -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, +If :mod:`!optparse`'s default error-handling behaviour does not suit your needs, you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` and/or :meth:`~OptionParser.error` methods. @@ -875,7 +872,7 @@ and/or :meth:`~OptionParser.error` methods. Putting it all together ^^^^^^^^^^^^^^^^^^^^^^^ -Here's what :mod:`optparse`\ -based scripts usually look like:: +Here's what :mod:`!optparse`\ -based scripts usually look like:: from optparse import OptionParser ... @@ -911,7 +908,7 @@ Reference Guide Creating the parser ^^^^^^^^^^^^^^^^^^^ -The first step in using :mod:`optparse` is to create an OptionParser instance. +The first step in using :mod:`!optparse` is to create an OptionParser instance. .. class:: OptionParser(...) @@ -921,7 +918,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``usage`` (default: ``"%prog [options]"``) The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands + help option. When :mod:`!optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the special value :const:`optparse.SUPPRESS_USAGE`. @@ -938,7 +935,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``version`` (default: ``None``) A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a + supply a true value for ``version``, :mod:`!optparse` automatically adds a version option with the single option string ``--version``. The substring ``%prog`` is expanded the same as for ``usage``. @@ -949,17 +946,17 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``description`` (default: ``None``) A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width + :mod:`!optparse` reformats this paragraph to fit the current terminal width and prints it when the user requests help (after ``usage``, but before the list of options). ``formatter`` (default: a new :class:`IndentedHelpFormatter`) An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: + text. :mod:`!optparse` provides two concrete classes for this purpose: IndentedHelpFormatter and TitledHelpFormatter. ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` + If true, :mod:`!optparse` will add a help option (with option strings ``-h`` and ``--help``) to the parser. ``prog`` @@ -997,7 +994,7 @@ the OptionParser constructor, as in:: (:func:`make_option` is a factory function for creating Option instances; currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` +:mod:`!optparse` may split Option into several classes, and :func:`make_option` will pick the right class to instantiate. Do not instantiate Option directly.) @@ -1027,12 +1024,12 @@ The canonical way to create an :class:`Option` instance is with the The keyword arguments define attributes of the new Option object. The most important option attribute is :attr:`~Option.action`, and it largely determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` + irrelevant option attributes, or fail to pass required ones, :mod:`!optparse` raises an :exc:`OptionError` exception explaining your mistake. - An option's *action* determines what :mod:`optparse` does when it encounters + An option's *action* determines what :mod:`!optparse` does when it encounters this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: + :mod:`!optparse` are: ``"store"`` store this option's argument (default) @@ -1066,7 +1063,7 @@ The canonical way to create an :class:`Option` instance is with the attributes; see :ref:`optparse-standard-option-actions`.) As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called +:mod:`!optparse` always creates a special object for this, conventionally called ``options``, which is an instance of :class:`optparse.Values`. .. class:: Values @@ -1084,7 +1081,7 @@ For example, when you call :: parser.parse_args() -one of the first things :mod:`optparse` does is create the ``options`` object:: +one of the first things :mod:`!optparse` does is create the ``options`` object:: options = Values() @@ -1099,7 +1096,7 @@ and the command-line being parsed includes any of the following:: --file=foo --file foo -then :mod:`optparse`, on seeing this option, will do the equivalent of :: +then :mod:`!optparse`, on seeing this option, will do the equivalent of :: options.filename = "foo" @@ -1124,13 +1121,13 @@ Option attributes The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. +:mod:`!optparse` raises :exc:`OptionError`. .. attribute:: Option.action (default: ``"store"``) - Determines :mod:`optparse`'s behaviour when this option is seen on the + Determines :mod:`!optparse`'s behaviour when this option is seen on the command line; the available options are documented :ref:`here `. @@ -1147,8 +1144,8 @@ relevant to a particular option, or fail to pass a required option attribute, (default: derived from option strings) If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses + tells :mod:`!optparse` where to write it: :attr:`~Option.dest` names an + attribute of the ``options`` object that :mod:`!optparse` builds as it parses the command line. .. attribute:: Option.default @@ -1161,7 +1158,7 @@ relevant to a particular option, or fail to pass a required option attribute, (default: 1) How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to + option is seen. If > 1, :mod:`!optparse` will store a tuple of values to :attr:`~Option.dest`. .. attribute:: Option.const @@ -1207,7 +1204,7 @@ Standard option actions The various option actions all have slightly different requirements and effects. Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you +guide :mod:`!optparse`'s behaviour; a few have required attributes, which you must specify for any option using that action. * ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, @@ -1225,9 +1222,9 @@ must specify for any option using that action. If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination + If :attr:`~Option.dest` is not supplied, :mod:`!optparse` derives a destination from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a + ``foo_bar``). If there are no long option strings, :mod:`!optparse` derives a destination from the first short option string (e.g., ``-f`` implies ``f``). Example:: @@ -1239,7 +1236,7 @@ must specify for any option using that action. -f foo.txt -p 1 -3.5 4 -fbar.txt - :mod:`optparse` will set :: + :mod:`!optparse` will set :: options.f = "foo.txt" options.point = (1.0, -3.5, 4.0) @@ -1259,7 +1256,7 @@ must specify for any option using that action. parser.add_option("--noisy", action="store_const", const=2, dest="verbose") - If ``--noisy`` is seen, :mod:`optparse` will set :: + If ``--noisy`` is seen, :mod:`!optparse` will set :: options.verbose = 2 @@ -1282,7 +1279,7 @@ must specify for any option using that action. The option must be followed by an argument, which is appended to the list in :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first + supplied, an empty list is automatically created when :mod:`!optparse` first encounters this option on the command-line. If :attr:`~Option.nargs` > 1, multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` is appended to :attr:`~Option.dest`. @@ -1294,7 +1291,7 @@ must specify for any option using that action. parser.add_option("-t", "--tracks", action="append", type="int") - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent + If ``-t3`` is seen on the command-line, :mod:`!optparse` does the equivalent of:: options.tracks = [] @@ -1333,7 +1330,7 @@ must specify for any option using that action. parser.add_option("-v", action="count", dest="verbosity") - The first time ``-v`` is seen on the command line, :mod:`optparse` does the + The first time ``-v`` is seen on the command line, :mod:`!optparse` does the equivalent of:: options.verbosity = 0 @@ -1364,7 +1361,7 @@ must specify for any option using that action. listed in the help message. To omit an option entirely, use the special value :const:`optparse.SUPPRESS_HELP`. - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all + :mod:`!optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. Example:: @@ -1382,7 +1379,7 @@ must specify for any option using that action. help="Input file to read data from") parser.add_option("--secret", help=SUPPRESS_HELP) - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, + If :mod:`!optparse` sees either ``-h`` or ``--help`` on the command line, it will print something like the following help message to stdout (assuming ``sys.argv[0]`` is ``"foo.py"``): @@ -1395,7 +1392,7 @@ must specify for any option using that action. -v Be moderately verbose --file=FILENAME Input file to read data from - After printing the help message, :mod:`optparse` terminates your process with + After printing the help message, :mod:`!optparse` terminates your process with ``sys.exit(0)``. * ``"version"`` @@ -1405,7 +1402,7 @@ must specify for any option using that action. ``print_version()`` method of OptionParser. Generally only relevant if the ``version`` argument is supplied to the OptionParser constructor. As with :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. + since :mod:`!optparse` automatically adds them when needed. .. _optparse-standard-option-types: @@ -1413,7 +1410,7 @@ must specify for any option using that action. Standard option types ^^^^^^^^^^^^^^^^^^^^^ -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, +:mod:`!optparse` has five built-in option types: ``"string"``, ``"int"``, ``"choice"``, ``"float"`` and ``"complex"``. If you need to add new option types, see section :ref:`optparse-extending-optparse`. @@ -1432,7 +1429,7 @@ Integer arguments (type ``"int"``) are parsed as follows: The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful +10, or 16). If this fails, so will :mod:`!optparse`, although with a more useful error message. ``"float"`` and ``"complex"`` option arguments are converted directly with @@ -1471,7 +1468,7 @@ The whole point of creating and populating an OptionParser is to call its ``options`` the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` + instance created by :mod:`!optparse` ``args`` the leftover positional arguments after all options have been processed @@ -1499,7 +1496,7 @@ provides several methods to help you out: .. method:: OptionParser.disable_interspersed_args() Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` + ``-b`` are both simple options that take no arguments, :mod:`!optparse` normally accepts this syntax:: prog -a arg1 -b arg2 @@ -1554,7 +1551,7 @@ strings:: (This is particularly true if you've defined your own OptionParser subclass with some standard options.) -Every time you add an option, :mod:`optparse` checks for conflicts with existing +Every time you add an option, :mod:`!optparse` checks for conflicts with existing options. If it finds any, it invokes the current conflict-handling mechanism. You can set the conflict-handling mechanism either in the constructor:: @@ -1581,7 +1578,7 @@ intelligently and add conflicting options to it:: parser.add_option("-n", "--dry-run", ..., help="do no harm") parser.add_option("-n", "--noisy", ..., help="be noisy") -At this point, :mod:`optparse` detects that a previously added option is already +At this point, :mod:`!optparse` detects that a previously added option is already using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, it resolves the situation by removing ``-n`` from the earlier option's list of option strings. Now ``--dry-run`` is the only way for the user to activate @@ -1594,14 +1591,14 @@ that option. If the user asks for help, the help message will reflect that:: It's possible to whittle away the option strings for a previously added option until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, +the command-line. In that case, :mod:`!optparse` removes that option completely, so it doesn't show up in help text or anywhere else. Carrying on with our existing OptionParser:: parser.add_option("--dry-run", ..., help="new dry-run option") At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: +accessible, so :mod:`!optparse` removes it, leaving this help text:: Options: ... @@ -1676,9 +1673,9 @@ OptionParser supports several other public methods: Option Callbacks ---------------- -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple +When :mod:`!optparse`'s built-in actions and types aren't quite enough for your +needs, you have two choices: extend :mod:`!optparse` or define a callback option. +Extending :mod:`!optparse` is more general, but overkill for a lot of simple cases. Quite often a simple callback is all you need. There are two steps to defining a callback option: @@ -1702,14 +1699,14 @@ only option attribute you must specify is ``callback``, the function to call:: ``callback`` is a function (or other callable object), so you must have already defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, +case, :mod:`!optparse` doesn't even know if ``-c`` takes any arguments, which usually means that the option takes no arguments---the mere presence of ``-c`` on the command-line is all it needs to know. In some circumstances, though, you might want your callback to consume an arbitrary number of command-line arguments. This is where writing callbacks gets tricky; it's covered later in this section. -:mod:`optparse` always passes four particular arguments to your callback, and it +:mod:`!optparse` always passes four particular arguments to your callback, and it will only pass additional arguments if you specify them via :attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the minimal callback function signature is:: @@ -1723,12 +1720,12 @@ callback option: :attr:`~Option.type` has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to + instructs :mod:`!optparse` to consume one argument and convert it to :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. + though, :mod:`!optparse` passes it to your callback function. :attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will + also has its usual meaning: if it is supplied and > 1, :mod:`!optparse` will consume :attr:`~Option.nargs` arguments, each of which must be convertible to :attr:`~Option.type`. It then passes a tuple of converted values to your callback. @@ -1762,7 +1759,7 @@ where ``"--foobar"``.) ``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will + is the argument to this option seen on the command-line. :mod:`!optparse` will only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be the type implied by the option's type. If :attr:`~Option.type` for this option is ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` @@ -1787,7 +1784,7 @@ where ``parser.values`` the object where option values are by default stored (an instance of optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess + rest of :mod:`!optparse` for storing option values; you don't need to mess around with globals or closures. You can also access or modify the value(s) of any options already encountered on the command-line. @@ -1806,7 +1803,7 @@ Raising errors in a callback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and +problems with the option or its argument(s). :mod:`!optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. Otherwise, the user will have a hard time figuring out what they did wrong. @@ -1906,7 +1903,7 @@ Here's an example that just emulates the standard ``"store"`` action:: action="callback", callback=store_value, type="int", nargs=3, dest="foo") -Note that :mod:`optparse` takes care of consuming 3 arguments and converting +Note that :mod:`!optparse` takes care of consuming 3 arguments and converting them to integers for you; all you have to do is store them. (Or whatever; obviously you don't need a callback for this example.) @@ -1917,9 +1914,9 @@ Callback example 6: variable arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any +For this case, you must write a callback, as :mod:`!optparse` doesn't provide any built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for +conventional Unix command-line parsing that :mod:`!optparse` normally handles for you. In particular, callbacks should implement the conventional rules for bare ``--`` and ``-`` arguments: @@ -1934,7 +1931,7 @@ you. In particular, callbacks should implement the conventional rules for bare If you want an option that takes a variable number of arguments, there are several subtle, tricky issues to worry about. The exact implementation you choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing +application (which is why :mod:`!optparse` doesn't support this sort of thing directly). Nevertheless, here's a stab at a callback for an option with variable @@ -1970,10 +1967,10 @@ arguments:: .. _optparse-extending-optparse: -Extending :mod:`optparse` -------------------------- +Extending :mod:`!optparse` +-------------------------- -Since the two major controlling factors in how :mod:`optparse` interprets +Since the two major controlling factors in how :mod:`!optparse` interprets command-line options are the action and type of each option, the most likely direction of extension is to add new actions and new types. @@ -1983,9 +1980,9 @@ direction of extension is to add new actions and new types. Adding new types ^^^^^^^^^^^^^^^^ -To add new types, you need to define your own subclass of :mod:`optparse`'s +To add new types, you need to define your own subclass of :mod:`!optparse`'s :class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. +:mod:`!optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. .. attribute:: Option.TYPES @@ -2015,7 +2012,7 @@ To add new types, you need to define your own subclass of :mod:`optparse`'s Here's a silly example that demonstrates adding a ``"complex"`` option type to parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for +than it used to be, because :mod:`!optparse` 1.3 added built-in support for complex numbers, but never mind.) First, the necessary imports:: @@ -2041,12 +2038,12 @@ Finally, the Option subclass:: TYPE_CHECKER["complex"] = check_complex (If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s +up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`!optparse`'s Option class. This being Python, nothing stops you from doing that except good manners and common sense.) That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your +any other :mod:`!optparse`\ -based script, except you have to instruct your OptionParser to use MyOption instead of Option:: parser = OptionParser(option_class=MyOption) @@ -2066,10 +2063,10 @@ Adding new actions ^^^^^^^^^^^^^^^^^^ Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: +:mod:`!optparse` has a couple of classifications for actions: "store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the + actions that result in :mod:`!optparse` storing a value to an attribute of the current OptionValues instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. @@ -2101,7 +2098,7 @@ of the following class attributes of Option (all are lists of strings): .. attribute:: Option.ALWAYS_TYPED_ACTIONS Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` + additionally listed here. The only effect of this is that :mod:`!optparse` assigns the default type, ``"string"``, to options with no explicit type whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. @@ -2144,12 +2141,12 @@ Features of note: somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and :attr:`~Option.TYPED_ACTIONS`. -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to +* to ensure that :mod:`!optparse` assigns the default type of ``"string"`` to ``"extend"`` actions, we put the ``"extend"`` action in :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. * :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` + control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. * ``values`` is an instance of the optparse_parser.Values class, which provides diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index c90e30ccf4fb8a9..808187061733be0 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -36,7 +36,7 @@ the :mod:`glob` module.) Since different operating systems have different path name conventions, there are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating + :mod:`!os.path` module is always the path module suitable for the operating system Python is running on, and therefore usable for local paths. However, you can also import and use the individual modules if you want to manipulate a path that is *always* in one of the different formats. They all have the @@ -57,8 +57,9 @@ the :mod:`glob` module.) .. function:: abspath(path) Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. + platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -96,15 +97,17 @@ the :mod:`glob` module.) .. function:: commonprefix(list, /) - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string + Return the longest string prefix (taken character-by-character) that is a + prefix of all strings in *list*. If *list* is empty, return the empty string (``''``). - .. note:: + .. warning:: This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. + character at a time. + If you need a **common path prefix**, then the algorithm + implemented in this function is not secure. Use + :func:`commonpath` for finding a common path prefix. :: @@ -117,6 +120,14 @@ the :mod:`glob` module.) .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. deprecated:: 3.15 + Deprecated in favor of :func:`os.path.commonpath` for path prefixes. + The :func:`os.path.commonprefix` function is being deprecated due to + having a misleading name and module. The function is not safe to use for + path prefixes despite being included in a module about path manipulation, + meaning it is easy to accidentally introduce path traversal + vulnerabilities into Python programs by using this function. + .. function:: dirname(path, /) @@ -243,6 +254,8 @@ the :mod:`glob` module.) begins with a slash, on Windows that it begins with two (back)slashes, or a drive letter, colon, and (back)slash together. + .. seealso:: :func:`abspath` + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -357,14 +370,28 @@ the :mod:`glob` module.) concatenation of *path* and all members of *\*paths*, with exactly one directory separator following each non-empty part, except the last. That is, the result will only end in a separator if the last part is either empty or - ends in a separator. If a segment is an absolute path (which on Windows - requires both a drive and a root), then all previous segments are ignored and - joining continues from the absolute path segment. + ends in a separator. + + If a segment is an absolute path (which on Windows requires both a drive and + a root), then all previous segments are ignored and joining continues from the + absolute path segment. On Linux, for example:: + + >>> os.path.join('/home/foo', 'bar') + '/home/foo/bar' + >>> os.path.join('/home/foo', '/home/bar') + '/home/bar' On Windows, the drive is not reset when a rooted path segment (e.g., ``r'\foo'``) is encountered. If a segment is on a different drive or is an - absolute path, all previous segments are ignored and the drive is reset. Note - that since there is a current directory for each drive, + absolute path, all previous segments are ignored and the drive is reset. For + example:: + + >>> os.path.join('c:\\', 'foo') + 'c:\\foo' + >>> os.path.join('c:\\foo', 'd:\\bar') + 'd:\\bar' + + Note that since there is a current directory for each drive, ``os.path.join("c:", "foo")`` represents a path relative to the current directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. @@ -527,8 +554,8 @@ the :mod:`glob` module.) *path* is empty, both *head* and *tail* are empty. Trailing slashes are stripped from *head* unless it is the root (one or more slashes only). In all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. + (but the strings may differ). Also see the functions :func:`join`, + :func:`dirname` and :func:`basename`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dbc3c92c8798b5e..5858d9733c57796 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -25,7 +25,7 @@ Notes on the availability of these functions: with the POSIX interface). * Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to + through the :mod:`!os` module, but using them is of course a threat to portability. * All functions accepting path or file names accept both bytes and string @@ -34,7 +34,7 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`!os` module are not available or behave differently. APIs related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are @@ -111,7 +111,7 @@ Python UTF-8 Mode .. versionchanged:: 3.15 Python UTF-8 mode is now enabled by default (:pep:`686`). - It may be disabled with by setting :envvar:`PYTHONUTF8=0 ` as + It may be disabled by setting :envvar:`PYTHONUTF8=0 ` as an environment variable or by using the :option:`-X utf8=0 <-X>` command line option. The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage @@ -185,7 +185,7 @@ process and user. of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. - This mapping is captured the first time the :mod:`os` module is imported, + This mapping is captured the first time the :mod:`!os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. @@ -219,6 +219,14 @@ process and user. :data:`os.environ`, and when one of the :meth:`~dict.pop` or :meth:`~dict.clear` methods is called. + If the :manpage:`clearenv(3)` function is available, the :meth:`~dict.clear` method + uses it and emits a single ``os._clearenv`` audit event. Otherwise, it emits + an ``os.unsetenv`` event on each deleted variable. + + .. audit-event:: os.unsetenv key os.unsetenv + + .. audit-event:: os._clearenv "" os._clearenv + .. seealso:: The :func:`os.reload_environ` function. @@ -226,6 +234,10 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.15 + The :meth:`~dict.clear` method can now emit an ``os._clearenv`` audit + event. + .. data:: environb @@ -254,7 +266,7 @@ process and user. .. warning:: This function is not thread-safe. Calling it while the environment is - being modified in an other thread is an undefined behavior. Reading from + being modified in another thread is an undefined behavior. Reading from :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` while reloading, may return an empty result. @@ -529,7 +541,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -539,7 +551,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -558,7 +570,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. @@ -713,7 +725,7 @@ process and user. Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -722,7 +734,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -788,29 +800,19 @@ process and user. single: gethostbyaddr() (in module socket) Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. + The return value is a :class:`uname_result`. On macOS, iOS and Android, this returns the *kernel* name and version (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` can be used to get the user-facing operating system name and version on iOS and Android. + .. seealso:: + :data:`sys.platform` which has finer granularity. + + The :mod:`platform` module provides detailed checks for the + system's identity. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -818,6 +820,41 @@ process and user. with named attributes. +.. class:: uname_result + + Name and information about the system returned by :func:`os.uname`. + These attributes correspond to the members described in :manpage:`uname(2)`. + + For backwards compatibility, this object is also iterable, behaving + like a five-tuple containing :attr:`~uname_result.sysname`, + :attr:`~uname_result.nodename`, :attr:`~uname_result.release`, + :attr:`~uname_result.version`, and :attr:`~uname_result.machine` + in that order. + + .. attribute:: sysname + + Operating system name. + + .. attribute:: nodename + + Name of machine on network. Some systems truncate + :attr:`~uname_result.nodename` to 8 characters or to the leading + component; a better way to get the hostname is :func:`socket.gethostname` + or even ``socket.gethostbyaddr(socket.gethostname())``. + + .. attribute:: release + + Operating system release. + + .. attribute:: version + + Operating system version. + + .. attribute:: machine + + Hardware identifier. + + .. function:: unsetenv(key, /) .. index:: single: environment variables; deleting @@ -1059,10 +1096,7 @@ as internal buffering of data. Force write of file with filedescriptor *fd* to disk. Does not force update of metadata. - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. + .. availability:: Unix, not macOS, not iOS. .. function:: fpathconf(fd, name, /) @@ -1100,8 +1134,8 @@ as internal buffering of data. .. function:: fstatvfs(fd, /) Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + file descriptor *fd* in a :class:`statvfs_result`, like :func:`statvfs`. + As of Python 3.3, this is equivalent to ``os.statvfs(fd)``. .. availability:: Unix. @@ -1279,7 +1313,7 @@ as internal buffering of data. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + the :mod:`!os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors @@ -1294,8 +1328,8 @@ as internal buffering of data. This function is intended for low-level I/O. For normal usage, use the built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. + :meth:`~io.BufferedIOBase.read` and :meth:`~io.BufferedIOBase.write` methods. + To wrap a file descriptor in a file object, use :func:`fdopen`. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -1414,7 +1448,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.3 @@ -1424,7 +1458,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1439,7 +1473,7 @@ or `the MSDN `_ on Windo :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1556,6 +1590,15 @@ or `the MSDN `_ on Windo .. versionadded:: 3.15 +.. data:: RWF_ATOMIC + + Write data atomically. Requires alignment to the device's atomic write unit. + + .. availability:: Linux >= 6.11 + + .. versionadded:: 3.15 + + .. function:: ptsname(fd, /) Return the name of the slave pseudo-terminal device associated with the @@ -1598,6 +1641,7 @@ or `the MSDN `_ on Windo - :data:`RWF_SYNC` - :data:`RWF_APPEND` - :data:`RWF_DONTCACHE` + - :data:`RWF_ATOMIC` Return the total number of bytes actually written. @@ -1660,7 +1704,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. + :meth:`~io.TextIOBase.read` or :meth:`~io.IOBase.readline` methods. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1895,7 +1939,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. + :meth:`~io.TextIOBase.write` method. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1969,7 +2013,8 @@ can be inherited by child processes. Since Python 3.4, file descriptors created by Python are non-inheritable by default. On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. +execution of a new program, other file descriptors are inherited. Note that +non-inheritable file descriptors are still *inherited* by child processes on :func:`os.fork`. On Windows, non-inheritable handles and file descriptors are closed in child processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout @@ -2013,7 +2058,7 @@ features: .. _path_fd: * **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module + Normally the *path* argument provided to functions in the :mod:`!os` module must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. @@ -2135,6 +2180,11 @@ features: .. audit-event:: os.chdir path os.chdir + .. seealso:: + + The :func:`contextlib.chdir` context manager, which changes the current + working directory on entering and restores the previous one on exit. + .. versionchanged:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -2398,6 +2448,10 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listdir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. function:: listdrives() @@ -2534,7 +2588,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2552,6 +2607,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2578,6 +2639,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -2928,6 +2994,10 @@ features: .. versionchanged:: 3.7 Added support for :ref:`file descriptors ` on Unix. + .. versionchanged:: 3.15 + ``os.scandir(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing the current directory. + .. class:: DirEntry @@ -2951,6 +3021,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name @@ -2964,10 +3037,16 @@ features: .. attribute:: path - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. If the :func:`scandir` *path* + The entry's path name: equivalent to ``os.path.join(scandir_path, + entry.name)`` where *scandir_path* is the original :func:`scandir` + *path* argument. Apart from the filename, the path preserves the + original :func:`scandir` argument. If the :func:`scandir` *path* + argument was relative, the :attr:`path` attribute is also relative. + Changing the current working directory after creating the + :func:`scandir` iterator may cause later uses of :attr:`path` to resolve + differently. On some platforms, the constructed path may not be valid + if the original :func:`scandir` argument was usable for enumeration but + not for joining with the entry name. If the :func:`scandir` *path* argument was a :ref:`file descriptor `, the :attr:`path` attribute is the same as the :attr:`name` attribute. @@ -3404,7 +3483,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: statx_result @@ -3661,7 +3740,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: STATX_TYPE @@ -3690,7 +3769,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_FORCE_SYNC @@ -3700,7 +3779,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_DONT_SYNC @@ -3709,7 +3788,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_SYNC_AS_STAT @@ -3721,7 +3800,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_NO_AUTOMOUNT @@ -3733,58 +3812,184 @@ features: .. availability:: Linux. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: statvfs(path) - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + Perform a :manpage:`statvfs(3)` system call on the given path. The return value + is a :class:`statvfs_result` whose attributes describe the filesystem + on the given path and correspond to the members of the :c:struct:`statvfs` + structure. This function can support :ref:`specifying a file descriptor `. .. availability:: Unix. - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.7 - Added the :attr:`f_fsid` attribute. + +.. class:: statvfs_result + + Filesystem statistics returned by :func:`os.statvfs` and :func:`os.fstatvfs`. + See :manpage:`statvfs(3)` for more details. + + .. attribute:: f_bsize + + Block size. + + .. attribute:: f_frsize + + Fragment size. + + .. attribute:: f_blocks + + Number of :attr:`~statvfs_result.f_frsize` sized blocks the filesystem + can contain. + + .. attribute:: f_bfree + + Number of free blocks. + + .. attribute:: f_bavail + + Number of free blocks for unprivileged users. + + .. attribute:: f_files + + Number of file entries, inodes, the filesystem can contain. + + .. attribute:: f_ffree + + Number of free files entries. + + .. attribute:: f_favail + + Number of free file entries for unprivileged users. + + .. attribute:: f_flag + + Bit-mask of mount flags. The following flags are defined: + :data:`ST_RDONLY`, :data:`ST_NOSUID`, :data:`ST_NODEV`, + :data:`ST_NOEXEC`, :data:`ST_SYNCHRONOUS`, :data:`ST_MANDLOCK`, + :data:`ST_WRITE`, :data:`ST_APPEND`, :data:`ST_IMMUTABLE`, + :data:`ST_NOATIME`, :data:`ST_NODIRATIME`, and :data:`ST_RELATIME`. + + .. attribute:: f_namemax + + Filesystem max filename length. OS specific limitations such as + :ref:`Windows MAX_PATH ` and those described in Linux + :manpage:`pathname(7)` may exist. + + .. attribute:: f_fsid + + Filesystem ID. + + .. versionadded:: 3.7 + + +The following flags are used in :attr:`statvfs_result.f_flag`. + +.. data:: ST_RDONLY + + Read-only filesystem. + + .. versionadded:: 3.2 + +.. data:: ST_NOSUID + + Setuid/setgid bits are disabled or not supported. + + .. versionadded:: 3.2 + +.. data:: ST_NODEV + + Disallow access to device special files. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOEXEC + + Disallow program execution. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_SYNCHRONOUS + + Writes are synced at once. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_MANDLOCK + + Allow mandatory locks on an FS. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_WRITE + + Write on file/directory/symlink. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_APPEND + + Append-only file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_IMMUTABLE + + Immutable file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOATIME + + Do not update access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NODIRATIME + + Do not update directory access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_RELATIME + + Update atime relative to mtime/ctime. + + .. availability:: Linux. + + .. versionadded:: 3.4 .. data:: supports_dir_fd - A :class:`set` object indicating which functions in the :mod:`os` + A :class:`set` object indicating which functions in the :mod:`!os` module accept an open file descriptor for their *dir_fd* parameter. Different platforms provide different features, and the underlying functionality Python uses to implement the *dir_fd* parameter is not @@ -3829,7 +4034,7 @@ features: .. data:: supports_fd A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file + :mod:`!os` module permit specifying their *path* parameter as an open file descriptor on the local platform. Different platforms provide different features, and the underlying functionality Python uses to accept open file descriptors as *path* arguments is not available on all platforms Python @@ -3848,7 +4053,7 @@ features: .. data:: supports_follow_symlinks - A :class:`set` object indicating which functions in the :mod:`os` module + A :class:`set` object indicating which functions in the :mod:`!os` module accept ``False`` for their *follow_symlinks* parameter on the local platform. Different platforms provide different features, and the underlying functionality Python uses to implement *follow_symlinks* is not available @@ -4251,7 +4456,7 @@ features: import os # semaphore with start value '1' - fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) + fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFD_CLOEXEC) try: # acquire semaphore v = os.eventfd_read(fd) @@ -4360,7 +4565,7 @@ Naturally, they are all only available on Linux. except it includes any time that the system is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_NONBLOCK` @@ -4392,7 +4597,7 @@ Naturally, they are all only available on Linux. *fd* must be a valid timer file descriptor. The timer's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_TIMER_ABSTIME` @@ -4461,7 +4666,7 @@ Naturally, they are all only available on Linux. Return a two-item tuple of floats (``next_expiration``, ``interval``). - ``next_expiration`` denotes the relative time until next the timer next fires, + ``next_expiration`` denotes the relative time until the timer next fires, regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. ``interval`` denotes the timer's interval. @@ -4563,6 +4768,10 @@ These functions are all available on Linux only. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + ``os.listxattr(-1)`` now fails with ``OSError(errno.EBADF)`` rather than + listing extended attributes of the current directory. + .. function:: removexattr(path, attribute, *, follow_symlinks=True) @@ -4697,7 +4906,7 @@ to be ignored. The current process is replaced immediately. Open file objects and descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an + :func:`~io.IOBase.flush` or :func:`os.fsync` before calling an :func:`exec\* ` function. The "l" and "v" variants of the :func:`exec\* ` functions differ in how @@ -5039,13 +5248,25 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Linux >= 5.10 .. versionadded:: 3.12 +.. function:: pidfd_getfd(pidfd, targetfd, *, flags=0) + + Duplicate *targetfd* from the process referred to by the process file + descriptor *pidfd*, into the calling process. The returned file descriptor + is :ref:`non-inheritable `. + + *flags* is reserved, and currently must be ``0``. + + See the :manpage:`pidfd_getfd(2)` man page for more details. + + .. availability:: Linux >= 5.6, Android >= :func:`build-time ` API level 31 + .. versionadded:: next .. function:: plock(op, /) Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not WASI, not iOS. + .. availability:: Unix, not WASI, not macOS, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -5087,9 +5308,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5317,9 +5537,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT @@ -5993,7 +6212,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overridden value *n*. + :func:`cpu_count` returns the override value *n*. .. function:: getloadavg() @@ -6015,7 +6234,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overridden value *n*. + :func:`process_cpu_count` returns the override value *n*. See also the :func:`sched_getaffinity` function. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 79e0b7f09eaa777..ab92c142c37a4f6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -486,6 +486,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').stem 'library' + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. method:: PurePath.as_posix() @@ -1262,6 +1266,8 @@ Reading and writing files >>> p.read_text() 'Text file contents' + Return the number of characters written. + An existing file of the same name is overwritten. The optional parameters have the same meaning as in :func:`open`. @@ -1282,6 +1288,8 @@ Reading and writing files >>> p.read_bytes() b'Binary file contents' + Return the number of bytes written. + An existing file of the same name is overwritten. .. versionadded:: 3.5 @@ -1331,6 +1339,10 @@ Reading directories PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + .. seealso:: :ref:`pathlib-pattern-language` documentation. @@ -1343,6 +1355,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1365,6 +1382,15 @@ Reading directories Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*. + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. @@ -1492,7 +1518,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1503,6 +1530,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1516,6 +1549,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) @@ -1875,7 +1911,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. ===================================== ============================================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` +:mod:`os` and :mod:`os.path` :mod:`!pathlib` ===================================== ============================================== :func:`os.path.dirname` :attr:`PurePath.parent` :func:`os.path.basename` :attr:`PurePath.name` @@ -1934,7 +1970,7 @@ Protocols :synopsis: pathlib types for static type checking -The :mod:`pathlib.types` module provides types for static type checking. +The :mod:`!pathlib.types` module provides types for static type checking. .. versionadded:: 3.14 diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 90dc6648045f27f..bfe017a5c8fe1c7 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -1,7 +1,7 @@ .. _debugger: -:mod:`pdb` --- The Python Debugger -================================== +:mod:`!pdb` --- The Python Debugger +=================================== .. module:: pdb :synopsis: The Python debugger for interactive interpreters. @@ -12,7 +12,7 @@ -------------- -The module :mod:`pdb` defines an interactive source code debugger for Python +The module :mod:`!pdb` defines an interactive source code debugger for Python programs. It supports setting (conditional) breakpoints and single stepping at the source line level, inspection of stack frames, source code listing, and evaluation of arbitrary Python code in the context of any stack frame. It also @@ -76,9 +76,13 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug .. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb -You can also invoke :mod:`pdb` from the command line to debug other scripts. For +You can also invoke :mod:`!pdb` from the command line to debug other scripts. For example:: python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] @@ -334,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be @@ -516,7 +520,8 @@ can be overridden by the local file. To remove all commands from a breakpoint, type ``commands`` and follow it immediately with ``end``; that is, give no commands. - With no *bpnumber* argument, ``commands`` refers to the last breakpoint set. + With no *bpnumber* argument, ``commands`` refers to the most recently set + breakpoint that still exists. You can use breakpoint commands to start your program up again. Simply use the :pdbcmd:`continue` command, or :pdbcmd:`step`, diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 3a9b66ec7e70886..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -4,9 +4,6 @@ .. module:: pickle :synopsis: Convert Python objects to streams of bytes and back. -.. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw - **Source code:** :source:`Lib/pickle.py` .. index:: @@ -19,7 +16,7 @@ -------------- -The :mod:`pickle` module implements binary protocols for serializing and +The :mod:`!pickle` module implements binary protocols for serializing and de-serializing a Python object structure. *"Pickling"* is the process whereby a Python object hierarchy is converted into a byte stream, and *"unpickling"* is the inverse operation, whereby a byte stream @@ -50,35 +47,22 @@ Comparison with ``marshal`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python +general :mod:`!pickle` should always be the preferred way to serialize Python objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` files. -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. +The :mod:`!pickle` module differs from :mod:`marshal` in several significant ways: * :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, + instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support :file:`.pyc` files, the Python implementers reserve the right to change the serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible + The :mod:`!pickle` serialization format is guaranteed to be backwards compatible across Python releases provided a compatible pickle protocol is chosen and pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. @@ -123,17 +107,17 @@ Data stream format .. index:: single: External Data Representation -The data format used by :mod:`pickle` is Python-specific. This has the +The data format used by :mod:`!pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary +By default, the :mod:`!pickle` data format uses a relatively compact binary representation. If you need optimal size characteristics, you can efficiently :doc:`compress ` pickled data. The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive +generated by :mod:`!pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. There are currently 6 different protocols which can be used for pickling. @@ -167,9 +151,9 @@ to read the pickle produced. .. note:: Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of + :mod:`!pickle` reads and writes file objects, it does not handle the issue of naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex + access to persistent objects. The :mod:`!pickle` module can transform a complex object into a byte stream and it can transform the byte stream into an object with the same internal structure. Perhaps the most obvious thing to do with these byte streams is to write them onto a file, but it is also conceivable to @@ -186,7 +170,7 @@ Similarly, to de-serialize a data stream, you call the :func:`loads` function. However, if you want more control over serialization and de-serialization, you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. -The :mod:`pickle` module provides the following constants: +The :mod:`!pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL @@ -217,7 +201,7 @@ The :mod:`pickle` module provides the following constants: The default protocol is 5. -The :mod:`pickle` module provides the following functions to make the pickling +The :mod:`!pickle` module provides the following functions to make the pickling process more convenient: .. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -275,7 +259,7 @@ process more convenient: The *buffers* argument was added. -The :mod:`pickle` module defines three exceptions: +The :mod:`!pickle` module defines three exceptions: .. exception:: PickleError @@ -300,7 +284,7 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports three classes, :class:`Pickler`, +The :mod:`!pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: .. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -532,7 +516,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`~object.__getstate__` +* instances of such classes for which the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -601,7 +585,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 2 and newer, classes that implements the + In protocols 2 and newer, classes that implement the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -709,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their @@ -773,13 +760,13 @@ Persistence of External Objects single: persistent_id (pickle protocol) single: persistent_load (pickle protocol) -For the benefit of object persistence, the :mod:`pickle` module supports the +For the benefit of object persistence, the :mod:`!pickle` module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). -The resolution of such persistent IDs is not defined by the :mod:`pickle` +The resolution of such persistent IDs is not defined by the :mod:`!pickle` module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. @@ -973,10 +960,10 @@ Out-of-band Buffers .. versionadded:: 3.8 -In some contexts, the :mod:`pickle` module is used to transfer massive amounts +In some contexts, the :mod:`!pickle` module is used to transfer massive amounts of data. Therefore, it can be important to minimize the number of memory copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure +operation of the :mod:`!pickle` module, as it transforms a graph-like structure of objects into a sequential stream of bytes, intrinsically involves copying data to and from the pickle stream. @@ -995,8 +982,8 @@ for any large data. A :class:`PickleBuffer` object *signals* that the underlying buffer is eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by +with normal usage of the :mod:`!pickle` module. However, consumers can also +opt-in to tell :mod:`!pickle` that they will handle those buffers by themselves. Consumer API @@ -1172,7 +1159,7 @@ Performance Recent versions of the pickle protocol (from protocol 2 and upwards) feature efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. +Also, the :mod:`!pickle` module has a transparent optimizer written in C. .. _pickle-example: @@ -1215,7 +1202,7 @@ The following example reads the resulting pickled data. :: Command-line interface ---------------------- -The :mod:`pickle` module can be invoked as a script from the command line, +The :mod:`!pickle` module can be invoked as a script from the command line, it will display contents of the pickle files. However, when the pickle file that you want to examine comes from an untrusted source, ``-m pickletools`` is a safer option because it does not execute pickle bytecode, see @@ -1243,7 +1230,7 @@ The following option is accepted: Tools for working with and analyzing pickled data. Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. + Indexed databases of objects; uses :mod:`!pickle`. Module :mod:`copy` Shallow and deep object copying. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 30fc2962e0bf785..769ca60af1837ea 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -15,7 +15,7 @@ This module contains various constants relating to the intimate details of the few useful functions for analyzing pickled data. The contents of this module are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. +:mod:`!pickletools` module relevant. .. _pickletools-cli: @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 47d24b6f7d06bbe..5473a367c49a3ab 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -151,26 +151,50 @@ support. :meth:`get_data ` API. The *package* argument should be the name of a package, in standard module format (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). + filename, using ``/`` as the path separator. The function returns a binary string that is the contents of the specified resource. + This function uses the :term:`loader` method + :func:`~importlib.abc.FileLoader.get_data` + to support modules installed in the filesystem, but also in zip files, + databases, or elsewhere. + For packages located in the filesystem, which have already been imported, this is the rough equivalent of:: d = os.path.dirname(sys.modules[package].__file__) data = open(os.path.join(d, resource), 'rb').read() + Like the :func:`open` function, :func:`!get_data` can follow parent + directories (``../``) and absolute paths (starting with ``/`` or ``C:/``, + for example). + It can open compilation/installation artifacts like ``.py`` and ``.pyc`` + files or files with :func:`reserved filenames `. + To be compatible with non-filesystem loaders, avoid using these features. + + .. warning:: + + This function is intended for trusted input. + It does not verify that *resource* "belongs" to *package*. + + If you use a user-provided *resource* path, consider verifying it. + For example, require an alphanumeric filename with a known extension, or + install and check a list of known resources. + If the package cannot be located or loaded, or it uses a :term:`loader` which does not support :meth:`get_data `, then ``None`` is returned. In particular, the :term:`loader` for :term:`namespace packages ` does not support :meth:`get_data `. + .. seealso:: -.. function:: resolve_name(name) + The :mod:`importlib.resources` module provides structured access to + module resources. + +.. function:: resolve_name(name, *, strict=False) Resolve a name to an object. @@ -184,6 +208,7 @@ support. * ``W(.W)*`` * ``W(.W)*:(W(.W)*)?`` + * ``W(.W)*:(W(.W)*)`` The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -198,6 +223,11 @@ support. hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. + The first two forms are accepted when ``strict=False`` (the default). + + The third form requires both the module name and callable, separated by + a colon. Only this form is accepted when ``strict=True``. + The function will return an object (which might be a module), or raise one of the following exceptions: @@ -209,3 +239,7 @@ support. hierarchy within the imported package to get to the desired object. .. versionadded:: 3.9 + + .. versionchanged:: 3.15 + + The optional keyword-only ``strict`` flag was added. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index d05c6e5a2aa22c2..1d30966794fd1bf 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -4,9 +4,6 @@ .. module:: platform :synopsis: Retrieves as much platform identifying data as possible. -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Bjorn Pettersen - **Source code:** :source:`Lib/platform.py` -------------- @@ -357,7 +354,7 @@ Android platform Command-line usage ------------------ -:mod:`platform` can also be invoked directly using the :option:`-m` +:mod:`!platform` can also be invoked directly using the :option:`-m` switch of the interpreter:: python -m platform [--terse] [--nonaliased] [{nonaliased,terse} ...] diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 415c4b45c4f100e..72140e41675c350 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -4,10 +4,6 @@ .. module:: plistlib :synopsis: Generate and parse Apple plist files. -.. moduleauthor:: Jack Jansen -.. sectionauthor:: Georg Brandl -.. (harvested from docstrings in the original file) - **Source code:** :source:`Lib/plistlib.py` .. index:: @@ -22,7 +18,7 @@ and XML plist files. The property list (``.plist``) file format is a simple serialization supporting basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. +top level object is a dictionary or a frozen dictionary. To write out and to parse a plist file, use the :func:`dump` and :func:`load` functions. @@ -184,7 +180,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -200,7 +196,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 23f20b00e6dc6d8..cd3a58016e9c12e 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -4,9 +4,6 @@ .. module:: poplib :synopsis: POP3 protocol client (requires sockets). -.. sectionauthor:: Andrew T. Csillag -.. revised by ESR, January 2000 - **Source code:** :source:`Lib/poplib.py` .. index:: pair: POP3; protocol @@ -30,7 +27,7 @@ mailserver supports IMAP, you would be better off using the .. include:: ../includes/wasm-notavail.rst -The :mod:`poplib` module provides two classes: +The :mod:`!poplib` module provides two classes: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -86,7 +83,7 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.12 The deprecated *keyfile* and *certfile* parameters have been removed. -One exception is defined as an attribute of the :mod:`poplib` module: +One exception is defined as an attribute of the :mod:`!poplib` module: .. exception:: error_proto diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 14ab3e91e8a8e4c..7d43b5d4cf7657e 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -2,7 +2,6 @@ ==================================================== .. module:: posix - :platform: Unix :synopsis: The most common POSIX system calls (normally used via module os). -------------- @@ -17,10 +16,10 @@ interface). **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always +module provides a superset of the :mod:`!posix` interface. On non-Unix operating +systems the :mod:`!posix` module is not available, but a subset is always available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, +*no* performance penalty in using it instead of :mod:`!posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling :func:`~os.putenv` when an entry in ``os.environ`` is changed. @@ -37,8 +36,6 @@ Large File Support single: large files single: file; large files -.. sectionauthor:: Steve Clift - Several operating systems (including AIX and Solaris) provide support for files that are larger than 2 GiB from a C programming model where :c:expr:`int` and :c:expr:`long` are 32-bit values. This is typically accomplished @@ -67,7 +64,7 @@ Notable Module Contents ----------------------- In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: +:mod:`!posix` defines the following data item: .. data:: environ @@ -91,4 +88,4 @@ In addition to many functions described in the :mod:`os` module documentation, which updates the environment on modification. Note also that updating :data:`os.environ` will render this dictionary obsolete. Use of the :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. + :mod:`!posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index f51892450798aeb..4f043fbb3a46dff 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -4,14 +4,11 @@ .. module:: pprint :synopsis: Data pretty printer. -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pprint.py` -------------- -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary +The :mod:`!pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects @@ -34,7 +31,8 @@ Functions --------- .. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=False, underscore_numbers=False) + compact=False, expand=False, sort_dicts=False, \ + underscore_numbers=False) Prints the formatted representation of *object*, followed by a newline. This function may be used in the interactive interpreter @@ -72,6 +70,13 @@ Functions each item of a sequence will be formatted on a separate line, otherwise as many items as will fit within the *width* will be formatted on each output line. + Incompatible with *expand*. + + :param bool expand: + If ``True``, + opening parentheses and brackets will be followed by a newline and the + following content will be indented by one level, similar to + pretty-printed JSON. Incompatible with *compact*. :param bool sort_dicts: If ``True``, dictionaries will be formatted with @@ -98,7 +103,8 @@ Functions .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, which would automatically sort the dictionaries' keys, @@ -106,10 +112,11 @@ Functions .. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + *width*, *depth*, *compact*, *expand*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters and their meanings are as described in the documentation above. @@ -153,7 +160,8 @@ PrettyPrinter Objects .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Construct a :class:`PrettyPrinter` instance. @@ -177,6 +185,22 @@ PrettyPrinter Objects 'knights', 'ni'], 'spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, expand=True, indent=3) + >>> pp.pprint(stuff) + [ + [ + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ], + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni', + ] >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', ... ('parrot', ('fresh fruit',)))))))) >>> pp = pprint.PrettyPrinter(depth=6) @@ -196,6 +220,9 @@ PrettyPrinter Objects .. versionchanged:: 3.11 No longer attempts to write to :data:`!sys.stdout` if it is ``None``. + .. versionchanged:: 3.15 + Added the *expand* parameter. + :class:`PrettyPrinter` instances have the following methods: @@ -418,3 +445,72 @@ cannot be split, the specified width will be exceeded:: 'requires_python': None, 'summary': 'A sample Python project', 'version': '1.2.0'} + +Lastly, we can format like pretty-printed JSON with the *expand* parameter. +Best results are achieved with a higher *indent* value:: + + >>> pprint.pp(project_info, indent=4, expand=True) + { + 'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Software Development :: Build Tools', + ], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ReStructured ' + 'Text. It\n' + 'will be used to generate the project webpage on PyPI, and should be ' + 'written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of the project, ' + 'basic\n' + 'usage examples, etc. Generally, including the project changelog in here ' + 'is not\n' + 'a good idea, although a simple "What\'s New" section for the most recent ' + 'version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, + 'dynamic': None, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'license_expression': None, + 'license_files': None, + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': { + 'Download': 'UNKNOWN', + 'Homepage': 'https://github.com/pypa/sampleproject', + }, + 'provides_extra': None, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0', + 'yanked': False, + 'yanked_reason': None, + } diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index faf8079db3ddd8e..f7e85d1598727f9 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -1,523 +1,64 @@ .. _profile: -******************** -The Python Profilers -******************** +**************************************** +:mod:`!profile` --- Pure Python profiler +**************************************** -**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py` +.. module:: profile + :synopsis: Pure Python profiler (deprecated). + :deprecated: --------------- +**Source code:** :source:`Lib/profile.py` -.. _profiler-introduction: +-------------- -Introduction to the profilers -============================= +.. deprecated-removed:: 3.15 3.17 -.. index:: - single: statistical profiling - single: profiling, statistical - single: deterministic profiling - single: profiling, deterministic +The :mod:`!profile` module is deprecated and will be removed in Python 3.17. +Use :mod:`profiling.tracing` instead. -Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. +The :mod:`!profile` module provides a pure Python implementation of a +deterministic profiler. While useful for understanding profiler internals or +extending profiler behavior through subclassing, its pure Python implementation +introduces significant overhead compared to the C-based :mod:`profiling.tracing` +module. -The Python standard library provides three different profiling implementations: +For most profiling tasks, use: -**Statistical Profiler:** +- :mod:`profiling.sampling` for production debugging with zero overhead +- :mod:`profiling.tracing` for development and testing -1. :mod:`!profiling.sampling` provides statistical profiling of running Python processes - using periodic stack sampling. It can attach to any running Python process without - requiring code modification or restart, making it ideal for production debugging. -**Deterministic Profilers:** +Migration +========= -2. :mod:`cProfile` is recommended for development and testing; it's a C extension with - reasonable overhead that makes it suitable for profiling long-running - programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted - Czotter. +Migrating from :mod:`!profile` to :mod:`profiling.tracing` is straightforward. +The APIs are compatible:: -3. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. + # Old (deprecated) + import profile + profile.run('my_function()') -.. note:: + # New (recommended) + import profiling.tracing + profiling.tracing.run('my_function()') - The profiler modules are designed to provide an execution profile for a given - program, not for benchmarking purposes (for that, there is :mod:`timeit` for - reasonably accurate results). This particularly applies to benchmarking - Python code against C code: the profilers introduce overhead for Python code, - but not for C-level functions, and so the C code would seem faster than any - Python one. - -**Profiler Comparison:** - -+-------------------+--------------------------+----------------------+----------------------+ -| Feature | Statistical | Deterministic | Deterministic | -| | (``profiling.sampling``) | (``cProfile``) | (``profile``) | -+===================+==========================+======================+======================+ -| **Target** | Running process | Code you run | Code you run | -+-------------------+--------------------------+----------------------+----------------------+ -| **Overhead** | Virtually none | Moderate | High | -+-------------------+--------------------------+----------------------+----------------------+ -| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts | -+-------------------+--------------------------+----------------------+----------------------+ -| **Setup** | Attach to any PID | Instrument code | Instrument code | -+-------------------+--------------------------+----------------------+----------------------+ -| **Use Case** | Production debugging | Development/testing | Profiler extension | -+-------------------+--------------------------+----------------------+----------------------+ -| **Implementation**| C extension | C extension | Pure Python | -+-------------------+--------------------------+----------------------+----------------------+ +For most code, replacing ``import profile`` with ``import profiling.tracing`` +(and using ``profiling.tracing`` instead of ``profile`` throughout) provides +a straightforward migration path. .. note:: - The statistical profiler (:mod:`!profiling.sampling`) is recommended for most production - use cases due to its extremely low overhead and ability to profile running processes - without modification. It can attach to any Python process and collect performance - data with minimal impact on execution speed, making it ideal for debugging - performance issues in live applications. - - -.. _statistical-profiling: - -What Is Statistical Profiling? -============================== - -:dfn:`Statistical profiling` works by periodically interrupting a running -program to capture its current call stack. Rather than monitoring every -function entry and exit like deterministic profilers, it takes snapshots at -regular intervals to build a statistical picture of where the program spends -its time. - -The sampling profiler uses process memory reading (via system calls like -``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on -Windows) to attach to a running Python process and extract stack trace -information without requiring any code modification or restart of the target -process. This approach provides several key advantages over traditional -profiling methods. - -The fundamental principle is that if a function appears frequently in the -collected stack samples, it is likely consuming significant CPU time. By -analyzing thousands of samples, the profiler can accurately estimate the -relative time spent in different parts of the program. The statistical nature -means that while individual measurements may vary, the aggregate results -converge to represent the true performance characteristics of the application. - -Since statistical profiling operates externally to the target process, it -introduces virtually no overhead to the running program. The profiler process -runs separately and reads the target process memory without interrupting its -execution. This makes it suitable for profiling production systems where -performance impact must be minimized. - -The accuracy of statistical profiling improves with the number of samples -collected. Short-lived functions may be missed or underrepresented, while -long-running functions will be captured proportionally to their execution time. -This characteristic makes statistical profiling particularly effective for -identifying the most significant performance bottlenecks rather than providing -exhaustive coverage of all function calls. - -Statistical profiling excels at answering questions like "which functions -consume the most CPU time?" and "where should I focus optimization efforts?" -rather than "exactly how many times was this function called?" The trade-off -between precision and practicality makes it an invaluable tool for performance -analysis in real-world applications. - -.. _profile-instant: - -Instant User's Manual -===================== - -This section is provided for users that "don't want to read the manual." It -provides a very brief overview, and allows a user to rapidly perform profiling -on an existing application. - -**Statistical Profiling (Recommended for Production):** - -To profile an existing running process:: - - python -m profiling.sampling 1234 - -To profile with custom settings:: - - python -m profiling.sampling -i 50 -d 30 1234 - -**Deterministic Profiling (Development/Testing):** - -To profile a function that takes a single argument, you can do:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")') - -(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on -your system.) - -The above action would run :func:`re.compile` and print profile results like -the following:: - - 214 function calls (207 primitive calls) in 0.002 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec} - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 __init__.py:250(compile) - 1 0.000 0.000 0.001 0.001 __init__.py:289(_compile) - 1 0.000 0.000 0.000 0.000 _compiler.py:759(compile) - 1 0.000 0.000 0.000 0.000 _parser.py:937(parse) - 1 0.000 0.000 0.000 0.000 _compiler.py:598(_code) - 1 0.000 0.000 0.000 0.000 _parser.py:435(_parse_sub) - -The first line indicates that 214 calls were monitored. Of those calls, 207 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time`` indicates the output is sorted -by the ``cumtime`` values. The column headings include: - -ncalls - for the number of calls. - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') - -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -.. _sampling-profiler-cli: - -Statistical Profiler Command Line Interface -=========================================== - -.. program:: profiling.sampling - -The :mod:`!profiling.sampling` module can be invoked as a script to profile running processes:: - - python -m profiling.sampling [options] PID - -**Basic Usage Examples:** - -Profile process 1234 for 10 seconds with default settings:: - - python -m profiling.sampling 1234 - -Profile with custom interval and duration, save to file:: - - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 - -Generate collapsed stacks to use with tools like `flamegraph.pl -`_:: - - python -m profiling.sampling --collapsed 1234 - -Profile all threads, sort by total time:: - - python -m profiling.sampling -a --sort-tottime 1234 - -Profile with real-time sampling statistics:: - - python -m profiling.sampling --realtime-stats 1234 - -**Command Line Options:** - -.. option:: PID - - Process ID of the Python process to profile (required) - -.. option:: -i, --interval INTERVAL - - Sampling interval in microseconds (default: 100) - -.. option:: -d, --duration DURATION - - Sampling duration in seconds (default: 10) - -.. option:: -a, --all-threads - - Sample all threads in the process instead of just the main thread - -.. option:: --realtime-stats - - Print real-time sampling statistics during profiling - -.. option:: --pstats - - Generate pstats output (default) - -.. option:: --collapsed - - Generate collapsed stack traces for flamegraphs - -.. option:: -o, --outfile OUTFILE - - Save output to a file - -**Sorting Options (pstats format only):** - -.. option:: --sort-nsamples - - Sort by number of direct samples - -.. option:: --sort-tottime - - Sort by total time - -.. option:: --sort-cumtime - - Sort by cumulative time (default) - -.. option:: --sort-sample-pct - - Sort by sample percentage - -.. option:: --sort-cumul-pct - - Sort by cumulative sample percentage - -.. option:: --sort-nsamples-cumul - - Sort by cumulative samples - -.. option:: --sort-name - - Sort by function name - -.. option:: -l, --limit LIMIT + The ``cProfile`` module remains available as a backward-compatible alias + to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will + continue to work without modification. - Limit the number of rows in the output (default: 15) -.. option:: --no-summary +:mod:`!profile` and :mod:`!profiling.tracing` module reference +============================================================== - Disable the summary section in the output - -**Understanding Statistical Profile Output:** - -The statistical profiler produces output similar to deterministic profilers but with different column meanings:: - - Profile Stats: - nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) - 45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data) - 23/23 6.4 15.230 6.4 15.230 :0(len) - -**Column Meanings:** - -- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack -- **sample%**: Percentage of total samples where function was directly executing -- **tottime**: Estimated time spent directly in this function -- **cumul%**: Percentage of samples where function was anywhere on call stack -- **cumtime**: Estimated cumulative time including called functions -- **filename:lineno(function)**: Location and name of the function - -.. _profile-cli: - -:mod:`!profiling.sampling` Module Reference -======================================================= - -.. module:: profiling.sampling - :synopsis: Python statistical profiler. - -This section documents the programmatic interface for the :mod:`!profiling.sampling` module. -For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information -about statistical profiling, see :ref:`statistical-profiling` - -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) - - Sample a Python process and generate profiling data. - - This is the main entry point for statistical profiling. It creates a - :class:`SampleProfiler`, collects stack traces from the target process, and - outputs the results in the specified format. - - :param int pid: Process ID of the target Python process - :param int sort: Sort order for pstats output (default: 2 for cumulative time) - :param int sample_interval_usec: Sampling interval in microseconds (default: 100) - :param int duration_sec: Duration to sample in seconds (default: 10) - :param str filename: Output filename (None for stdout/default naming) - :param bool all_threads: Whether to sample all threads (default: False) - :param int limit: Maximum number of functions to display (default: None) - :param bool show_summary: Whether to show summary statistics (default: True) - :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') - :param bool realtime_stats: Whether to display real-time statistics (default: False) - - :raises ValueError: If output_format is not 'pstats' or 'collapsed' - - Examples:: - - # Basic usage - profile process 1234 for 10 seconds - import profiling.sampling - profiling.sampling.sample(1234) - - # Profile with custom settings - profiling.sampling.sample(1234, duration_sec=30, sample_interval_usec=50, all_threads=True) - - # Generate collapsed stack traces for flamegraph.pl - profiling.sampling.sample(1234, output_format='collapsed', filename='profile.collapsed') - -.. class:: SampleProfiler(pid, sample_interval_usec, all_threads) - - Low-level API for the statistical profiler. - - This profiler uses periodic stack sampling to collect performance data - from running Python processes with minimal overhead. It can attach to - any Python process by PID and collect stack traces at regular intervals. - - :param int pid: Process ID of the target Python process - :param int sample_interval_usec: Sampling interval in microseconds - :param bool all_threads: Whether to sample all threads or just the main thread - - .. method:: sample(collector, duration_sec=10) - - Sample the target process for the specified duration. - - Collects stack traces from the target process at regular intervals - and passes them to the provided collector for processing. - - :param collector: Object that implements ``collect()`` method to process stack traces - :param int duration_sec: Duration to sample in seconds (default: 10) - - The method tracks sampling statistics and can display real-time - information if realtime_stats is enabled. - -.. seealso:: - - :ref:`sampling-profiler-cli` - Command-line interface documentation for the statistical profiler. - -Deterministic Profiler Command Line Interface -============================================= - -.. program:: cProfile - -The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to -profile another script. For example:: - - python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) - -.. option:: -o - - Writes the profile results to a file instead of to stdout. - -.. option:: -s - - Specifies one of the :func:`~pstats.Stats.sort_stats` sort values - to sort the output by. - This only applies when :option:`-o ` is not supplied. - -.. option:: -m - - Specifies that a module is being profiled instead of a script. - - .. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. - - .. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. - -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: - - import pstats - from pstats import SortKey - p = pstats.Stats('restats') - p.strip_dirs().sort_stats(-1).print_stats() - -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: - - p.sort_stats(SortKey.NAME) - p.print_stats() - -The first call will actually sort the list by function name, and the second call -will print out the statistics. The following are some interesting calls to -experiment with:: - - p.sort_stats(SortKey.CUMULATIVE).print_stats(10) - -This sorts the profile by cumulative time in a function, and then only prints -the ten most significant lines. If you want to understand what algorithms are -taking time, the above line is what you would use. - -If you were looking to see what functions were looping a lot, and taking a lot -of time, you would do:: - - p.sort_stats(SortKey.TIME).print_stats(10) - -to sort according to time spent within each function, and then print the -statistics for the top ten functions. - -You might also try:: - - p.sort_stats(SortKey.FILENAME).print_stats('__init__') - -This will sort all the statistics by file name, and then print out statistics -for only the class init methods (since they are spelled with ``__init__`` in -them). As one final example, you could try:: - - p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init') - -This line sorts statistics with a primary key of time, and a secondary key of -cumulative time, and then prints out some of the statistics. To be specific, the -list is first culled down to 50% (re: ``.5``) of its original size, then only -lines containing ``init`` are maintained, and that sub-sub-list is printed. - -If you wondered what functions called the above functions, you could now (``p`` -is still sorted according to the last criteria) do:: - - p.print_callers(.5, 'init') - -and you would get a list of callers for each of the listed functions. - -If you want more functionality, you're going to have to read the manual, or -guess what the following functions do:: - - p.print_callees() - p.add('restats') - -Invoked as a script, the :mod:`pstats` module is a statistics browser for -reading and examining profile dumps. It has a simple line-oriented interface -(implemented using :mod:`cmd`) and interactive help. - -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: +Both the :mod:`!profile` and :mod:`profiling.tracing` modules provide the +following functions: .. function:: run(command, filename=None, sort=-1) @@ -545,7 +86,7 @@ functions: .. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. + needed than what the :func:`profiling.tracing.run` function provides. A custom timer can be supplied for measuring how long code takes to run via the *timer* argument. This must be a function that returns a single number @@ -557,9 +98,12 @@ functions: Directly using the :class:`Profile` class allows formatting profile results without writing the profile data to a file:: - import cProfile, pstats, io + import profiling.tracing + import pstats + import io from pstats import SortKey - pr = cProfile.Profile() + + pr = profiling.tracing.Profile() pr.enable() # ... do something ... pr.disable() @@ -570,11 +114,12 @@ functions: print(s.getvalue()) The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: + only in :mod:`profiling.tracing`, not in the deprecated :mod:`!profile` + module; see :ref:`typecontextmanager`):: - import cProfile + import profiling.tracing - with cProfile.Profile() as pr: + with profiling.tracing.Profile() as pr: # ... do something ... pr.print_stats() @@ -584,11 +129,11 @@ functions: .. method:: enable() - Start collecting profiling data. Only in :mod:`cProfile`. + Start collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: disable() - Stop collecting profiling data. Only in :mod:`cProfile`. + Stop collecting profiling data. Only in :mod:`profiling.tracing`. .. method:: create_stats() @@ -602,7 +147,7 @@ functions: The *sort* parameter specifies the sorting order of the displayed statistics. It accepts a single key or a tuple of keys to enable - multi-level sorting, as in :func:`Stats.sort_stats `. + multi-level sorting, as in :meth:`pstats.Stats.sort_stats`. .. versionadded:: 3.13 :meth:`~Profile.print_stats` now accepts a tuple of keys. @@ -629,237 +174,43 @@ returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call during the called command/function execution) no profiling results will be printed. -.. _profile-stats: -The :class:`Stats` Class -======================== +Differences from :mod:`!profiling.tracing` +========================================== -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. +The :mod:`!profile` module differs from :mod:`profiling.tracing` in several +ways: -.. module:: pstats - :synopsis: Statistics object for use with the profiler. +**Higher overhead.** The pure Python implementation is significantly slower +than the C implementation, making it unsuitable for profiling long-running +programs or performance-sensitive code. -.. class:: Stats(*filenames or profile, stream=sys.stdout) +**Calibration support.** The :mod:`!profile` module supports calibration to +compensate for profiling overhead. This is not needed in :mod:`profiling.tracing` +because the C implementation has negligible overhead. - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. +**Custom timers.** Both modules support custom timers, but :mod:`!profile` +accepts timer functions that return tuples (like :func:`os.times`), while +:mod:`profiling.tracing` requires a function returning a single number. - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers, or the same profiler run on a different operating system. If - several files are provided, all the statistics for identical functions will - be coalesced, so that an overall view of several processes can be considered - in a single report. If additional files need to be combined with data in an - existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method - can be used. +**Subclassing.** The pure Python implementation is easier to subclass and +extend for custom profiling behavior. - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument can be either a string or a SortKey - enum identifying the basis of a sort (example: ``'time'``, ``'name'``, - ``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have - advantage over the string argument in that it is more robust and less - error prone. - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort - all the entries according to their function name, and resolve all ties - (identical function names) by sorting by file name. - - For the string argument, abbreviations can be used for any key names, as - long as the abbreviation is unambiguous. - - The following are the valid string and SortKey: - - +------------------+---------------------+----------------------+ - | Valid String Arg | Valid enum Arg | Meaning | - +==================+=====================+======================+ - | ``'calls'`` | SortKey.CALLS | call count | - +------------------+---------------------+----------------------+ - | ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time | - +------------------+---------------------+----------------------+ - | ``'cumtime'`` | N/A | cumulative time | - +------------------+---------------------+----------------------+ - | ``'file'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'filename'`` | SortKey.FILENAME | file name | - +------------------+---------------------+----------------------+ - | ``'module'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'ncalls'`` | N/A | call count | - +------------------+---------------------+----------------------+ - | ``'pcalls'`` | SortKey.PCALLS | primitive call count | - +------------------+---------------------+----------------------+ - | ``'line'`` | SortKey.LINE | line number | - +------------------+---------------------+----------------------+ - | ``'name'`` | SortKey.NAME | function name | - +------------------+---------------------+----------------------+ - | ``'nfl'`` | SortKey.NFL | name/file/line | - +------------------+---------------------+----------------------+ - | ``'stdname'`` | SortKey.STDNAME | standard name | - +------------------+---------------------+----------------------+ - | ``'time'`` | SortKey.TIME | internal time | - +------------------+---------------------+----------------------+ - | ``'tottime'`` | N/A | internal time | - +------------------+---------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a - sort of the name as printed, which means that the embedded line numbers - get compared in an odd way. For example, lines 3, 20, and 40 would (if - the file names were the same) appear in the string order 20, 3 and 40. - In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers. - In fact, ``sort_stats(SortKey.NFL)`` is the same as - ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - .. versionadded:: 3.7 - Added the SortKey enum. - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will be interpreted as a - regular expression (to pattern match the standard name that is printed). - If several restrictions are provided, then they are applied sequentially. - For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - - - .. method:: get_stats_profile() - - This method returns an instance of StatsProfile, which contains a mapping - of function names to instances of FunctionProfile. Each FunctionProfile - instance holds information related to the function's profile such as how - long the function took to run, how many times it was called, etc... - - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. .. _deterministic-profiling: -What Is Deterministic Profiling? +What is deterministic profiling? ================================ :dfn:`Deterministic profiling` is meant to reflect the fact that all *function call*, *function return*, and *exception* events are monitored, and precise timings are made for the intervals between these events (during which time the user's code is executing). In contrast, :dfn:`statistical profiling` (which is -provided by the :mod:`!profiling.sampling` module) periodically samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. +provided by the :mod:`profiling.sampling` module) periodically samples the +effective instruction pointer, and deduces where time is being spent. The +latter technique traditionally involves less overhead (as the code does not +need to be instrumented), but provides only relative indications of where time +is being spent. In Python, since there is an interpreter active during execution, the presence of instrumented code is not required in order to do deterministic profiling. @@ -903,15 +254,16 @@ this error. The error that accumulates in this fashion is typically less than the accuracy of the clock (less than one clock tick), but it *can* accumulate and become very significant. -The problem is more important with :mod:`profile` than with the lower-overhead -:mod:`cProfile`. For this reason, :mod:`profile` provides a means of -calibrating itself for a given platform so that this error can be -probabilistically (on the average) removed. After the profiler is calibrated, it -will be more accurate (in a least square sense), but it will sometimes produce -negative numbers (when call counts are exceptionally low, and the gods of -probability work against you :-). ) Do *not* be alarmed by negative numbers in -the profile. They should *only* appear if you have calibrated your profiler, -and the results are actually better than without calibration. +The problem is more important with the deprecated :mod:`!profile` module than +with the lower-overhead :mod:`profiling.tracing`. For this reason, +:mod:`!profile` provides a means of calibrating itself for a given platform so +that this error can be probabilistically (on the average) removed. After the +profiler is calibrated, it will be more accurate (in a least square sense), but +it will sometimes produce negative numbers (when call counts are exceptionally +low, and the gods of probability work against you :-). ) Do *not* be alarmed +by negative numbers in the profile. They should *only* appear if you have +calibrated your profiler, and the results are actually better than without +calibration. .. _profile-calibration: @@ -919,7 +271,7 @@ and the results are actually better than without calibration. Calibration =========== -The profiler of the :mod:`profile` module subtracts a constant from each event +The profiler of the :mod:`!profile` module subtracts a constant from each event handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see @@ -957,6 +309,7 @@ When you have a consistent answer, there are three ways you can use it:: If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. + .. _profile-timers: Using a custom timer @@ -969,7 +322,7 @@ to the :class:`Profile` class constructor:: pr = profile.Profile(your_time_func) The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, +you are using :class:`profile.Profile` or :class:`profiling.tracing.Profile`, ``your_time_func``'s return value will be interpreted differently: :class:`profile.Profile` @@ -988,20 +341,32 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`, replacement dispatch method that best handles your timer call, along with the appropriate calibration constant. -:class:`cProfile.Profile` +:class:`profiling.tracing.Profile` ``your_time_func`` should return a single number. If it returns integers, you can also invoke the class constructor with a second argument specifying the real duration of one unit of time. For example, if ``your_integer_time_func`` returns times measured in thousands of seconds, you would construct the :class:`Profile` instance as follows:: - pr = cProfile.Profile(your_integer_time_func, 0.001) + pr = profiling.tracing.Profile(your_integer_time_func, 0.001) - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`!_lsprof` module. + As the :class:`profiling.tracing.Profile` class cannot be calibrated, custom + timer functions should be used with care and should be as fast as possible. + For the best results with a custom timer, it might be necessary to hard-code + it in the C source of the internal :mod:`!_lsprof` module. Python 3.3 adds several new functions in :mod:`time` that can be used to make precise measurements of process or wall-clock time. For example, see :func:`time.perf_counter`. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Recommended replacement for this module. + + :mod:`pstats` + Statistical analysis and formatting for profile data. diff --git a/Doc/library/profiling-sampling-visualization.html b/Doc/library/profiling-sampling-visualization.html new file mode 100644 index 000000000000000..0cbd0f2374deaa0 --- /dev/null +++ b/Doc/library/profiling-sampling-visualization.html @@ -0,0 +1 @@ +
diff --git a/Doc/library/profiling.rst b/Doc/library/profiling.rst new file mode 100644 index 000000000000000..f4ac47826b28ef0 --- /dev/null +++ b/Doc/library/profiling.rst @@ -0,0 +1,270 @@ +.. highlight:: sh + +.. _profiling-module: + +************************************** +:mod:`!profiling` --- Python profilers +************************************** + +.. module:: profiling + :synopsis: Python profiling tools for performance analysis. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/` + +-------------- + +.. index:: + single: statistical profiling + single: profiling, statistical + single: deterministic profiling + single: profiling, deterministic + + +Introduction to profiling +========================= + +A :dfn:`profile` is a set of statistics that describes how often and for how +long various parts of a program execute. These statistics help identify +performance bottlenecks and guide optimization efforts. Python provides two +fundamentally different approaches to collecting this information: statistical +sampling and deterministic tracing. + +The :mod:`!profiling` package organizes Python's built-in profiling tools under +a single namespace. It contains two submodules, each implementing a different +profiling methodology: + +:mod:`profiling.sampling` + A statistical profiler that periodically samples the call stack. Run scripts + directly or attach to running processes by PID. Provides multiple output + formats (flame graphs, heatmaps, Firefox Profiler), GIL analysis, GC tracking, + and multiple profiling modes (wall-clock, CPU, GIL) with virtually no overhead. + +:mod:`profiling.tracing` + A deterministic profiler that traces every function call, return, and + exception event. Provides exact call counts and precise timing information, + capturing every invocation including very fast functions. + +.. note:: + + The profiler modules are designed to provide an execution profile for a + given program, not for benchmarking purposes. For benchmarking, use the + :mod:`timeit` module, which provides reasonably accurate timing + measurements. This distinction is particularly important when comparing + Python code against C code: deterministic profilers introduce overhead for + Python code but not for C-level functions, which can skew comparisons. + + +.. _choosing-a-profiler: + +Choosing a profiler +=================== + +For most performance analysis, use the statistical profiler +(:mod:`profiling.sampling`). It has minimal overhead, works for both development +and production, and provides rich visualization options including flame graphs, +heatmaps, GIL analysis, and more. + +Use the deterministic profiler (:mod:`profiling.tracing`) when you need **exact +call counts** and cannot afford to miss any function calls. Since it instruments +every function call and return, it will capture even very fast functions that +complete between sampling intervals. The tradeoff is higher overhead. + +The following table summarizes the key differences: + ++--------------------+------------------------------+------------------------------+ +| Feature | Statistical sampling | Deterministic | +| | (:mod:`profiling.sampling`) | (:mod:`profiling.tracing`) | ++====================+==============================+==============================+ +| **Overhead** | Virtually none | Moderate | ++--------------------+------------------------------+------------------------------+ +| **Accuracy** | Statistical estimate | Exact call counts | ++--------------------+------------------------------+------------------------------+ +| **Output formats** | pstats, flame graph, heatmap,| pstats | +| | gecko, collapsed | | ++--------------------+------------------------------+------------------------------+ +| **Profiling modes**| Wall-clock, CPU, GIL | Wall-clock | ++--------------------+------------------------------+------------------------------+ +| **Special frames** | GC, native (C extensions) | N/A | ++--------------------+------------------------------+------------------------------+ +| **Attach to PID** | Yes | No | ++--------------------+------------------------------+------------------------------+ + + +When to use statistical sampling +-------------------------------- + +The statistical profiler (:mod:`profiling.sampling`) is recommended for most +performance analysis tasks. Use it the same way you would use +:mod:`profiling.tracing`:: + + python -m profiling.sampling run script.py + +One of the main strengths of the sampling profiler is its variety of output +formats. Beyond traditional pstats tables, it can generate interactive +flame graphs that visualize call hierarchies, line-level source heatmaps that +show exactly where time is spent in your code, and Firefox Profiler output for +timeline-based analysis. + +The profiler also provides insight into Python interpreter behavior that +deterministic profiling cannot capture. Use ``--mode gil`` to identify GIL +contention in multi-threaded code, ``--mode cpu`` to measure actual CPU time +excluding I/O waits, or inspect ```` frames to understand garbage collection +overhead. The ``--native`` option reveals time spent in C extensions, helping +distinguish Python overhead from library performance. + +For multi-threaded applications, the ``-a`` option samples all threads +simultaneously, showing how work is distributed. And for production debugging, +the ``attach`` command connects to any running Python process by PID without +requiring a restart or code changes. + + +When to use deterministic tracing +--------------------------------- + +The deterministic profiler (:mod:`profiling.tracing`) instruments every function +call and return. This approach has higher overhead than sampling, but guarantees +complete coverage of program execution. + +The primary reason to choose deterministic tracing is when you need exact call +counts. Statistical profiling estimates frequency based on sampling, which may +undercount short-lived functions that complete between samples. If you need to +verify that an optimization actually reduced the number of function calls, or +if you want to trace the complete call graph to understand caller-callee +relationships, deterministic tracing is the right choice. + +Deterministic tracing also excels at capturing functions that execute in +microseconds. Such functions may not appear frequently enough in statistical +samples, but deterministic tracing records every invocation regardless of +duration. + + +Quick start +=========== + +This section provides the minimal steps needed to start profiling. For complete +documentation, see the dedicated pages for each profiler. + + +Statistical profiling +--------------------- + +To profile a script, use the :mod:`profiling.sampling` module with the ``run`` +command:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +This runs the script under the profiler and prints a summary of where time was +spent. For an interactive flame graph:: + + python -m profiling.sampling run --flamegraph script.py + +To profile an already-running process, use the ``attach`` command with the +process ID:: + + python -m profiling.sampling attach 1234 + +For custom settings, specify the sampling interval (in microseconds) and +duration (in seconds):: + + python -m profiling.sampling run -i 50 -d 30 script.py + + +Deterministic profiling +----------------------- + +To profile a script from the command line:: + + python -m profiling.tracing script.py + +To profile a piece of code programmatically: + +.. code-block:: python + + import profiling.tracing + profiling.tracing.run('my_function()') + +This executes the given code under the profiler and prints a summary showing +exact function call counts and timing. + + +.. _profile-output: + +Understanding profile output +============================ + +Both profilers collect function-level statistics, though they present them in +different formats. The sampling profiler offers multiple visualizations +(flame graphs, heatmaps, Firefox Profiler, pstats tables), while the +deterministic profiler produces pstats-compatible output. Regardless of format, +the underlying concepts are the same. + +Key profiling concepts: + +**Direct time** (also called *self time* or *tottime*) + Time spent executing code in the function itself, excluding time spent in + functions it called. High direct time indicates the function contains + expensive operations. + +**Cumulative time** (also called *total time* or *cumtime*) + Time spent in the function and all functions it called. This measures the + total cost of calling a function, including its entire call subtree. + +**Call count** (also called *ncalls* or *samples*) + How many times the function was called (deterministic) or sampled + (statistical). In deterministic profiling, this is exact. In statistical + profiling, it represents the number of times the function appeared in a + stack sample. + +**Primitive calls** + Calls that are not induced by recursion. When a function recurses, the total + call count includes recursive invocations, but primitive calls counts only + the initial entry. Displayed as ``total/primitive`` (for example, ``3/1`` + means three total calls, one primitive). + +**Caller/Callee relationships** + Which functions called a given function (callers) and which functions it + called (callees). Flame graphs visualize this as nested rectangles; pstats + can display it via the :meth:`~pstats.Stats.print_callers` and + :meth:`~pstats.Stats.print_callees` methods. + + +Legacy compatibility +==================== + +For backward compatibility, the ``cProfile`` module remains available as an +alias to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will +continue to work without modification in all future Python versions. + +.. deprecated:: 3.15 + + The pure Python :mod:`profile` module is deprecated and will be removed in + Python 3.17. Use :mod:`profiling.tracing` (or its alias ``cProfile``) + instead. See :mod:`profile` for migration guidance. + + +.. seealso:: + + :mod:`profiling.sampling` + Statistical sampling profiler with flame graphs, heatmaps, and GIL analysis. + Recommended for most users. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`timeit` + Module for measuring execution time of small code snippets. + + +.. rubric:: Submodules + +.. toctree:: + :maxdepth: 1 + + profiling.tracing.rst + profiling.sampling.rst diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst new file mode 100644 index 000000000000000..aeb1a429b58515a --- /dev/null +++ b/Doc/library/profiling.sampling.rst @@ -0,0 +1,1735 @@ +.. highlight:: sh + +.. _profiling-sampling: + +*************************************************** +:mod:`!profiling.sampling` --- Statistical profiler +*************************************************** + +.. module:: profiling.sampling + :synopsis: Statistical sampling profiler for Python processes. + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/sampling/` + +.. program:: profiling.sampling + +-------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon logo + :align: center + :width: 300px + +The :mod:`!profiling.sampling` module, named **Tachyon**, provides statistical +profiling of Python programs through periodic stack sampling. Tachyon can +run scripts directly or attach to any running Python process without requiring +code changes or restarts. Because sampling occurs externally to the target +process, overhead is virtually zero, making Tachyon suitable for both +development and production environments. + + +What is statistical profiling? +============================== + +Statistical profiling builds a picture of program behavior by periodically +capturing snapshots of the call stack. Rather than instrumenting every function +call and return as deterministic profilers do, Tachyon reads the call stack at +regular intervals to record what code is currently running. + +This approach rests on a simple principle: functions that consume significant +CPU time will appear frequently in the collected samples. By gathering thousands +of samples over a profiling session, Tachyon constructs an accurate statistical +estimate of where time is spent. The more samples collected, the +more precise this estimate becomes. + +.. only:: html + + The following interactive visualization demonstrates how sampling profiling + works. Press **Play** to watch a Python program execute, and observe how the + profiler periodically captures snapshots of the call stack. Adjust the + **sample interval** to see how sampling frequency affects the results. + + .. raw:: html + :file: profiling-sampling-visualization.html + +.. only:: not html + + .. note:: + + An interactive visualization of sampling profiling is available in the + HTML version of this documentation. + + +How time is estimated +--------------------- + +The time values shown in Tachyon's output are **estimates derived from sample +counts**, not direct measurements. Tachyon counts how many times each function +appears in the collected samples, then multiplies by the sampling interval to +estimate time. + +For example, with a 10 kHz sampling rate over a 10-second profile, +Tachyon collects approximately 100,000 samples. If a function appears in 5,000 +samples (5% of total), Tachyon estimates it consumed 5% of the 10-second +duration, or about 500 milliseconds. This is a statistical estimate, not a +precise measurement. + +The accuracy of these estimates depends on sample count. With 100,000 samples, +a function showing 5% has a margin of error of roughly ±0.5%. With only 1,000 +samples, the same 5% measurement could actually represent anywhere from 3% to +7% of real time. + +This is why longer profiling durations and shorter sampling intervals produce +more reliable results---they collect more samples. For most performance +analysis, the default settings provide sufficient accuracy to identify +bottlenecks and guide optimization efforts. + +Because sampling is statistical, results will vary slightly between runs. A +function showing 12% in one run might show 11% or 13% in the next. This is +normal and expected. Focus on the overall pattern rather than exact percentages, +and don't worry about small variations between runs. + + +When to use a different approach +-------------------------------- + +Statistical sampling is not ideal for every situation. + +For very short scripts that complete in under one second, the profiler may not +collect enough samples for reliable results. Use :mod:`profiling.tracing` +instead, or run the script in a loop to extend profiling time. + +When you need exact call counts, sampling cannot provide them. Sampling +estimates frequency from snapshots, so if you need to know precisely how many +times a function was called, use :mod:`profiling.tracing`. + +When comparing two implementations where the difference might be only 1-2%, +sampling noise can obscure real differences. Use :mod:`timeit` for +micro-benchmarks or :mod:`profiling.tracing` for precise measurements. + + +The key difference from :mod:`profiling.tracing` is how measurement happens. +A tracing profiler instruments your code, recording every function call and +return. This provides exact call counts and precise timing but adds overhead +to every function call. A sampling profiler, by contrast, observes the program +from outside at fixed intervals without modifying its execution. Think of the +difference like this: tracing is like having someone follow you and write down +every step you take, while sampling is like taking photographs every second +and inferring your path from those snapshots. + +This external observation model is what makes sampling profiling practical for +production use. The profiled program runs at full speed because there is no +instrumentation code running inside it, and the target process is never stopped +or paused during sampling---Tachyon reads the call stack directly from the +process's memory while it continues to run. You can attach to a live server, +collect data, and detach without the application ever knowing it was observed. +The trade-off is that very short-lived functions may be missed if they happen +to complete between samples. + +Statistical profiling excels at answering the question, "Where is my program +spending time?" It reveals hotspots and bottlenecks in production code where +deterministic profiling overhead would be unacceptable. For exact call counts +and complete call graphs, use :mod:`profiling.tracing` instead. + + +Quick examples +============== + +Profile a script and see the results immediately:: + + python -m profiling.sampling run script.py + +Profile a module with arguments:: + + python -m profiling.sampling run -m mypackage.module arg1 arg2 + +Generate an interactive flame graph:: + + python -m profiling.sampling run --flamegraph -o profile.html script.py + +Attach to a running process by PID:: + + python -m profiling.sampling attach 12345 + +Print a single snapshot of a running process's stack:: + + python -m profiling.sampling dump 12345 + +Use live mode for real-time monitoring (press ``q`` to quit):: + + python -m profiling.sampling run --live script.py + +Profile for 60 seconds with a faster sampling rate:: + + python -m profiling.sampling run -d 60 -r 20khz script.py + +Generate a line-by-line heatmap:: + + python -m profiling.sampling run --heatmap script.py + +Enable opcode-level profiling to see which bytecode instructions are executing:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + + +Commands +======== + +Tachyon operates through several subcommands. ``run`` and ``attach`` collect +samples over time; ``dump`` captures a single snapshot; ``replay`` converts +binary profiles to other formats. + + +The ``run`` command +------------------- + +The ``run`` command launches a Python script or module and profiles it from +startup:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run -m mypackage.module + +When profiling a script, the profiler starts the target in a subprocess, waits +for it to initialize, then begins collecting samples. The ``-m`` flag +indicates that the target should be run as a module (equivalent to +``python -m``). Arguments after the target are passed through to the +profiled program:: + + python -m profiling.sampling run script.py --config settings.yaml + + +The ``attach`` command +---------------------- + +The ``attach`` command connects to an already-running Python process by its +process ID:: + + python -m profiling.sampling attach 12345 + +This command is particularly valuable for investigating performance issues in +production systems. The target process requires no modification and need not +be restarted. The profiler attaches, collects samples for the specified +duration, then detaches and produces output. + +:: + + python -m profiling.sampling attach --live 12345 + python -m profiling.sampling attach --flamegraph -d 30 -o profile.html 12345 + +On most systems, attaching to another process requires appropriate permissions. +See :ref:`profiling-permissions` for platform-specific requirements. + + +.. _dump-command: + +The ``dump`` command +-------------------- + +The ``dump`` command prints a single snapshot of a running process's Python +stack and exits, similar to a traceback:: + + python -m profiling.sampling dump 12345 + +Unlike ``attach``, ``dump`` does not run a sampling loop: it reads the +stack once. This is useful for investigating hung or unresponsive +processes, or for answering "what is this process doing right now?". + +The output mirrors a traceback (most recent call last) and annotates each +thread with its current state (main thread, has GIL, on CPU, waiting for +GIL, has exception, or idle): + +.. code-block:: text + + Stack dump for PID 12345, thread 140735 (main thread, has GIL, on CPU; most recent call last): + File "server.py", line 28, in serve + await handle_request(req) + File "handler.py", line 91, in handle_request + result = expensive_call(req) + +When the target's source files are readable, ``dump`` prints the source +line for each frame and highlights the executing expression. + +Like ``attach``, ``dump`` requires permission to read the target process's +memory. See :ref:`profiling-permissions`. + +The ``dump`` command supports the following options: + +``-a``, ``--all-threads`` + Dump every thread in the target process. Without this flag only the main + thread is shown. + +``--native`` + Include synthetic ```` frames marking transitions into C + extensions or other non-Python code. + +``--no-gc`` + Hide the synthetic ```` frames that mark active garbage collection. + +``--opcodes`` + Annotate each frame with the bytecode opcode the thread is currently + executing (for example, ``opcode=CALL_KW``). Useful for + instruction-level investigation, including identifying specializations + chosen by the adaptive interpreter. + +``--async-aware`` + Reconstruct stacks across ``await`` boundaries. ``dump`` walks the task + graph and emits one section per task, with ```` markers separating + coroutines awaiting each other. + +``--async-mode {running,all}`` + Controls which tasks are included when ``--async-aware`` is enabled. + ``running`` shows only the task currently executing on each thread; + ``all`` (the default for ``dump``) also includes tasks suspended on a + wait. ``attach``'s default for this flag is ``running``; ``dump`` + defaults to ``all`` because a single snapshot is most useful when it + shows the full task graph. + +``--blocking`` + Pause every thread in the target while reading its stack and resume + them after. Guarantees a fully consistent snapshot at the cost of + briefly stopping the target. Without it, ``dump`` reads memory while + the target keeps running, which is faster but can occasionally produce + a torn stack. + + +.. _replay-command: + +The ``replay`` command +---------------------- + +The ``replay`` command converts binary profile files to other output formats:: + + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + +This command is useful when you have captured profiling data in binary format +and want to analyze it later or convert it to a visualization format. Binary +profiles can be replayed multiple times to different formats without +re-profiling. + +:: + + # Convert binary to pstats (default, prints to stdout) + python -m profiling.sampling replay profile.bin + + # Convert binary to flame graph + python -m profiling.sampling replay --flamegraph -o output.html profile.bin + + # Convert binary to gecko format for Firefox Profiler + python -m profiling.sampling replay --gecko -o profile.json profile.bin + + # Convert binary to heatmap + python -m profiling.sampling replay --heatmap -o my_heatmap profile.bin + + +Profiling in production +----------------------- + +The sampling profiler is designed for production use. It imposes no measurable +overhead on the target process because it reads memory externally rather than +instrumenting code. The target application continues running at full speed and +is unaware it is being profiled. + +When profiling production systems, keep these guidelines in mind: + +Start with shorter durations (10-30 seconds) to get quick results, then extend +if you need more statistical accuracy. By default, profiling runs until the +target process completes, which is usually sufficient to identify major hotspots. + +If possible, profile during representative load rather than peak traffic. +Profiles collected during normal operation are easier to interpret than those +collected during unusual spikes. + +The profiler itself consumes some CPU on the machine where it runs (not on the +target process). On the same machine, this is typically negligible. When +profiling remote processes, network latency does not affect the target. + +Results from production may differ from development due to different data +sizes, concurrent load, or caching effects. This is expected and is often +exactly what you want to capture. + + +.. _profiling-permissions: + +Platform requirements +--------------------- + +The profiler reads the target process's memory to capture stack traces. This +requires elevated permissions on most operating systems. + +**Linux** + +On Linux, the profiler uses ``ptrace`` or ``process_vm_readv`` to read the +target process's memory. This typically requires one of: + +- Running as root +- Having the ``CAP_SYS_PTRACE`` capability +- Adjusting the Yama ptrace scope: ``/proc/sys/kernel/yama/ptrace_scope`` + +The default ptrace_scope of 1 restricts ptrace to parent processes only. To +allow attaching to any process owned by the same user, set it to 0:: + + echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + +**macOS** + +On macOS, the profiler uses ``task_for_pid()`` to access the target process. +This requires one of: + +- Running as root +- The profiler binary having the ``com.apple.security.cs.debugger`` entitlement +- System Integrity Protection (SIP) being disabled (not recommended) + +**Windows** + +On Windows, the profiler requires administrative privileges or the +``SeDebugPrivilege`` privilege to read another process's memory. + + +Version compatibility +--------------------- + +The profiler and target process must run the same Python minor version (for +example, both Python 3.15). Attaching from Python 3.14 to a Python 3.15 process +is not supported. + +Additional restrictions apply to pre-release Python versions: if either the +profiler or target is running a pre-release (alpha, beta, or release candidate), +both must run the exact same version. + +On free-threaded Python builds, the profiler cannot attach from a free-threaded +build to a standard build, or vice versa. + + +Sampling configuration +====================== + +Before exploring the various output formats and visualization options, it is +important to understand how to configure the sampling process itself. The +profiler offers several options that control how frequently samples are +collected, how long profiling runs, which threads are observed, and what +additional context is captured in each sample. + +The default configuration works well for most use cases: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Option + - Default + * - Default for ``--sampling-rate`` / ``-r`` + - 1 kHz + * - Default for ``--duration`` / ``-d`` + - Run to completion + * - Default for ``--all-threads`` / ``-a`` + - Main thread only + * - Default for ``--native`` + - No ```` frames (C code time attributed to caller) + * - Default for ``--no-gc`` + - ```` frames included when garbage collection is active + * - Default for ``--mode`` + - Wall-clock mode (all samples recorded) + * - Default for ``--realtime-stats`` + - Disabled + * - Default for ``--subprocesses`` + - Disabled + * - Default for ``--blocking`` + - Disabled (non-blocking sampling) + + +Sampling rate and duration +-------------------------- + +The two most fundamental parameters are the sampling rate and duration. +Together, these determine how many samples will be collected during a profiling +session. + +The :option:`--sampling-rate` option (:option:`-r`) sets how frequently samples +are collected. The default is 1 kHz (1,000 samples per second):: + + python -m profiling.sampling run -r 20khz script.py + +Higher rates capture more samples and provide finer-grained data at the +cost of slightly higher profiler CPU usage. Lower rates reduce profiler +overhead but may miss short-lived functions. For most applications, the +default rate provides a good balance between accuracy and overhead. + +The :option:`--duration` option (:option:`-d`) sets how long to profile in seconds. By +default, profiling continues until the target process exits or is interrupted:: + + python -m profiling.sampling run -d 60 script.py + +Specifying a duration is useful when attaching to long-running processes or when +you want to limit profiling to a specific time window. When profiling a script, +the default behavior of running to completion is usually what you want. + + +Thread selection +---------------- + +Python programs often use multiple threads, whether explicitly through the +:mod:`threading` module or implicitly through libraries that manage thread +pools. + +By default, the profiler samples only the main thread. The :option:`--all-threads` +option (:option:`-a`) enables sampling of all threads in the process:: + + python -m profiling.sampling run -a script.py + +Multi-thread profiling reveals how work is distributed across threads and can +identify threads that are blocked or starved. Each thread's samples are +combined in the output, with the ability to filter by thread in some formats. +This option is particularly useful when investigating concurrency issues or +when work is distributed across a thread pool. + + +.. _blocking-mode: + +Blocking mode +------------- + +By default, Tachyon reads the target process's memory without stopping it. +This non-blocking approach is ideal for most profiling scenarios because it +imposes virtually zero overhead on the target application: the profiled +program runs at full speed and is unaware it is being observed. + +However, non-blocking sampling can occasionally produce incomplete or +inconsistent stack traces in applications with many generators or coroutines +that rapidly switch between yield points, or in programs with very fast-changing +call stacks where functions enter and exit between the start and end of a single +stack read, resulting in reconstructed stacks that mix frames from different +execution states or that never actually existed. + +For these cases, the :option:`--blocking` option stops the target process during +each sample:: + + python -m profiling.sampling run --blocking script.py + python -m profiling.sampling attach --blocking 12345 + +When blocking mode is enabled, the profiler suspends the target process, +reads its stack, then resumes it. This guarantees that each captured stack +represents a real, consistent snapshot of what the process was doing at that +instant. The trade-off is that the target process runs slower because it is +repeatedly paused. + +.. warning:: + + Do not use very high sample rates (low ``--interval`` values) with blocking + mode. Suspending and resuming a process takes time, and if the sampling + interval is too short, the target will spend more time stopped than running. + For blocking mode, intervals of 1000 microseconds (1 millisecond) or higher + are recommended. The default 100 microsecond interval may cause noticeable + slowdown in the target application. + +Use blocking mode only when you observe inconsistent stacks in your profiles, +particularly with generator-heavy or coroutine-heavy code. For most +applications, the default non-blocking mode provides accurate results with +zero impact on the target process. + + +Special frames +-------------- + +The profiler can inject artificial frames into the captured stacks to provide +additional context about what the interpreter is doing at the moment each +sample is taken. These synthetic frames help distinguish different types of +execution that would otherwise be invisible. + +The :option:`--native` option adds ```` frames to indicate when Python has +called into C code (extension modules, built-in functions, or the interpreter +itself):: + + python -m profiling.sampling run --native script.py + +These frames help distinguish time spent in Python code versus time spent in +native libraries. Without this option, native code execution appears as time +in the Python function that made the call. This is useful when optimizing +code that makes heavy use of C extensions like NumPy or database drivers. + +By default, the profiler includes ```` frames when garbage collection is +active. The :option:`--no-gc` option suppresses these frames:: + + python -m profiling.sampling run --no-gc script.py + +GC frames help identify programs where garbage collection consumes significant +time, which may indicate memory allocation patterns worth optimizing. If you +see substantial time in ```` frames, consider investigating object +allocation rates or using object pooling. + + +Opcode-aware profiling +---------------------- + +The :option:`--opcodes` option enables instruction-level profiling that captures +which Python bytecode instructions are executing at each sample:: + + python -m profiling.sampling run --opcodes --flamegraph script.py + +This feature provides visibility into Python's bytecode execution, including +adaptive specialization optimizations. When a generic instruction like +``LOAD_ATTR`` is specialized at runtime into a more efficient variant like +``LOAD_ATTR_INSTANCE_VALUE``, the profiler shows both the specialized name +and the base instruction. + +Opcode information appears in several output formats: + +- **Flame graphs**: Hovering over a frame displays a tooltip with a bytecode + instruction breakdown, showing which opcodes consumed time in that function +- **Heatmap**: Expandable bytecode panels per source line show instruction + breakdown with specialization percentages +- **Live mode**: An opcode panel shows instruction-level statistics for the + selected function, accessible via keyboard navigation +- **Gecko format**: Opcode transitions are emitted as interval markers in the + Firefox Profiler timeline + +This level of detail is particularly useful for: + +- Understanding the performance impact of Python's adaptive specialization +- Identifying hot bytecode instructions that might benefit from optimization +- Analyzing the effectiveness of different code patterns at the instruction level +- Debugging performance issues that occur at the bytecode level + +The :option:`--opcodes` option is compatible with :option:`--live`, :option:`--flamegraph`, +:option:`--heatmap`, and :option:`--gecko` formats. It requires additional memory to store +opcode information and may slightly reduce sampling performance, but provides +unprecedented visibility into Python's execution model. + + +Real-time statistics +-------------------- + +The :option:`--realtime-stats` option displays sampling rate statistics during +profiling:: + + python -m profiling.sampling run --realtime-stats script.py + +This shows the actual achieved sampling rate, which may be lower than requested +if the profiler cannot keep up. The statistics help verify that profiling is +working correctly and that sufficient samples are being collected. See +:ref:`sampling-efficiency` for details on interpreting these metrics. + + +Subprocess profiling +-------------------- + +The :option:`--subprocesses` option enables automatic profiling of subprocesses +spawned by the target:: + + python -m profiling.sampling run --subprocesses script.py + python -m profiling.sampling attach --subprocesses 12345 + +When enabled, the profiler monitors the target process for child process +creation. When a new Python child process is detected, a separate profiler +instance is automatically spawned to profile it. This is useful for +applications that use :mod:`multiprocessing`, :mod:`subprocess`, +:mod:`concurrent.futures` with :class:`~concurrent.futures.ProcessPoolExecutor`, +or other process spawning mechanisms. + +.. code-block:: python + :caption: worker_pool.py + + from concurrent.futures import ProcessPoolExecutor + import math + + def compute_factorial(n): + total = 0 + for i in range(50): + total += math.factorial(n) + return total + + if __name__ == "__main__": + numbers = [5000 + i * 100 for i in range(50)] + with ProcessPoolExecutor(max_workers=4) as executor: + results = list(executor.map(compute_factorial, numbers)) + print(f"Computed {len(results)} factorials") + +:: + + python -m profiling.sampling run --subprocesses --flamegraph worker_pool.py + +This produces separate flame graphs for the main process and each worker +process: ``flamegraph_.html``, ``flamegraph_.html``, +and so on. + +Each subprocess receives its own output file. The filename is derived from +the specified output path (or the default) with the subprocess's process ID +appended: + +- If you specify ``-o profile.html``, subprocesses produce ``profile_12345.html``, + ``profile_12346.html``, and so on +- With default output, subprocesses produce files like ``flamegraph_12345.html`` + or directories like ``heatmap_12345`` +- For pstats format (which defaults to stdout), subprocesses produce files like + ``profile_12345.pstats`` + +The subprocess profilers inherit most sampling options from the parent (sampling +rate, duration, thread selection, native frames, GC frames, async-aware mode, +and output format). All Python descendant processes are profiled recursively, +including grandchildren and further descendants. + +Subprocess detection works by periodically scanning for new descendants of +the target process and checking whether each new process is a Python process +by probing the process memory for Python runtime structures. Non-Python +subprocesses (such as shell commands or external tools) are ignored. + +There is a limit of 100 concurrent subprocess profilers to prevent resource +exhaustion in programs that spawn many processes. If this limit is reached, +additional subprocesses are not profiled and a warning is printed. + +The :option:`--subprocesses` option is incompatible with :option:`--live` mode +because live mode uses an interactive terminal interface that cannot +accommodate multiple concurrent profiler displays. + + +.. _sampling-efficiency: + +Sampling efficiency +------------------- + +Sampling efficiency metrics help assess the quality of the collected data. +These metrics appear in the profiler's terminal output and in the flame graph +sidebar. + +**Sampling efficiency** is the percentage of sample attempts that succeeded. +Each sample attempt reads the target process's call stack from memory. An +attempt can fail if the process is in an inconsistent state at the moment of +reading, such as during a context switch or while the interpreter is updating +its internal structures. A low efficiency may indicate that the profiler could +not keep up with the requested sampling rate, often due to system load or an +overly aggressive interval setting. + +**Missed samples** is the percentage of expected samples that were not +collected. Based on the configured interval and duration, the profiler expects +to collect a certain number of samples. Some samples may be missed if the +profiler falls behind schedule, for example when the system is under heavy +load. A small percentage of missed samples is normal and does not significantly +affect the statistical accuracy of the profile. + +Both metrics are informational. Even with some failed attempts or missed +samples, the profile remains statistically valid as long as enough samples +were collected. The profiler reports the actual number of samples captured, +which you can use to judge whether the data is sufficient for your analysis. + + +Profiling modes +=============== + +The sampling profiler supports four modes that control which samples are +recorded. The mode determines what the profile measures: total elapsed time, +CPU execution time, time spent holding the global interpreter lock, or +exception handling. + + +Wall-clock mode +--------------- + +Wall-clock mode (:option:`--mode`\ ``=wall``) captures all samples regardless of what the +thread is doing. This is the default mode and provides a complete picture of +where time passes during program execution:: + + python -m profiling.sampling run --mode=wall script.py + +In wall-clock mode, samples are recorded whether the thread is actively +executing Python code, waiting for I/O, blocked on a lock, or sleeping. +This makes wall-clock profiling ideal for understanding the overall time +distribution in your program, including time spent waiting. + +If your program spends significant time in I/O operations, network calls, or +sleep, wall-clock mode will show these waits as time attributed to the calling +function. This is often exactly what you want when optimizing end-to-end +latency. + + +CPU mode +-------- + +CPU mode (:option:`--mode`\ ``=cpu``) records samples only when the thread is actually +executing on a CPU core:: + + python -m profiling.sampling run --mode=cpu script.py + +Samples taken while the thread is sleeping, blocked on I/O, or waiting for +a lock are discarded. The resulting profile shows where CPU cycles are consumed, +filtering out idle time. + +CPU mode is useful when you want to focus on computational hotspots without +being distracted by I/O waits. If your program alternates between computation +and network calls, CPU mode reveals which computational sections are most +expensive. + + +Comparing wall-clock and CPU profiles +------------------------------------- + +Running both wall-clock and CPU mode profiles can reveal whether a function's +time is spent computing or waiting. + +If a function appears prominently in both profiles, it is a true computational +hotspot---actively using the CPU. Optimization should focus on algorithmic +improvements or more efficient code. + +If a function is high in wall-clock mode but low or absent in CPU mode, it is +I/O-bound or waiting. The function spends most of its time waiting for network, +disk, locks, or sleep. CPU optimization won't help here; consider async I/O, +connection pooling, or reducing wait time instead. + +.. code-block:: python + + import time + + def do_sleep(): + time.sleep(2) + + def do_compute(): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + do_sleep() + do_compute() + +:: + + python -m profiling.sampling run --mode=wall script.py # do_sleep ~98%, do_compute ~1% + python -m profiling.sampling run --mode=cpu script.py # do_sleep absent, do_compute dominates + + +GIL mode +-------- + +GIL mode (:option:`--mode`\ ``=gil``) records samples only when the thread holds Python's +global interpreter lock:: + + python -m profiling.sampling run --mode=gil script.py + +The GIL is held only while executing Python bytecode. When Python calls into +C extensions, performs I/O operations, or executes native code, the GIL is +typically released. This means GIL mode effectively measures time spent +running Python code specifically, filtering out time in native libraries. + +In multi-threaded programs, GIL mode reveals which code is preventing other +threads from running Python bytecode. Since only one thread can hold the GIL +at a time, functions that appear frequently in GIL mode profiles are +monopolizing the interpreter. + +GIL mode helps answer questions like "which functions are monopolizing the +GIL?" and "why are my other threads starving?" It can also be useful in +single-threaded programs to distinguish Python execution time from time spent +in C extensions or I/O. + +.. code-block:: python + + import hashlib + + def hash_work(): + # C extension - releases GIL during computation + for _ in range(200): + hashlib.sha256(b"data" * 250000).hexdigest() + + def python_work(): + # Pure Python - holds GIL during computation + for _ in range(3): + sum(i**2 for i in range(1000000)) + + if __name__ == "__main__": + hash_work() + python_work() + +:: + + python -m profiling.sampling run --mode=cpu script.py # hash_work ~42%, python_work ~38% + python -m profiling.sampling run --mode=gil script.py # hash_work ~5%, python_work ~60% + + +Exception mode +-------------- + +Exception mode (``--mode=exception``) records samples only when a thread has +an active exception:: + + python -m profiling.sampling run --mode=exception script.py + +Samples are recorded in two situations: when an exception is being propagated +up the call stack (after ``raise`` but before being caught), or when code is +executing inside an ``except`` block where exception information is still +present in the thread state. + +The following example illustrates which code regions are captured: + +.. code-block:: python + + def example(): + try: + raise ValueError("error") # Captured: exception being raised + except ValueError: + process_error() # Captured: inside except block + finally: + cleanup() # NOT captured: exception already handled + + def example_propagating(): + try: + try: + raise ValueError("error") + finally: + cleanup() # Captured: exception propagating through + except ValueError: + pass + + def example_no_exception(): + try: + do_work() + finally: + cleanup() # NOT captured: no exception involved + +Note that ``finally`` blocks are only captured when an exception is actively +propagating through them. Once an ``except`` block finishes executing, Python +clears the exception information before running any subsequent ``finally`` +block. Similarly, ``finally`` blocks that run during normal execution (when no +exception was raised) are not captured because no exception state is present. + +This mode is useful for understanding where your program spends time handling +errors. Exception handling can be a significant source of overhead in code +that uses exceptions for flow control (such as ``StopIteration`` in iterators) +or in applications that process many error conditions (such as network servers +handling connection failures). + +Exception mode helps answer questions like "how much time is spent handling +exceptions?" and "which exception handlers are the most expensive?" It can +reveal hidden performance costs in code that catches and processes many +exceptions, even when those exceptions are handled gracefully. For example, +if a parsing library uses exceptions internally to signal format errors, this +mode will capture time spent in those handlers even if the calling code never +sees the exceptions. + + +Output formats +============== + +The profiler produces output in several formats, each suited to different +analysis workflows. The format is selected with a command-line flag, and +output goes to stdout, a file, or a directory depending on the format. + + +pstats format +------------- + +The pstats format (:option:`--pstats`) produces a text table similar to what +deterministic profilers generate. This is the default output format:: + + python -m profiling.sampling run script.py + python -m profiling.sampling run --pstats script.py + +.. figure:: tachyon-pstats.png + :alt: Tachyon pstats terminal output + :align: center + :width: 100% + + The pstats format displays profiling results in a color-coded table showing + function hotspots, sample counts, and timing estimates. + +Output appears on stdout by default:: + + Profile Stats (Mode: wall): + nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function) + 234/892 11.7% 234.00 44.6% 892.00 server.py:145(handle_request) + 156/156 7.8% 156.00 7.8% 156.00 :0(socket.recv) + 98/421 4.9% 98.00 21.1% 421.00 parser.py:67(parse_message) + +The columns show sampling counts and estimated times: + +- **nsamples**: Displayed as ``direct/cumulative`` (for example, ``10/50``). + Direct samples are when the function was at the top of the stack, actively + executing. Cumulative samples are when the function appeared anywhere on the + stack, including when it was waiting for functions it called. If a function + shows ``10/50``, it was directly executing in 10 samples and was on the call + stack in 50 samples total. + +- **sample%** and **cumul%**: Percentages of total samples for direct and + cumulative counts respectively. + +- **tottime** and **cumtime**: Estimated wall-clock time based on sample counts + and the profiling duration. Time units are selected automatically based on + the magnitude: seconds for large values, milliseconds for moderate values, + or microseconds for small values. + +The output includes a legend explaining each column and a summary of +interesting functions that highlights: + +- **Hot spots**: Functions with high direct/cumulative sample ratio (ratio + close to 1.0). These functions spend most of their time executing their own + code rather than waiting for callees. High ratios indicate where CPU time + is actually consumed. + +- **Indirect calls**: Functions with large differences between cumulative and + direct samples. These are orchestration functions that delegate work to + other functions. They appear frequently on the stack but rarely at the top. + +- **Call magnification**: Functions where cumulative samples far exceed direct + samples (high cumulative/direct multiplier). These are frequently-nested + functions that appear deep in many call chains. + +Use :option:`--no-summary` to suppress both the legend and summary sections. + +To save pstats output to a binary file instead of stdout:: + + python -m profiling.sampling run -o profile.pstats script.py + +The pstats format supports several options for controlling the display. +The :option:`--sort` option determines the column used for ordering results:: + + python -m profiling.sampling run --sort=tottime script.py + python -m profiling.sampling run --sort=cumtime script.py + python -m profiling.sampling run --sort=nsamples script.py + +The :option:`--limit` option restricts output to the top N entries:: + + python -m profiling.sampling run --limit=30 script.py + +The :option:`--no-summary` option suppresses the header summary that precedes the +statistics table. + + +Collapsed stacks format +----------------------- + +Collapsed stacks format (:option:`--collapsed`) produces one line per unique call +stack, with a count of how many times that stack was sampled:: + + python -m profiling.sampling run --collapsed script.py + +The output looks like: + +.. code-block:: text + + main;process_data;parse_json;decode_utf8 42 + main;process_data;parse_json 156 + main;handle_request;send_response 89 + +Each line contains semicolon-separated function names representing the call +stack from bottom to top, followed by a space and the sample count. This +format is designed for compatibility with external flame graph tools, +particularly Brendan Gregg's ``flamegraph.pl`` script. + +To generate a flame graph from collapsed stacks:: + + python -m profiling.sampling run --collapsed script.py > stacks.txt + flamegraph.pl stacks.txt > profile.svg + +The resulting SVG can be viewed in any web browser and provides an interactive +visualization where you can click to zoom into specific call paths. + + +Flame graph format +------------------ + +Flame graph format (:option:`--flamegraph`) produces a self-contained HTML file with +an interactive flame graph visualization:: + + python -m profiling.sampling run --flamegraph script.py + python -m profiling.sampling run --flamegraph -o profile.html script.py + +.. figure:: tachyon-flamegraph.png + :alt: Tachyon interactive flame graph + :align: center + :width: 100% + + The flame graph visualization shows call stacks as nested rectangles, with + width proportional to time spent. The sidebar displays runtime statistics, + GIL metrics, and hotspot functions. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-flamegraph.html>`__! + +If no output file is specified, the profiler generates a filename based on +the process ID (for example, ``flamegraph.12345.html``). + +The generated HTML file requires no external dependencies and can be opened +directly in a web browser. The visualization displays call stacks as nested +rectangles, with width proportional to time spent. Hovering over a rectangle +shows details about that function including source code context, and clicking +zooms into that portion of the call tree. + +The flame graph interface includes: + +- A sidebar showing profile summary, thread statistics, sampling efficiency + metrics (see :ref:`sampling-efficiency`), and top hotspot functions +- Search functionality supporting both function name matching and + ``file.py:42`` line patterns +- Per-thread filtering via dropdown +- Dark/light theme toggle (preference saved across sessions) +- SVG export for saving the current view + +The thread statistics section shows runtime behavior metrics: + +- **GIL Held**: percentage of samples where a thread held the global interpreter + lock (actively running Python code) +- **GIL Released**: percentage of samples where no thread held the GIL +- **Waiting GIL**: percentage of samples where a thread was waiting to acquire + the GIL +- **GC**: percentage of samples during garbage collection + +These statistics help identify GIL contention and understand how time is +distributed between Python execution, native code, and waiting. + +Flame graphs are particularly effective for identifying deep call stacks and +understanding the hierarchical structure of time consumption. Wide rectangles +at the top indicate functions that consume significant time either directly +or through their callees. + + +Differential flame graphs +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Differential flame graphs compare two profiling runs to highlight where +performance changed. This helps identify regressions introduced by code changes +and validate that optimizations achieved their intended effect:: + + # Capture baseline profile + python -m profiling.sampling run --binary -o baseline.bin script.py + + # After modifying code, generate differential flamegraph + python -m profiling.sampling run --diff-flamegraph baseline.bin -o diff.html script.py + +The visualization draws the current profile with frame widths showing current +time consumption, then applies color to indicate how each function changed +relative to the baseline. + +**Color coding**: + +- **Red**: Functions consuming more time (regressions). Lighter shades indicate + modest increases, while darker shades show severe regressions. + +- **Blue**: Functions consuming less time (improvements). Lighter shades for + modest reductions, darker shades for significant speedups. + +- **Gray**: Minimal or no change. + +- **Purple**: New functions not present in the baseline. + +Frame colors indicate changes in **direct time** (time when the function was at +the top of the stack, actively executing), not cumulative time including callees. +Hovering over a frame shows comparison details including baseline time, current +time, and the percentage change. + +Some call paths may disappear entirely between profiles. These are called +**elided stacks** and occur when optimizations eliminate code paths or certain +branches stop executing. If elided stacks are present, an elided toggle appears +allowing you to switch between the main differential view and an elided-only +view that shows just the removed paths (colored purple). + + +Gecko format +------------ + +Gecko format (:option:`--gecko`) produces JSON output compatible with the Firefox +Profiler:: + + python -m profiling.sampling run --gecko script.py + python -m profiling.sampling run --gecko -o profile.json script.py + +The `Firefox Profiler `__ is a sophisticated +web-based tool originally built for profiling Firefox itself. It provides +features beyond basic flame graphs, including a timeline view, call tree +exploration, and marker visualization. See the +`Firefox Profiler documentation `__ for +detailed usage instructions. + +To use the output, open the Firefox Profiler in your browser and load the +JSON file. The profiler runs entirely client-side, so your profiling data +never leaves your machine. + +Gecko format automatically collects additional metadata about GIL state and +CPU activity, enabling analysis features specific to Python's threading model. +The profiler emits interval markers that appear as colored bands in the +Firefox Profiler timeline: + +- **GIL markers**: show when threads hold or release the global interpreter lock +- **CPU markers**: show when threads are executing on CPU versus idle +- **Code type markers**: distinguish Python code from native (C extension) code +- **GC markers**: indicate garbage collection activity + +For this reason, the :option:`--mode` option is not available with Gecko format; +all relevant data is captured automatically. + +.. figure:: tachyon-gecko-calltree.png + :alt: Firefox Profiler Call Tree view + :align: center + :width: 100% + + The Call Tree view shows the complete call hierarchy with sample counts + and percentages. The sidebar displays detailed statistics for the + selected function including running time and sample distribution. + +.. figure:: tachyon-gecko-flamegraph.png + :alt: Firefox Profiler Flame Graph view + :align: center + :width: 100% + + The Flame Graph visualization shows call stacks as nested rectangles. + Functions names are visible in the call hierarchy. + +.. figure:: tachyon-gecko-opcodes.png + :alt: Firefox Profiler Marker Chart with opcodes + :align: center + :width: 100% + + The Marker Chart displays interval markers including CPU state, GIL + status, and opcodes. With ``--opcodes`` enabled, bytecode instructions + like ``BINARY_OP_ADD_FLOAT``, ``CALL_PY_EXACT_ARGS``, and + ``CALL_LIST_APPEND`` appear as markers showing execution over time. + + +Heatmap format +-------------- + +Heatmap format (:option:`--heatmap`) generates an interactive HTML visualization +showing sample counts at the source line level:: + + python -m profiling.sampling run --heatmap script.py + python -m profiling.sampling run --heatmap -o my_heatmap script.py + +.. figure:: tachyon-heatmap.png + :alt: Tachyon heatmap visualization + :align: center + :width: 100% + + The heatmap overlays sample counts directly on your source code. Lines are + color-coded from cool (few samples) to hot (many samples). Navigation + buttons (▲▼) let you jump between callers and callees. + +Unlike other formats that produce a single file, heatmap output creates a +directory containing HTML files for each profiled source file. If no output +path is specified, the directory is named ``heatmap_PID``. + +The heatmap visualization displays your source code with a color gradient +indicating how many samples were collected at each line. Hot lines (many +samples) appear in warm colors, while cold lines (few or no samples) appear +in cool colors. This view helps pinpoint exactly which lines of code are +responsible for time consumption. + +The heatmap interface provides several interactive features: + +- **Coloring modes**: toggle between "Self Time" (direct execution) and + "Total Time" (cumulative, including time in called functions) +- **Cold code filtering**: show all lines or only lines with samples +- **Call graph navigation**: each line shows navigation buttons (▲ for callers, + ▼ for callees) that let you trace execution paths through your code. When + multiple functions called or were called from a line, a menu appears showing + all options with their sample counts. +- **Scroll minimap**: a vertical overview showing the heat distribution across + the entire file +- **Hierarchical index**: files organized by type (stdlib, site-packages, + project) with aggregate sample counts per folder +- **Dark/light theme**: toggle with preference saved across sessions +- **Line linking**: click line numbers to create shareable URLs + +When opcode-level profiling is enabled with :option:`--opcodes`, each hot line +can be expanded to show which bytecode instructions consumed time: + +.. figure:: tachyon-heatmap-with-opcodes.png + :alt: Heatmap with expanded bytecode panel + :align: center + :width: 100% + + Expanding a hot line reveals the bytecode instructions executed, including + specialized variants. The panel shows sample counts per instruction and the + overall specialization percentage for the line. + +.. only:: html + + `Try the interactive example <../_static/tachyon-example-heatmap.html>`__! + +Heatmaps are especially useful when you know which file contains a performance +issue but need to identify the specific lines. Many developers prefer this +format because it maps directly to their source code, making it easy to read +and navigate. For smaller scripts and focused analysis, heatmaps provide an +intuitive view that shows exactly where time is spent without requiring +interpretation of hierarchical visualizations. + + +Binary format +------------- + +Binary format (:option:`--binary`) produces a compact binary file for efficient +storage of profiling data:: + + python -m profiling.sampling run --binary -o profile.bin script.py + python -m profiling.sampling attach --binary -o profile.bin 12345 + +The :option:`--compression` option controls data compression: + +- ``auto`` (default): Use zstd compression if available, otherwise no + compression +- ``zstd``: Force zstd compression (requires :mod:`compression.zstd` support) +- ``none``: Disable compression + +:: + + python -m profiling.sampling run --binary --compression=zstd -o profile.bin script.py + +To analyze binary profiles, use the :ref:`replay-command` to convert them to +other formats like flame graphs or pstats output. + + +Record and replay workflow +========================== + +The binary format combined with the replay command enables a record-and-replay +workflow that separates data capture from analysis. Rather than generating +visualizations during profiling, you capture raw data to a compact binary file +and convert it to different formats later. + +This approach has three main benefits: + +- Sampling runs faster because the work of building data structures for + visualization is deferred until replay. +- A single binary capture can be converted to multiple output formats + without re-profiling: pstats for a quick overview, flame graph for visual + exploration, heatmap for line-level detail. +- Binary files are compact and easy to share with colleagues who can convert + them to their preferred format. + +A typical workflow:: + + # Capture profile in production or during tests + python -m profiling.sampling attach --binary -o profile.bin 12345 + + # Later, analyze with different formats + python -m profiling.sampling replay profile.bin + python -m profiling.sampling replay --flamegraph -o profile.html profile.bin + python -m profiling.sampling replay --heatmap -o heatmap profile.bin + + +Live mode +========= + +Live mode (:option:`--live`) provides a terminal-based real-time view of profiling +data, similar to the ``top`` command for system processes:: + + python -m profiling.sampling run --live script.py + python -m profiling.sampling attach --live 12345 + +.. only:: not latex + + .. figure:: tachyon-live-mode-2.gif + :alt: Tachyon live mode showing all threads + :align: center + :width: 100% + + Live mode displays real-time profiling statistics, showing combined + data from multiple threads in a multi-threaded application. + +The display updates continuously as new samples arrive, showing the current +hottest functions. This mode requires the :mod:`curses` module, which is +available on Unix-like systems but not on Windows. The terminal must be at +least 60 columns wide and 12 lines tall; larger terminals display more columns. + +The header displays the top 3 hottest functions, sampling efficiency metrics, +and thread status statistics (GIL held percentage, CPU usage, GC time). The +main table shows function statistics with the currently sorted column indicated +by an arrow (▼). + +When :option:`--opcodes` is enabled, an additional opcode panel appears below the +main table, showing instruction-level statistics for the currently selected +function. This panel displays which bytecode instructions are executing most +frequently, including specialized variants and their base opcodes. + +.. only:: not latex + + .. figure:: tachyon-live-mode-1.gif + :alt: Tachyon live mode with opcode panel + :align: center + :width: 100% + + Live mode with ``--opcodes`` enabled shows an opcode panel with a bytecode + instruction breakdown for the selected function. + + +Keyboard commands +----------------- + +Within live mode, keyboard commands control the display: + +:kbd:`q` + Quit the profiler and return to the shell. + +:kbd:`s` / :kbd:`S` + Cycle through sort orders forward/backward (sample count, percentage, + total time, cumulative percentage, cumulative time). + +:kbd:`p` + Pause or resume display updates. Sampling continues in the background + while the display is paused, so you can freeze the view to examine results + without stopping data collection. + +:kbd:`r` + Reset all statistics and start fresh. This is disabled after profiling + finishes to prevent accidental data loss. + +:kbd:`/` + Enter filter mode to search for functions by name. The filter uses + case-insensitive substring matching against the filename and function name. + Type a pattern and press Enter to apply, or Escape to cancel. Glob patterns + and regular expressions are not supported. + +:kbd:`c` + Clear the current filter and show all functions again. + +:kbd:`t` + Toggle between viewing all threads combined or per-thread statistics. + In per-thread mode, a thread counter (for example, ``1/4``) appears showing + your position among the available threads. + +:kbd:`←` :kbd:`→` or :kbd:`↑` :kbd:`↓` + In per-thread view, navigate between threads. Navigation wraps around + from the last thread to the first and vice versa. + +:kbd:`+` / :kbd:`-` + Increase or decrease the display refresh rate. The range is 0.05 seconds + (20 Hz, very responsive) to 1.0 second (1 Hz, lower overhead). Faster refresh + rates use more CPU. The default is 0.1 seconds (10 Hz). + +:kbd:`x` + Toggle trend indicators that show whether functions are becoming hotter + or cooler over time. When enabled, increasing metrics appear in green and + decreasing metrics appear in red, comparing each update to the previous one. + +:kbd:`h` or :kbd:`?` + Show the help screen with all available commands. + +:kbd:`j` / :kbd:`k` (or :kbd:`Up` / :kbd:`Down`) + Navigate through opcode entries in the opcode panel (when ``--opcodes`` is + enabled). These keys scroll through the instruction-level statistics for the + currently selected function. + +When profiling finishes (duration expires or target process exits), the display +shows a "PROFILING COMPLETE" banner and freezes the final results. You can +still navigate, sort, and filter the results before pressing :kbd:`q` to exit. + +Live mode is incompatible with output format options (:option:`--collapsed`, +:option:`--flamegraph`, and so on) because it uses an interactive terminal +interface rather than producing file output. + + +Async-aware profiling +===================== + +For programs using :mod:`asyncio`, the profiler offers async-aware mode +(:option:`--async-aware`) that reconstructs call stacks based on the task structure +rather than the raw Python frames:: + + python -m profiling.sampling run --async-aware async_script.py + +Standard profiling of async code can be confusing because the physical call +stack often shows event loop internals rather than the logical flow of your +coroutines. Async-aware mode addresses this by tracking which task is running +and presenting stacks that reflect the ``await`` chain. + +.. code-block:: python + + import asyncio + + async def fetch(url): + await asyncio.sleep(0.1) + return url + + async def main(): + for _ in range(50): + await asyncio.gather(fetch("a"), fetch("b"), fetch("c")) + + if __name__ == "__main__": + asyncio.run(main()) + +:: + + python -m profiling.sampling run --async-aware --flamegraph -o out.html script.py + +.. note:: + + Async-aware profiling requires the target process to have the :mod:`asyncio` + module loaded. If you profile a script before it imports asyncio, async-aware + mode will not be able to capture task information. + + +Async modes +----------- + +The :option:`--async-mode` option controls which tasks appear in the profile:: + + python -m profiling.sampling run --async-aware --async-mode=running async_script.py + python -m profiling.sampling run --async-aware --async-mode=all async_script.py + +With :option:`--async-mode`\ ``=running`` (the default), only the task currently executing +on the CPU is profiled. This shows where your program is actively spending time +and is the typical choice for performance analysis. + +With :option:`--async-mode`\ ``=all``, tasks that are suspended (awaiting I/O, locks, or +other tasks) are also included. This mode is useful for understanding what your +program is waiting on, but produces larger profiles since every suspended task +appears in each sample. + + +Task markers and stack reconstruction +------------------------------------- + +In async-aware profiles, you will see ```` frames that mark boundaries +between asyncio tasks. These are synthetic frames inserted by the profiler to +show the task structure. The task name appears as the function name in these +frames. + +When a task awaits another task, the profiler reconstructs the logical call +chain by following the ``await`` relationships. Only "leaf" tasks (tasks that +no other task is currently awaiting) generate their own stack entries. Tasks +being awaited by other tasks appear as part of their awaiter's stack instead. + +If a task has multiple awaiters (a diamond pattern in the task graph), the +profiler deterministically selects one parent and annotates the task marker +with the number of parents, for example ``MyTask (2 parents)``. This indicates +that alternate execution paths exist but are not shown in this particular stack. + + +Option restrictions +------------------- + +Async-aware mode uses a different stack reconstruction mechanism and is +incompatible with: :option:`--native`, :option:`--no-gc`, :option:`--all-threads`, and +:option:`--mode`\ ``=cpu`` or :option:`--mode`\ ``=gil``. + + +Command-line interface +====================== + +.. program:: profiling.sampling + +The complete command-line interface for reference. + + +Global options +-------------- + +.. option:: run + + Run and profile a Python script or module. + +.. option:: attach + + Attach to and profile a running process by PID. + +.. option:: dump + + Print a single one-shot snapshot of a running process's Python stack. + +.. option:: replay + + Convert a binary profile file to another output format. + + +Dump options +------------ + +The following options apply to the ``dump`` subcommand: + +.. option:: -a, --all-threads + + Dump all threads in the target process instead of just the main thread. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for active garbage collection. + +.. option:: --opcodes + + Show bytecode opcode names when available. + +.. option:: --async-aware + + Reconstruct the stack across ``await`` boundaries for asyncio + applications. + +.. option:: --async-mode + + Async stack mode: ``running`` (only the running task) or ``all`` + (all tasks including waiting). Defaults to ``all`` for ``dump``. + Requires :option:`--async-aware`. + +.. option:: --blocking + + Pause all threads in the target process while reading the stack. + + +Sampling options +---------------- + +.. option:: -r , --sampling-rate + + Sampling rate (for example, ``10000``, ``10khz``, ``10k``). Default: ``1khz``. + +.. option:: -d , --duration + + Profiling duration in seconds. Default: run to completion. + +.. option:: -a, --all-threads + + Sample all threads, not just the main thread. + +.. option:: --realtime-stats + + Display sampling statistics during profiling. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for garbage collection. + +.. option:: --async-aware + + Enable async-aware profiling for asyncio programs. + +.. option:: --opcodes + + Gather bytecode opcode information for instruction-level profiling. Shows + which bytecode instructions are executing, including specializations. + Compatible with ``--live``, ``--flamegraph``, ``--heatmap``, and ``--gecko`` + formats only. + +.. option:: --subprocesses + + Also profile subprocesses. Each subprocess gets its own profiler + instance and output file. Incompatible with ``--live``. + +.. option:: --blocking + + Pause the target process during each sample. This ensures consistent + stack traces at the cost of slowing down the target. Use with longer + intervals (1000 µs or higher) to minimize impact. See :ref:`blocking-mode` + for details. + + +Mode options +------------ + +.. option:: --mode + + Sampling mode: ``wall`` (default), ``cpu``, ``gil``, or ``exception``. + The ``cpu``, ``gil``, and ``exception`` modes are incompatible with + ``--async-aware``. + +.. option:: --async-mode + + Async profiling mode: ``running`` (default) or ``all``. + Requires ``--async-aware``. + + +Output options +-------------- + +.. option:: --pstats + + Generate pstats statistics. This is the default. + When written to stdout, the output is a text table; with :option:`-o`, + it is a binary pstats file. + +.. option:: --collapsed + + Generate collapsed stack format for external flame graph tools. + +.. option:: --flamegraph + + Generate self-contained HTML flame graph. + +.. option:: --diff-flamegraph + + Generate differential flamegraph comparing to a baseline binary profile. + +.. option:: --gecko + + Generate Gecko JSON format for Firefox Profiler. + +.. option:: --heatmap + + Generate HTML heatmap with line-level sample counts. + +.. option:: --binary + + Generate high-performance binary format for later conversion with the + ``replay`` command. + +.. option:: --compression + + Compression for binary format: ``auto`` (use zstd if available, default), + ``zstd``, or ``none``. + +.. option:: -o , --output + + Output file or directory path. Default behavior varies by format: + :option:`--pstats` prints a text table to stdout, while ``-o`` writes a + binary pstats file. Other formats generate a file named + ``_.`` (for example, ``flamegraph_12345.html``). + :option:`--heatmap` creates a directory named ``heatmap_``. + +.. option:: --browser + + Automatically open HTML output (:option:`--flamegraph` and + :option:`--heatmap`) in your default web browser after generation. + When profiling with :option:`--subprocesses`, only the main process + opens the browser; subprocess outputs are never auto-opened. + + +pstats display options +---------------------- + +These options apply only to pstats format output. + +.. option:: --sort + + Sort order: ``nsamples``, ``tottime``, ``cumtime``, ``sample-pct``, + ``cumul-pct``, ``nsamples-cumul``, or ``name``. Default: ``nsamples``. + +.. option:: -l , --limit + + Maximum number of entries to display. Default: 15. + +.. option:: --no-summary + + Omit the Legend and Summary of Interesting Functions sections from output. + + +Run command options +------------------- + +.. option:: -m, --module + + Treat the target as a module name rather than a script path. + +.. option:: --live + + Start interactive terminal interface instead of batch profiling. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.tracing` + Deterministic tracing profiler for exact call counts and timing. + + :mod:`pstats` + Statistics analysis for profile data. + + `Firefox Profiler `__ + Web-based profiler that accepts Gecko format output. See the + `documentation `__ for usage details. + + `FlameGraph `__ + Tools for generating flame graphs from collapsed stack format. diff --git a/Doc/library/profiling.tracing.rst b/Doc/library/profiling.tracing.rst new file mode 100644 index 000000000000000..d45423cf0d8a72d --- /dev/null +++ b/Doc/library/profiling.tracing.rst @@ -0,0 +1,331 @@ +.. _profiling-tracing: + +**************************************************** +:mod:`!profiling.tracing` --- Deterministic profiler +**************************************************** + +.. module:: profiling.tracing + :synopsis: Deterministic tracing profiler for Python programs. + +.. module:: cProfile + :synopsis: Alias for profiling.tracing (backward compatibility). + :noindex: + +.. versionadded:: 3.15 + +**Source code:** :source:`Lib/profiling/tracing/` + +-------------- + +The :mod:`!profiling.tracing` module provides deterministic profiling of Python +programs. It monitors every function call, function return, and exception event, +recording precise timing for each. This approach provides exact call counts and +complete visibility into program execution, making it ideal for development and +testing scenarios. + +.. note:: + + This module is also available as ``cProfile`` for backward compatibility. + The ``cProfile`` name will continue to work in all future Python versions. + Use whichever import style suits your codebase:: + + # Preferred (new style) + import profiling.tracing + profiling.tracing.run('my_function()') + + # Also works (backward compatible) + import cProfile + cProfile.run('my_function()') + + +What is deterministic profiling? +================================ + +:dfn:`Deterministic profiling` captures every function call, function return, +and exception event during program execution. The profiler measures the precise +time intervals between these events, providing exact statistics about how the +program behaves. + +In contrast to :ref:`statistical profiling `, which samples +the call stack periodically to estimate where time is spent, deterministic +profiling records every event. This means you get exact call counts rather than +statistical approximations. The trade-off is that instrumenting every event +introduces overhead that can slow down program execution. + +Python's interpreted nature makes deterministic profiling practical. The +interpreter already dispatches events for function calls and returns, so the +profiler can hook into this mechanism without requiring code modification. The +overhead tends to be moderate relative to the inherent cost of interpretation, +making deterministic profiling suitable for most development workflows. + +Deterministic profiling helps answer questions like: + +- How many times was this function called? +- What is the complete call graph of my program? +- Which functions are called by a particular function? +- Are there unexpected function calls happening? + +Call count statistics can identify bugs (surprising counts) and inline +expansion opportunities (high call counts). Internal time statistics reveal +"hot loops" that warrant optimization. Cumulative time statistics help identify +algorithmic inefficiencies. The handling of cumulative times in this profiler +allows direct comparison of recursive and iterative implementations. + + +.. _profiling-tracing-cli: + +Command-line interface +====================== + +.. program:: profiling.tracing + +The :mod:`!profiling.tracing` module can be invoked as a script to profile +another script or module: + +.. code-block:: shell-session + + python -m profiling.tracing [-o output_file] [-s sort_order] (-m module | script.py) + +This runs the specified script or module under the profiler and prints the +results to standard output (or saves them to a file). + +.. option:: -o + + Write the profile results to a file instead of standard output. The output + file can be read by the :mod:`pstats` module for later analysis. + +.. option:: -s + + Sort the output by the specified key. This accepts any of the sort keys + recognized by :meth:`pstats.Stats.sort_stats`, such as ``cumulative``, + ``time``, ``calls``, or ``name``. This option only applies when + :option:`-o ` is not specified. + +.. option:: -m + + Profile a module instead of a script. The module is located using the + standard import mechanism. + + .. versionadded:: 3.7 + The ``-m`` option for ``cProfile``. + + .. versionadded:: 3.8 + The ``-m`` option for :mod:`profile`. + + +Programmatic usage examples +=========================== + +For more control over profiling, use the module's functions and classes +directly. + + +Basic profiling +--------------- + +The simplest approach uses the :func:`!run` function:: + + import profiling.tracing + profiling.tracing.run('my_function()') + +This profiles the given code string and prints a summary to standard output. +To save results for later analysis:: + + profiling.tracing.run('my_function()', 'output.prof') + + +Using the :class:`!Profile` class +--------------------------------- + +The :class:`!Profile` class provides fine-grained control:: + + import profiling.tracing + import pstats + from io import StringIO + + pr = profiling.tracing.Profile() + pr.enable() + # ... code to profile ... + pr.disable() + + # Print results + s = StringIO() + ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE) + ps.print_stats() + print(s.getvalue()) + +The :class:`!Profile` class also works as a context manager:: + + import profiling.tracing + + with profiling.tracing.Profile() as pr: + # ... code to profile ... + + pr.print_stats() + + +Module reference +================ + +.. currentmodule:: profiling.tracing + +.. function:: run(command, filename=None, sort=-1) + + Profile execution of a command and print or save the results. + + This function executes the *command* string using :func:`exec` in the + ``__main__`` module's namespace:: + + exec(command, __main__.__dict__, __main__.__dict__) + + If *filename* is not provided, the function creates a :class:`pstats.Stats` + instance and prints a summary to standard output. If *filename* is + provided, the raw profile data is saved to that file for later analysis + with :mod:`pstats`. + + The *sort* argument specifies the sort order for printed output, accepting + any value recognized by :meth:`pstats.Stats.sort_stats`. + + +.. function:: runctx(command, globals, locals, filename=None, sort=-1) + + Profile execution of a command with explicit namespaces. + + Like :func:`run`, but executes the command with the specified *globals* + and *locals* mappings instead of using the ``__main__`` module's namespace:: + + exec(command, globals, locals) + + +.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) + + A profiler object that collects execution statistics. + + The optional *timer* argument specifies a custom timing function. If not + provided, the profiler uses a platform-appropriate default timer. When + supplying a custom timer, it must return a single number representing the + current time. If the timer returns integers, use *timeunit* to specify the + duration of one time unit (for example, ``0.001`` for milliseconds). + + The *subcalls* argument controls whether the profiler tracks call + relationships between functions. The *builtins* argument controls whether + built-in functions are profiled. + + .. versionchanged:: 3.8 + Added context manager support. + + .. method:: enable() + + Start collecting profiling data. + + .. method:: disable() + + Stop collecting profiling data. + + .. method:: create_stats() + + Stop collecting data and record the results internally as the current + profile. + + .. method:: print_stats(sort=-1) + + Create a :class:`pstats.Stats` object from the current profile and print + the results to standard output. + + The *sort* argument specifies the sorting order. It accepts a single + key or a tuple of keys for multi-level sorting, using the same values + as :meth:`pstats.Stats.sort_stats`. + + .. versionadded:: 3.13 + Support for a tuple of sort keys. + + .. method:: dump_stats(filename) + + Write the current profile data to *filename*. The file can be read by + :class:`pstats.Stats` for later analysis. + + .. method:: run(cmd) + + Profile the command string via :func:`exec`. + + .. method:: runctx(cmd, globals, locals) + + Profile the command string via :func:`exec` with the specified + namespaces. + + .. method:: runcall(func, /, *args, **kwargs) + + Profile a function call. Returns whatever *func* returns:: + + result = pr.runcall(my_function, arg1, arg2, keyword=value) + +.. note:: + + Profiling requires that the profiled code returns normally. If the + interpreter terminates (for example, via :func:`sys.exit`) during + profiling, no results will be available. + + +Using a custom timer +==================== + +The :class:`Profile` class accepts a custom timing function, allowing you to +measure different aspects of execution such as wall-clock time or CPU time. +Pass the timing function to the constructor:: + + pr = profiling.tracing.Profile(my_timer_function) + +The timer function must return a single number representing the current time. +If it returns integers, also specify *timeunit* to indicate the duration of +one unit:: + + # Timer returns time in milliseconds + pr = profiling.tracing.Profile(my_ms_timer, 0.001) + +For best performance, the timer function should be as fast as possible. The +profiler calls it frequently, so timer overhead directly affects profiling +overhead. + +The :mod:`time` module provides several functions suitable for use as custom +timers: + +- :func:`time.perf_counter` for high-resolution wall-clock time +- :func:`time.process_time` for CPU time (excluding sleep) +- :func:`time.monotonic` for monotonic clock time + + +Limitations +=========== + +Deterministic profiling has inherent limitations related to timing accuracy. + +The underlying timer typically has a resolution of about one millisecond. +Measurements cannot be more accurate than this resolution. With enough +measurements, timing errors tend to average out, but individual measurements +may be imprecise. + +There is also latency between when an event occurs and when the profiler +captures the timestamp. Similarly, there is latency after reading the +timestamp before user code resumes. Functions called frequently accumulate +this latency, which can make them appear slower than they actually are. This +error is typically less than one clock tick per call but can become +significant for functions called many times. + +The :mod:`!profiling.tracing` module (and its ``cProfile`` alias) is +implemented as a C extension with low overhead, so these timing issues are +less pronounced than with the deprecated pure Python :mod:`profile` module. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools and guidance on choosing a profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler for production use. + + :mod:`pstats` + Statistics analysis and formatting for profile data. + + :mod:`profile` + Deprecated pure Python profiler (includes calibration documentation). diff --git a/Doc/library/pstats.rst b/Doc/library/pstats.rst new file mode 100644 index 000000000000000..585f17bdb99a709 --- /dev/null +++ b/Doc/library/pstats.rst @@ -0,0 +1,362 @@ +.. _pstats-module: + +******************************************* +:mod:`!pstats` --- Statistics for profilers +******************************************* + +.. module:: pstats + :synopsis: Statistics object for analyzing profiler output. + +**Source code:** :source:`Lib/pstats.py` + +-------------- + +The :mod:`!pstats` module provides tools for reading, manipulating, and +displaying profiling statistics generated by Python's profilers. It reads +output from both :mod:`profiling.tracing` (deterministic profiler) and +:mod:`profiling.sampling` (statistical profiler). + + +Reading and displaying profile data +=================================== + +The :class:`Stats` class is the primary interface for working with profile +data. It can read statistics from files or directly from a +:class:`~profiling.tracing.Profile` object. + +Load statistics from a file and print a basic report:: + + import pstats + + p = pstats.Stats('profile_output.prof') + p.print_stats() + +The :class:`Stats` object provides methods for sorting and filtering the +data before printing. For example, to see the ten functions with the highest +cumulative time:: + + from pstats import SortKey + + p = pstats.Stats('profile_output.prof') + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + +Working with statistics +----------------------- + +The :class:`Stats` class supports method chaining, making it convenient to +perform multiple operations:: + + p = pstats.Stats('restats') + p.strip_dirs().sort_stats(-1).print_stats() + +The :meth:`~Stats.strip_dirs` method removes directory paths from filenames, +making the output more compact. The :meth:`~Stats.sort_stats` method accepts +various keys to control the sort order. + +Different sort keys highlight different aspects of performance:: + + from pstats import SortKey + + # Functions that consume the most cumulative time + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) + + # Functions that consume the most time in their own code + p.sort_stats(SortKey.TIME).print_stats(10) + + # Functions sorted by name + p.sort_stats(SortKey.NAME).print_stats() + + +Filtering output +---------------- + +The :meth:`~Stats.print_stats` method accepts restrictions that filter +which functions are displayed. Restrictions can be integers (limiting the +count), floats between 0 and 1 (selecting a percentage), or strings (matching +function names via regular expression). + +Print only the top 10%:: + + p.print_stats(.1) + +Print only functions whose names contain "init":: + + p.print_stats('init') + +Combine restrictions (they apply sequentially):: + + # Top 10%, then only those containing "init" + p.print_stats(.1, 'init') + + # Functions in files matching "foo:", limited to top 50% + p.sort_stats(SortKey.FILENAME).print_stats('foo:', .5) + + +Analyzing call relationships +---------------------------- + +The :meth:`~Stats.print_callers` method shows which functions called each +displayed function:: + + p.print_callers() + +The :meth:`~Stats.print_callees` method shows the opposite relationship, +listing which functions each displayed function called:: + + p.print_callees() + +Both methods accept the same restriction arguments as :meth:`~Stats.print_stats`. + + +Combining multiple profiles +--------------------------- + +Statistics from multiple profiling runs can be combined into a single +:class:`Stats` object:: + + # Load multiple files at once + p = pstats.Stats('run1.prof', 'run2.prof', 'run3.prof') + + # Or add files incrementally + p = pstats.Stats('run1.prof') + p.add('run2.prof') + p.add('run3.prof') + +When files are combined, statistics for identical functions (same file, line, +and name) are accumulated, giving an aggregate view across all profiling runs. + + +The :class:`!Stats` class +========================= + +.. class:: Stats(*filenames_or_profile, stream=sys.stdout) + + Create a statistics object from profile data. + + The arguments can be filenames (strings or path-like objects) or + :class:`~profiling.tracing.Profile` objects. If multiple sources are + provided, their statistics are combined. + + The *stream* argument specifies where output from :meth:`print_stats` and + related methods is written. It defaults to :data:`sys.stdout`. + + The profile data format is specific to the Python version that created it. + There is no compatibility guarantee between Python versions or between + different profilers. + + .. method:: strip_dirs() + + Remove leading path information from all filenames. + + This method modifies the object in place and returns it for method + chaining. After stripping, the statistics are considered to be in + random order. + + If stripping causes two functions to become indistinguishable (same + filename, line number, and function name), their statistics are + combined into a single entry. + + .. method:: add(*filenames) + + Add profiling data from additional files. + + The files must have been created by the same profiler type. Statistics + for identical functions are accumulated. + + .. method:: dump_stats(filename) + + Save the current statistics to a file. + + The file is created if it does not exist and overwritten if it does. + The saved data can be loaded by creating a new :class:`Stats` object. + + .. method:: sort_stats(*keys) + + Sort the statistics according to the specified criteria. + + Each key can be a string or a :class:`SortKey` enum member. When + multiple keys are provided, later keys break ties in earlier keys. + + Using :class:`SortKey` enum members is preferred over strings as it + provides better error checking:: + + from pstats import SortKey + p.sort_stats(SortKey.CUMULATIVE) + + Valid sort keys: + + +------------------+------------------------+----------------------+ + | String | Enum | Meaning | + +==================+========================+======================+ + | ``'calls'`` | ``SortKey.CALLS`` | call count | + +------------------+------------------------+----------------------+ + | ``'cumulative'`` | ``SortKey.CUMULATIVE`` | cumulative time | + +------------------+------------------------+----------------------+ + | ``'cumtime'`` | N/A | cumulative time | + +------------------+------------------------+----------------------+ + | ``'file'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'filename'`` | ``SortKey.FILENAME`` | file name | + +------------------+------------------------+----------------------+ + | ``'module'`` | N/A | file name | + +------------------+------------------------+----------------------+ + | ``'ncalls'`` | N/A | call count | + +------------------+------------------------+----------------------+ + | ``'pcalls'`` | ``SortKey.PCALLS`` | primitive call count | + +------------------+------------------------+----------------------+ + | ``'line'`` | ``SortKey.LINE`` | line number | + +------------------+------------------------+----------------------+ + | ``'name'`` | ``SortKey.NAME`` | function name | + +------------------+------------------------+----------------------+ + | ``'nfl'`` | ``SortKey.NFL`` | name/file/line | + +------------------+------------------------+----------------------+ + | ``'stdname'`` | ``SortKey.STDNAME`` | standard name | + +------------------+------------------------+----------------------+ + | ``'time'`` | ``SortKey.TIME`` | internal time | + +------------------+------------------------+----------------------+ + | ``'tottime'`` | N/A | internal time | + +------------------+------------------------+----------------------+ + + All sorts on statistics are in descending order (most time consuming + first), while name, file, and line number sorts are ascending + (alphabetical). + + The difference between ``SortKey.NFL`` and ``SortKey.STDNAME`` is that + NFL sorts line numbers numerically while STDNAME sorts them as strings. + ``sort_stats(SortKey.NFL)`` is equivalent to + ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. + + For backward compatibility, the numeric arguments ``-1``, ``0``, ``1``, + and ``2`` are also accepted, meaning ``'stdname'``, ``'calls'``, + ``'time'``, and ``'cumulative'`` respectively. + + .. versionadded:: 3.7 + The :class:`SortKey` enum. + + .. method:: reverse_order() + + Reverse the current sort order. + + By default, the sort direction is chosen appropriately for the sort key + (descending for time-based keys, ascending for name-based keys). This + method inverts that choice. + + .. method:: print_stats(*restrictions) + + Print a report of the profiling statistics. + + The output includes a header line summarizing the data, followed by a + table of function statistics sorted according to the last + :meth:`sort_stats` call. + + Restrictions filter the output. Each restriction is either: + + - An integer: limits output to that many entries + - A float between 0.0 and 1.0: selects that fraction of entries + - A string: matches function names via regular expression + + Restrictions are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + First limits to the top 10%, then filters to functions matching 'foo:'. + + .. method:: print_callers(*restrictions) + + Print the callers of each function in the statistics. + + For each function in the filtered results, shows which functions called + it and how often. + + With :mod:`profiling.tracing` (or ``cProfile``), each caller line + shows three numbers: the number of calls from that caller, and the + total and cumulative times for those specific calls. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: print_callees(*restrictions) + + Print the functions called by each function in the statistics. + + This is the inverse of :meth:`print_callers`, showing which functions + each listed function called. + + Accepts the same restriction arguments as :meth:`print_stats`. + + .. method:: get_stats_profile() + + Return a ``StatsProfile`` object containing the statistics. + + The returned object provides programmatic access to the profile data, + with function names mapped to ``FunctionProfile`` objects + containing timing and call count information. + + .. versionadded:: 3.9 + + +.. class:: SortKey + + An enumeration of valid sort keys for :meth:`Stats.sort_stats`. + + .. attribute:: CALLS + + Sort by call count. + + .. attribute:: CUMULATIVE + + Sort by cumulative time. + + .. attribute:: FILENAME + + Sort by file name. + + .. attribute:: LINE + + Sort by line number. + + .. attribute:: NAME + + Sort by function name. + + .. attribute:: NFL + + Sort by name, then file, then line number (numeric line sort). + + .. attribute:: PCALLS + + Sort by primitive (non-recursive) call count. + + .. attribute:: STDNAME + + Sort by standard name (string-based line sort). + + .. attribute:: TIME + + Sort by internal time (time in function excluding subcalls). + + +.. _pstats-cli: + +Command-line interface +====================== + +The :mod:`!pstats` module can be invoked as a script to interactively browse +profile data:: + + python -m pstats profile_output.prof + +This opens a line-oriented interface (built on :mod:`cmd`) for examining the +statistics. Type ``help`` at the prompt for available commands. + + +.. seealso:: + + :mod:`profiling` + Overview of Python profiling tools. + + :mod:`profiling.tracing` + Deterministic tracing profiler. + + :mod:`profiling.sampling` + Statistical sampling profiler. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 2912c9e16c6149b..a7be5779fb26208 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -2,17 +2,13 @@ ========================================= .. module:: pty - :platform: Unix :synopsis: Pseudo-Terminal Handling for Unix. -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/pty.py` -------------- -The :mod:`pty` module defines operations for handling the pseudo-terminal +The :mod:`!pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. @@ -22,7 +18,7 @@ Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). -The :mod:`pty` module defines the following functions: +The :mod:`!pty` module defines the following functions: .. function:: fork() @@ -93,8 +89,6 @@ The :mod:`pty` module defines the following functions: Example ------- -.. sectionauthor:: Steen Lumholt - The following program acts like the Unix command :manpage:`script(1)`, using a pseudo-terminal to record all input and output of a terminal session in a "typescript". :: diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index e1ff32912132f7d..7691fed2c7cb835 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -2,7 +2,6 @@ ===================================== .. module:: pwd - :platform: Unix :synopsis: The password database (getpwnam() and friends). -------------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 75aa739d1003b8b..7aa960de3f23454 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -4,16 +4,13 @@ .. module:: py_compile :synopsis: Generate byte-code files from Python source files. -.. sectionauthor:: Fred L. Drake, Jr. -.. documentation based on module docstrings - **Source code:** :source:`Lib/py_compile.py` .. index:: pair: file; byte-code -------------- -The :mod:`py_compile` module provides a function to generate a byte-code file +The :mod:`!py_compile` module provides a function to generate a byte-code file from a source file, and another function used when the module source file is invoked as a script. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 5efb11d89dd143d..ed9fc6d0b5cf8c0 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -4,13 +4,11 @@ .. module:: pyclbr :synopsis: Supports information extraction for a Python module browser. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/pyclbr.py` -------------- -The :mod:`pyclbr` module provides limited information about the +The :mod:`!pyclbr` module provides limited information about the functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The information is extracted from the Python source code rather than by diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e8f153ee1b35ce0..a0cfb440a36ffa9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -4,9 +4,6 @@ .. module:: pydoc :synopsis: Documentation generator and online help system. -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - **Source code:** :source:`Lib/pydoc.py` .. index:: @@ -71,6 +68,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2f5db81955c235d..c88411ce0b7b91f 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -4,8 +4,6 @@ .. module:: xml.parsers.expat :synopsis: An interface to the Expat non-validating XML parser. -.. moduleauthor:: Paul Prescod - -------------- .. Markup notes: @@ -24,7 +22,7 @@ .. index:: single: Expat -The :mod:`xml.parsers.expat` module is a Python interface to the Expat +The :mod:`!xml.parsers.expat` module is a Python interface to the Expat non-validating XML parser. The module provides a single extension type, :class:`xmlparser`, that represents the current state of an XML parser. After an :class:`xmlparser` object has been created, various attributes of the object @@ -55,7 +53,7 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. -The :mod:`xml.parsers.expat` module contains two functions: +The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ErrorString(errno) @@ -65,12 +63,26 @@ The :mod:`xml.parsers.expat` module contains two functions: .. function:: ParserCreate(encoding=None, namespace_separator=None) - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. + Creates and returns a new :class:`xmlparser` object. + *encoding* [1]_, if specified, must be a string naming the encoding + used by the XML data. + If it is given it will override the implicit or explicit encoding + of the document. + + .. impl-detail:: + + Expat natively understands and processes UTF-8, UTF-16, UTF-16BE, + UTF-16LE, ISO-8859-1, and US-ASCII. + For other encodings (including aliases like Latin1 and ASCII) it + falls back to Python. + It supports most of 8-bit encodings and many multi-byte encodings + like Shift_JIS, although only BMP characters (``U+0000-U+FFFF``) + are supported with non-native encodings (this restriction is also + applied to aliases like UTF8). + These restrictions only apply if *encoding* is not given. + + .. versionchanged:: next + Added support for multi-byte encodings. .. _xmlparser-non-root: @@ -115,7 +127,6 @@ The :mod:`xml.parsers.expat` module contains two functions: XML document. Call ``ParserCreate`` for each document to provide unique parser instances. - .. seealso:: `The Expat XML Parser `_ @@ -485,7 +496,7 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains and internal document declaration subset. This requires Expat version + contains an internal document declaration subset. This requires Expat version 1.2 or newer. @@ -666,9 +677,6 @@ otherwise stated. ExpatError Exceptions --------------------- -.. sectionauthor:: Fred L. Drake, Jr. - - :exc:`ExpatError` exceptions have a number of interesting attributes: @@ -752,14 +760,12 @@ Content Model Descriptions .. module:: xml.parsers.expat.model -.. sectionauthor:: Fred L. Drake, Jr. - Content models are described using nested tuples. Each tuple contains four values: the type, the quantifier, the name, and a tuple of children. Children are simply additional content model descriptions. The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two +:mod:`!xml.parsers.expat.model` module. These constants can be collected in two groups: the model type group and the quantifier group. The constants in the model type group are: @@ -833,7 +839,7 @@ Expat error constants .. module:: xml.parsers.expat.errors -The following constants are provided in the :mod:`xml.parsers.expat.errors` +The following constants are provided in the :mod:`!xml.parsers.expat.errors` module. These constants are useful in interpreting some of the attributes of the :exc:`ExpatError` exception objects raised when an error has occurred. Since for backwards compatibility reasons, the constants' value is the error @@ -980,7 +986,7 @@ The ``errors`` module has the following attributes: An operation was requested that requires DTD support to be compiled in, but Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. + standard build of the :mod:`!xml.parsers.expat` module. .. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING @@ -1090,9 +1096,11 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl +.. [1] The encoding string included in XML output should conform to + the appropriate standards. For example, "UTF-8" is valid, but + "UTF8" is not valid in an XML document's declaration, even though + Python accepts it as an encoding name. + See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1b75582f0cf45b8..79da2a3db719e78 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`queue` module implements multi-producer, multi-consumer queues. +The :mod:`!queue` module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming when information must be exchanged safely between multiple threads. The :class:`Queue` class in this module implements all the required locking semantics. @@ -30,7 +30,7 @@ In addition, the module implements a "simple" specific implementation provides additional guarantees in exchange for the smaller functionality. -The :mod:`queue` module defines the following classes and exceptions: +The :mod:`!queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) @@ -76,6 +76,8 @@ The :mod:`queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 @@ -171,7 +173,7 @@ provide the public methods described below. .. method:: Queue.get_nowait() - Equivalent to ``get(False)``. + Equivalent to ``get(block=False)``. Two methods are offered to support tracking whether enqueued tasks have been fully processed by daemon consumer threads. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 4e55e301b89095f..73a37e189ddf2a8 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -37,7 +37,7 @@ Class :class:`Random` can also be subclassed if you want to use a different basic generator of your own devising: see the documentation on that class for more details. -The :mod:`random` module also provides the :class:`SystemRandom` class which +The :mod:`!random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. @@ -78,7 +78,7 @@ Bookkeeping functions instead of the system time (see the :func:`os.urandom` function for details on availability). - If *a* is an int, it is used directly. + If *a* is an int, its absolute value is used directly. With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` object gets converted to an :class:`int` and all of its bits are used. @@ -410,7 +410,7 @@ Alternative Generator .. class:: Random([seed]) Class that implements the default pseudo-random number generator used by the - :mod:`random` module. + :mod:`!random` module. .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: @@ -634,11 +634,12 @@ from the combinatoric iterators in the :mod:`itertools` module or the :pypi:`more-itertools` project: .. testcode:: + import random - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat + def random_product(*iterables, repeat=1): + "Random selection from itertools.product(*iterables, repeat=repeat)" + pools = tuple(map(tuple, iterables)) * repeat return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): @@ -663,15 +664,89 @@ or the :pypi:`more-itertools` project: return tuple(pool[i] for i in indices) def random_derangement(iterable): - "Choose a permutation where no element is in its original position." + "Choose a permutation where no element stays in its original position." seq = tuple(iterable) if len(seq) < 2: - raise ValueError('derangements require at least two values') - perm = list(seq) + if not seq: + return () + raise IndexError('No derangments to choose from') + perm = list(range(len(seq))) + start = tuple(perm) while True: random.shuffle(perm) - if all(p != q for p, q in zip(seq, perm)): - return tuple(perm) + if all(p != q for p, q in zip(start, perm)): + return tuple([seq[i] for i in perm]) + +.. doctest:: + :hide: + + >>> import random + + + >>> random.seed(8675309) + >>> random_product('ABCDEFG', repeat=5) + ('D', 'B', 'E', 'F', 'E') + + + >>> random.seed(8675309) + >>> random_permutation('ABCDEFG') + ('D', 'B', 'E', 'C', 'G', 'A', 'F') + >>> random_permutation('ABCDEFG', 5) + ('A', 'G', 'D', 'C', 'B') + + + >>> random.seed(8675309) + >>> random_combination('ABCDEFG', 7) + ('A', 'B', 'C', 'D', 'E', 'F', 'G') + >>> random_combination('ABCDEFG', 6) + ('A', 'B', 'C', 'D', 'F', 'G') + >>> random_combination('ABCDEFG', 5) + ('A', 'B', 'C', 'E', 'F') + >>> random_combination('ABCDEFG', 4) + ('B', 'C', 'D', 'G') + >>> random_combination('ABCDEFG', 3) + ('B', 'E', 'G') + >>> random_combination('ABCDEFG', 2) + ('E', 'G') + >>> random_combination('ABCDEFG', 1) + ('C',) + >>> random_combination('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_combination_with_replacement('ABCDEFG', 7) + ('B', 'C', 'D', 'E', 'E', 'E', 'G') + >>> random_combination_with_replacement('ABCDEFG', 3) + ('A', 'B', 'E') + >>> random_combination_with_replacement('ABCDEFG', 2) + ('A', 'G') + >>> random_combination_with_replacement('ABCDEFG', 1) + ('E',) + >>> random_combination_with_replacement('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_derangement('') + () + >>> random_derangement('A') + Traceback (most recent call last): + ... + IndexError: No derangments to choose from + >>> random_derangement('AB') + ('B', 'A') + >>> random_derangement('ABC') + ('C', 'A', 'B') + >>> random_derangement('ABCD') + ('B', 'A', 'D', 'C') + >>> random_derangement('ABCDE') + ('B', 'C', 'A', 'E', 'D') + >>> # Identical inputs treated as distinct + >>> identical = 20 + >>> random_derangement((10, identical, 30, identical)) + (20, 30, 10, 20) + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 75ebbf11c8e47c2..1aefb91e0ceb405 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -4,9 +4,6 @@ .. module:: re :synopsis: Regular expression operations. -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Andrew M. Kuchling - **Source code:** :source:`Lib/re/` -------------- @@ -49,13 +46,13 @@ fine-tuning parameters. .. seealso:: The third-party :pypi:`regex` module, - which has an API compatible with the standard library :mod:`re` module, + which has an API compatible with the standard library :mod:`!re` module, but offers additional functionality and a more thorough Unicode support. .. _re-syntax: -Regular Expression Syntax +Regular expression syntax ------------------------- A regular expression (or RE) specifies a set of strings that matches it; the @@ -208,7 +205,7 @@ The special characters are: *without* establishing any backtracking points. This is the possessive version of the quantifier above. For example, on the 6-character string ``'aaaaaa'``, ``a{3,5}+aa`` - attempt to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, + attempts to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, will need more characters than available and thus fail, while ``a{3,5}aa`` will match with ``a{3,5}`` capturing 5, then 4 ``'a'``\ s by backtracking and then the final 2 ``'a'``\ s are matched by the final @@ -282,25 +279,47 @@ The special characters are: ``[]()[{}]`` will match a right bracket, as well as left bracket, braces, and parentheses. - .. .. index:: single: --; in regular expressions - .. .. index:: single: &&; in regular expressions - .. .. index:: single: ~~; in regular expressions - .. .. index:: single: ||; in regular expressions - - * Support of nested sets and set operations as in `Unicode Technical - Standard #18`_ might be added in the future. This would change the - syntax, so to facilitate this change a :exc:`FutureWarning` will be raised - in ambiguous cases for the time being. - That includes sets starting with a literal ``'['`` or containing literal - character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To - avoid a warning escape them with a backslash. + .. index:: + single: --; in regular expressions + single: &&; in regular expressions + single: ||; in regular expressions + + * A character set may contain a nested set written in square brackets, and + two sets may be combined with a set operator, as in `Unicode Technical + Standard #18`_: + + * ``[A--B]`` (*difference*) matches a character that is in *A* but not + in *B*; for example ``[a-z--[aeiou]]`` matches an ASCII lowercase + consonant. + * ``[A&&B]`` (*intersection*) matches a character that is in both *A* + and *B*; for example ``[\w&&[a-z]]`` matches an ASCII lowercase letter. + * ``[A||B]`` (*union*) matches a character that is in *A* or in *B*; this + is the same as listing the members of both sets in a single set, but + allows combining nested sets. + + Operators have no precedence and are applied from left to right. To + group, write a nested set as the operand after an operator, as in + ``[a-z--[aeiou]]``. A leading ``'^'`` complements the whole result. + A ``'['`` begins a nested set only immediately after a set operator; + anywhere else -- including at the start of a character set -- it is an + ordinary character, so existing patterns keep their meaning. Escape it + as ``'\['`` to include a literal ``'['`` right after an operator. .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + .. note:: + + Symmetric difference (``A~~B``) is not yet supported; a literal ``'~~'`` + in a character set still raises a :exc:`FutureWarning`. + .. versionchanged:: 3.7 :exc:`FutureWarning` is raised if a character set contains constructs that will change semantically in the future. + .. versionchanged:: next + Added support for nested sets and the set operators ``--``, ``&&`` + and ``||``. + .. index:: single: | (vertical bar); in regular expressions ``|`` @@ -512,8 +531,9 @@ The special characters are: *name* exists, and with ``no-pattern`` if it doesn't. ``no-pattern`` is optional and can be omitted. For example, ``(<)?(\w+@\w+(?:\.\w+)+)(?(1)>|$)`` is a poor email matching pattern, which - will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. + matches ``''`` as well as ``'user@host.com'``, but does not + match ``''`` in their entirety + (:func:`re.search` finds only ``'user@host.com'`` in the former). .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. @@ -594,7 +614,7 @@ character ``'$'``. Matches ``[0-9]`` if the :py:const:`~re.ASCII` flag is used. - __ https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G134153 + __ https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-4/#G124142 For 8-bit (bytes) patterns: Matches any decimal digit in the ASCII character set; @@ -661,6 +681,51 @@ character ``'$'``. matches characters which are neither alphanumeric in the current locale nor the underscore. +.. index:: single: \p; in regular expressions + single: \P; in regular expressions + +``\p{property=value}``, ``\p{value}`` + Matches any character with the given Unicode property + (see `Unicode Technical Standard #18 + `_, requirement RL1.2 "Properties"). + Property and value names are matched loosely: + case, whitespace, ``'-'`` and ``'_'`` are ignored. + The following properties are supported: + + * The ``General_Category`` property (short name ``gc``), + spelled ``\p{Lu}``, ``\p{gc=Lu}`` or, for a one-letter group, ``\p{L}``. + The supported values are the groups ``L``, ``N``, ``Z`` and ``C`` and the + values ``Lu``, ``Lt``, ``Lm``, ``Nd``, ``Nl``, ``No``, ``Zs``, ``Zl``, + ``Zp``, ``Cc``, ``Cf``, ``Cs``, ``Co`` and ``Cn``. + * The binary properties ``XID_Start``, ``XID_Continue``, ``Alphabetic``, + ``Lowercase``, ``Uppercase``, ``Numeric``, ``Printable``, ``Cased`` and + ``Case_Ignorable``. A binary property may also be spelled + ``\p{name=yes}`` or ``\p{name=no}``. + * The POSIX compatibility classes ``alpha``, ``alnum``, ``blank``, + ``cntrl``, ``digit``, ``graph``, ``lower``, ``print``, ``space``, + ``upper``, ``word`` and ``xdigit``. + * The properties ``ASCII``, ``Any``, ``Assigned``, + ``Noncharacter_Code_Point``, ``Join_Control``, ``Regional_Indicator``, + ``ASCII_Hex_Digit``, ``Hex_Digit``, ``Pattern_Syntax`` and + ``Pattern_White_Space``. + + Where a supported property corresponds to a :mod:`unicodedata` accessor or + :class:`str` method, the set of characters it matches is exactly the one + they report. For consistency with these, ``space`` follows + :py:meth:`str.isspace` (like ``\s``) and ``xdigit`` matches only the ASCII + hexadecimal digits. + + This is only recognized in Unicode (str) patterns. + In bytes patterns it is an error. + + .. versionadded:: next + +``\P{...}`` + Matches any character which does *not* have the given Unicode property. + This is the opposite of ``\p``. + + .. versionadded:: next + .. index:: single: \z; in regular expressions single: \Z; in regular expressions @@ -720,7 +785,7 @@ three digits in length. .. _contents-of-module-re: -Module Contents +Module contents --------------- The module defines several functions, constants, and an exception. Some of the @@ -836,8 +901,8 @@ Flags will be conditionally ORed with other flags. Example of use as a default value:: - def myfunc(text, flag=re.NOFLAG): - return re.match(text, flag) + def myfunc(pattern, text, flag=re.NOFLAG): + return re.search(pattern, text, flag) .. versionadded:: 3.11 @@ -893,8 +958,8 @@ Functions Compile a regular expression pattern into a :ref:`regular expression object `, which can be used for matching using its - :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described - below. + :func:`~Pattern.prefixmatch`, + :func:`~Pattern.search`, and other methods, described below. The expression's behaviour can be modified by specifying a *flags* value. Values can be any of the `flags`_ variables, combined using bitwise OR @@ -903,11 +968,11 @@ Functions The sequence :: prog = re.compile(pattern) - result = prog.match(string) + result = prog.search(string) is equivalent to :: - result = re.match(pattern, string) + result = re.search(pattern, string) but using :func:`re.compile` and saving the resulting regular expression object for reuse is more efficient when the expression will be used several @@ -933,15 +998,17 @@ Functions (the ``|`` operator). -.. function:: match(pattern, string, flags=0) +.. function:: prefixmatch(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. - Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match - at the beginning of the string and not at the beginning of each line. + .. note:: + + Even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. If you want to locate a match anywhere in *string*, use :func:`search` instead (see also :ref:`search-vs-match`). @@ -950,6 +1017,23 @@ Functions Values can be any of the `flags`_ variables, combined using bitwise OR (the ``|`` operator). + This function now has two names and has long been known as + :func:`~re.match`. Use that name when you need to retain compatibility with + older Python versions. + + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 + :func:`~re.match` has been :term:`soft deprecated` in favor of + the alternate :func:`~re.prefixmatch` name of this API which is + more explicitly descriptive. Use it to better + express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :func:`~re.search`. + See :ref:`prefixmatch-vs-match`. + .. function:: fullmatch(pattern, string, flags=0) @@ -1234,13 +1318,16 @@ Exceptions .. _re-objects: -Regular Expression Objects +Regular expression objects -------------------------- .. class:: Pattern Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1271,24 +1358,44 @@ Regular Expression Objects >>> pattern.search("dog", 1) # No match; search doesn't include the "d" -.. method:: Pattern.match(string[, pos[, endpos]]) +.. method:: Pattern.prefixmatch(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the string does not match the pattern; note that this is different from a zero-length match. + Note that even in :const:`MULTILINE` mode, this will only match at the + beginning of the string and not at the beginning of each line. + The optional *pos* and *endpos* parameters have the same meaning as for the :meth:`~Pattern.search` method. :: >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". + >>> pattern.prefixmatch("dog") # No match as "o" is not at the start of "dog". + >>> pattern.prefixmatch("dog", 1) # Match as "o" is the 2nd character of "dog". If you want to locate a match anywhere in *string*, use :meth:`~Pattern.search` instead (see also :ref:`search-vs-match`). + This method now has two names and has long been known as + :meth:`~Pattern.match`. Use that name when you need to retain compatibility + with older Python versions. + + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 + :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of + the alternate :meth:`~Pattern.prefixmatch` name of this API which is + more explicitly descriptive. Use it to + better express intent. The norm in other languages and regular expression + implementations is to use the term *match* to refer to the behavior of + what Python has always called :meth:`~Pattern.search`. + See :ref:`prefixmatch-vs-match`. + .. method:: Pattern.fullmatch(string[, pos[, endpos]]) @@ -1368,7 +1475,7 @@ Regular Expression Objects .. _match-objects: -Match Objects +Match objects ------------- Match objects always have a boolean value of ``True``. @@ -1376,14 +1483,16 @@ Since :meth:`~Pattern.match` and :meth:`~Pattern.search` return ``None`` when there is no match, you can test whether there was a match with a simple ``if`` statement:: - match = re.search(pattern, string) - if match: + if match := re.search(pattern, string): process(match) .. class:: Match Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. @@ -1407,23 +1516,23 @@ when there is no match, you can test whether there was a match with a simple result is a single string; if there are multiple arguments, the result is a tuple with one item per argument. Without arguments, *group1* defaults to zero (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a + return value is the entire matching string; if it is a positive integer, it is + the string matching the corresponding parenthesized group. If a group number is + negative or larger than the number of groups defined in the pattern, an + :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, the last match is returned. :: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"\A(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m.group(0) # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m.group(1) # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m.group(2) # The second parenthesized subgroup. - 'Newton' + 'Blue' >>> m.group(1, 2) # Multiple arguments give us a tuple. - ('Isaac', 'Newton') + ('Norwegian', 'Blue') If the regular expression uses the ``(?P...)`` syntax, the *groupN* arguments may also be strings identifying groups by their group name. If a @@ -1432,23 +1541,23 @@ when there is no match, you can test whether there was a match with a simple A moderately complicated example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.group('first_name') - 'Malcolm' - >>> m.group('last_name') - 'Reynolds' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m.group('adjective') + 'killer' + >>> m.group('animal') + 'rabbit' Named groups can also be referred to by their index:: >>> m.group(1) - 'Malcolm' + 'killer' >>> m.group(2) - 'Reynolds' + 'rabbit' If a group matches multiple times, only the last match is accessible:: - >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. - >>> m.group(1) # Returns only the last match. + >>> m = re.search(r"(..)+", "a1b2c3") # Matches 3 times. + >>> m.group(1) # Returns only the last match. 'c3' @@ -1457,21 +1566,21 @@ when there is no match, you can test whether there was a match with a simple This is identical to ``m.group(g)``. This allows easier access to an individual group from a match:: - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") + >>> m = re.search(r"(\w+) (\w+)", "Norwegian Blue, pining for the fjords") >>> m[0] # The entire match - 'Isaac Newton' + 'Norwegian Blue' >>> m[1] # The first parenthesized subgroup. - 'Isaac' + 'Norwegian' >>> m[2] # The second parenthesized subgroup. - 'Newton' + 'Blue' Named groups are supported as well:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Isaac Newton") - >>> m['first_name'] - 'Isaac' - >>> m['last_name'] - 'Newton' + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") + >>> m['adjective'] + 'killer' + >>> m['animal'] + 'rabbit' .. versionadded:: 3.6 @@ -1484,7 +1593,7 @@ when there is no match, you can test whether there was a match with a simple For example:: - >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") + >>> m = re.search(r"(\d+)\.(\d+)", "24.1632") >>> m.groups() ('24', '1632') @@ -1492,7 +1601,7 @@ when there is no match, you can test whether there was a match with a simple might participate in the match. These groups will default to ``None`` unless the *default* argument is given:: - >>> m = re.match(r"(\d+)\.?(\d+)?", "24") + >>> m = re.search(r"(\d+)\.?(\d+)?", "24") >>> m.groups() # Second group defaults to None. ('24', None) >>> m.groups('0') # Now, the second group defaults to '0'. @@ -1505,9 +1614,9 @@ when there is no match, you can test whether there was a match with a simple the subgroup name. The *default* argument is used for groups that did not participate in the match; it defaults to ``None``. For example:: - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") + >>> m = re.search(r"(?P\w+) (?P\w+)", "killer rabbit") >>> m.groupdict() - {'first_name': 'Malcolm', 'last_name': 'Reynolds'} + {'adjective': 'killer', 'animal': 'rabbit'} .. method:: Match.start([group]) @@ -1588,11 +1697,11 @@ when there is no match, you can test whether there was a match with a simple .. _re-examples: -Regular Expression Examples +Regular expression examples --------------------------- -Checking for a Pair +Checking for a pair ^^^^^^^^^^^^^^^^^^^ In this example, we'll use the following helper function to display match @@ -1610,42 +1719,41 @@ representing the card with that value. To see if a given string is a valid hand, one could do the following:: - >>> valid = re.compile(r"^[a2-9tjqk]{5}$") - >>> displaymatch(valid.match("akt5q")) # Valid. + >>> valid_hand_re = re.compile(r"^[a2-9tjqk]{5}$") + >>> displaymatch(valid_hand_re.search("akt5q")) # Valid. "" - >>> displaymatch(valid.match("akt5e")) # Invalid. - >>> displaymatch(valid.match("akt")) # Invalid. - >>> displaymatch(valid.match("727ak")) # Valid. + >>> displaymatch(valid_hand_re.search("akt5e")) # Invalid. + >>> displaymatch(valid_hand_re.search("akt")) # Invalid. + >>> displaymatch(valid_hand_re.search("727ak")) # Valid. "" That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. To match this with a regular expression, one could use backreferences as such:: - >>> pair = re.compile(r".*(.).*\1") - >>> displaymatch(pair.match("717ak")) # Pair of 7s. + >>> pair_re = re.compile(r".*(.).*\1") + >>> displaymatch(pair_re.prefixmatch("717ak")) # Pair of 7s. "" - >>> displaymatch(pair.match("718ak")) # No pairs. - >>> displaymatch(pair.match("354aa")) # Pair of aces. + >>> displaymatch(pair_re.prefixmatch("718ak")) # No pairs. + >>> displaymatch(pair_re.prefixmatch("354aa")) # Pair of aces. "" To find out what card the pair consists of, one could use the :meth:`~Match.group` method of the match object in the following manner:: - >>> pair = re.compile(r".*(.).*\1") - >>> pair.match("717ak").group(1) + >>> pair_re = re.compile(r".*(.).*\1") + >>> pair_re.prefixmatch("717ak").group(1) '7' - # Error because re.match() returns None, which doesn't have a group() method: - >>> pair.match("718ak").group(1) + # Error because prefixmatch() returns None, which doesn't have a group() method: + >>> pair_re.prefixmatch("718ak").group(1) Traceback (most recent call last): File "", line 1, in - re.match(r".*(.).*\1", "718ak").group(1) + pair_re.prefixmatch("718ak").group(1) AttributeError: 'NoneType' object has no attribute 'group' - >>> pair.match("354aa").group(1) + >>> pair_re.prefixmatch("354aa").group(1) 'a' - Simulating scanf() ^^^^^^^^^^^^^^^^^^ @@ -1679,38 +1787,41 @@ expressions. | ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | +--------------------------------+---------------------------------------------+ -To extract the filename and numbers from a string like :: +To extract the filename and numbers from a string like: + +.. code-block:: text /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`!scanf` format like :: +you would use a :c:func:`!scanf` format like: + +.. code-block:: text %s - %d errors, %d warnings -The equivalent regular expression would be :: +The equivalent regular expression would be: + +.. code-block:: text (\S+) - (\d+) errors, (\d+) warnings .. _search-vs-match: -search() vs. match() -^^^^^^^^^^^^^^^^^^^^ - -.. sectionauthor:: Fred L. Drake, Jr. +search() vs. prefixmatch() +^^^^^^^^^^^^^^^^^^^^^^^^^^ Python offers different primitive operations based on regular expressions: -+ :func:`re.match` checks for a match only at the beginning of the string ++ :func:`re.prefixmatch` checks for a match only at the beginning of the string + :func:`re.search` checks for a match anywhere in the string (this is what Perl does by default) + :func:`re.fullmatch` checks for entire string to be a match - For example:: - >>> re.match("c", "abcdef") # No match - >>> re.search("c", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("c", "abcdef") # Match >>> re.fullmatch("p.*n", "python") # Match @@ -1719,21 +1830,54 @@ For example:: Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: - >>> re.match("c", "abcdef") # No match - >>> re.search("^c", "abcdef") # No match - >>> re.search("^a", "abcdef") # Match + >>> re.prefixmatch("c", "abcdef") # No match + >>> re.search("^c", "abcdef") # No match + >>> re.search("^a", "abcdef") # Match -Note however that in :const:`MULTILINE` mode :func:`match` only matches at the +Note however that in :const:`MULTILINE` mode :func:`prefixmatch` only matches at the beginning of the string, whereas using :func:`search` with a regular expression beginning with ``'^'`` will match at the beginning of each line. :: - >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match + >>> re.prefixmatch("X", "A\nB\nX", re.MULTILINE) # No match >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match +.. _prefixmatch-vs-match: + +prefixmatch() vs. match() +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Why is the :func:`~re.match` function and method discouraged in +favor of the longer :func:`~re.prefixmatch` spelling? + +Many other languages have gained regex support libraries since regular +expressions were added to Python. However in the most popular of those, they +use the term *match* in their APIs to mean the unanchored behavior provided in +Python by :func:`~re.search`. Thus use of the plain term *match* can be +unclear to those used to other languages when reading or writing code and +not familiar with the Python API's divergence from what otherwise become the +industry norm. + +Quoting from the Zen Of Python (``python3 -m this``): *"Explicit is better than +implicit"*. Anyone reading the name :func:`!prefixmatch` is likely to +understand the intended semantics. When reading :func:`!match` there remains +a seed of doubt about the intended behavior to anyone not already familiar with +this old Python gotcha. + +We **do not** plan to remove the older :func:`!match` name, +as it has been used in code for over 30 years. +It has been :term:`soft deprecated`: +code supporting older versions of Python should continue to use :func:`!match`, +while new code should prefer :func:`!prefixmatch`. + +.. versionadded:: 3.15 + :func:`!prefixmatch` + +.. soft-deprecated:: 3.15 + :func:`!match` -Making a Phonebook +Making a phonebook ^^^^^^^^^^^^^^^^^^ :func:`split` splits a string into a list delimited by the passed pattern. The @@ -1794,7 +1938,7 @@ house number from the street name: ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] -Text Munging +Text munging ^^^^^^^^^^^^ :func:`sub` replaces every occurrence of a pattern with a string or the @@ -1814,7 +1958,7 @@ in each word of a sentence except for the first and last characters:: 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' -Finding all Adverbs +Finding all adverbs ^^^^^^^^^^^^^^^^^^^ :func:`findall` matches *all* occurrences of a pattern, not just the first @@ -1827,7 +1971,7 @@ the following manner:: ['carefully', 'quickly'] -Finding all Adverbs and their Positions +Finding all adverbs and their positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched @@ -1843,7 +1987,7 @@ to find all of the adverbs *and their positions* in some text, they would use 40-47: quickly -Raw String Notation +Raw string notation ^^^^^^^^^^^^^^^^^^^ Raw string notation (``r"text"``) keeps regular expressions sane. Without it, @@ -1851,9 +1995,9 @@ every backslash (``'\'``) in a regular expression would have to be prefixed with another one to escape it. For example, the two following lines of code are functionally identical:: - >>> re.match(r"\W(.)\1\W", " ff ") + >>> re.search(r"\W(.)\1\W", " ff ") - >>> re.match("\\W(.)\\1\\W", " ff ") + >>> re.search("\\W(.)\\1\\W", " ff ") When one wants to match a literal backslash, it must be escaped in the regular @@ -1861,13 +2005,13 @@ expression. With raw string notation, this means ``r"\\"``. Without raw string notation, one must use ``"\\\\"``, making the following lines of code functionally identical:: - >>> re.match(r"\\", r"\\") + >>> re.search(r"\\", r"\\") - >>> re.match("\\\\", r"\\") + >>> re.search("\\\\", r"\\") -Writing a Tokenizer +Writing a tokenizer ^^^^^^^^^^^^^^^^^^^ A `tokenizer or scanner `_ @@ -1883,7 +2027,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 75db832c546b64f..234af8d191e3e35 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -2,14 +2,11 @@ =========================================== .. module:: readline - :platform: Unix :synopsis: GNU readline support for Python. -.. sectionauthor:: Skip Montanaro - -------------- -The :mod:`readline` module defines a number of functions to facilitate +The :mod:`!readline` module defines a number of functions to facilitate completion and reading/writing of history files from the Python interpreter. This module can be used directly, or via the :mod:`rlcompleter` module, which supports completion of Python identifiers at the interactive prompt. Settings @@ -28,11 +25,13 @@ Readline library in general. .. include:: ../includes/optional-module.rst +.. availability:: Unix. + .. note:: The underlying Readline library API may be implemented by the ``editline`` (``libedit``) library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used + On macOS the :mod:`!readline` module detects which library is being used at run time. The configuration file for ``editline`` is different from that @@ -246,6 +245,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: 3.15 + + .. _readline-completion: Completion @@ -255,7 +263,7 @@ The following functions relate to implementing a custom word completion function. This is typically operated by the Tab key, and can suggest and automatically complete a word being typed. By default, Readline is set up to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used +the interactive interpreter. If the :mod:`!readline` module is to be used with a custom completer, a different set of word delimiters should be set. @@ -324,7 +332,7 @@ with a custom completer, a different set of word delimiters should be set. Example ------- -The following example demonstrates how to use the :mod:`readline` module's +The following example demonstrates how to use the :mod:`!readline` module's history reading and writing functions to automatically load and save a history file named :file:`.python_history` from the user's home directory. The code below would normally be executed automatically during interactive sessions @@ -394,3 +402,9 @@ support history save/restore. :: def save_history(self, histfile): readline.set_history_length(1000) readline.write_history_file(histfile) + +.. note:: + + The new :term:`REPL` introduced in version 3.13 doesn't support readline. + However, readline can still be used by setting the :envvar:`PYTHON_BASIC_REPL` + environment variable. diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 28c7855dfeeef3b..d269d8bbaa55da5 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -4,8 +4,6 @@ .. module:: reprlib :synopsis: Alternate repr() implementation with size limits. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/reprlib.py` -------------- diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index c58dc4243ecb19f..561b2976ecea22b 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -2,12 +2,8 @@ =============================================== .. module:: resource - :platform: Unix :synopsis: An interface to provide resource usage information on the current process. -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Jeremy Hylton - -------------- This module provides basic mechanisms for measuring and controlling system @@ -356,9 +352,9 @@ These functions are used to retrieve resource usage information: print(getrusage(RUSAGE_SELF)) The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times + has been used, e.g. amount of time spent running in user mode or number of times the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. + clock tick interval, e.g. the amount of memory the process is using. For backward compatibility, the return value is also accessible as a tuple of 16 elements. diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst index 91779feb5250139..2acd1df3c49007a 100644 --- a/Doc/library/rlcompleter.rst +++ b/Doc/library/rlcompleter.rst @@ -4,8 +4,6 @@ .. module:: rlcompleter :synopsis: Python identifier completion, suitable for the GNU readline library. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/rlcompleter.py` -------------- diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index b07ec6e93f80aba..d764a98c8419ed3 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -4,13 +4,11 @@ .. module:: runpy :synopsis: Locate and run Python modules without importing them first. -.. moduleauthor:: Nick Coghlan - **Source code:** :source:`Lib/runpy.py` -------------- -The :mod:`runpy` module is used to locate and run Python modules without +The :mod:`!runpy` module is used to locate and run Python modules without importing them first. Its main use is to implement the :option:`-m` command line switch that allows scripts to be located using the Python module namespace rather than the filesystem. @@ -20,11 +18,11 @@ current process, and any side effects (such as cached imports of other modules) will remain in place after the functions have returned. Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. +guaranteed to work correctly after a :mod:`!runpy` function has returned. If that limitation is not acceptable for a given use case, :mod:`importlib` is likely to be a more suitable choice than this module. -The :mod:`runpy` module provides two functions: +The :mod:`!runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) @@ -50,10 +48,10 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_module`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None`, to ``mod_name + '.__main__'`` if the named module is a @@ -63,7 +61,7 @@ The :mod:`runpy` module provides two functions: module (that is, ``__spec__.name`` will always be *mod_name* or ``mod_name + '.__main__'``, never *run_name*). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are + ``__file__``, ``__loader__`` and ``__package__`` are :ref:`set as normal ` based on the module spec. If the argument *alter_sys* is supplied and evaluates to :const:`True`, @@ -98,6 +96,9 @@ The :mod:`runpy` module provides two functions: ``__package__`` are deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. function:: run_path(path_name, init_globals=None, run_name=None) .. index:: @@ -105,7 +106,7 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module's globals dictionary. As with a script name supplied to the CPython - command line, *file_path* may refer to a Python source file, a + command line, *path_name* may refer to a Python source file, a compiled bytecode file or a valid :data:`sys.path` entry containing a :mod:`__main__` module (e.g. a zipfile containing a top-level :file:`__main__.py` file). @@ -125,28 +126,28 @@ The :mod:`runpy` module provides two functions: overridden by :func:`run_path`. The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed. (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail.) + ``__loader__`` and ``__package__`` are set in the globals dictionary before + the module code is executed. (Note that this is a minimal set of variables - + other variables may be set implicitly as an interpreter implementation + detail.) ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. - If *file_path* directly references a script file (whether as source + If *path_name* directly references a script file (whether as source or as precompiled byte code), then ``__file__`` will be set to - *file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and + *path_name*, and ``__spec__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If *file_path* is a reference to a valid :data:`sys.path` entry, then + If *path_name* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be + ``__file__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of *file_path* and ``sys.modules[__name__]`` is updated + with the value of *path_name* and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function returns. @@ -173,6 +174,9 @@ The :mod:`runpy` module provides two functions: The setting of ``__cached__``, ``__loader__``, and ``__package__`` are deprecated. + .. versionchanged:: 3.15 + ``__cached__`` is no longer set. + .. seealso:: :pep:`338` -- Executing modules as scripts diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c3218989..037e27f031d0c82 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -4,15 +4,13 @@ .. module:: sched :synopsis: General purpose event scheduler. -.. sectionauthor:: Moshe Zadka - **Source code:** :source:`Lib/sched.py` .. index:: single: event scheduling -------------- -The :mod:`sched` module defines a class which implements a general purpose event +The :mod:`!sched` module defines a class which implements a general purpose event scheduler: .. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) @@ -119,9 +117,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 75dafc54d40ca58..3b5b57fb1c21702 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -4,8 +4,6 @@ .. module:: secrets :synopsis: Generate secure random numbers for managing secrets. -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano .. versionadded:: 3.6 .. testsetup:: @@ -17,11 +15,11 @@ ------------- -The :mod:`secrets` module is used for generating cryptographically strong +The :mod:`!secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particular, :mod:`secrets` should be used in preference to the +In particular, :mod:`!secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. @@ -33,7 +31,7 @@ is designed for modelling and simulation, not security or cryptography. Random numbers -------------- -The :mod:`secrets` module provides access to the most secure source of +The :mod:`!secrets` module provides access to the most secure source of randomness that your operating system provides. .. class:: SystemRandom @@ -58,43 +56,48 @@ randomness that your operating system provides. Generating tokens ----------------- -The :mod:`secrets` module provides functions for generating secure +The :mod:`!secrets` module provides functions for generating secure tokens, suitable for applications such as password resets, hard-to-guess URLs, and similar. -.. function:: token_bytes([nbytes=None]) +.. function:: token_bytes(nbytes=None) Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_bytes(16) #doctest:+SKIP + >>> token_bytes(16) # doctest: +SKIP b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' -.. function:: token_hex([nbytes=None]) +.. function:: token_hex(nbytes=None) Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. + random bytes, each byte converted to two hex digits. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_hex(16) #doctest:+SKIP + >>> token_hex(16) # doctest: +SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' -.. function:: token_urlsafe([nbytes=None]) +.. function:: token_urlsafe(nbytes=None) Return a random URL-safe text string, containing *nbytes* random bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. + in approximately 1.3 characters. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_urlsafe(16) #doctest:+SKIP + >>> token_urlsafe(16) # doctest: +SKIP 'Drmhze6EPcv0fN_81Bj-nA' @@ -107,7 +110,7 @@ tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. +the typical use-case expected for the :mod:`!secrets` module. For those who want to manage their own token length, you can explicitly specify how much randomness is used for tokens by giving an :class:`int` @@ -115,11 +118,13 @@ argument to the various ``token_*`` functions. That argument is taken as the number of bytes of randomness to use. Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. +the ``token_*`` functions use :const:`DEFAULT_ENTROPY` instead. -.. note:: +.. data:: DEFAULT_ENTROPY + + Default number of bytes of randomness used by the ``token_*`` functions. - That default is subject to change at any time, including during + The exact value is subject to change at any time, including during maintenance releases. @@ -139,7 +144,7 @@ Other functions Recipes and best practices -------------------------- -This section shows recipes and best practices for using :mod:`secrets` +This section shows recipes and best practices for using :mod:`!secrets` to manage a basic level of security. Generate an eight-character alphanumeric password: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index e821cb01d941b2f..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -18,7 +18,7 @@ it was last read. .. note:: The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are + multiplexing, built upon the :mod:`!select` module primitives. Users are encouraged to use the :mod:`selectors` module instead, unless they want precise control over the OS-level primitives used. @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,15 +54,17 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. *sizehint* informs epoll about the expected number of events to be registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`!epoll_create1` is not available; + used on older systems where :manpage:`epoll_create1(2)` is not available; otherwise it has no effect (though its value is still checked). *flags* is deprecated and completely ignored. However, when supplied, its @@ -89,18 +91,27 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. versionchanged:: 3.15 + + When CPython is built, this function may be disabled using + :option:`--disable-epoll`. + + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -108,14 +119,18 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -131,7 +146,7 @@ The module defines the following: platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out in seconds; it may be a non-integer to specify fractions of seconds. - When the *timeout* argument is omitted the function blocks until + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -174,7 +189,7 @@ The module defines the following: The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. + to other kinds of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -185,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -226,7 +241,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` + check for. The constants are the same as with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -241,7 +256,7 @@ object. .. method:: devpoll.modify(fd[, eventmask]) This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same + :meth:`register`. It is (a bit) more efficient than doing the same explicitly. @@ -280,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -357,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -391,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -416,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -478,11 +493,13 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. versionchanged:: 3.15 Accepts any real number as *timeout*, not only integer or float. + If ``ppoll()`` function is available, *timeout* has a resolution + of ``1`` ns (``1e-6`` ms) instead of ``1`` ms. .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -526,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -546,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -613,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -633,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -656,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index ee556f1f3cecae6..2d523a9d2ea440b 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -54,7 +54,7 @@ Classes hierarchy:: In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules +be waited for on a given file object. It can be a combination of the module's constants below: +-----------------------+-----------------------------------------------+ diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 51bae2fce30138d..bd3d56f6af595ab 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -42,7 +42,7 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). - By default, :mod:`shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` + By default, :mod:`!shelve` uses :func:`pickle.dumps` and :func:`pickle.loads` for serializing and deserializing. This can be changed by supplying *serializer* and *deserializer*, respectively. @@ -51,7 +51,7 @@ lots of shared sub-objects. The keys are ordinary strings. :term:`bytes-like object`; the *protocol* value may be ignored by the serializer. - The *deserializer* argument must be callable which takes a serialized object + The *deserializer* argument must be a callable which takes a serialized object given as a :class:`bytes` object and returns the corresponding object. A :exc:`ShelveError` is raised if *serializer* is given but *deserializer* @@ -81,11 +81,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. warning:: - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + Because the :mod:`!shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support most of methods and operations supported by dictionaries +Shelf objects support most of the methods and operations supported by dictionaries (except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. @@ -133,7 +133,7 @@ Restrictions database should be fairly small, and in rare cases key collisions may cause the database to refuse updates. -* The :mod:`shelve` module does not support *concurrent* read/write access to +* The :mod:`!shelve` module does not support *concurrent* read/write access to shelved objects. (Multiple simultaneous read accesses are safe.) When a program has a shelf open for writing, no other program should have it open for reading or writing. Unix file locking can be used to solve this, but this @@ -283,5 +283,5 @@ Exceptions Generic interface to ``dbm``-style databases. Module :mod:`pickle` - Object serialization used by :mod:`shelve`. + Object serialization used by :mod:`!shelve`. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc12604..2dfb0246d5d90c0 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -4,11 +4,6 @@ .. module:: shlex :synopsis: Simple lexical analysis for Unix shell-like languages. -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Gustavo Niemeyer -.. sectionauthor:: Eric S. Raymond -.. sectionauthor:: Gustavo Niemeyer - **Source code:** :source:`Lib/shlex.py` -------------- @@ -18,7 +13,7 @@ simple syntaxes resembling that of the Unix shell. This will often be useful for writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. -The :mod:`shlex` module defines the following functions: +The :mod:`!shlex` module defines the following functions: .. function:: split(s, comments=False, posix=True) @@ -49,12 +44,15 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s) +.. function:: quote(s, *, force=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for cases where you cannot use a list. + If *force* is :const:`True`, then *s* is unconditionally quoted, + even if it is already safe for a shell without being quoted. + .. _shlex-quote-warning: .. warning:: @@ -96,9 +94,24 @@ The :mod:`shlex` module defines the following functions: >>> command ['ls', '-l', 'somefile; rm -rf ~'] + The *force* keyword can be used to produce consistent behavior when + escaping multiple strings: + + >>> from shlex import quote + >>> filenames = ['my first file', 'file2', 'file 3'] + >>> filenames_some_escaped = [quote(f) for f in filenames] + >>> filenames_some_escaped + ["'my first file'", 'file2', "'file 3'"] + >>> filenames_all_escaped = [quote(f, force=True) for f in filenames] + >>> filenames_all_escaped + ["'my first file'", "'file2'", "'file 3'"] + .. versionadded:: 3.3 -The :mod:`shlex` module defines the following class: + .. versionchanged:: next + The *force* keyword was added. + +The :mod:`!shlex` module defines the following class: .. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) @@ -214,7 +227,7 @@ A :class:`~shlex.shlex` instance has the following methods: with the name of the current source file and the ``%d`` with the current input line number (the optional arguments can be used to override these). - This convenience is provided to encourage :mod:`shlex` users to generate error + This convenience is provided to encourage :mod:`!shlex` users to generate error messages in the standard, parseable format understood by Emacs and other Unix tools. @@ -343,7 +356,7 @@ variables which either control lexical analysis or can be used for debugging: Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -366,7 +379,7 @@ following rules. * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in POSIX mode, :class:`~shlex.shlex` will try to obey the following parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is @@ -382,7 +395,7 @@ following parsing rules. * Enclosing characters in quotes which are part of :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its + mentioned in :attr:`~shlex.escape`. The escape characters retain their special meaning only when followed by the quote in use, or the escape character itself. Otherwise the escape character will be considered a normal character. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3a4631e7c657fe7..2822e979990c4de 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -4,9 +4,6 @@ .. module:: shutil :synopsis: High-level file operations, including copying. -.. sectionauthor:: Fred L. Drake, Jr. -.. partly based on the docstrings - **Source code:** :source:`Lib/shutil.py` .. index:: @@ -15,7 +12,7 @@ -------------- -The :mod:`shutil` module offers a number of high-level operations on files and +The :mod:`!shutil` module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal. For operations on individual files, see also the :mod:`os` module. @@ -67,8 +64,8 @@ Directory and files operations The destination location must be writable; otherwise, an :exc:`OSError` exception will be raised. If *dst* already exists, it will be replaced. - Special files such as character or block devices and pipes cannot be - copied with this function. + Special files such as character or block devices, pipes, and sockets cannot + be copied with this function. If *follow_symlinks* is false and *src* is a symbolic link, a new symbolic link will be created instead of copying the @@ -90,10 +87,13 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. + .. versionchanged:: 3.15 + :exc:`SpecialFileError` is now also raised for sockets and device files. + .. exception:: SpecialFileError This exception is raised when :func:`copyfile` or :func:`copytree` attempt - to copy a named pipe. + to copy a named pipe, socket, or device file. .. versionadded:: 2.7 @@ -388,10 +388,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -515,7 +519,7 @@ Directory and files operations .. exception:: Error - This exception collects exceptions that are raised during a multi-file + Subclass of :exc:`OSError` collecting exceptions raised during a multi-file operation. For :func:`copytree`, the exception argument is a list of 3-tuples (*srcname*, *dstname*, *exception*). @@ -540,10 +544,12 @@ On Solaris :func:`os.sendfile` is used. On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB instead of 64 KiB) and a :func:`memoryview`-based variant of -:func:`shutil.copyfileobj` is used. +:func:`shutil.copyfileobj` is used, which still reads and writes in a loop. +:func:`shutil.copy2` uses the native ``CopyFile2`` call on Windows, which is the most +efficient method, supports copy-on-write, and preserves metadata. If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient +file then shutil will silently fall back to less efficient :func:`copyfileobj` function internally. .. versionchanged:: 3.8 @@ -619,8 +625,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Create an archive file (such as zip or tar) and return its name. - *base_name* is the name of the file to create, including the path, minus - any format-specific extension. + *base_name* is a string or :term:`path-like object` specifying the name of + the file to create, including the path, minus any format-specific extension. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the @@ -628,13 +634,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" (if the :mod:`compression.zstd` module is available). - *root_dir* is a directory that will be the root directory of the - archive, all paths in the archive will be relative to it; for example, - we typically chdir into *root_dir* before creating the archive. + *root_dir* is a string or :term:`path-like object` specifying a directory + that will be the root directory of the archive, all paths in the archive + will be relative to it; for example, we typically chdir into *root_dir* + before creating the archive. - *base_dir* is the directory where we start archiving from; - i.e. *base_dir* will be the common prefix of all files and - directories in the archive. *base_dir* must be given relative + *base_dir* is a string or :term:`path-like object` specifying a directory + where we start archiving from; i.e. *base_dir* will be the common prefix of + all files and directories in the archive. *base_dir* must be given relative to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to use *base_dir* and *root_dir* together. @@ -669,12 +676,16 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. This function is now made thread-safe during creation of standard ``.zip`` and tar archives. + .. versionchanged:: 3.15 + Accepts a :term:`path-like object` for *base_name*, *root_dir* and + *base_dir*. + .. function:: get_archive_formats() Return a list of supported formats for archiving. Each element of the returned sequence is a tuple ``(name, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (if the :mod:`zlib` module is available). - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. @@ -692,7 +703,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -745,8 +756,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, @@ -791,7 +802,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Each element of the returned sequence is a tuple ``(name, extensions, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (unpacking compressed files works only if the corresponding module is available). diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index cdefcd29ef7fd5c..12ad45f557e6db5 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -36,7 +36,7 @@ Execution of Python signal handlers A Python signal handler does not get executed inside the low-level (C) signal handler. Instead, the low-level signal handler sets a flag which tells the :term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). +at a later point (for example, at the next :term:`bytecode` instruction). This has consequences: * It makes little sense to catch synchronous errors like :const:`SIGFPE` or @@ -68,6 +68,11 @@ the synchronization primitives from the :mod:`threading` module instead. Besides, only the main thread of the main interpreter is allowed to set a new signal handler. +.. warning:: + + Synchronization primitives such as :class:`threading.Lock` should not be used + within signal handlers. Doing so can lead to unexpected deadlocks. + Module contents --------------- @@ -92,13 +97,13 @@ The signal module defines three enums: .. class:: Handlers - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_DFL` and :const:`SIG_IGN`. .. versionadded:: 3.5 .. class:: Sigmasks - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. .. availability:: Unix. @@ -108,7 +113,7 @@ The signal module defines three enums: .. versionadded:: 3.5 -The variables defined in the :mod:`signal` module are: +The variables defined in the :mod:`!signal` module are: .. data:: SIG_DFL @@ -225,6 +230,8 @@ The variables defined in the :mod:`signal` module are: Stop executing (cannot be caught or ignored). + .. availability:: Unix. + .. data:: SIGSTKFLT Stack fault on coprocessor. The Linux kernel does not raise this signal: it @@ -350,7 +357,7 @@ The variables defined in the :mod:`signal` module are: .. versionadded:: 3.3 -The :mod:`signal` module defines one exception: +The :mod:`!signal` module defines one exception: .. exception:: ItimerError @@ -364,7 +371,7 @@ The :mod:`signal` module defines one exception: alias of :exc:`OSError`. -The :mod:`signal` module defines the following functions: +The :mod:`!signal` module defines the following functions: .. function:: alarm(time) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60eb600..11a5484c2b13362 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -17,7 +17,7 @@ import can be suppressed using the interpreter's :option:`-S` option. Importing this module normally appends site-specific paths to the module search path and adds :ref:`callables `, including :func:`help` to the built-in -namespace. However, Python startup option :option:`-S` blocks this and this module +namespace. However, Python startup option :option:`-S` blocks this, and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -34,7 +34,7 @@ For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or :file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The -optional suffix "t" indicates the :term:`free threading` build, and is +optional suffix "t" indicates the :term:`free-threaded build`, and is appended if ``"t"`` is present in the :data:`sys.abiflags` constant.) For each of the distinct head-tail combinations, it sees if it refers to an existing @@ -51,11 +51,11 @@ added path for configuration files. .. versionchanged:: 3.14 - :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :mod:`!site` is no longer responsible for updating :data:`sys.prefix` and :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`!site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: @@ -64,46 +64,128 @@ When running under a :ref:`virtual environment ` :file:`{name}.start` file exists. + + Errors on individual lines no longer abort processing of the rest of the + file. Each error is reported and the remaining lines continue to be + processed. + +.. deprecated-removed:: 3.15 3.20 + + Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig`` + is deprecated in Python 3.15, and support for decoding from the locale + encoding will be removed in Python 3.20. + + ``import`` lines in :file:`{name}.pth` files are deprecated and will be + silently ignored in Python 3.18 and 3.19. In Python 3.20 a warning will be + produced for ``import`` lines in :file:`{name}.pth` files. + + +.. _site-start-files: + +Startup entry points (:file:`.start` files) +------------------------------------------- + +.. versionadded:: 3.15 + +A startup entry point file is a file whose name has the form +:file:`{name}.start` and exists in one of the site-packages directories +described above. Each file specifies entry points to be called during +interpreter startup, using the ``pkg.mod:callable`` syntax understood by +:func:`pkgutil.resolve_name`. + +Each non-blank line that does not begin with ``#`` must contain an entry +point reference in the form ``pkg.mod:callable``. The colon and callable +portion are mandatory. Each callable is invoked with no arguments, and +any return value is discarded. + +:file:`.start` files are processed after all :file:`.pth` path extensions +have been applied to :data:`sys.path`, ensuring that paths are available +before any startup code runs. + +Unlike :data:`sys.path` extensions from :file:`.pth` files, duplicate entry +points are **not** de-duplicated --- if an entry point appears more than once, +it will be called more than once. + +If an exception occurs during resolution or invocation of an entry point, +a traceback is printed to :data:`sys.stderr` and processing continues with +the remaining entry points. + +:file:`.start` files must be encoded in UTF-8. + +:pep:`829` defined the original specification for these features. + +.. note:: + + If a :file:`{name}.start` file exists alongside a :file:`{name}.pth` file + with the same base name, any ``import`` lines in the :file:`.pth` file are + ignored in favor of the entry points in the :file:`.start` file. + +.. note:: + + Executable lines (``import`` lines in :file:`{name}.pth` files and + :file:`{name}.start` file entry points) are always run at Python startup + (unless :option:`-S` is given to disable the ``site.py`` module entirely), + regardless of whether a particular module is actually going to be used. + +.. note:: + + :file:`{name}.start` files invoke :func:`pkgutil.resolve_name` with + ``strict=True``, which requires the full ``pkg.mod:callable`` form. + .. index:: single: package triple: path; configuration; file + +Startup file examples +--------------------- + For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to :file:`/usr/local`. The Python X.Y library is then installed in :file:`/usr/local/lib/python{X.Y}`. Suppose this has a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path +sub-subdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume :file:`foo.pth` contains the following:: @@ -130,38 +212,77 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -:mod:`sitecustomize` --------------------- +Let's say that there is also a :file:`foo.start` file containing the +following:: + + # foo package startup code + + foo.submod:initialize + +Now, after ``sys.path`` has been extended as above, and before Python turns +control over to user code, the ``foo.submod`` module is imported and the +``initialize()`` function from that module is called. + + +.. _site-migration-guide: + +Migrating from ``import`` lines in ``.pth`` files to ``.start`` files +--------------------------------------------------------------------- + +If your package currently ships a :file:`{name}.pth` file, you can keep all +``sys.path`` extension lines unchanged. Only ``import`` lines need to be +migrated. + +To migrate, create a callable (taking zero arguments) within an importable +module in your package. Reference it as a ``pkg.mod:callable`` entry point +in a matching :file:`{name}.start` file. Move everything on your ``import`` +line after the first semi-colon into the ``callable()`` function. + +If your package must straddle older Pythons that do not support :pep:`829` +and newer Pythons that do, change the ``import`` lines in your +:file:`{name}.pth` to use the following form: + +.. code-block:: python + + import pkg.mod; pkg.mod.callable() + +Older Pythons will execute these ``import`` lines, while newer Pythons will +ignore them in favor of the :file:`{name}.start` file. After the straddling +period, remove all ``import`` lines from your :file:`.pth` files. + + +:mod:`!sitecustomize` +--------------------- .. module:: sitecustomize After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. +:mod:`!sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'sitecustomize'``, +attribute equals ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any other exception +attempted output from :mod:`!sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -:mod:`usercustomize` --------------------- +:mod:`!usercustomize` +--------------------- .. module:: usercustomize -After this, an attempt is made to import a module named :mod:`usercustomize`, +After this, an attempt is made to import a module named :mod:`!usercustomize`, which can perform arbitrary user-specific customizations, if :data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'usercustomize'``, it is silently ignored. +attribute equals ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +:mod:`sitecustomize` and :mod:`!usercustomize` is still attempted. .. currentmodule:: site @@ -173,7 +294,7 @@ Readline configuration On systems that support :mod:`readline`, this module will also import and configure the :mod:`rlcompleter` module, if Python is started in :ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use +The default behavior is to enable tab completion and to use :file:`~/.python_history` as the history save file. To disable it, delete (or override) the :data:`sys.__interactivehook__` attribute in your :mod:`sitecustomize` or :mod:`usercustomize` module or your @@ -235,10 +356,97 @@ Module contents This function used to be called unconditionally. +.. function:: makepath(*paths) + + Join *paths* with :func:`os.path.join`, attempt to make the result + absolute with :func:`os.path.abspath`, and return a 2-tuple containing + the absolute path and its case-normalized form as produced by + :func:`os.path.normcase`. If :func:`os.path.abspath` raises + :exc:`OSError`, the joined path is used unchanged for the + case-normalization step. + + The second element of the returned tuple is the form used throughout the + :mod:`!site` module to compare paths on case-insensitive file systems, and + is what populates the ``known_paths`` sets that prevent duplicate + :data:`sys.path` entries in various APIs within this module. + + +.. class:: StartupState(known_paths=None) + + Instances of this class accumulate interpreter startup configuration data + from one or more site directories. They are the preferred interface for + batching the processing of :file:`.pth` and :file:`.start` files across + multiple site directories, so that every :data:`sys.path` extension is + visible before any startup code runs. + + The optional *known_paths* argument is a set of case-normalized paths + (which can be produced by :func:`makepath`) used to prevent duplicate + :data:`sys.path` entries. When ``None`` (the default), the set is built + from the current :data:`sys.path`. :func:`main` implicitly uses an + instance of this class. + + Typical use: + + .. code-block:: python + + state = site.StartupState() + for sitedir in site_dirs: + state.addsitedir(sitedir) + state.process() + + .. versionadded:: 3.15 + + .. method:: addsitedir(sitedir) + + Read the :file:`.pth` and :file:`.start` files in *sitedir* and + record their :data:`sys.path` extensions, deprecated :file:`.pth` + ``import`` lines, and :file:`.start` entry points on this state. + The recorded data is not applied until :meth:`process` is called. + + .. method:: addusersitepackages() + + Add the per-user site-packages directory, if enabled and if it exists. + The directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: addsitepackages(prefixes=None) + + Add global site-packages directories, computed from *prefixes* or from + the global :data:`PREFIXES` when *prefixes* is ``None``. Each + directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: process() + + Apply the accumulated state by first adding the path extensions to + :data:`sys.path`, then executing the :file:`.start` file entry points + and :file:`.pth` file ``import`` lines (:ref:`deprecated + `). + + This method is not idempotent and must not be called more than once + on the same instance. Doing so will apply the accumulated state + more than once, re-running entry points and ``import`` lines. + + .. function:: addsitedir(sitedir, known_paths=None) - Add a directory to sys.path and process its :file:`.pth` files. Typically - used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). + Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` + files found in that directory. Typically used in :mod:`sitecustomize` or + :mod:`usercustomize` (see above). + + The *known_paths* argument is an optional set of case-normalized paths + used to prevent duplicate :data:`sys.path` entries. When ``None`` (the + default), the set is built from the current :data:`sys.path`. + + For batched processing across multiple site directories, build a + :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir` + on it; that defers :file:`.pth` and :file:`.start` processing until a + single :meth:`StartupState.process` call, ensuring every :data:`sys.path` + extension is visible before any startup code runs. + + .. versionchanged:: 3.15 + + Also processes :file:`.start` files. See :ref:`site-start-files`. .. function:: getsitepackages() @@ -270,12 +478,12 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site -The :mod:`site` module also provides a way to get the user directories from the +The :mod:`!site` module also provides a way to get the user directories from the command line: .. code-block:: shell-session @@ -307,5 +515,5 @@ value greater than 2 if there is an error. .. seealso:: * :pep:`370` -- Per user site-packages directory + * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 3ee8b82a1880f38..5c97199bc453e85 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -4,8 +4,6 @@ .. module:: smtplib :synopsis: SMTP protocol client (requires sockets). -.. sectionauthor:: Eric S. Raymond - **Source code:** :source:`Lib/smtplib.py` .. index:: @@ -14,7 +12,7 @@ -------------- -The :mod:`smtplib` module defines an SMTP client session object that can be used +The :mod:`!smtplib` module defines an SMTP client session object that can be used to send mail to any internet machine with an SMTP or ESMTP listener daemon. For details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer Protocol) and :rfc:`1869` (SMTP Service Extensions). @@ -354,7 +352,7 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in + Each of the authentication methods supported by :mod:`!smtplib` are tried in turn if they are advertised as supported by the server. See :meth:`auth` for a list of supported authentication methods. *initial_response_ok* is passed through to :meth:`auth`. @@ -406,7 +404,7 @@ An :class:`SMTP` instance has the following methods: call the :meth:`login` method, which will try each of the above mechanisms in turn, in the order listed. ``auth`` is exposed to facilitate the implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. + directly by :mod:`!smtplib`. .. versionadded:: 3.5 @@ -458,9 +456,11 @@ An :class:`SMTP` instance has the following methods: Send mail. The required arguments are an :rfc:`822` from-address string, a list of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. + (such as ``"8bitmime"``) to be used in ``MAIL FROM`` commands as *mail_options*. ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP + commands can be passed as *rcpt_options*. Each option should be passed as a string + containing the full text of the option, including any potential key + (for instance, ``"NOTIFY=SUCCESS,FAILURE"``). (If you need to use different ESMTP options to different recipients you have to use the low-level methods such as :meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 89bca9b5b20df7c..836aa91bb0885b1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -39,6 +39,8 @@ is implicit on send operations. A TLS/SSL wrapper for socket objects. +.. _socket-addresses: + Socket families --------------- @@ -83,7 +85,7 @@ created. Socket addresses are represented as follows: - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + :mod:`!socket` module methods, *flowinfo* and *scope_id* can be omitted just for backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. @@ -118,10 +120,10 @@ created. Socket addresses are represented as follows: ``'can0'``. The network interface name ``''`` can be used to receive packets from all network interfaces of this family. - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` + - :const:`CAN_ISOTP` protocol requires a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + - :const:`CAN_J1939` protocol requires a tuple ``(interface, name, pgn, addr)`` where additional parameters are 64-bit unsigned integer representing the ECU name, a 32-bit unsigned integer representing the Parameter Group Number (PGN), and an 8-bit integer representing the address. @@ -302,7 +304,7 @@ generalization of this based on timeouts is supported through Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`!socket` exports the following elements. Exceptions @@ -482,6 +484,10 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and .. versionchanged:: 3.14 Added support for ``TCP_QUICKACK`` on Windows platforms when available. + .. versionchanged:: 3.15 + ``IPV6_HDRINCL`` was added. + Added support for ``SO_PASSRIGHTS`` on Linux platforms when available. + .. data:: AF_CAN PF_CAN @@ -836,71 +842,14 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. +The :class:`socket ` class constructor creates a new socket +directly; see :ref:`socket-objects` for its parameters and full description. .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -996,8 +945,8 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`~socket.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- + family, socket type and protocol number are as for the :func:`~socket.socket` function. + The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on a socket passed to a program as standard input or output (such as a server @@ -1019,22 +968,16 @@ The following functions all create :ref:`socket objects `. .. versionadded:: 3.3 -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - Other functions ''''''''''''''' -The :mod:`socket` module also offers various network-related services: +The :mod:`!socket` module also offers various network-related services: .. function:: close(fd) Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` + sockets. On some platforms (most notably Windows) :func:`os.close` does not work for socket file descriptors. .. versionadded:: 3.7 @@ -1069,10 +1012,16 @@ The :mod:`socket` module also offers various network-related services: a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. + format depends on the returned *family* and flags Python was compiled with, + and is meant to be passed to the :meth:`socket.connect` method. + + *sockaddr* can be one of the following: + + * a ``(address, port)`` 2-tuple for :const:`AF_INET` + * a ``(address, port, flowinfo, scope_id)`` 4-tuple for :const:`AF_INET6` if + Python was compiled with ``--enable-ipv6`` (the default) + * a 2-tuple containing raw data for :const:`AF_INET6` if Python was + compiled with ``--disable-ipv6`` .. note:: @@ -1526,644 +1475,709 @@ The :mod:`socket` module also offers various network-related services: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using + :meth:`socket.close`. -.. method:: socket.accept() + The newly created socket is :ref:`non-inheritable `. - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. .. versionchanged:: 3.4 - The socket is now non-inheritable. + The CAN_BCM protocol was added. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. versionchanged:: 3.7 + The CAN_ISOTP protocol was added. -.. method:: socket.bind(address) + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system ``socket()`` call. Therefore, - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) + :: - .. audit-event:: socket.bind self,address socket.socket.bind + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - .. availability:: not WASI. + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. -.. method:: socket.close() + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). + Socket objects have the following methods. Except for + :meth:`~socket.makefile`, these correspond to Unix system calls applicable + to sockets. - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. + .. versionchanged:: 3.2 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`~socket.close`. - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - .. note:: + .. method:: accept() - :meth:`close` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown` - before :meth:`close`. + Accept a connection. The socket must be bound to an address and listening for + connections. The return value is a pair ``(conn, address)`` where *conn* is a + *new* socket object usable to send and receive data on the connection, and + *address* is the address bound to the socket on the other end of the connection. + The newly created socket is :ref:`non-inheritable `. -.. method:: socket.connect(address) + .. versionchanged:: 3.4 + The socket is now non-inheritable. - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect self,address socket.socket.connect + .. method:: bind(address) - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. + + .. audit-event:: socket.bind self,address socket.socket.bind + + .. availability:: not WASI. + + + .. method:: close() + + Mark the socket closed. The underlying system resource (e.g. a file + descriptor) is also closed when all file objects from :meth:`makefile` + are closed. Once that happens, all future operations on the socket + object will fail. The remote end will receive no more data (after + queued data is flushed). + + Sockets are automatically closed when they are garbage-collected, but + it is recommended to :meth:`close` them explicitly, or to use a + :keyword:`with` statement around them. + + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`!close` call is made. + + .. note:: + + :meth:`close` releases the resource associated with a connection but + does not necessarily close the connection immediately. If you want + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. + + + .. method:: connect(address) + + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. + + If the connection is interrupted by a signal, the method waits until the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the + signal handler doesn't raise an exception and the socket is blocking or has + a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). + signal (or the exception raised by the signal handler). - .. availability:: not WASI. + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 + The method now waits until the connection completes instead of raising an + :exc:`InterruptedError` exception if the connection is interrupted by a + signal, the signal handler doesn't raise an exception and the socket is + blocking or has a timeout (see the :pep:`475` for the rationale). -.. method:: socket.connect_ex(address) + .. availability:: not WASI. - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - .. audit-event:: socket.connect self,address socket.socket.connect_ex + .. method:: connect_ex(address) - .. availability:: not WASI. + Like ``connect(address)``, but return an error indicator instead of raising an + exception for errors returned by the C-level :c:func:`!connect` call (other + problems, such as "host not found," can still raise exceptions). The error + indicator is ``0`` if the operation succeeded, otherwise the value of the + :c:data:`errno` variable. This is useful to support, for example, asynchronous + connects. -.. method:: socket.detach() + .. audit-event:: socket.connect self,address socket.socket.connect_ex - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. + .. availability:: not WASI. - .. versionadded:: 3.2 + .. method:: detach() + Put the socket object into closed state without actually closing the + underlying file descriptor. The file descriptor is returned, and can + be reused for other purposes. -.. method:: socket.dup() + .. versionadded:: 3.2 - Duplicate the socket. - The newly created socket is :ref:`non-inheritable `. + .. method:: dup() - .. versionchanged:: 3.4 - The socket is now non-inheritable. + Duplicate the socket. - .. availability:: not WASI. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.fileno() + .. availability:: not WASI. - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. + .. method:: fileno() -.. method:: socket.get_inheritable() + Return the socket's file descriptor (a small integer), or -1 on failure. This + is useful with :func:`select.select`. - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. + Under Windows the small integer returned by this method cannot be used where a + file descriptor can be used (such as :func:`os.fdopen`). Unix does not have + this limitation. - .. versionadded:: 3.4 + .. method:: get_inheritable() + Get the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. -.. method:: socket.getpeername() + .. versionadded:: 3.4 - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. + .. method:: getpeername() -.. method:: socket.getsockname() + Return the remote address to which the socket is connected. This is useful to + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) + .. method:: getsockname() -.. method:: socket.getsockopt(level, optname[, buflen]) + Return the socket's own address. This is useful to find out the port number of + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - .. availability:: not WASI. + .. method:: getsockopt(level, optname[, buflen]) + Return the value of the given socket option (see the Unix man page + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) + are defined in this module. If *buflen* is absent, an integer option is assumed + and its integer value is returned by the function. If *buflen* is present, it + specifies the maximum length of the buffer used to receive the option in, and + this buffer is returned as a bytes object. It is up to the caller to decode the + contents of the buffer (see the optional built-in module :mod:`struct` for a way + to decode C structures encoded as byte strings). -.. method:: socket.getblocking() + .. availability:: not WASI. - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - This is equivalent to checking ``socket.gettimeout() != 0``. + .. method:: getblocking() - .. versionadded:: 3.7 + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + This is equivalent to checking ``socket.gettimeout() != 0``. -.. method:: socket.gettimeout() + .. versionadded:: 3.7 - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. + .. method:: gettimeout() -.. method:: socket.ioctl(control, option) + Return the timeout in seconds (float) associated with socket operations, + or ``None`` if no timeout is set. This reflects the last call to + :meth:`setblocking` or :meth:`settimeout`. - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. + .. method:: ioctl(control, option) - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + The :meth:`ioctl` method is a limited interface to the WSAIoctl system + interface. Please refer to the `Win32 documentation + `_ for more + information. - .. availability:: Windows + On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` + functions may be used; they accept a socket object as their first argument. - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. + Currently only the following control codes are supported: + ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows -.. method:: socket.listen([backlog]) + .. versionchanged:: 3.6 + ``SIO_LOOPBACK_FAST_PATH`` was added. - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - .. availability:: not WASI. + .. method:: listen([backlog]) - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. + Enable a server to accept connections. If *backlog* is specified, it must + be at least 0 (if it is lower, it is set to 0); it specifies the number of + unaccepted connections that the system will allow before refusing new + connections. If not specified, a default reasonable value is chosen. + .. availability:: not WASI. -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) + .. versionchanged:: 3.5 + The *backlog* parameter is now optional. - .. index:: single: I/O control; buffering - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or - a combination of those. + .. method:: makefile(mode='r', buffering=None, *, encoding=None, \ + errors=None, newline=None) - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. + .. index:: single: I/O control; buffering - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Return a :term:`file object` associated with the socket. The exact returned + type depends on the arguments given to :meth:`makefile`. These arguments are + interpreted the same way as by the built-in :func:`open` function, except + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. - .. note:: + The socket must be in blocking mode; it can have a timeout, but the file + object's internal buffer may end up in an inconsistent state if a timeout + occurs. - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. + Closing the file object returned by :meth:`makefile` won't close the + original socket unless all other file objects have been closed and + :meth:`socket.close` has been called on the socket object. + .. note:: -.. method:: socket.recv(bufsize[, flags]) + On Windows, the file-like object created by :meth:`makefile` cannot be + used where a file object with a file descriptor is expected, such as the + stream arguments of :meth:`subprocess.Popen`. - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. A returned empty bytes object indicates that the client has disconnected. - See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument - *flags*; it defaults to zero. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recv(bufsize[, flags]) + Receive data from the socket. The return value is a bytes object representing the + data received. The maximum amount of data to be received at once is specified + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. -.. method:: socket.recvfrom(bufsize[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recvfrom(bufsize[, flags]) - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. + Receive data from the socket. The return value is a pair ``(bytes, address)`` + where *bytes* is a bytes object representing the data received and *address* is the + address of the socket sending the data. See the Unix manual page + :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. - Most Unix platforms. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope_id`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. + .. method:: recvmsg(bufsize[, ancbufsize[, flags]]) + + Receive normal data (up to *bufsize* bytes) and ancillary data from + the socket. The *ancbufsize* argument sets the size in bytes of + the internal buffer used to receive the ancillary data; it defaults + to 0, meaning that no ancillary data will be received. Appropriate + buffer sizes for ancillary data can be calculated using + :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit + into the buffer might be truncated or discarded. The *flags* + argument defaults to 0 and has the same meaning as for + :meth:`recv`. + + The return value is a 4-tuple: ``(data, ancdata, msg_flags, + address)``. The *data* item is a :class:`bytes` object holding the + non-ancillary data received. The *ancdata* item is a list of zero + or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing + the ancillary data (control messages) received: *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + :class:`bytes` object holding the associated data. The *msg_flags* + item is the bitwise OR of various flags indicating conditions on + the received message; see your system documentation for details. + If the receiving socket is unconnected, *address* is the address of + the sending socket, if available; otherwise, its value is + unspecified. + + On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to + pass file descriptors between processes over an :const:`AF_UNIX` + socket. When this facility is used (it is often restricted to + :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its + ancillary data, items of the form ``(socket.SOL_SOCKET, + socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object + representing the new file descriptors as a binary array of the + native C :c:expr:`int` type. If :meth:`recvmsg` raises an + exception after the system call returns, it will first attempt to + close any file descriptors received via this mechanism. + + Some systems do not indicate the truncated length of ancillary data + items which have been only partially received. If an item appears + to extend beyond the end of the buffer, :meth:`recvmsg` will issue + a :exc:`RuntimeWarning`, and will return the part of it which is + inside the buffer provided it has not been truncated before the + start of its associated data. + + On systems which support the :const:`!SCM_RIGHTS` mechanism, the + following function will receive up to *maxfds* file descriptors, + returning the message data and a list containing the descriptors + (while ignoring unexpected conditions such as unrelated control + messages being received). See also :meth:`sendmsg`. :: + + import socket, array + + def recv_fds(sock, msglen, maxfds): + fds = array.array("i") # Array of ints + msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: + # Append data, ignoring any truncated integers at the end. + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + return msg, list(fds) + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + + .. method:: recvmsg_into(buffers[, ancbufsize[, flags]]) + + Receive normal data and ancillary data from the socket, behaving as + :meth:`recvmsg` would, but scatter the non-ancillary data into a + series of buffers instead of returning a new bytes object. The + *buffers* argument must be an iterable of objects that export + writable buffers (e.g. :class:`bytearray` objects); these will be + filled with successive chunks of the non-ancillary data until it + has all been written or there are no more buffers. The operating + system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) + on the number of buffers that can be used. The *ancbufsize* and + *flags* arguments have the same meaning as for :meth:`recvmsg`. + + The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, + address)``, where *nbytes* is the total number of bytes of + non-ancillary data written into the buffers, and *ancdata*, + *msg_flags* and *address* are the same as for :meth:`recvmsg`. + + Example:: + + >>> import socket + >>> s1, s2 = socket.socketpair() + >>> b1 = bytearray(b'----') + >>> b2 = bytearray(b'0123456789') + >>> b3 = bytearray(b'--------------') + >>> s1.send(b'Mary had a little lamb') + 22 + >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) + (22, [], 0, None) + >>> [b1, b2, b3] + [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + + .. method:: recvfrom_into(buffer[, nbytes[, flags]]) + + Receive data from the socket, writing it into *buffer* instead of creating a + new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is + the number of bytes received and *address* is the address of the socket sending + the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. + + + .. method:: recv_into(buffer[, nbytes[, flags]]) + + Receive up to *nbytes* bytes from the socket, storing the data into a buffer + rather than creating a new bytestring. If *nbytes* is not specified (or 0), + receive up to the size available in the given buffer. Returns the number of + bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning + of the optional argument *flags*; it defaults to zero. - Most Unix platforms. - .. versionadded:: 3.3 + .. method:: send(bytes[, flags]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Returns the number of bytes sent. Applications are responsible for checking that + all data has been sent; if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. For further + information on this topic, consult the :ref:`socket-howto`. -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) + .. method:: sendall(bytes[, flags]) -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Unlike :meth:`send`, this method continues to send data from *bytes* until + either all data has been sent or an error occurs. ``None`` is returned on + success. On error, an exception is raised, and there is no way to determine how + much data, if any, was successfully sent. - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. + .. versionchanged:: 3.5 + The socket timeout is no longer reset each time data is sent successfully. + The socket timeout is now the maximum total duration to send all data. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. method:: socket.send(bytes[, flags]) - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. + .. method:: sendto(bytes, address) + sendto(bytes, flags, address) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send data to the socket. The socket should not be connected to a remote socket, + since the destination socket is specified by *address*. The optional *flags* + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. + .. audit-event:: socket.sendto self,address socket.socket.sendto -.. method:: socket.sendall(bytes[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. + .. method:: sendmsg(buffers[, ancdata[, flags[, address]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send normal and ancillary data to the socket, gathering the + non-ancillary data from a series of buffers and concatenating it + into a single message. The *buffers* argument specifies the + non-ancillary data as an iterable of + :term:`bytes-like objects ` + (e.g. :class:`bytes` objects); the operating system may set a limit + (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers + that can be used. The *ancdata* argument specifies the ancillary + data (control messages) as an iterable of zero or more tuples + ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + bytes-like object holding the associated data. Note that + some systems (in particular, systems without :func:`CMSG_SPACE`) + might support sending only one control message per call. The + *flags* argument defaults to 0 and has the same meaning as for + :meth:`send`. If *address* is supplied and not ``None``, it sets a + destination address for the message. The return value is the + number of bytes of non-ancillary data sent. + The following function sends the list of file descriptors *fds* + over an :const:`AF_UNIX` socket, on systems which support the + :const:`!SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) + import socket, array - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) + def send_fds(sock, msg, fds): + return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - .. audit-event:: socket.sendto self,address socket.socket.sendto + .. availability:: Unix, not WASI. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) + Most Unix platforms. - .. availability:: Unix, not WASI. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - Most Unix platforms. + .. versionadded:: 3.3 - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. method:: sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. + Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) + .. availability:: Linux >= 2.6.38. - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. + .. versionadded:: 3.6 - .. availability:: Linux >= 2.6.38. + .. method:: sendfile(file, offset=0, count=None) - .. versionadded:: 3.6 + Send a file until EOF is reached by using high-performance + :mod:`os.sendfile` and return the total number of bytes which were sent. + *file* must be a regular file object opened in binary mode. If + :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a + regular file :meth:`send` will be used instead. *offset* tells from where to + start reading the file. If specified, *count* is the total number of bytes + to transmit as opposed to sending the file until EOF is reached. File + position is updated on return or also in case of error in which case + :meth:`file.tell() ` can be used to figure out the number of + bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. + Non-blocking sockets are not supported. -.. method:: socket.sendfile(file, offset=0, count=None) + .. versionadded:: 3.5 - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. + .. method:: set_inheritable(inheritable) - .. versionadded:: 3.5 + Set the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle. -.. method:: socket.set_inheritable(inheritable) + .. versionadded:: 3.4 - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - .. versionadded:: 3.4 + .. method:: setblocking(flag) + Set blocking or non-blocking mode of the socket: if *flag* is false, the + socket is set to non-blocking, else to blocking mode. -.. method:: socket.setblocking(flag) + This method is a shorthand for certain :meth:`~socket.settimeout` calls: - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. + * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - This method is a shorthand for certain :meth:`~socket.settimeout` calls: + * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: settimeout(value) + Set a timeout on blocking socket operations. The *value* argument can be a + nonnegative real number expressing seconds, or ``None``. + If a non-zero value is given, subsequent socket operations will raise a + :exc:`timeout` exception if the timeout period *value* has elapsed before + the operation has completed. If zero is given, the socket is put in + non-blocking mode. If ``None`` is given, the socket is put in blocking mode. -.. method:: socket.settimeout(value) + For further information, please consult the :ref:`notes on socket timeouts `. - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative real number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - For further information, please consult the :ref:`notes on socket timeouts `. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. - .. versionchanged:: 3.15 - Accepts any real number, not only integer or float. + .. method:: setsockopt(level, optname, value: int | Buffer) + setsockopt(level, optname, None, optlen: int) + .. index:: pair: module; struct -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: + Set the value of the given socket option (see the Unix manual page + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter + case it is up to the caller to ensure that the bytestring contains the + proper bits (see the optional built-in module :mod:`struct` for a way to + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to calling :c:func:`!setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. - .. index:: pair: module; struct + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this - module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. + .. versionchanged:: 3.6 + setsockopt(level, optname, None, optlen: int) form added. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. availability:: not WASI. - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - .. availability:: not WASI. + .. method:: shutdown(how) + Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, + further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends + are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are + disallowed. -.. method:: socket.shutdown(how) + .. availability:: not WASI. - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - .. availability:: not WASI. + .. method:: share(process_id) + Duplicate a socket and prepare it for sharing with a target process. The + target process must be provided with *process_id*. The resulting bytes object + can then be passed to the target process using some form of interprocess + communication and the socket can be recreated there using :func:`fromshare`. + Once this method has been called, it is safe to close the socket since + the operating system has already duplicated it for the target process. -.. method:: socket.share(process_id) + .. availability:: Windows. - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. + .. versionadded:: 3.3 - .. availability:: Windows. - .. versionadded:: 3.3 + Note that there are no methods :meth:`!read` or :meth:`!write`; use + :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. + Socket objects also have these (read-only) attributes that correspond to the + values given to the :class:`~socket.socket` constructor. -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. + .. attribute:: family + The socket family. -.. attribute:: socket.family - The socket family. + .. attribute:: type + The socket type. -.. attribute:: socket.type - The socket type. + .. attribute:: proto + The socket protocol. -.. attribute:: socket.proto - The socket protocol. +.. class:: SocketType + The base class of the :class:`~socket.socket` type, re-exported from + :mod:`!_socket`. An instance check such as + ``isinstance(socket(...), SocketType)`` is true, but ``SocketType`` is not + the same as ``type(socket(...))``, which is :class:`~socket.socket` itself. .. _socket-timeouts: @@ -2421,7 +2435,7 @@ lead to this error:: This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. -There is a :mod:`socket` flag to set, in order to prevent this, +There is a :mod:`!socket` flag to set, in order to prevent this, :const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7bc2f7afbbb0b1e..4c98bb8e3b9c9bf 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`socketserver` module simplifies the task of writing network servers. +The :mod:`!socketserver` module simplifies the task of writing network servers. .. include:: ../includes/wasm-notavail.rst @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 3b1a9c2f6eefe9f..3a75d44f3f7d21b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -4,8 +4,6 @@ .. module:: sqlite3 :synopsis: A DB-API 2.0 implementation using SQLite 3.x. -.. sectionauthor:: Gerhard Häring - **Source code:** :source:`Lib/sqlite3/` .. Make sure we always doctest the tutorial with an empty database. @@ -57,7 +55,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -291,7 +289,7 @@ Module functions Set it to any combination (using ``|``, bitwise or) of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to enable this. - Column names takes precedence over declared types if both flags are set. + Column names take precedence over declared types if both flags are set. By default (``0``), type detection is disabled. :param isolation_level: @@ -1419,6 +1417,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1428,6 +1429,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: next + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1711,6 +1715,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break @@ -2287,7 +2294,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2302,21 +2309,21 @@ This section shows recipes for common adapters and converters. """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2325,17 +2332,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index fa0a52347204226..66fe6c7aee48626 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -4,9 +4,6 @@ .. module:: ssl :synopsis: TLS/SSL wrapper for socket objects -.. moduleauthor:: Bill Janssen -.. sectionauthor:: Bill Janssen - **Source code:** :source:`Lib/ssl.py` .. index:: single: OpenSSL; (use in module ssl) @@ -70,7 +67,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -130,7 +127,7 @@ purposes. cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, + the given *purpose*. The settings are chosen by the :mod:`!ssl` module, and usually represent a higher security level than when calling the :class:`SSLContext` constructor directly. @@ -149,9 +146,9 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. - When :attr:`~SSLContext.keylog_filename` is supported and the environment - variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` - enables key logging. + When the environment variable :envvar:`!SSLKEYLOGFILE` is set, + :func:`create_default_context` enables key logging by setting + :attr:`~SSLContext.keylog_filename` to the variable's value. The default settings for this context include :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. @@ -377,7 +374,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -387,12 +384,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -1075,7 +1072,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1124,7 +1121,7 @@ SSL Sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -1465,7 +1462,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -1510,7 +1507,7 @@ to speed up repeated connections from the same clients. TLS 1.3. .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose + :func:`create_default_context` lets the :mod:`!ssl` module choose security settings for a given purpose. .. versionchanged:: 3.6 @@ -2079,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2656,7 +2653,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 @@ -2882,11 +2879,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. @@ -2959,16 +2956,16 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Steve Kent :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller + Donald E. Eastlake, Jeffrey I. Schiller, Steve Crocker :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper + David Cooper et al. :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. + Tim Dierks and Eric Rescorla. :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake + Donald E. Eastlake `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 1cbec3ab847c5fa..5c5f1858ba44768 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -5,13 +5,11 @@ :synopsis: Utilities for interpreting the results of os.stat(), os.lstat() and os.fstat(). -.. sectionauthor:: Skip Montanaro - **Source code:** :source:`Lib/stat.py` -------------- -The :mod:`stat` module defines constants and functions for interpreting the +The :mod:`!stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and :c:func:`!lstat` calls, consult the documentation for your system. @@ -19,7 +17,7 @@ exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and .. versionchanged:: 3.4 The stat module is backed by a C implementation. -The :mod:`stat` module defines the following functions to test for specific file +The :mod:`!stat` module defines the following functions to test for specific file types: @@ -511,4 +509,4 @@ meaning of these constants. STATX_ATTR_DAX STATX_ATTR_WRITE_ATOMIC - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 614f5b905a4a2eb..dba0e26787d9516 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -4,9 +4,6 @@ .. module:: statistics :synopsis: Mathematical statistics functions -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - .. versionadded:: 3.4 **Source code:** :source:`Lib/statistics.py` @@ -716,7 +713,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: covariance(x, y, /) - Return the sample covariance of two inputs *x* and *y*. Covariance + Return the sample covariance of two sequence inputs *x* and *y*. Covariance is a measure of the joint variability of two inputs. Both inputs must be of the same length (no less than two), otherwise @@ -742,7 +739,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the `Pearson's correlation coefficient `_ - for two inputs. Pearson's correlation coefficient *r* takes values + for two sequence inputs. Pearson's correlation coefficient *r* takes values between -1 and +1. It measures the strength and direction of a linear relationship. @@ -805,7 +802,7 @@ However, for reading convenience, most of the examples show sorted sequences. (it is equal to the difference between predicted and actual values of the dependent variable). - Both inputs must be of the same length (no less than two), and + Both inputs must be sequences of the same length (no less than two), and the independent variable *x* cannot be constant; otherwise a :exc:`StatisticsError` is raised. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c539345e5987771..886648e820f071d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -46,8 +46,10 @@ Any object can be tested for truth value, for use in an :keyword:`if` or By default, an object is considered true unless its class defines either a :meth:`~object.__bool__` method that returns ``False`` or a :meth:`~object.__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: +returns zero, when called with the object. [1]_ If one of the methods raises an +exception when called, the exception is propagated and the object does +not have a truth value (for example, :data:`NotImplemented`). +Here are most of the built-in objects considered false: .. index:: single: None (Built-in object) @@ -164,7 +166,7 @@ This table summarizes the comparison operations: pair: object; numeric pair: objects; comparing -Objects of different types, except different numeric types, never compare equal. +Unless stated otherwise, objects of different types never compare equal. The ``==`` operator is always defined but for some object types (for example, class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense; for example, they raise a @@ -263,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: @@ -1091,11 +1101,14 @@ Notes: still ``0``. (4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. + The slice of *s* from *i* to *j* is defined as the sequence of items with + index *k* such that ``i <= k < j``. + + * If *i* is omitted or ``None``, use ``0``. + * If *j* is omitted or ``None``, use ``len(s)``. + * If *i* or *j* is less than ``-len(s)``, use ``0``. + * If *i* or *j* is greater than ``len(s)``, use ``len(s)``. + * If *i* is greater than or equal to *j*, the slice is empty. (5) The slice of *s* from *i* to *j* with step *k* is defined as the sequence of @@ -1150,13 +1163,13 @@ Sequence types also support the following methods: Return the total number of occurrences of *value* in *sequence*. -.. method:: list.index(value[, start[, stop]) - range.index(value[, start[, stop]) - tuple.index(value[, start[, stop]) +.. method:: list.index(value[, start[, stop]]) + range.index(value[, start[, stop]]) + tuple.index(value[, start[, stop]]) :no-contents-entry: :no-index-entry: :no-typesetting: -.. method:: sequence.index(value[, start[, stop]) +.. method:: sequence.index(value[, start[, stop]]) Return the index of the first occurrence of *value* in *sequence*. @@ -1273,7 +1286,7 @@ Mutable sequence types also support the following methods: :no-typesetting: .. method:: sequence.append(value, /) - Append *value* to the end of the sequence + Append *value* to the end of the sequence. This is equivalent to writing ``seq[len(seq):len(seq)] = [value]``. .. method:: bytearray.clear() @@ -1390,6 +1403,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1436,6 +1451,11 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`list` + objects, see :ref:`thread-safety-list`. + .. _typesseq-tuple: @@ -1476,6 +1496,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -1842,6 +1866,14 @@ expression support in the :mod:`re` module). lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` converts it to ``"ss"``. + For example: + + .. doctest:: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' The casefolding algorithm is `described in section 3.13.3 'Default Case Folding' of the Unicode Standard @@ -1994,10 +2026,16 @@ expression support in the :mod:`re` module). ``{}``. Each replacement field contains either the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of - the corresponding argument. + the corresponding argument. For example: + + .. doctest:: >>> "The sum of 1 + 2 is {0}".format(1+2) 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -2037,7 +2075,20 @@ expression support in the :mod:`re` module). .. method:: str.index(sub[, start[, end]]) Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. + not found. For example: + + .. doctest:: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`rindex`. .. method:: str.isalnum() @@ -2045,7 +2096,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are alphanumeric and there is at least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. + ``c.isdigit()``, or ``c.isnumeric()``. For example: + + .. doctest:: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False .. method:: str.isalpha() @@ -2057,13 +2119,32 @@ expression support in the :mod:`re` module). from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard `__. + For example: + + .. doctest:: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See :ref:`unicode-properties`. .. method:: str.isascii() Return ``True`` if the string is empty or all characters in the string are ASCII, ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. + ASCII characters have code points in the range U+0000-U+007F. For example: + + .. doctest:: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False .. versionadded:: 3.7 @@ -2073,9 +2154,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are decimal characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT + numbers in base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". + General Category "Nd". For example: + + .. doctest:: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False .. method:: str.isdigit() @@ -2084,9 +2174,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2121,6 +2227,20 @@ expression support in the :mod:`re` module). that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. + For example: + + .. doctest:: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2139,17 +2259,43 @@ expression support in the :mod:`re` module). Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + .. doctest:: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + + See also :meth:`isspace`. + .. method:: str.isspace() Return ``True`` if there are only whitespace characters in the string and there is at least one character, ``False`` otherwise. + For example: + + .. doctest:: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see :mod:`unicodedata`), either its general category is ``Zs`` ("Separator, space"), or its bidirectional class is one of ``WS``, ``B``, or ``S``. + See also :meth:`isprintable`. + .. method:: str.istitle() @@ -2194,7 +2340,16 @@ expression support in the :mod:`re` module). Return a string which is the concatenation of the strings in *iterable*. A :exc:`TypeError` will be raised if there are any non-string values in *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. + elements is the string providing this method. For example: + + .. doctest:: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also :meth:`split`. .. method:: str.ljust(width, fillchar=' ', /) @@ -2203,11 +2358,29 @@ expression support in the :mod:`re` module). done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also :meth:`rjust`. + .. method:: str.lower() Return a copy of the string with all the cased characters [4]_ converted to - lowercase. + lowercase. For example: + + .. doctest:: + + >>> 'Lower Method Example'.lower() + 'lower method example' The lowercasing algorithm used is `described in section 3.13.2 'Default Case Conversion' of the Unicode Standard @@ -2218,7 +2391,8 @@ expression support in the :mod:`re` module). Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a prefix; rather, all combinations of its values are stripped:: >>> ' spacious '.lstrip() @@ -2250,6 +2424,10 @@ expression support in the :mod:`re` module). the same position in *to*. If there is a third argument, it must be a string, whose characters will be mapped to ``None`` in the result. + .. versionchanged:: 3.15 + + *dict* can now be a :class:`frozendict`. + .. method:: str.partition(sep, /) @@ -2258,12 +2436,27 @@ expression support in the :mod:`re` module). after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + .. doctest:: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also :meth:`rpartition`. + .. method:: str.removeprefix(prefix, /) If the string starts with the *prefix* string, return ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: + string: + + .. doctest:: >>> 'TestHook'.removeprefix('Test') 'Hook' @@ -2272,12 +2465,16 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removesuffix` and :meth:`startswith`. + .. method:: str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is not empty, return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: + original string: + + .. doctest:: >>> 'MiscTests'.removesuffix('Tests') 'Misc' @@ -2286,12 +2483,22 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removeprefix` and :meth:`endswith`. + .. method:: str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or ``-1``, then all occurrences are replaced. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' .. versionchanged:: 3.13 *count* is now supported as a keyword argument. @@ -2302,12 +2509,36 @@ expression support in the :mod:`re` module). Return the highest index in the string where substring *sub* is found, such that *sub* is contained within ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` on failure. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also :meth:`find` and :meth:`rindex`. .. method:: str.rindex(sub[, start[, end]]) Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not found. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`index` and :meth:`find`. .. method:: str.rjust(width, fillchar=' ', /) @@ -2316,6 +2547,19 @@ expression support in the :mod:`re` module). done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also :meth:`ljust` and :meth:`zfill`. + .. method:: str.rpartition(sep, /) @@ -2324,6 +2568,19 @@ expression support in the :mod:`re` module). after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + .. doctest:: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also :meth:`partition`. + .. method:: str.rsplit(sep=None, maxsplit=-1) @@ -2338,15 +2595,19 @@ expression support in the :mod:`re` module). Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* + argument is not a suffix; rather, all combinations of its values are stripped. + For example: + + .. doctest:: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See :meth:`str.removesuffix` for a method that will remove a single suffix + See :meth:`removesuffix` for a method that will remove a single suffix string rather than all of a set of characters. For example:: >>> 'Monty Python'.rstrip(' Python') @@ -2354,6 +2615,9 @@ expression support in the :mod:`re` module). >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also :meth:`strip`. + + .. method:: str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the delimiter @@ -2369,7 +2633,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2387,7 +2653,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2399,7 +2667,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2408,6 +2678,8 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] + See also :meth:`join` and :meth:`rsplit`. + .. index:: single: universal newlines; str.splitlines method @@ -2482,14 +2754,31 @@ expression support in the :mod:`re` module). test string beginning at that position. With optional *end*, stop comparing string at that position. + For example: + + .. doctest:: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also :meth:`endswith` and :meth:`removeprefix`. + .. method:: str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + If omitted or ``None``, the *chars* argument defaults to removing whitespace, + that is characters for which :meth:`str.isspace` is true. The *chars* argument + is not a prefix or suffix; rather, all combinations of its values are stripped. + + For example: + + .. doctest:: >>> ' spacious '.strip() 'spacious' @@ -2500,18 +2789,37 @@ expression support in the :mod:`re` module). from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the trailing end. - For example:: + + For example: + + .. doctest:: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also :meth:`rstrip`. + .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -2564,6 +2872,14 @@ expression support in the :mod:`re` module). You can use :meth:`str.maketrans` to create a translation map from character-to-character mappings in different formats. + The following example uses a mapping to replace ``'a'`` with ``'X'``, + ``'b'`` with ``'Y'``, and delete ``'c'``: + + .. doctest:: + + >>> 'abc123'.translate({ord('a'): 'X', ord('b'): 'Y', ord('c'): None}) + 'XY123' + See also the :mod:`codecs` module for a more flexible approach to custom character mappings. @@ -2589,13 +2905,17 @@ expression support in the :mod:`re` module). than before. The original string is returned if *width* is less than or equal to ``len(s)``. - For example:: + For example: + + .. doctest:: >>> "42".zfill(5) '00042' >>> "-42".zfill(5) '-0042' + See also :meth:`rjust`. + .. index:: single: ! formatted string literal @@ -2611,6 +2931,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2619,123 +2941,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: - - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' - -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: - -.. doctest:: - - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' +.. versionadded:: 3.8 -To include a literal ``{`` or ``}``, use a double bracket: - -.. doctest:: +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -Functions can also be used, and :ref:`format specifiers `: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: -.. doctest:: + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' -Any non-string expression is converted using :func:`str`, by default: +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' - -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: -.. doctest:: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' - >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' - -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" + + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" -.. doctest:: + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' + +.. _stdtypes-tstrings: + +Template String Literals (t-strings) +------------------------------------ + +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: @@ -2899,6 +3245,10 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ +For floating-point formats, the result should be correctly rounded to a given +precision ``p`` of digits after the decimal point. The rounding mode matches +that of the :func:`round` builtin. + Notes: (1) @@ -3191,11 +3541,11 @@ objects. Taking all bytes is a zero-copy operation. - .. versionadded:: next + .. versionadded:: 3.15 See the :ref:`What's New ` entry for common code patterns which can be optimized with - :func:`bytearray.take_bytes`. + :meth:`bytearray.take_bytes`. Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be @@ -3207,6 +3557,11 @@ The representation of bytearray objects uses the bytes literal format ``bytearray([46, 46, 46])``. You can always convert a bytearray object into a list of integers using ``list(b)``. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`bytearray` + objects, see :ref:`thread-safety-bytearray`. + .. _bytes-methods: @@ -3423,12 +3778,13 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new, count=-1, /) - bytearray.replace(old, new, count=-1, /) +.. method:: bytes.replace(old, new, /, count=-1) + bytearray.replace(old, new, /, count=-1) Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. + replaced by *new*. If *count* is given, only the first *count* occurrences + are replaced. If *count* is not specified or ``-1``, then all occurrences + are replaced. The subsequence to search for and its replacement may be any :term:`bytes-like object`. @@ -3438,6 +3794,9 @@ arbitrary binary data. The bytearray version of this method does *not* operate in place - it always produces a new object, even if no changes were made. + .. versionchanged:: 3.15 + *count* is now supported as a keyword argument. + .. method:: bytes.rfind(sub[, start[, end]]) bytearray.rfind(sub[, start[, end]]) @@ -4254,7 +4613,10 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`, which + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -4604,7 +4966,7 @@ copying. .. versionadded:: 3.14 - .. method:: index(value, start=0, stop=sys.maxsize, /) + .. method:: index(value, start=0, stop=sys.maxsize, /) Return the index of the first occurrence of *value* (at or after index *start* and before index *stop*). @@ -4733,6 +5095,9 @@ copying. .. versionadded:: 3.3 +For information on the thread safety of :class:`memoryview` objects in +the :term:`free-threaded build`, see :ref:`thread-safety-memoryview`. + .. _types-set: @@ -4755,11 +5120,12 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. +Since it is mutable, it has no hash value and cannot be used as +either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; +it can therefore be used as a dictionary key or as an element of another set. Non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the @@ -4776,170 +5142,185 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other, /) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other, /) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other, /) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other, /) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other, /) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem, /) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem, /) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem, /) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. + +Sets and frozensets are :ref:`generic ` over the type of their elements. + +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`set` + objects, see :ref:`thread-safety-set`. .. _typesmapping: -Mapping Types --- :class:`dict` -=============================== +Mapping types --- :class:`!dict`, :class:`!frozendict` +====================================================== .. index:: pair: object; mapping @@ -4950,8 +5331,9 @@ Mapping Types --- :class:`dict` pair: built-in function; len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in +There are currently two standard mapping types, the :dfn:`dictionary` and +:class:`frozendict`. +(For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.) @@ -4992,9 +5374,6 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - Dictionaries compare equal if and only if they have the same ``(key, value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise :exc:`TypeError`. To illustrate dictionary creation and equality, @@ -5035,6 +5414,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5224,9 +5606,14 @@ can be used interchangeably to index the same dictionary entry. Dictionaries are now reversible. + .. seealso:: + :class:`frozendict` and :class:`types.MappingProxyType` can be used to + create a read-only view of a :class:`dict`. + .. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. + + For detailed information on thread-safety guarantees for :class:`dict` + objects, see :ref:`thread-safety-dict`. .. _dict-views: @@ -5335,6 +5722,51 @@ An example of dictionary view usage:: 500 +Frozen dictionaries +------------------- + +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + + Return a new frozen dictionary initialized from an optional positional + argument and a possibly empty set of keyword arguments. + + A :class:`!frozendict` has a similar API to the :class:`dict` API, with the + following differences: + + * :class:`!dict` has more methods than :class:`!frozendict`: + + * :meth:`!__delitem__` + * :meth:`!__setitem__` + * :meth:`~dict.clear` + * :meth:`~dict.pop` + * :meth:`~dict.popitem` + * :meth:`~dict.setdefault` + * :meth:`~dict.update` + + * A :class:`!frozendict` can be hashed with ``hash(frozendict)`` if all keys and + values can be hashed. + + * ``frozendict |= other`` does not modify the :class:`!frozendict` in-place but + creates a new frozen dictionary. + + :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly + from ``object``. + + Like dictionaries, frozendicts are :ref:`generic ` over two types, + signifying (respectively) the types of the frozendict's keys and values. + + .. classmethod:: fromkeys(iterable, value=None, /) + + Similar to :meth:`dict.fromkeys`, but call again the type constructor + with an initialized :class:`frozendict` if the type is a + :class:`frozendict` subclass or if the constructor returned a + :class:`frozendict`. + + .. versionadded:: 3.15 + + .. _typecontextmanager: Context Manager Types @@ -5380,9 +5812,11 @@ before the statement body is executed and exited when the statement ends: Returning a true value from this method will cause the :keyword:`with` statement to suppress the exception and continue execution with the statement immediately following the :keyword:`!with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`!with` statement. + propagating after this method has finished executing. + + If this method raises an exception while handling an earlier exception from the + :keyword:`with` block, the new exception is raised, and the original exception + is stored in its :attr:`~BaseException.__context__` attribute. The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed @@ -5471,7 +5905,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5520,6 +5955,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) @@ -5556,6 +6000,7 @@ list is non-exhaustive. * :class:`list` * :class:`dict` * :class:`set` +* :class:`frozendict` * :class:`frozenset` * :class:`type` * :class:`asyncio.Future` diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 58c836c73823306..be968a3c53d8430 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -82,12 +82,12 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows +:pep:`3101`. The :class:`Formatter` class in the :mod:`!string` module allows you to create and customize your own string formatting behaviors using the same implementation as the built-in :meth:`~str.format` method. @@ -192,7 +192,7 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same @@ -304,7 +304,7 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -546,6 +553,9 @@ The available presentation types for :class:`float` and | | :class:`float`, and shows all coefficient digits | | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | | | point is omitted unless the ``#`` option is used. | + | | | + | | For :class:`float`, the exponent always contains at | + | | least two digits, and is zero if the value is zero. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -600,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | @@ -756,8 +771,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' @@ -837,7 +852,7 @@ Template strings support ``$``-based substitutions, using the following rules: Any other appearance of ``$`` in the string will result in a :exc:`ValueError` being raised. -The :mod:`string` module provides a :class:`Template` class that implements +The :mod:`!string` module provides a :class:`Template` class that implements these rules. The methods of :class:`Template` are: diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index a5b2d796aaf4b83..6e91850fdf59ca4 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -245,6 +245,8 @@ Types ... 3.0 | 1. + 2. | None | .2f + Interpolations are :ref:`generic ` over the types of their values. + .. rubric:: Attributes .. attribute:: value diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 37d5adf0fa95411..325ac9ae7c5293a 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -4,9 +4,6 @@ .. module:: stringprep :synopsis: String preparation, as per RFC 3453 -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - **Source code:** :source:`Lib/stringprep.py` -------------- @@ -26,14 +23,14 @@ define which tables it uses, and what other optional parts of the ``stringprep`` procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these +The module :mod:`!stringprep` only exposes the tables from :rfc:`3454`. As these tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that +:mod:`!stringprep` provides the "characteristic function", i.e. a function that returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 17fc479fd0c8c93..ed5a719028d60ef 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -36,7 +36,7 @@ and the C layer. responsible for defining byte ordering and padding between elements. See :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* +Several :mod:`!struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and provide either a readable or read-writable buffer. The most common types used for that purpose are :class:`bytes` and :class:`bytearray`, but many other types @@ -111,7 +111,7 @@ Format Strings -------------- Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, +packing and unpacking data. They are built up from :ref:`type codes `, which specify the type of data being packed/unpacked. In addition, special characters control the :ref:`byte order, size and alignment`. Each format string consists of an optional prefix character which @@ -183,8 +183,8 @@ Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. +Standard size depends only on the type code; see the table in +the :ref:`type-codes` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. @@ -208,12 +208,13 @@ Notes: count of zero. See :ref:`struct-examples`. +.. _type-codes: .. _format-characters: -Format Characters -^^^^^^^^^^^^^^^^^ +Type Codes +^^^^^^^^^^ -Format characters have the following meaning; the conversion between C and +Type codes (or format codes) have the following meaning; the conversion between C and Python values should be obvious given their types. The 'Standard size' column refers to the size of the packed value in bytes when using standard size; that is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or @@ -227,48 +228,48 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | +| ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ | ``f`` | :c:expr:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ | ``d`` | :c:expr:`double` | float | 8 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``F`` | :c:expr:`float complex` | complex | 8 | \(10) | +| ``Zf`` | :c:expr:`float complex` | complex | 8 | \(10) | +--------+--------------------------+--------------------+----------------+------------+ -| ``D`` | :c:expr:`double complex` | complex | 16 | \(10) | +| ``Zd`` | :c:expr:`double complex` | complex | 16 | \(10) | +--------+--------------------------+--------------------+----------------+------------+ | ``s`` | :c:expr:`char[]` | bytes | | \(9) | +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -280,6 +281,18 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. versionchanged:: 3.15 + Added support for the ``'Zf'`` and ``'Zd'`` formats. + +.. versionchanged:: 3.16 + ``'F'`` and ``'D'`` formats are deprecated. + +.. seealso:: + + The :mod:`array` and :ref:`ctypes ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Notes: @@ -311,7 +324,7 @@ Notes: format used by the platform. (5) - The ``'P'`` format character is only available for the native byte ordering + The ``'P'`` type code is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native @@ -322,7 +335,9 @@ Notes: revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a + at full precision. This type is not widely supported by C compilers: + it's available as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. On a typical machine, an unsigned short can be used for storage, but not for math operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. @@ -331,42 +346,48 @@ Notes: When packing, ``'x'`` inserts one NUL byte. (8) - The ``'p'`` format character encodes a "Pascal string", meaning a short + The ``'p'`` type code encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + are used. Note that for :func:`unpack`, the ``'p'`` type code consumes + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) - For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + For the ``'s'`` type code, the count is interpreted as the length of the + byte string, not a repeat count like for the other type codes; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) - For the ``'F'`` and ``'D'`` format characters, the packed representation uses + For the ``'Zf'`` and ``'Zd'`` type codes, the packed representation uses the IEEE 754 binary32 and binary64 format for components of the complex number, regardless of the floating-point format used by the platform. - Note that complex types (``F`` and ``D``) are available unconditionally, + Note that complex types are available unconditionally, despite complex types being an optional feature in C. As specified in the C11 standard, each complex type is represented by a two-element C array containing, respectively, the real and imaginary parts. + The ``'F'`` and ``'D'`` (for ``'Zf'`` and ``'Zd'``, respectively) format + characters are supported for compatibility. -A format character may be preceded by an integral repeat count. For example, +A type code may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must @@ -383,7 +404,7 @@ then :exc:`struct.error` is raised. .. index:: single: ? (question mark); in struct format strings -For the ``'?'`` format character, the return value is either :const:`True` or +For the ``'?'`` type code, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and any non-zero value will be ``True`` when unpacking. @@ -438,7 +459,7 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size in native +The ordering of type codes may have an impact on size in native mode since padding is implicit. In standard mode, the user is responsible for inserting any desired padding. Note in @@ -479,7 +500,7 @@ at the end, assuming the platform's longs are aligned on 4-byte boundaries:: Applications ------------ -Two main applications for the :mod:`struct` module exist, data +Two main applications for the :mod:`!struct` module exist, data interchange between Python and C code within an application or another application compiled using the same compiler (:ref:`native formats`), and data interchange between applications using agreed upon data layout @@ -496,7 +517,7 @@ When constructing format strings which mimic native layouts, the compiler and machine architecture determine byte ordering and padding. In such cases, the ``@`` format character should be used to specify native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be +automatically. It is possible that a zero-repeat type code will be needed at the end of a format string to round up to the correct byte boundary for proper alignment of consecutive chunks of data. @@ -515,7 +536,7 @@ code solves that problem:: >>> calcsize('@llh0l') 24 -The ``'x'`` format code can be used to specify the repeat, but for +The ``'x'`` type code can be used to specify the repeat, but for native formats it is better to use a zero-repeat format like ``'0l'``. By default, native byte ordering and alignment is used, but it is @@ -571,7 +592,7 @@ below were executed on a 32-bit machine:: Classes ------- -The :mod:`struct` module also defines the following type: +The :mod:`!struct` module also defines the following type: .. class:: Struct(format) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 1aade881745f21f..fe64daa3291d670 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -4,21 +4,18 @@ .. module:: subprocess :synopsis: Subprocess management. -.. moduleauthor:: Peter Åstrand -.. sectionauthor:: Peter Åstrand - **Source code:** :source:`Lib/subprocess.py` -------------- -The :mod:`subprocess` module allows you to spawn new processes, connect to their +The :mod:`!subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:: os.system os.spawn* -Information about how the :mod:`subprocess` module can be used to replace these +Information about how the :mod:`!subprocess` module can be used to replace these modules and functions can be found in the following sections. .. seealso:: @@ -27,8 +24,8 @@ modules and functions can be found in the following sections. .. include:: ../includes/wasm-mobile-notavail.rst -Using the :mod:`subprocess` Module ----------------------------------- +Using the :mod:`!subprocess` Module +----------------------------------- The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the @@ -630,6 +627,12 @@ functions. the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) + .. note:: + + Specifying *user* will not drop existing supplementary group memberships! + The caller must also pass ``extra_groups=()`` to reduce the group membership + of the child process for security purposes. + .. availability:: POSIX .. versionadded:: 3.9 @@ -803,14 +806,29 @@ Instances of the :class:`Popen` class have the following methods: .. note:: - When the ``timeout`` parameter is not ``None``, then (on POSIX) the - function is implemented using a busy loop (non-blocking call and short - sleeps). Use the :mod:`asyncio` module for an asynchronous wait: see + When ``timeout`` is not ``None`` and the platform supports it, an + efficient event-driven mechanism is used to wait for process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll` + - macOS and other BSD variants use :func:`select.kqueue` + + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT`` + - Windows uses ``WaitForSingleObject`` + + If none of these mechanisms are available, the function falls back to a + busy loop (non-blocking call and short sleeps). + + .. note:: + + Use the :mod:`asyncio` module for an asynchronous wait: see :class:`asyncio.create_subprocess_exec`. .. versionchanged:: 3.3 *timeout* was added. + .. versionchanged:: 3.15 + if *timeout* is not ``None``, use efficient event-driven implementation + on Linux >= 5.3 and macOS / BSD. + .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, @@ -831,7 +849,9 @@ Instances of the :class:`Popen` class have the following methods: If the process does not terminate after *timeout* seconds, a :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. + retrying communication will not lose any output. Supplying *input* to a + subsequent post-timeout :meth:`communicate` call is in undefined behavior + and may become an error in the future. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and @@ -844,6 +864,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data @@ -945,6 +970,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- @@ -1034,7 +1064,7 @@ on Windows. Windows Constants ^^^^^^^^^^^^^^^^^ -The :mod:`subprocess` module exposes the following constants. +The :mod:`!subprocess` module exposes the following constants. .. data:: STD_INPUT_HANDLE @@ -1323,8 +1353,8 @@ calls these functions. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the :mod:`!subprocess` Module +------------------------------------------------------------ In this section, "a becomes b" means that b can be used as a replacement for a. @@ -1340,7 +1370,7 @@ In this section, "a becomes b" means that b can be used as a replacement for a. :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the :mod:`!subprocess` module. Replacing :program:`/bin/sh` shell command substitution @@ -1400,7 +1430,7 @@ Notes: * The :func:`os.system` function ignores SIGINT and SIGQUIT signals while the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. + using the :mod:`!subprocess` module. A more realistic example would look like this:: @@ -1584,7 +1614,7 @@ runtime): Disable use of ``posix_spawn()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call +On Linux, :mod:`!subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index d120c6acf621e3f..b6b2f28d067299b 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -1,7 +1,7 @@ .. _superseded: ****************** -Superseded Modules +Superseded modules ****************** The modules described in this chapter have been superseded by other modules @@ -24,3 +24,4 @@ currently no modules in this latter category. :maxdepth: 1 getopt.rst + profile.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index c0d9e79197de7c0..95f20b06b5aa1e7 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -8,13 +8,9 @@ -------------- -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Benjamin Peterson - - Symbol tables are generated by the compiler from AST just before bytecode is generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these +identifier in the code. :mod:`!symtable` provides an interface to examine these tables. @@ -30,7 +26,7 @@ Generating Symbol Tables It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. @@ -180,61 +176,16 @@ Examining Symbol Tables Return a tuple containing names of :term:`free (closure) variables ` in this function. + .. method:: get_cells() -.. class:: Class - - A namespace of a class. This class inherits from :class:`SymbolTable`. - - .. method:: get_methods() - - Return a tuple containing the names of method-like functions declared - in the class. - - Here, the term 'method' designates *any* function defined in the class - body via :keyword:`def` or :keyword:`async def`. - - Functions defined in a deeper scope (e.g., in an inner class) are not - picked up by :meth:`get_methods`. - - For example: + Return a tuple containing names of :term:`cell variables ` in this table. - .. testsetup:: symtable.Class.get_methods + .. versionadded:: 3.15 - import warnings - context = warnings.catch_warnings() - context.__enter__() - warnings.simplefilter("ignore", category=DeprecationWarning) - .. testcleanup:: symtable.Class.get_methods - - context.__exit__() - - .. doctest:: symtable.Class.get_methods - - >>> import symtable - >>> st = symtable.symtable(''' - ... def outer(): pass - ... - ... class A: - ... def f(): - ... def w(): pass - ... - ... def g(self): pass - ... - ... @classmethod - ... async def h(cls): pass - ... - ... global outer - ... def outer(self): pass - ... ''', 'test', 'exec') - >>> class_A = st.get_children()[2] - >>> class_A.get_methods() - ('f', 'g', 'h') - - Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still - considered as a method-like function. +.. class:: Class - .. deprecated-removed:: 3.14 3.16 + A namespace of a class. This class inherits from :class:`SymbolTable`. .. class:: Symbol @@ -291,6 +242,12 @@ Examining Symbol Tables Return ``True`` if the symbol is referenced in its block, but not assigned to. + .. method:: is_cell() + + Return ``True`` if the symbol is referenced but not assigned in a nested block. + + .. versionadded:: 3.15 + .. method:: is_free_class() Return *True* if a class-scoped symbol is free from @@ -361,7 +318,7 @@ Command-Line Usage .. versionadded:: 3.13 -The :mod:`symtable` module can be executed as a script from the command line. +The :mod:`!symtable` module can be executed as a script from the command line. .. code-block:: sh diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 303655fb128b37a..7cca6f2bcdae912 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,17 +10,17 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`sys.monitoring` namespace provides means to +monitor execution. The :mod:`!sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -226,7 +228,7 @@ Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled via :data:`DISABLE`. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -234,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -247,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -314,15 +321,14 @@ location by returning :data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -If :data:`DISABLE` is returned by a callback for a -:ref:`global event `, :exc:`ValueError` will be raised -by the interpreter in a non-specific location (that is, no traceback will be -provided). +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a0621d4b0dbd091..d0fe0625deb5133 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -560,7 +560,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of + line syntax errors and 1 for all other kinds of errors. If another type of object is passed, ``None`` is equivalent to passing zero, and any other object is printed to :data:`stderr` and results in an exit code of 1. In particular, ``sys.exit("some error message")`` is a quick way to exit a @@ -879,7 +879,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionchanged:: 3.6 Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. + for more information. .. versionchanged:: 3.7 Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is @@ -911,6 +911,33 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: get_lazy_imports() + + Returns the current lazy imports mode as a string. + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports are potentially lazy + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: get_lazy_imports_filter() + + Returns the current lazy imports filter function, or ``None`` if no + filter is set. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. See + :func:`set_lazy_imports_filter` for details on the filter function + signature. + + .. versionadded:: 3.15 + + .. function:: getrefcount(object) Return the reference count of the *object*. The count returned is generally one @@ -1454,6 +1481,21 @@ always available. Unless explicitly noted otherwise, all variables are read-only They hold the legacy representation of ``sys.last_exc``, as returned from :func:`exc_info` above. + +.. data:: lazy_modules + + A :class:`set` of fully qualified module name strings that have been lazily + imported in the current interpreter but not yet loaded. When a + lazily imported module is accessed for the first time, its name is removed + from this set. + + This attribute is intended for debugging and introspection. + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + .. data:: maxsize An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can @@ -1719,6 +1761,61 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.11 + +.. function:: set_lazy_imports(mode) + + Sets the global lazy imports mode. The *mode* parameter must be one of + the following strings: + + * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword + are lazy + * ``"all"``: All top-level imports become potentially lazy + + This function is intended for advanced users who need to control lazy + imports across their entire application. Library developers should + generally not use this function as it affects the runtime execution of + applications. + + In addition to the mode, lazy imports can be controlled via the filter + provided by :func:`set_lazy_imports_filter`. + + See also :func:`get_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + +.. function:: set_lazy_imports_filter(filter) + + Sets the lazy imports filter callback. The *filter* parameter must be a + callable or ``None`` to clear the filter. + + The filter function is called for every potentially lazy import to + determine whether it should actually be lazy. It must have the following + signature:: + + def filter(importing_module: str, imported_module: str, + fromlist: tuple[str, ...] | None) -> bool + + Where: + + * *importing_module* is the name of the module doing the import + * *imported_module* is the resolved name of the module being imported + (for example, ``lazy from .spam import eggs`` passes + ``package.spam``) + * *fromlist* is the tuple of names being imported (for ``from ... import`` + statements), or ``None`` for regular imports + + The filter should return ``True`` to allow the import to be lazy, or + ``False`` to force an eager import. + + This is an advanced feature intended for specialized users who need + fine-grained control over lazy import behavior. + + See also :func:`get_lazy_imports_filter` and :pep:`810`. + + .. versionadded:: 3.15 + + .. function:: setprofile(profilefunc) .. index:: @@ -1997,6 +2094,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only interpreter is pre-release (alpha, beta, or release candidate) then the local and remote interpreters must be the same exact version. + See :ref:`remote-debugging` for more information about the remote debugging + mechanism. + .. audit-event:: sys.remote_exec pid script_path When the code is executed in the remote process, an @@ -2015,33 +2115,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. availability:: Unix, Windows. .. versionadded:: 3.14 + See :pep:`768` for more details. -.. function:: _enablelegacywindowsfsencoding() - - Changes the :term:`filesystem encoding and error handler` to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to - 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - See also :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors`. - - .. availability:: Windows. - - .. note:: - Changing the filesystem encoding after Python startup is risky because - the old fsencoding or paths encoded by the old fsencoding may be cached - somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - - .. versionadded:: 3.6 - See :pep:`529` for more details. - - .. deprecated-removed:: 3.13 3.16 - Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - .. data:: stdin stdout stderr @@ -2277,7 +2353,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The version number used to form registry keys on Windows platforms. This is stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` + major and minor versions of the running Python interpreter. It is provided in the :mod:`!sys` module for informational purposes; modifying this value has no effect on the registry keys used by Python. diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a37bb59e7cec760..e6c2cddbe842487 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -57,15 +57,19 @@ otherwise they are set to the same value as :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`, respectively. This is used by :ref:`sys-path-init-virtual-environments`. -Finally, the :mod:`site` module is processed and :file:`site-packages` directories -are added to the module search path. A common way to customize the search path is -to create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in -the :mod:`site` module documentation. +Finally, the :mod:`site` module is processed and :file:`site-packages` +directories are added to the module search path. The :envvar:`PYTHONUSERBASE` +environment variable controls where is searched for user site-packages and the +:envvar:`PYTHONNOUSERSITE` environment variable prevents searching for user +site-packages all together. A common way to customize the search path is to +create :mod:`sitecustomize` or :mod:`usercustomize` modules as described in the +:mod:`site` module documentation. .. note:: - Certain command line options may further affect path calculations. - See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. + The command line options :option:`-E`, :option:`-P`, :option:`-I`, + :option:`-S` and :option:`-s` further affect path calculations, see their + documentation for details. .. versionchanged:: 3.14 @@ -96,11 +100,10 @@ Please refer to :mod:`site`'s .. note:: - There are other ways how "virtual environments" could be implemented, this - documentation refers implementations based on the ``pyvenv.cfg`` mechanism, - such as :mod:`venv`. Most virtual environment implementations follow the - model set by :mod:`venv`, but there may be exotic implementations that - diverge from it. + There are other ways "virtual environments" could be implemented. + This documentation refers to implementations based on the ``pyvenv.cfg`` + mechanism, such as :mod:`venv`, that many virtual environment implementations + follow. _pth files ---------- diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 3b0bfb85da72afe..8aa912d99ba756e 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -4,9 +4,6 @@ .. module:: sysconfig :synopsis: Python's configuration information -.. moduleauthor:: Tarek Ziadé -.. sectionauthor:: Tarek Ziadé - .. versionadded:: 3.2 **Source code:** :source:`Lib/sysconfig` @@ -16,7 +13,7 @@ -------------- -The :mod:`sysconfig` module provides access to Python's configuration +The :mod:`!sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. @@ -28,7 +25,7 @@ A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` header file that are necessary to build both the Python binary itself and third-party C extensions compiled using ``setuptools``. -:mod:`sysconfig` puts all variables found in these files in a dictionary that +:mod:`!sysconfig` puts all variables found in these files in a dictionary that can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. @@ -68,7 +65,7 @@ Installation paths ------------------ Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under +the installation options. These schemes are stored in :mod:`!sysconfig` under unique identifiers based on the value returned by :const:`os.name`. The schemes are used by package installers to determine where to copy files to. @@ -258,12 +255,12 @@ Path Installation directory Installation path functions --------------------------- -:mod:`sysconfig` provides some functions to determine these installation paths. +:mod:`!sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_default_scheme() @@ -285,7 +282,7 @@ Installation path functions *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + can be passed to :mod:`!sysconfig` functions that take a *scheme* argument, such as :func:`get_paths`. .. versionadded:: 3.10 @@ -313,7 +310,7 @@ Installation path functions .. function:: get_path_names() Return a tuple containing all path names currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_path(name, [scheme, [vars, [expand]]]) @@ -323,7 +320,7 @@ Installation path functions *name* has to be a value from the list returned by :func:`get_path_names`. - :mod:`sysconfig` stores installation paths corresponding to each path name, + :mod:`!sysconfig` stores installation paths corresponding to each path name, for each platform, with variables to be expanded. For instance the *stdlib* path for the *nt* scheme is: ``{base}/Lib``. @@ -434,7 +431,7 @@ Other functions Command-line usage ------------------ -You can use :mod:`sysconfig` as a script with Python's *-m* option: +You can use :mod:`!sysconfig` as a script with Python's *-m* option: .. code-block:: shell-session diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 548898a37bc6ea7..b6bf01240951ebd 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -2,7 +2,6 @@ =============================================== .. module:: syslog - :platform: Unix :synopsis: An interface to the Unix syslog library routines. -------------- diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst index 4f61b3dd761400f..570cc7fd93beef9 100644 --- a/Doc/library/tabnanny.rst +++ b/Doc/library/tabnanny.rst @@ -5,11 +5,6 @@ :synopsis: Tool for detecting white space related problems in Python source files in a directory tree. -.. moduleauthor:: Tim Peters -.. sectionauthor:: Peter Funk - -.. rudimentary documentation based on module comments - **Source code:** :source:`Lib/tabnanny.py` -------------- diff --git a/Doc/library/tachyon-flamegraph.png b/Doc/library/tachyon-flamegraph.png new file mode 100644 index 000000000000000..a17cd304f8bb287 Binary files /dev/null and b/Doc/library/tachyon-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-calltree.png b/Doc/library/tachyon-gecko-calltree.png new file mode 100644 index 000000000000000..71b096940e8fcd0 Binary files /dev/null and b/Doc/library/tachyon-gecko-calltree.png differ diff --git a/Doc/library/tachyon-gecko-flamegraph.png b/Doc/library/tachyon-gecko-flamegraph.png new file mode 100644 index 000000000000000..d427ed85ac04e76 Binary files /dev/null and b/Doc/library/tachyon-gecko-flamegraph.png differ diff --git a/Doc/library/tachyon-gecko-opcodes.png b/Doc/library/tachyon-gecko-opcodes.png new file mode 100644 index 000000000000000..9741eb659120b87 Binary files /dev/null and b/Doc/library/tachyon-gecko-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap-with-opcodes.png b/Doc/library/tachyon-heatmap-with-opcodes.png new file mode 100644 index 000000000000000..5ad67d131548e49 Binary files /dev/null and b/Doc/library/tachyon-heatmap-with-opcodes.png differ diff --git a/Doc/library/tachyon-heatmap.png b/Doc/library/tachyon-heatmap.png new file mode 100644 index 000000000000000..47ac1119f4e5725 Binary files /dev/null and b/Doc/library/tachyon-heatmap.png differ diff --git a/Doc/library/tachyon-live-mode-1.gif b/Doc/library/tachyon-live-mode-1.gif new file mode 100644 index 000000000000000..2d58e807d6a7c1d Binary files /dev/null and b/Doc/library/tachyon-live-mode-1.gif differ diff --git a/Doc/library/tachyon-live-mode-2.gif b/Doc/library/tachyon-live-mode-2.gif new file mode 100644 index 000000000000000..bbc2163fe603eb0 Binary files /dev/null and b/Doc/library/tachyon-live-mode-2.gif differ diff --git a/Doc/library/tachyon-pstats.png b/Doc/library/tachyon-pstats.png new file mode 100644 index 000000000000000..d0281ade660914d Binary files /dev/null and b/Doc/library/tachyon-pstats.png differ diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 5ff8502bbe219fd..fc352e901f31dc7 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -4,14 +4,11 @@ .. module:: tarfile :synopsis: Read and write tar-format archive files. -.. moduleauthor:: Lars Gustäbel -.. sectionauthor:: Lars Gustäbel - **Source code:** :source:`Lib/tarfile.py` -------------- -The :mod:`tarfile` module makes it possible to read and write tar +The :mod:`!tarfile` module makes it possible to read and write tar archives, including those using gzip, bz2 and lzma compression. Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the higher-level functions in :ref:`shutil `. @@ -145,6 +142,12 @@ Some facts and figures: a Zstandard dictionary used to improve compression of smaller amounts of data. + For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the + keyword argument *mtime* to create a gzip archive header with that mtime. By + default, the mtime is set to the time of creation of the archive. Use + *mtime* ``0`` to generate a compressed stream that does not depend on + creation time, for reproducible output. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -220,25 +223,25 @@ Some facts and figures: .. function:: is_tarfile(name) - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` + Return :const:`True` if *name* is a tar archive file, that the :mod:`!tarfile` module can read. *name* may be a :class:`str`, file, or file-like object. .. versionchanged:: 3.9 Support for file and file-like objects. -The :mod:`tarfile` module defines the following exceptions: +The :mod:`!tarfile` module defines the following exceptions: .. exception:: TarError - Base class for all :mod:`tarfile` exceptions. + Base class for all :mod:`!tarfile` exceptions. .. exception:: ReadError Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. + :mod:`!tarfile` module or is somehow invalid. .. exception:: CompressionError @@ -359,7 +362,7 @@ The following constants are available at the module level: Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for +:mod:`!tarfile` module is able to create. See section :ref:`tar-formats` for details. @@ -667,7 +670,7 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod` to prevent injection of a ``self`` argument. + :deco:`staticmethod` to prevent injection of a ``self`` argument. .. versionchanged:: 3.14 @@ -1192,6 +1195,8 @@ Here is an incomplete list of things to consider: * Check filenames against an allow-list of characters (to filter out control characters, confusables, foreign path separators, and so on). +* Check for platform-specific filename semantics. For example, on Windows + some names can have reserved meanings. * Check that filenames have expected extensions (discouraging files that execute when you “click on them”, or extension-less files like Windows special device names). @@ -1289,7 +1294,7 @@ Command-Line Interface .. versionadded:: 3.4 -The :mod:`tarfile` module provides a simple command-line interface to interact +The :mod:`!tarfile` module provides a simple command-line interface to interact with tar archives. If you want to create a new tar archive, specify its name after the :option:`-c` @@ -1450,7 +1455,7 @@ parameter in :meth:`TarFile.add`:: Supported tar formats --------------------- -There are three tar formats that can be created with the :mod:`tarfile` module: +There are three tar formats that can be created with the :mod:`!tarfile` module: * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. @@ -1459,7 +1464,7 @@ There are three tar formats that can be created with the :mod:`tarfile` module: * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar + standard on GNU/Linux systems. :mod:`!tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. * The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible @@ -1504,7 +1509,7 @@ Unfortunately, there is no way to autodetect the encoding of an archive. The pax format was designed to solve this problem. It stores non-ASCII metadata using the universal character encoding *UTF-8*. -The details of character conversion in :mod:`tarfile` are controlled by the +The details of character conversion in :mod:`!tarfile` are controlled by the *encoding* and *errors* keyword arguments of the :class:`TarFile` class. *encoding* defines the character encoding to use for the metadata in the diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f0a81a093b435b6..bf9198e175a0e11 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -4,8 +4,6 @@ .. module:: tempfile :synopsis: Generate temporary files and directories. -.. sectionauthor:: Zack Weinberg - **Source code:** :source:`Lib/tempfile.py` .. index:: @@ -225,8 +223,9 @@ The module defines the following user-callable items: properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The file is readable and writable only by the creating user ID. If the platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. + the file is executable by no one. + + The file descriptor is :ref:`not inherited by child processes `. Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible for deleting the temporary file when done with it. @@ -385,7 +384,7 @@ not surprise other unsuspecting code by changing global API behavior. Examples -------- -Here are some examples of typical usage of the :mod:`tempfile` module:: +Here are some examples of typical usage of the :mod:`!tempfile` module:: >>> import tempfile diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 0c6f3059fe71d16..537dfcedd8cf5a3 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -2,7 +2,6 @@ =========================================== .. module:: termios - :platform: Unix :synopsis: POSIX style tty control. .. index:: @@ -38,7 +37,7 @@ The module defines the following functions: items with indices :const:`VMIN` and :const:`VTIME`, which are integers when these fields are defined). The interpretation of the flags and the speeds as well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. + constants defined in the :mod:`!termios` module. .. function:: tcsetattr(fd, when, attributes) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 395cde21ccf4490..4e21e1ded82724c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -4,10 +4,8 @@ .. module:: test :synopsis: Regression tests package containing the testing suite for Python. -.. sectionauthor:: Brett Cannon - .. note:: - The :mod:`test` package is meant for internal use by Python only. It is + The :mod:`!test` package is meant for internal use by Python only. It is documented for the benefit of the core developers of Python. Any use of this package outside of Python's standard library is discouraged as code mentioned here can change or be removed without notice between releases of @@ -15,12 +13,12 @@ -------------- -The :mod:`test` package contains all regression tests for Python as well as the +The :mod:`!test` package contains all regression tests for Python as well as the modules :mod:`test.support` and :mod:`test.regrtest`. :mod:`test.support` is used to enhance your tests while :mod:`test.regrtest` drives the testing suite. -Each module in the :mod:`test` package whose name starts with ``test_`` is a +Each module in the :mod:`!test` package whose name starts with ``test_`` is a testing suite for a specific module or feature. All new tests should be written using the :mod:`unittest` or :mod:`doctest` module. Some older tests are written using a "traditional" testing style that compares output printed to @@ -38,8 +36,8 @@ written using a "traditional" testing style that compares output printed to .. _writing-tests: -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- +Writing Unit Tests for the :mod:`!test` package +----------------------------------------------- It is preferred that tests that use the :mod:`unittest` module follow a few guidelines. One is to name the test module by starting it with ``test_`` and end @@ -162,12 +160,12 @@ Running tests using the command-line interface .. module:: test.regrtest :synopsis: Drives the regression test suite. -The :mod:`test` package can be run as a script to drive Python's regression +The :mod:`!test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m +the hood, it uses :mod:`!test.regrtest`; the call :program:`python -m test.regrtest` used in previous Python versions still works. Running the script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose +:mod:`!test` package. It does this by finding all modules in the package whose name starts with ``test_``, importing them, and executing the function :func:`test_main` if present or loading the tests via unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The @@ -175,14 +173,14 @@ names of tests to execute may also be passed to the script. Specifying a single regression test (:program:`python -m test test_spam`) will minimize output and only print whether the test passed or failed. -Running :mod:`test` directly allows what resources are available for +Running :mod:`!test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line option. Specifying ``all`` as the value for the ``-u`` option enables all possible resources: :program:`python -m test -uall`. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and +will run :mod:`!test` with all resources except the ``audio`` and ``largefile`` resources. For a list of all resources and more command-line options, run :program:`python -m test -h`. @@ -197,19 +195,19 @@ regression tests. :ref:`controlled using environment variables `. -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== +:mod:`!test.support` --- Utilities for the Python test suite +============================================================ .. module:: test.support :synopsis: Support for Python's regression test suite. -The :mod:`test.support` module provides support for Python's regression +The :mod:`!test.support` module provides support for Python's regression test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help + :mod:`!test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -230,7 +228,7 @@ This module defines the following exceptions: function. -The :mod:`test.support` module defines the following constants: +The :mod:`!test.support` module defines the following constants: .. data:: verbose @@ -363,7 +361,7 @@ The :mod:`test.support` module defines the following constants: .. data:: TEST_SUPPORT_DIR - Set to the top level directory that contains :mod:`test.support`. + Set to the top level directory that contains :mod:`!test.support`. .. data:: TEST_HOME_DIR @@ -438,7 +436,7 @@ The :mod:`test.support` module defines the following constants: Used to test mixed type comparison. -The :mod:`test.support` module defines the following functions: +The :mod:`!test.support` module defines the following functions: .. function:: busy_retry(timeout, err_msg=None, /, *, error=True) @@ -492,6 +490,12 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: get_resource_value(resource) + + Return the value specified for *resource* (as :samp:`-u {resource}={value}`). + Return ``None`` if *resource* is disabled or no value is specified. + + .. function:: python_is_optimized() Return ``True`` if Python was not built with ``-O0`` or ``-Og``. @@ -714,7 +718,7 @@ The :mod:`test.support` module defines the following functions: .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should + :deco:`unittest.expectedFailure`. Any use of this decorator should have an associated comment identifying the relevant tracker issue. @@ -1037,7 +1041,7 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.11 -The :mod:`test.support` module defines the following classes: +The :mod:`!test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1083,14 +1087,14 @@ The :mod:`test.support` module defines the following classes: Try to match a single stored value (*dv*) with a supplied value (*v*). -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ +:mod:`!test.support.socket_helper` --- Utilities for socket tests +================================================================= .. module:: test.support.socket_helper :synopsis: Support for socket tests. -The :mod:`test.support.socket_helper` module provides support for socket tests. +The :mod:`!test.support.socket_helper` module provides support for socket tests. .. versionadded:: 3.9 @@ -1161,14 +1165,14 @@ The :mod:`test.support.socket_helper` module provides support for socket tests. exceptions. -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== +:mod:`!test.support.script_helper` --- Utilities for the Python execution tests +=============================================================================== .. module:: test.support.script_helper :synopsis: Support for Python's script execution tests. -The :mod:`test.support.script_helper` module provides support for Python's +The :mod:`!test.support.script_helper` module provides support for Python's script execution tests. .. function:: interpreter_requires_environment() @@ -1272,13 +1276,13 @@ script execution tests. path and the archive name for the zip file. -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= +:mod:`!test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================== .. module:: test.support.bytecode_helper :synopsis: Support tools for testing correct bytecode generation. -The :mod:`test.support.bytecode_helper` module provides support for testing +The :mod:`!test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. .. versionadded:: 3.9 @@ -1304,13 +1308,13 @@ The module defines the following class: Throws :exc:`AssertionError` if *opname* is found. -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== +:mod:`!test.support.threading_helper` --- Utilities for threading tests +======================================================================= .. module:: test.support.threading_helper :synopsis: Support for threading tests. -The :mod:`test.support.threading_helper` module provides support for threading tests. +The :mod:`!test.support.threading_helper` module provides support for threading tests. .. versionadded:: 3.10 @@ -1391,13 +1395,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t finished. -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== +:mod:`!test.support.os_helper` --- Utilities for os tests +========================================================= .. module:: test.support.os_helper :synopsis: Support for os tests. -The :mod:`test.support.os_helper` module provides support for os tests. +The :mod:`!test.support.os_helper` module provides support for os tests. .. versionadded:: 3.10 @@ -1586,13 +1590,13 @@ The :mod:`test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ +:mod:`!test.support.import_helper` --- Utilities for import tests +================================================================= .. module:: test.support.import_helper :synopsis: Support for import tests. -The :mod:`test.support.import_helper` module provides support for import tests. +The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 @@ -1700,13 +1704,13 @@ The :mod:`test.support.import_helper` module provides support for import tests. will be reverted at the end of the block. -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== +:mod:`!test.support.warnings_helper` --- Utilities for warnings tests +===================================================================== .. module:: test.support.warnings_helper :synopsis: Support for warnings tests. -The :mod:`test.support.warnings_helper` module provides support for warnings tests. +The :mod:`!test.support.warnings_helper` module provides support for warnings tests. .. versionadded:: 3.10 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a58b460fef409c6..d12968dee91f3c3 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -4,14 +4,11 @@ .. module:: textwrap :synopsis: Text wrapping and filling -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - **Source code:** :source:`Lib/textwrap.py` -------------- -The :mod:`textwrap` module provides some convenience functions, +The :mod:`!textwrap` module provides some convenience functions, as well as :class:`TextWrapper`, the class that does all the work. If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of @@ -102,6 +99,10 @@ functions should be good enough; otherwise, you should use an instance of print(repr(s)) # prints ' hello\n world\n ' print(repr(dedent(s))) # prints 'hello\n world\n' + .. versionchanged:: 3.14 + The :func:`!dedent` function now correctly normalizes blank lines containing + only whitespace characters. Previously, the implementation only normalized + blank lines containing tabs and spaces. .. function:: indent(text, prefix, predicate=None) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 19cc4f191dff8d9..5d9a7b6314b1668 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -515,7 +515,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. @@ -1436,3 +1436,159 @@ is equivalent to:: Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as :keyword:`with` statement context managers. + + +Iterator synchronization +------------------------ + +By default, Python iterators do not support concurrent access. Most iterators make +no guarantees when accessed simultaneously from multiple threads. Generator +iterators, for example, raise :exc:`ValueError` if one of their iterator methods +is called while the generator is already executing. The tools in this section +allow reliable concurrency support to be added to ordinary iterators and +iterator-producing callables. + +The :class:`serialize_iterator` wrapper lets multiple threads share a single iterator and +take turns consuming from it. While one thread is running ``__next__()``, the +others block until the iterator becomes available. Each value produced by the +underlying iterator is delivered to exactly one caller. + +The :func:`concurrent_tee` function lets multiple threads each receive the full +stream of values from one underlying iterator. It creates independent iterators +that all draw from the same source. Values are buffered until consumed by all +of the derived iterators. + +.. class:: serialize_iterator(iterable) + + Return an iterator wrapper that serializes concurrent calls to + :meth:`~iterator.__next__` using a lock. + + If the wrapped iterator also defines :meth:`~generator.send`, + :meth:`~generator.throw`, or :meth:`~generator.close`, those calls + are serialized as well. + + This makes it possible to share a single iterator, including a generator + iterator, between multiple threads. A lock ensures that calls are handled + one at a time. No values are duplicated or skipped by the wrapper itself. + Each item from the underlying iterator is given to exactly one caller. + + This wrapper does not copy or buffer values. Threads that call + :func:`next` while another thread is already advancing the iterator will + block until the active call completes. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = threading.serialize_iterator(squares(5)) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, each number is printed exactly once, but the work is shared + between the two threads. + + .. versionadded:: 3.15 + + +.. function:: synchronized_iterator(func) + + Wrap an iterator-producing callable so that each iterator it returns is + automatically passed through :class:`serialize_iterator`. + + This is especially useful as a :term:`decorator` for generator functions, + allowing their generator-iterators to be consumed from multiple threads. + + Example: + + .. code-block:: python + + import threading + + @threading.synchronized_iterator + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + The returned wrapper preserves the metadata of *func*, such as its name and + wrapped function reference. + + .. versionadded:: 3.15 + + +.. function:: concurrent_tee(iterable, n=2) + + Return *n* independent iterators from a single input *iterable*, with + guaranteed behavior when the derived iterators are consumed concurrently. + + This function is similar to :func:`itertools.tee`, but is intended for cases + where the source iterator may feed consumers running in different threads. + Each returned iterator yields every value from the underlying iterable, in + the same order. + + Internally, values are buffered until every derived iterator has consumed + them. + + The returned iterators share the same underlying synchronization lock. Each + individual derived iterator is intended to be consumed by one thread at a + time. If a single derived iterator must itself be shared by multiple + threads, wrap it with :class:`serialize_iterator`. + + If *n* is ``0``, return an empty tuple. If *n* is negative, raise + :exc:`ValueError`. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + left, right = threading.concurrent_tee(source) + + t1 = threading.Thread(target=consume, args=("left", left)) + t2 = threading.Thread(target=consume, args=("right", right)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, both consumer threads see the full sequence of squares + from a single generator expression. + + .. versionadded:: 3.15 diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst new file mode 100644 index 000000000000000..a529f7803affbc2 --- /dev/null +++ b/Doc/library/threadsafety.rst @@ -0,0 +1,606 @@ +.. _threadsafety: + +************************ +Thread Safety Guarantees +************************ + +This page documents thread-safety guarantees for built-in types in Python's +free-threaded build. The guarantees described here apply when using Python with +the :term:`GIL` disabled (free-threaded mode). When the GIL is enabled, most +operations are implicitly serialized. + +For general guidance on writing thread-safe code in free-threaded Python, see +:ref:`freethreading-python-howto`. + + +.. _threadsafety-levels: + +Thread safety levels +==================== + +The C API documentation uses the following levels to describe the thread +safety guarantees of each function. The levels are listed from least to +most safe. + +.. _threadsafety-level-incompatible: + +Incompatible +------------ + +A function or operation that cannot be made safe for concurrent use even +with external synchronization. Incompatible code typically accesses +global state in an unsynchronized way and must only be called from a single +thread throughout the program's lifetime. + +Example: a function that modifies process-wide state such as signal handlers +or environment variables, where concurrent calls from any threads, even with +external locking, can conflict with the runtime or other libraries. + +.. _threadsafety-level-compatible: + +Compatible +---------- + +A function or operation that is safe to call from multiple threads +*provided* the caller supplies appropriate external synchronization, for +example by holding a :term:`lock` for the duration of each call. Without +such synchronization, concurrent calls may produce :term:`race conditions +` or :term:`data races `. + +Example: a function that reads from or writes to an object whose internal +state is not protected by a lock. Callers must ensure that no two threads +access the same object at the same time. + +.. _threadsafety-level-distinct: + +Safe on distinct objects +------------------------ + +A function or operation that is safe to call from multiple threads without +external synchronization, as long as each thread operates on a **different** +object. Two threads may call the function at the same time, but they must +not pass the same object (or objects that share underlying state) as +arguments. + +Example: a function that modifies fields of a struct using non-atomic +writes. Two threads can each call the function on their own struct +instance safely, but concurrent calls on the *same* instance require +external synchronization. + +.. _threadsafety-level-shared: + +Safe on shared objects +---------------------- + +A function or operation that is safe for concurrent use on the **same** +object. The implementation uses internal synchronization (such as +:term:`per-object locks ` or +:ref:`critical sections `) to protect shared +mutable state, so callers do not need to supply their own locking. + +Example: :c:func:`PyList_GetItemRef` can be called from multiple threads on the +same :c:type:`PyListObject` - it uses internal synchronization to serialize +access. + +.. _threadsafety-level-atomic: + +Atomic +------ + +A function or operation that appears :term:`atomic ` with +respect to other threads - it executes instantaneously from the perspective +of other threads. This is the strongest form of thread safety. + +Example: :c:func:`PyMutex_IsLocked` performs an atomic read of the mutex +state and can be called from any thread at any time. + + +.. _thread-safety-list: + +Thread safety for list objects +============================== + +Reading a single element from a :class:`list` is +:term:`atomic `: + +.. code-block:: + :class: good + + lst[i] # list.__getitem__ + +The following methods traverse the list and use :term:`atomic ` +reads of each item to perform their function. That means that they may +return results affected by concurrent modifications: + +.. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + +All of the above operations avoid acquiring :term:`per-object locks +`. They do not block concurrent modifications. Other +operations that hold a lock will not block these from observing intermediate +states. + +All other operations from here on block using the :term:`per-object lock`. + +Writing a single item via ``lst[i] = x`` is safe to call from multiple +threads and will not corrupt the list. + +The following operations return new objects and appear +:term:`atomic ` to other threads: + +.. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +The following methods that only operate on a single element with no shifting +required are :term:`atomic `: + +.. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The :meth:`~list.clear` method is also :term:`atomic `. +Other threads cannot observe elements being removed. + +The :meth:`~list.sort` method is not :term:`atomic `. +Other threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort. + +The following operations may allow :term:`lock-free` operations to observe +intermediate states since they modify multiple elements in place: + +.. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The :meth:`~list.remove` method may allow concurrent modifications since +element comparison may execute arbitrary Python code (via +:meth:`~object.__eq__`). + +:meth:`~list.extend` is safe to call from multiple threads. However, its +guarantees depend on the iterable passed to it. If it is a :class:`list`, a +:class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a +:ref:`dictionary view object ` (but not their subclasses), the +``extend`` operation is safe from concurrent modifications to the iterable. +Otherwise, an iterator is created which can be concurrently modified by +another thread. The same applies to inplace concatenation of a list with +other iterables when using ``lst += iterable``. + +Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe +to call from multiple threads, but ``iterable`` is only locked when it is +also a :class:`list` (but not its subclasses). + +Operations that involve multiple accesses, as well as iteration, are never +atomic. For example: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing :class:`list` instances +across threads. + + +.. _thread-safety-dict: + +Thread safety for dict objects +============================== + +Creating a dictionary with the :class:`dict` constructor is atomic when the +argument to it is a :class:`dict` or a :class:`tuple`. When using the +:meth:`dict.fromkeys` method, dictionary creation is atomic when the +argument is a :class:`dict`, :class:`tuple`, :class:`set` or +:class:`frozenset`. + +The following operations and functions are :term:`lock-free` and +:term:`atomic `. + +.. code-block:: + :class: good + + d[key] # dict.__getitem__ + d.get(key) # dict.get + key in d # dict.__contains__ + len(d) # dict.__len__ + +All other operations from here on hold the :term:`per-object lock`. + +Writing or removing a single item is safe to call from multiple threads +and will not corrupt the dictionary: + +.. code-block:: + :class: good + + d[key] = value # write + del d[key] # delete + d.pop(key) # remove and return + d.popitem() # remove and return last item + d.setdefault(key, v) # insert if missing + +These operations may compare keys using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the dictionary may +be modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, that implement :meth:`~object.__eq__` in C, +the underlying lock is not released during comparisons and this is not a +concern. + +The following operations return new objects and hold the :term:`per-object lock` +for the duration of the operation: + +.. code-block:: + :class: good + + d.copy() # returns a shallow copy of the dictionary + d | other # merges two dicts into a new dict + d.keys() # returns a new dict_keys view object + d.values() # returns a new dict_values view object + d.items() # returns a new dict_items view object + +The :meth:`~dict.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations lock both dictionaries. For :meth:`~dict.update` +and ``|=``, this applies only when the other operand is a :class:`dict` +that uses the standard dict iterator (but not subclasses that override +iteration). For equality comparison, this applies to :class:`dict` and +its subclasses: + +.. code-block:: + :class: good + + d.update(other_dict) # both locked when other_dict is a dict + d |= other_dict # both locked when other_dict is a dict + d == other_dict # both locked for dict and subclasses + +All comparison operations also compare values using :meth:`~object.__eq__`, +so for non-built-in types the lock may be released during comparison. + +:meth:`~dict.fromkeys` locks both the new dictionary and the iterable +when the iterable is exactly a :class:`dict`, :class:`set`, or +:class:`frozenset` (not subclasses): + +.. code-block:: + :class: good + + dict.fromkeys(a_dict) # locks both + dict.fromkeys(a_set) # locks both + dict.fromkeys(a_frozenset) # locks both + +When updating from a non-dict iterable, only the target dictionary is +locked. The iterable may be concurrently modified by another thread: + +.. code-block:: + :class: maybe + + d.update(iterable) # iterable is not a dict: only d locked + d |= iterable # iterable is not a dict: only d locked + dict.fromkeys(iterable) # iterable is not a dict/set/frozenset: only result locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + d[key] = d[key] + 1 + + # NOT atomic: check-then-act (TOCTOU) + if key in d: + del d[key] + + # NOT thread-safe: iteration while modifying + for key, value in d.items(): + process(key) # another thread may modify d + +To avoid time-of-check to time-of-use (TOCTOU) issues, use atomic +operations or handle exceptions: + +.. code-block:: + :class: good + + # Use pop() with default instead of check-then-delete + d.pop(key, None) + + # Or handle the exception + try: + del d[key] + except KeyError: + pass + +To safely iterate over a dictionary that may be modified by another +thread, iterate over a copy: + +.. code-block:: + :class: good + + # Make a copy to iterate safely + for key, value in d.copy().items(): + process(key) + +Consider external synchronization when sharing :class:`dict` instances +across threads. + + +.. _thread-safety-set: + +Thread safety for set objects +============================== + +The :func:`len` function is lock-free and :term:`atomic `. + +The following read operation is lock-free. It does not block concurrent +modifications and may observe intermediate states from operations that +hold the per-object lock: + +.. code-block:: + :class: good + + elem in s # set.__contains__ + +This operation may compare elements using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the set may be +modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, :meth:`!__eq__` does not release the +underlying lock during comparisons and this is not a concern. + +All other operations from here on hold the per-object lock. + +Adding or removing a single element is safe to call from multiple threads +and will not corrupt the set: + +.. code-block:: + :class: good + + s.add(elem) # add element + s.remove(elem) # remove element, raise if missing + s.discard(elem) # remove element if present + s.pop() # remove and return arbitrary element + +These operations also compare elements, so the same :meth:`~object.__eq__` +considerations as above apply. + +The :meth:`~set.copy` method returns a new object and holds the per-object lock +for the duration so that it is always atomic. + +The :meth:`~set.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations only accept :class:`set` or :class:`frozenset` +as operands and always lock both objects: + +.. code-block:: + :class: good + + s |= other # other must be set/frozenset + s &= other # other must be set/frozenset + s -= other # other must be set/frozenset + s ^= other # other must be set/frozenset + s & other # other must be set/frozenset + s | other # other must be set/frozenset + s - other # other must be set/frozenset + s ^ other # other must be set/frozenset + +:meth:`set.update`, :meth:`set.union`, :meth:`set.intersection` and +:meth:`set.difference` can take multiple iterables as arguments. They all +iterate through all the passed iterables and do the following: + + * :meth:`set.update` and :meth:`set.union` lock both objects only when + the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. + * :meth:`set.intersection` and :meth:`set.difference` always try to lock + all objects. + +:meth:`set.symmetric_difference` tries to lock both objects. + +The update variants of the above methods also have some differences between +them: + + * :meth:`set.difference_update` and :meth:`set.intersection_update` try + to lock all objects one-by-one. + * :meth:`set.symmetric_difference_update` only locks the arguments if it is + of type :class:`set`, :class:`frozenset`, or :class:`dict`. + +The following methods always try to lock both objects: + +.. code-block:: + :class: good + + s.isdisjoint(other) # both locked + s.issubset(other) # both locked + s.issuperset(other) # both locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: check-then-act + if elem in s: + s.remove(elem) + + # NOT thread-safe: iteration while modifying + for elem in s: + process(elem) # another thread may modify s + +Consider external synchronization when sharing :class:`set` instances +across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-bytearray: + +Thread safety for bytearray objects +=================================== + + The :func:`len` function is lock-free and :term:`atomic `. + + Concatenation and comparisons use the buffer protocol, which prevents + resizing but does not hold the per-object lock. These operations may + observe intermediate states from concurrent modifications: + + .. code-block:: + :class: maybe + + ba + other # may observe concurrent writes + ba == other # may observe concurrent writes + ba < other # may observe concurrent writes + + All other operations from here on hold the per-object lock. + + Reading a single element or slice is safe to call from multiple threads: + + .. code-block:: + :class: good + + ba[i] # bytearray.__getitem__ + ba[i:j] # slice + + The following operations are safe to call from multiple threads and will + not corrupt the bytearray: + + .. code-block:: + :class: good + + ba[i] = x # write single byte + ba[i:j] = values # write slice + ba.append(x) # append single byte + ba.extend(other) # extend with iterable + ba.insert(i, x) # insert single byte + ba.pop() # remove and return last byte + ba.pop(i) # remove and return byte at index + ba.remove(x) # remove first occurrence + ba.reverse() # reverse in place + ba.clear() # remove all bytes + + Slice assignment locks both objects when *values* is a :class:`bytearray`: + + .. code-block:: + :class: good + + ba[i:j] = other_bytearray # both locked + + The following operations return new objects and hold the per-object lock + for the duration: + + .. code-block:: + :class: good + + ba.copy() # returns a shallow copy + ba * n # repeat into new bytearray + + The membership test holds the lock for its duration: + + .. code-block:: + :class: good + + x in ba # bytearray.__contains__ + + All other bytearray methods (such as :meth:`~bytearray.find`, + :meth:`~bytearray.replace`, :meth:`~bytearray.split`, + :meth:`~bytearray.decode`, etc.) hold the per-object lock for their + duration. + + Operations that involve multiple accesses, as well as iteration, are never + atomic: + + .. code-block:: + :class: bad + + # NOT atomic: check-then-act + if x in ba: + ba.remove(x) + + # NOT thread-safe: iteration while modifying + for byte in ba: + process(byte) # another thread may modify ba + + To safely iterate over a bytearray that may be modified by another + thread, iterate over a copy: + + .. code-block:: + :class: good + + # Make a copy to iterate safely + for byte in ba.copy(): + process(byte) + + Consider external synchronization when sharing :class:`bytearray` instances + across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-memoryview: + +Thread safety for memoryview objects +==================================== + +:class:`memoryview` objects provide access to the internal data of an +underlying object without copying. Thread safety depends on both the +memoryview itself and the underlying buffer exporter. + +The memoryview implementation uses atomic operations to track its own +exports in the :term:`free-threaded build`. Creating and +releasing a memoryview are thread-safe. Attribute access (e.g., +:attr:`~memoryview.shape`, :attr:`~memoryview.format`) reads fields that +are immutable for the lifetime of the memoryview, so concurrent reads +are safe as long as the memoryview has not been released. + +However, the actual data accessed through the memoryview is owned by the +underlying object. Concurrent access to this data is only safe if the +underlying object supports it: + +* For immutable objects like :class:`bytes`, concurrent reads through + multiple memoryviews are safe. + +* For mutable objects like :class:`bytearray`, reading and writing the + same memory region from multiple threads without external + synchronization is not safe and may result in data corruption. + Note that even read-only memoryviews of mutable objects do not + prevent data races if the underlying object is modified from + another thread. + +.. code-block:: + :class: bad + + # NOT safe: concurrent writes to the same buffer + data = bytearray(1000) + view = memoryview(data) + # Thread 1: view[0:500] = b'x' * 500 + # Thread 2: view[0:500] = b'y' * 500 + +.. code-block:: + :class: good + + # Safe: use a lock for concurrent access + import threading + lock = threading.Lock() + data = bytearray(1000) + view = memoryview(data) + + with lock: + view[0:500] = b'x' * 500 + +Resizing or reallocating the underlying object (such as calling +:meth:`bytearray.resize`) while a memoryview is exported raises +:exc:`BufferError`. This is enforced regardless of threading. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 548a3ee05405069..dd574fce09f5fc8 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -19,7 +19,7 @@ See also Tim Peters' introduction to the "Algorithms" chapter in the second edition of *Python Cookbook*, published by O'Reilly. -Basic Examples +Basic examples -------------- The following example shows how the :ref:`timeit-command-line-interface` @@ -56,7 +56,7 @@ repetitions only when the command-line interface is used. In the .. _python-interface: -Python Interface +Python interface ---------------- The module defines three convenience functions and a public class: @@ -143,21 +143,24 @@ The module defines three convenience functions and a public class: timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - .. method:: Timer.autorange(callback=None) + .. method:: Timer.autorange(callback=None, target_time=None) Automatically determine how many times to call :meth:`.timeit`. This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual + so that the total time >= *Timer.target_time* seconds, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 seconds. + 10, 20, 50, ... until the time taken is at least *target_time* seconds. If *callback* is given and is not ``None``, it will be called after each trial with two arguments: ``callback(number, time_taken)``. .. versionadded:: 3.6 + .. versionchanged:: 3.15 + The optional *target_time* parameter was added. + .. method:: Timer.repeat(repeat=5, number=1000000) @@ -203,7 +206,7 @@ The module defines three convenience functions and a public class: .. _timeit-command-line-interface: -Command-Line Interface +Command-line interface ---------------------- When called as a program from the command line, the following form is used:: @@ -239,6 +242,13 @@ Where the following options are understood: .. versionadded:: 3.5 +.. option:: -t, --target-time=T + + if :option:`--number` is 0, the code will run until it takes at + least this many seconds (default: 0.2) + + .. versionadded:: 3.15 + .. option:: -v, --verbose print raw timing results; repeat for more digits precision @@ -254,7 +264,7 @@ similarly. If :option:`-n` is not given, a suitable number of loops is calculated by trying increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total -time is at least 0.2 seconds. +time is at least :option:`--target-time` seconds (default: 0.2). :func:`default_timer` measurements can be affected by other programs running on the same machine, so the best thing to do when accurate timing is necessary is @@ -269,6 +279,9 @@ most cases. You can use :func:`time.process_time` to measure CPU time. baseline overhead can be measured by invoking the program without arguments, and it might differ between Python versions. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. .. _timeit-examples: @@ -355,7 +368,7 @@ to test for missing and present object attributes: 0.08588060699912603 -To give the :mod:`timeit` module access to functions you define, you can pass a +To give the :mod:`!timeit` module access to functions you define, you can pass a *setup* parameter which contains an import statement:: def test(): diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index df2b324fd5d3a72..e1e04d04a08b80e 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -2,28 +2,39 @@ ====================================================== .. module:: tkinter.colorchooser - :platform: Tk :synopsis: Color choosing dialog **Source code:** :source:`Lib/tkinter/colorchooser.py` -------------- -The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class +The :mod:`!tkinter.colorchooser` module provides the :class:`Chooser` class as an interface to the native color picker dialog. ``Chooser`` implements a modal color choosing dialog window. The ``Chooser`` class inherits from the :class:`~tkinter.commondialog.Dialog` class. .. class:: Chooser(master=None, **options) + The class implementing the modal color-choosing dialog. + Most applications use the :func:`askcolor` convenience function rather than + instantiating this class directly. + .. function:: askcolor(color=None, **options) - Create a color choosing dialog. A call to this method will show the window, - wait for the user to make a selection, and return the selected color (or - ``None``) to the caller. + Show a modal color-choosing dialog and return the chosen color. + *color* is the color selected when the dialog opens. + The return value is a tuple ``((r, g, b), hexstr)``, where ``r``, ``g`` and + ``b`` are the red, green and blue components as integers in the range 0–255 + and *hexstr* is the equivalent Tk color string, such as ``'#ff8000'``. + If the user cancels the dialog, ``(None, None)`` is returned. + + .. versionchanged:: 3.10 + The RGB values in the returned color are now integers in the range 0–255 + instead of floats. .. seealso:: Module :mod:`tkinter.commondialog` Tkinter standard dialog module + diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 62298d96c264598..ba267c0e3840fe2 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -2,7 +2,6 @@ ============================================= .. module:: tkinter.dnd - :platform: Tk :synopsis: Tkinter drag-and-drop interface **Source code:** :source:`Lib/tkinter/dnd.py` @@ -12,7 +11,7 @@ .. note:: This is experimental and due to be deprecated when it is replaced with the Tk DND. -The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within +The :mod:`!tkinter.dnd` module provides drag-and-drop support for objects within a single application, within the same window or between windows. To enable an object to be dragged, you must create an event binding for it that starts the drag-and-drop process. Typically, you bind a ButtonPress event to a callback @@ -49,7 +48,8 @@ Selection of a target object occurs as follows: .. method:: on_motion(event) - Inspect area below mouse for target objects while drag is performed. + Inspect area below mouse for target objects while a drag + is performed. .. method:: on_release(event) @@ -57,7 +57,9 @@ Selection of a target object occurs as follows: .. function:: dnd_start(source, event) - Factory function for drag-and-drop process. + Factory function for the drag-and-drop process. + Return the :class:`DndHandler` instance managing the drag, or ``None`` if a + drag could not be started. .. seealso:: diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index ed01bd5f4839438..2a9563fffab1f71 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -2,14 +2,13 @@ ============================================= .. module:: tkinter.font - :platform: Tk :synopsis: Tkinter font-wrapping class **Source code:** :source:`Lib/tkinter/font.py` -------------- -The :mod:`tkinter.font` module provides the :class:`Font` class for creating +The :mod:`!tkinter.font` module provides the :class:`Font` class for creating and using named fonts. The different font weights and slants are: @@ -27,6 +26,10 @@ The different font weights and slants are: fonts as a single object, rather than specifying a font by its attributes with each occurrence. + .. versionchanged:: 3.10 + Two fonts now compare equal (``==``) only when both are :class:`Font` + instances with the same name belonging to the same Tcl interpreter. + arguments: | *font* - font specifier tuple (family, size, options) @@ -35,7 +38,7 @@ The different font weights and slants are: additional keyword options (ignored if *font* is specified): - | *family* - font family i.e. Courier, Times + | *family* - font family, for example, Courier, Times | *size* - font size | If *size* is positive it is interpreted as size in points. | If *size* is a negative number its absolute value is treated @@ -47,15 +50,24 @@ The different font weights and slants are: .. method:: actual(option=None, displayof=None) - Return the attributes of the font. + Return the actual attributes of the font, which may differ from the + requested ones because of platform limitations. + With no *option*, return a dictionary of all the attributes; if *option* + is given, return the value of that single attribute. .. method:: cget(option) Retrieve an attribute of the font. .. method:: config(**options) + :no-typesetting: + + .. method:: configure(**options) + + Modify one or more attributes of the font. + With no arguments, return a dictionary of the current attributes. - Modify attributes of the font. + :meth:`config` is an alias of :meth:`!configure`. .. method:: copy() @@ -64,12 +76,15 @@ The different font weights and slants are: .. method:: measure(text, displayof=None) Return amount of space the text would occupy on the specified display - when formatted in the current font. If no display is specified then the - main application window is assumed. + when formatted in the current font, as an integer number of pixels. + If no display is specified then the main application window is assumed. .. method:: metrics(*options, **kw) Return font-specific data. + With no options, return a dictionary mapping each metric name to its + integer value; if one option name is given, return that metric's value as + an integer. Options include: *ascent* - distance between baseline and highest point that a @@ -85,15 +100,17 @@ The different font weights and slants are: .. function:: families(root=None, displayof=None) - Return the different font families. + Return a tuple of the names of the available font families. .. function:: names(root=None) - Return the names of defined fonts. + Return a tuple of the names of all the defined fonts. .. function:: nametofont(name, root=None) - Return a :class:`Font` representation of a tk named font. + Return a :class:`Font` representation of the existing named font *name*. + *root* is the widget whose Tcl interpreter owns the font; if omitted, the + default root window is used. .. versionchanged:: 3.10 The *root* parameter was added. diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 0dc9632ca73304c..7fd561ade2f8bc5 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -2,19 +2,21 @@ ====================================================== .. module:: tkinter.messagebox - :platform: Tk :synopsis: Various types of alert dialogs **Source code:** :source:`Lib/tkinter/messagebox.py` -------------- -The :mod:`tkinter.messagebox` module provides a template base class as well as -a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (``True``, ``False``, ``None``, -:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on -the user's selection. Common message box styles and layouts include but are not -limited to: +The :mod:`!tkinter.messagebox` module provides a template base class as well as +a variety of convenience methods for commonly used configurations. +The message boxes are modal: each blocks until the user responds, then returns +a value that depends on the function. +The ``show*`` functions and :meth:`Message.show` return the symbolic name of +the button the user pressed, as a string (such as :data:`OK` or :data:`YES`), +while the ``ask*`` functions return a :class:`bool` or ``None`` (see each +function below). +Common message box styles and layouts include but are not limited to: .. figure:: tk_msg.png @@ -67,6 +69,10 @@ limited to: Arranges for a :ref:`predefined set of buttons ` to be displayed. + .. note:: + + Tk 8.6 added the *command* option. + .. method:: show(**options) Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. @@ -107,7 +113,7 @@ limited to: .. function:: askretrycancel(title=None, message=None, **options) Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. - Return ``True`` if the answer is yes and ``False`` otherwise. + Return ``True`` if the answer is retry and ``False`` otherwise. .. function:: askyesno(title=None, message=None, **options) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 81177533be84c98..ca034fcdc09b439 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -4,25 +4,28 @@ .. module:: tkinter :synopsis: Interface to Tcl/Tk for graphical user interfaces -.. moduleauthor:: Guido van Rossum - **Source code:** :source:`Lib/tkinter/__init__.py` -------------- -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +The :mod:`!tkinter` package ("Tk interface") is the standard Python interface to +the Tcl/Tk GUI toolkit. Both Tk and :mod:`!tkinter` are available on most Unix platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is +demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. +Tkinter supports a range of Tcl/Tk versions, built either with or without +thread support. +Tcl/Tk 8.5.12 is the minimum supported version; the official Python binary +release bundles Tcl/Tk 9.0. +See the source code for the :mod:`_tkinter` module for more information about +supported versions. + +.. versionchanged:: 3.11 + Support for Tcl/Tk versions older than 8.5.12 was removed. Tkinter is not a thin wrapper, but adds a fair amount of its own logic to make the experience more pythonic. This documentation will concentrate on these @@ -49,7 +52,7 @@ details that are unchanged. Tcl/Tk Resources: - * `Tk commands `_ + * `Tk commands `_ Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. * `Tcl/Tk Home Page `_ @@ -105,20 +108,20 @@ Ttk bindings are provided in a separate module, :mod:`tkinter.ttk`. Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. +that is, Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. -When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +When your Python application uses a class in Tkinter, for example, to create a widget, +the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. -Tkinter Modules +Tkinter modules --------------- Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +main :mod:`!tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides the modern themed widget set and API:: @@ -126,85 +129,9 @@ the modern themed widget set and API:: from tkinter import ttk -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) - - Construct a toplevel Tk widget, which is usually the main window of an - application, and initialize a Tcl interpreter for this widget. Each - instance has its own associated Tcl interpreter. - - The :class:`Tk` class is typically instantiated using all default values. - However, the following keyword arguments are currently recognized: - - *screenName* - When given (as a string), sets the :envvar:`DISPLAY` environment - variable. (X11 only) - *baseName* - Name of the profile file. By default, *baseName* is derived from the - program name (``sys.argv[0]``). - *className* - Name of the widget class. Used as a profile file and also as the name - with which Tcl is invoked (*argv0* in *interp*). - *useTk* - If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` - function sets this to ``False``. - *sync* - If ``True``, execute all X server commands synchronously, so that errors - are reported immediately. Can be used for debugging. (X11 only) - *use* - Specifies the *id* of the window in which to embed the application, - instead of it being created as an independent toplevel window. *id* must - be specified in the same way as the value for the -use option for - toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). - - Note that on some platforms this will only work correctly if *id* refers - to a Tk frame or toplevel that has its -container option enabled. - - :class:`Tk` reads and interprets profile files, named - :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl - interpreter and calls :func:`exec` on the contents of - :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the - profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :data:`os.curdir`. - - .. attribute:: tk - - The Tk application object created by instantiating :class:`Tk`. This - provides access to the Tcl interpreter. Each widget that is attached - the same instance of :class:`Tk` has the same value for its :attr:`tk` - attribute. - - .. attribute:: master - - The widget object that contains this widget. For :class:`Tk`, the - *master* is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master*/*slave* reflects the container structure. - - .. attribute:: children - - The immediate descendants of this widget as a :class:`dict` with the - child widget names as the keys and the child instance objects as the - values. - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - The modules that provide Tk support include: -:mod:`tkinter` +:mod:`!tkinter` Main Tkinter module. :mod:`tkinter.colorchooser` @@ -230,7 +157,7 @@ The modules that provide Tk support include: :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. + for many of the classic widgets in the main :mod:`!tkinter` module. Additional modules: @@ -239,29 +166,31 @@ Additional modules: :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, + It is automatically imported by the main :mod:`!tkinter` module, and should never be used directly by application programmers. It is usually a shared library (or DLL), but might in some cases be statically linked with the Python interpreter. :mod:`idlelib` Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. + on :mod:`!tkinter`. -:mod:`tkinter.constants` +:mod:`!tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. + main :mod:`!tkinter` module. :mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will + (experimental) Drag-and-drop support for :mod:`!tkinter`. This will become deprecated when it is replaced with the Tk DND. :mod:`turtle` Turtle graphics in a Tk window. +.. currentmodule:: tkinter + -Tkinter Life Preserver +Tkinter life preserver ---------------------- This section is not designed to be an exhaustive tutorial on either Tk or @@ -276,7 +205,7 @@ find more detailed documentation on them, including in the official Tcl/Tk reference manual. -A Hello World Program +A Hello World program ^^^^^^^^^^^^^^^^^^^^^ We'll start by walking through a "Hello World" application in Tkinter. This @@ -304,19 +233,20 @@ The following line creates a frame widget, which in this case will contain a label and a button we'll create next. The frame is fit inside the root window. -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. +The next line creates a label widget holding a static text string. +The :meth:`~Grid.grid` method is used to specify the relative layout (position) +of the label within its containing frame widget, similar to how tables in HTML +work. -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. +A button widget is then created, and placed to the right of the label. +When pressed, it will call the :meth:`~Misc.destroy` method of the root window. Finally, the :meth:`mainloop` method puts everything on the display, and responds to user input until the program terminates. -Important Tk Concepts +Important Tk concepts ^^^^^^^^^^^^^^^^^^^^^ Even this simple program illustrates the following key Tk concepts: @@ -348,7 +278,7 @@ event loop isn't running the event loop, your user interface won't update. -Understanding How Tkinter Wraps Tcl/Tk +Understanding how Tkinter wraps Tcl/Tk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When your application uses Tkinter's classes and methods, internally Tkinter @@ -416,9 +346,9 @@ interactive Python shell or with :func:`print`, can help you identify what you need. To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. +:meth:`~Misc.configure` method, which returns a dictionary containing a variety +of information about each object, including its default and current values. +Use :meth:`~Misc.keys` to get just the names of each option. :: @@ -445,18 +375,20 @@ is helpful. print(set(dir(btn)) - set(dir(frm))) -Navigating the Tcl/Tk Reference Manual +Navigating the Tcl/Tk reference manual ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. +As noted, the official +`Tk commands `_ reference +manual (man pages) is often the most accurate description of what specific +operations on widgets do. +Even when you know the name of the option or method that you need, you may +still have a few places to look. While all operations in Tkinter are implemented as method calls on widget objects, you've seen that many Tcl/Tk operations appear as commands that take a widget pathname as its first parameter, followed by optional -parameters, e.g. +parameters, for example :: @@ -475,22 +407,23 @@ name of a method to call). In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ +that look like method calls on the man page for a specific widget (for example, +you'll find the :meth:`~tkinter.ttk.Button.invoke` method on the +`ttk::button `_ man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). +their own man page (for example, +`grid `_). You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man +`options `_ or +`ttk::widget `_ man pages, while others are found in the man page for a specific widget class. -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. +You'll also find that many Tkinter methods have compound names, for example, +:meth:`~Misc.winfo_x`, :meth:`~Misc.winfo_height`, +:meth:`~Misc.winfo_viewable`. +You'd find documentation for all of these in the +`winfo `_ man page. .. note:: Somewhat confusingly, there are also methods on all Tkinter widgets @@ -504,7 +437,7 @@ documentation for all of these in the Threading model --------------- -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` +Python and Tcl/Tk have very different threading models, which :mod:`!tkinter` tries to bridge. If you use threads, you may need to be aware of this. A Python interpreter may have many threads associated with it. In Tcl, multiple @@ -512,34 +445,38 @@ threads can be created, but each thread has a separate Tcl interpreter instance associated with it. Threads can also create more than one interpreter instance, though each interpreter instance can be used only by the one thread that created it. -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. +Each :class:`Tk` object created by :mod:`!tkinter` contains a Tcl interpreter. It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes +:mod:`!tkinter` can be made from any Python thread. Internally, if a call comes from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. +Tcl/Tk applications are normally event-driven, meaning that after +initialization, the interpreter runs an event loop (that is, +:meth:`Tk.mainloop `) and responds to events. +Because it is single-threaded, event handlers must respond quickly, otherwise +they will block other events from being processed. +To avoid this, any long-running computations should not run in an event +handler, but are either broken into smaller pieces using timers, or run in +another thread. +This is different from many GUI toolkits where the GUI runs in a completely +separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl +:mod:`!tkinter` calls made from threads other than the one running the Tcl interpreter will fail. A number of special cases exist: -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even +* Tcl/Tk libraries built without thread support are now rare: Tcl/Tk 9.0 (the + bundled version) is always thread-aware, so this case only arises with some + older 8.x builds. When the library is not thread-aware, + :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` +* While :mod:`!tkinter` allows you to create more than one instance of a :class:`Tk` object (with its own interpreter), all interpreters that are part of the same thread share a common event queue, which gets ugly fast. In practice, don't create more than one instance of :class:`Tk` at a time. Otherwise, it's best to create @@ -550,17 +487,17 @@ A number of special cases exist: or abandon the event loop entirely. If you're doing anything tricky when it comes to events or threads, be aware of these possibilities. -* There are a few select :mod:`tkinter` functions that presently work only when +* There are a few select :mod:`!tkinter` functions that presently work only when called from the thread that created the Tcl interpreter. -Handy Reference +Handy reference --------------- .. _tkinter-setting-options: -Setting Options +Setting options ^^^^^^^^^^^^^^^ Options control things like the color and border width of a widget. Options can @@ -596,11 +533,11 @@ document. Some options don't apply to some kinds of widgets. Whether a given widget responds to a particular option depends on the class of the widget; buttons have a ``command`` option, labels do not. -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. +The options supported by a given widget are listed in that widget's man page, +or can be queried at runtime by calling the :meth:`~Misc.config` method without +arguments, or by calling the :meth:`~Misc.keys` method on that widget. +The return value of these calls is a dictionary whose key is the name of the +option as a string (for example, ``'relief'``) and whose values are 5-tuples. Some options, like ``bg`` are synonyms for common options with long names (``bg`` is shorthand for "background"). Passing the ``config()`` method the name @@ -632,30 +569,33 @@ Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. -The Packer +.. _pack-the-packer: + +The packer ^^^^^^^^^^ .. index:: single: packing (widgets) The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container - -their mutual *master*. In contrast to the more cumbersome *placer* (which is +are used to specify the relative positioning of widgets within their container. +In contrast to the more cumbersome *placer* (which is used less commonly, and we do not cover here), the packer takes qualitative relationship specification - *above*, *to the left of*, *filling*, etc - and works everything out to determine the exact placement coordinates for you. -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames +The size of any container widget is determined by the size of the "content widgets" +inside. The packer is used to control where content widgets appear inside the +container into which they are packed. You can pack widgets into frames, and frames into other frames, in order to achieve the kind of layout you desire. Additionally, the arrangement is dynamically adjusted to accommodate incremental changes to the configuration, once it is packed. Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. +with a geometry manager. +It's a common early mistake to leave out the geometry specification, and then +be surprised when the widget is created but nothing appears. +A widget will appear only after it has had, for example, the packer's +:meth:`~Pack.pack` method applied to it. The pack() method can be called with keyword-option/value pairs that control where the widget is to appear within its container, and how it is to behave when @@ -666,32 +606,34 @@ the main application window is resized. Here are some examples:: fred.pack(expand=1) -Packer Options +Packer options ^^^^^^^^^^^^^^ For more extensive information on the packer and the options that it can take, see the man pages and page 183 of John Ousterhout's book. anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. + Anchor type. Denotes where the packer is to place each content in its parcel. expand - Boolean, ``0`` or ``1``. + boolean, ``0`` or ``1``. fill Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. ipadx and ipady - A distance - designating internal padding on each side of the slave widget. + A distance - designating internal padding on each side of the content. padx and pady - A distance - designating external padding on each side of the slave widget. + A distance - designating external padding on each side of the content. side Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. -Coupling Widget Variables +.. _coupling-widget-variables: + +Coupling widget variables ^^^^^^^^^^^^^^^^^^^^^^^^^ The current-value setting of some widgets (like text entry widgets) can be @@ -700,18 +642,22 @@ options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and ``value``. This connection works both ways: if the variable changes for any reason, the widget it's connected to will be updated to reflect the new value. -Unfortunately, in the current implementation of :mod:`tkinter` it is not +Unfortunately, in the current implementation of :mod:`!tkinter` it is not possible to hand over an arbitrary Python variable to a widget through a ``variable`` or ``textvariable`` option. The only kinds of variables for which this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. +defined in :mod:`!tkinter`. There are many useful subclasses of Variable already defined: :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. +:class:`BooleanVar`. +To read the current value of such a variable, call the :meth:`~Variable.get` +method on it, and to change its value you call the :meth:`!set` method. +If you follow this protocol, the widget will always track the value of the +variable, with no further intervention on your part. + +Keep a reference to the variable for as long as a widget uses it, for example +by storing it as an attribute (see :class:`Variable`). For example:: @@ -745,24 +691,29 @@ For example:: myapp = App(root) myapp.mainloop() -The Window Manager +The window manager ^^^^^^^^^^^^^^^^^^ .. index:: single: window manager (widgets) In Tk, there is a utility command, ``wm``, for interacting with the window manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have +placement, icon bitmaps, and the like. In :mod:`!tkinter`, these commands have been implemented as methods on the :class:`Wm` class. Toplevel widgets are subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods directly. To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. +refer to the widget's :attr:`~Tk.master`. +Of course if the widget has been packed inside of a frame, the :attr:`!master` +won't represent a toplevel window. +To get at the toplevel window that contains an arbitrary widget, you can call +the :meth:`~Misc.winfo_toplevel` method. +There is also a :meth:`!_root` method; it begins with an underscore to denote +the fact that this function is part of the implementation, and not an interface +to Tk functionality. +It returns the application's root window rather than the nearest enclosing +toplevel. Here are some examples of typical usage:: @@ -786,7 +737,9 @@ Here are some examples of typical usage:: myapp.mainloop() -Tk Option Data Types +.. _Tk-option-data-types: + +Tk option data types ^^^^^^^^^^^^^^^^^^^^ .. index:: single: Tk Option Data Types @@ -796,10 +749,11 @@ anchor ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. + There are ten built-in, named bitmaps: ``'error'``, ``'gray12'``, + ``'gray25'``, ``'gray50'``, ``'gray75'``, ``'hourglass'``, ``'info'``, + ``'questhead'``, ``'question'``, ``'warning'``. To specify an X bitmap + filename, give the full path to the file, preceded with an ``@``, as in + ``"@/usr/contrib/bitmap/gumby.bit"``. boolean You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. @@ -818,10 +772,16 @@ color represent any legal hex digit. See page 160 of Ousterhout's book for details. cursor - The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the - string ``"hand2"``. You can also specify a bitmap and mask file of your own. - See page 179 of Ousterhout's book. + The name of the mouse cursor to display while the pointer is over the widget. + Tk provides a portable set of cursor names available on all platforms + (for example ``"arrow"``, ``"watch"``, ``"cross"``, or ``"hand2"``); + the standard X cursor names from :file:`cursorfont.h` may also be used, + without the ``XC_`` prefix (so ``XC_hand2`` becomes ``"hand2"``). + The full list of names, including the platform-specific ones, + is given in the :manpage:`cursors(3tk)` manual page. + You can also specify a bitmap and mask file of your own. + On Windows a cursor file (:file:`.cur` or :file:`.ani`) may be used directly, + giving its path preceded with an ``@``, as in ``"@C:/cursors/bart.ani"``. distance Screen distances can be specified in either pixels or absolute distances. @@ -831,9 +791,11 @@ distance as ``"3.5i"``. font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. + Tk uses a font description such as ``{courier 10 bold}``; in + :mod:`!tkinter` this is most naturally passed as a tuple of + ``(family, size, *styles)`` (or as the equivalent string + ``"Courier 10 bold"``). Font sizes with positive numbers are measured in + points; sizes with negative numbers are measured in pixels. geometry This is a string of the form ``widthxheight``, where width and height are @@ -850,7 +812,8 @@ region relief Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. + ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, ``"ridge"``, and + ``"solid"``. scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can @@ -861,7 +824,7 @@ wrap .. _Bindings-and-Events: -Bindings and Events +Bindings and events ^^^^^^^^^^^^^^^^^^^ .. index:: @@ -877,7 +840,10 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the + is a string that denotes the target kind of event. Physical events use the + ```` form (for example ``""`` or + ``""``); application-defined virtual events use double angle + brackets, as in ``"<>"``. (See the :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). @@ -923,9 +889,27 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages. +----+---------------------+----+---------------------+ | %y | y | %Y | y_root | +----+---------------------+----+---------------------+ +| %# | serial | %b | num | ++----+---------------------+----+---------------------+ +| %d | detail | %D | delta | ++----+---------------------+----+---------------------+ + +The ``add`` parameter above only affects the bindings you make yourself. +Every widget also inherits *class bindings* +that implement its standard behavior -- +for example a :class:`Text` widget binds :kbd:`Control-t` +to transpose two characters. +These are described in the bindings section of the widget's Tk man page +(such as :manpage:`text(3tk)` or :manpage:`entry(3tk)`). + +Class bindings are processed separately from your own, +so binding an event yourself does not replace the default; both run. +To suppress an unwanted default binding, +bind the event on the widget +and return the string ``"break"`` from your callback. -The index Parameter +The index parameter ^^^^^^^^^^^^^^^^^^^ A number of widgets require "index" parameters to be passed. These are used to @@ -934,7 +918,7 @@ Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special + displayed. You can use these :mod:`!tkinter` functions to access these special points in text widgets: Text widget indexes @@ -990,7 +974,7 @@ option (other options are available as well). :meth:`!copy`. The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a +some widget (for example, labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. @@ -1000,51 +984,5832 @@ wherever the image was used. The `Pillow `_ package adds support for formats such as BMP, JPEG, TIFF, and WebP, among others. -.. _tkinter-file-handlers: -File Handlers -------------- +Reference +--------- -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: +.. currentmodule:: tkinter - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) +This section documents the classes, methods, functions and constants of the +:mod:`!tkinter` module. +Most of them wrap Tcl/Tk commands; consult the official Tcl/Tk manual pages for +the full list of widget options and further details. -This feature is not available on Windows. +.. exception:: TclError -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. + The exception raised when a call into the Tcl interpreter fails, for example + when a widget is given an unknown option or an invalid value. +Base and mixin classes +^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Widget.tk.createfilehandler(file, mask, func) +.. class:: Misc() - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: + The :class:`!Misc` class is a mix-in inherited by :class:`Tk` and, through + :class:`BaseWidget`, by every widget. + It provides the large set of methods common to all Tk objects: querying + window information, managing event bindings and the event loop, controlling + the keyboard focus and pointer grabs, accessing the selection, clipboard and + option database, and assorted utility and introspection services. + Because they are inherited, these methods are available on every widget and + on the :class:`Tk` application object, and are documented here once rather + than repeated for each widget. - callback(file, mask) + .. method:: cget(key) + Return the current value of the configuration option named *key* for this + widget, as a string. + The expression ``widget[key]`` is equivalent and may be used instead. -.. method:: Widget.tk.deletefilehandler(file) + .. method:: config(cnf=None, **kw) + :no-typesetting: - Unregisters a file handler. + .. method:: configure(cnf=None, **kw) + Query or modify the configuration options of the widget. + With no arguments, return a dictionary mapping every available option + name to a tuple describing it (its name, X resource name, X resource + class, default value and current value). + If a single option name is given as a string, return the tuple for just + that option. + If one or more keyword arguments are given, or a dictionary is passed as + *cnf*, set each named option to the corresponding value; the expression + ``widget[key] = value`` sets a single option in the same way. -.. data:: READABLE - WRITABLE - EXCEPTION + :meth:`config` is an alias of :meth:`!configure`. - Constants used in the *mask* arguments. + .. method:: keys() + + Return a list of the names of all configuration options of this widget. + + .. method:: getboolean(s) + + Interpret the string *s* as a Tcl boolean and return the corresponding + :class:`bool`. + Tcl accepts values such as ``'1'``, ``'0'``, ``'yes'``, ``'no'``, + ``'true'`` and ``'false'``. + Raise :exc:`ValueError` if *s* is not a valid boolean. + + .. method:: getdouble(s) + + Interpret the string *s* as a Tcl floating-point number and return it as + a :class:`float`. + Raise :exc:`ValueError` if *s* is not a valid number. + + .. versionadded:: 3.5 + + + .. method:: getint(s) + + Interpret the string *s* as a Tcl integer and return it as an + :class:`int`. + Raise :exc:`ValueError` if *s* is not a valid integer. + + .. method:: getvar(name) + + Return the value of the Tcl global variable named *name*. + + .. method:: setvar(name, value) + + Set the Tcl global variable named *name* to *value*. + + The :meth:`!getvar` and :meth:`!setvar` methods give direct access to Tcl + variables. + In most code you will instead use a :class:`Variable` subclass such as + :class:`StringVar` or :class:`IntVar`, which wraps a Tcl variable and + converts its value to and from a Python type. + + .. method:: register(func, subst=None, needcleanup=1) + + Register the Python callable *func* as a Tcl command and return the name + of the new command as a string. + Whenever Tcl invokes that command, *func* is called; if *subst* is given, + it is applied to the command's arguments first. + This is the mechanism used internally to turn Python callbacks into the + command names passed to Tk options such as *command*. + Unless *needcleanup* is false, the command is deleted automatically when + the widget is destroyed. + + .. versionchanged:: 3.13 + The arguments passed to *func* are no longer converted to strings. + + .. method:: deletecommand(name) + + Delete the Tcl command named *name*, such as one previously returned by + :meth:`register`. + + .. method:: nametowidget(name) + + Return the widget instance corresponding to the Tk pathname *name*. + + .. method:: send(interp, cmd, *args) + + Send the Tcl command *cmd*, with the given *args*, to the Tcl interpreter + registered under the name *interp*, and return its result. + This is not available on all platforms. + + .. method:: destroy() + + Destroy this widget and all of its descendant widgets, and delete the Tcl + commands associated with them. + + .. method:: lift(aboveThis=None) + :no-typesetting: + + .. method:: tkraise(aboveThis=None) + + Raise this widget in the stacking order so that it is drawn on top of its + siblings. + If *aboveThis* is given, the widget is moved to be just above it in the + stacking order instead. + + :meth:`lift` is an alias of :meth:`!tkraise`. + + .. method:: lower(belowThis=None) + + Lower this widget in the stacking order so that it is drawn beneath its + siblings. + If *belowThis* is given, the widget is moved to be just below it in the + stacking order instead. + + :meth:`tkraise`/:meth:`lift` and :meth:`lower` are overridden by the + :class:`Canvas` widget, + where they restack canvas items instead. + + .. method:: image_names() + + Return the names of all images that currently exist in the Tcl + interpreter. + + This is overridden by the :class:`Text` widget, + where :meth:`!image_names` returns the names of its embedded images + instead. + + .. method:: image_types() + + Return the available image types, such as ``'photo'`` and ``'bitmap'``. + + .. method:: anchor(anchor=None) + :no-typesetting: + + .. method:: grid_anchor(anchor=None) + + Set the anchor that controls where the grid is placed inside this + container when the container is larger than the grid and no row or column + has a non-zero weight. + *anchor* is one of the usual anchor strings, such as ``'nw'`` (the + default) or ``'center'``. + Called with no argument, this method has no effect. + + :meth:`anchor` is an alias of :meth:`!grid_anchor`. + + .. versionadded:: 3.3 + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Return the bounding box, in pixels, of a region of the grid laid out in + this container, as a 4-tuple ``(xoffset, yoffset, width, height)``. + With no arguments the bounding box of the whole grid is returned. + If *column* and *row* are given, the box spans from the cell at row and + column 0 to that cell; if *col2* and *row2* are also given, it spans from + the cell (*column*, *row*) to the cell (*col2*, *row2*). + + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Query or set the properties of the column (or columns) *index* of the + grid managed by this container. + *index* may be a column number; when setting options it may also be a + list of column numbers, the string ``'all'`` to affect every column, or + a child widget whose occupied columns are affected. + The supported options are: + + *minsize* + The column's minimum size, in pixels. + + *weight* + An integer setting how much of any extra space is apportioned to the + column. + A weight of ``0`` keeps the column at its requested size, and a column + of weight two grows twice as fast as a column of weight one. + + *uniform* + The name of a uniform group. + Columns sharing a non-empty group name are kept in sizes that are + strictly proportional to their weights. + + *pad* + Extra space, in pixels, added to the largest widget in the column when + computing the column's size. + + With a single option name, return that option's value; with no options, + return a dictionary of all of them. + + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Query or set the properties of the row (or rows) *index* of the grid + managed by this container. + *index* is interpreted as for :meth:`grid_columnconfigure`, and the + supported options (*minsize*, *weight*, *uniform* and *pad*) are the + same, applied to a row instead of a column. + + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: grid_location(x, y) + + Return the ``(column, row)`` of the grid cell that contains the pixel at + position (*x*, *y*), given in pixels relative to this container. + For locations above or to the left of the grid, ``-1`` is returned for + the corresponding coordinate. + + .. method:: grid_propagate() + grid_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the grid geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Return the size of the grid managed by this container as a + ``(columns, rows)`` tuple. + + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: grid_slaves(row=None, column=None) + + Return a list of the child widgets managed in this container's grid, most + recently managed first. + If *row* or *column* is given, only the children in that row or column + are returned. + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`grid_slaves`: return the child widgets managed in this + container's grid, optionally restricted to a given *row* or *column*. + + .. versionadded:: 3.15 + + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the pack geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Return a list of the child widgets managed by this container with the + pack geometry manager, in packing order. + + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`pack_slaves`: return the child widgets managed by this + container with the pack geometry manager, in packing order. + + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + + .. method:: place_slaves() + + Return a list of the child widgets managed by this container with the + place geometry manager. + + .. method:: place_content() + + Same as :meth:`place_slaves`: return the child widgets managed by this + container with the place geometry manager. + + .. versionadded:: 3.15 + + The methods with the ``bind`` and ``unbind`` prefixes associate event + patterns with callbacks and remove those associations. + + .. method:: bind(sequence=None, func=None, add=None) + + Bind the event pattern *sequence* on this widget to the callable *func*. + + *sequence* is an event pattern, such as ``''`` (a mouse click) + or ``''``, optionally a concatenation of several such + patterns that must occur shortly after one another. + When the event occurs, *func* is called with an :class:`Event` instance + describing it as its only argument; if *func* returns the string + ``'break'``, no further bindings for the event are invoked. + + If *add* is true, *func* is added to any functions already bound to + *sequence*; otherwise it replaces them. + The binding applies only to this widget. + + :meth:`!bind` returns a string identifier (a *funcid*) that can later be + passed to :meth:`unbind` to remove the binding without leaking the + associated Tcl command. + + If *func* is omitted, return the binding currently associated with + *sequence*; if *sequence* is also omitted, return a list of all the + sequences for which bindings exist on this widget. + + .. method:: bind_class(className, sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the binding tag *className* rather + than to a single widget, so that the binding applies to every widget + having that tag. + *className* is usually the name of a widget class, such as ``'Button'``, + in which case the binding affects all widgets of that class. + The set of binding tags for a widget can be inspected and changed with + :meth:`bindtags`. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: bind_all(sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the special binding tag ``'all'``, + so that the binding applies to every widget in the application. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: unbind(sequence, funcid=None) + + Remove bindings for the event pattern *sequence* on this widget. + + If *funcid* is given, only the function identified by it (a value + returned from a previous call to :meth:`bind`) is removed, and its + associated Tcl command is deleted. + Otherwise all bindings for *sequence* are destroyed, leaving it unbound. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound; other callbacks + bound to *sequence* are kept. + + + .. method:: unbind_class(className, sequence) + + Remove all bindings for the event pattern *sequence* from the binding tag + *className*. + See :meth:`bind_class`. + + .. method:: unbind_all(sequence) + + Remove all bindings for the event pattern *sequence* from the special + binding tag ``'all'``. + See :meth:`bind_all`. + + .. method:: bindtags(tagList=None) + + If *tagList* is omitted, return a tuple of the binding tags associated + with this widget. + When an event occurs in a widget, it is applied to each of the widget's + binding tags in order, and for each tag the most specific matching + binding is executed. + By default a widget has four binding tags: its own pathname, its widget + class, the pathname of its nearest toplevel ancestor, and ``'all'``, in + that order. + + If *tagList* is given, it must be a sequence of strings; the widget's + binding tags are set to its elements, which determines the order in which + bindings are evaluated. + + The methods with the ``event_`` prefix define virtual events and generate + events programmatically. + + .. method:: event_add(virtual, *sequences) + + Associate the virtual event *virtual*, whose name has the form + ``'<>'``, with each of the physical event patterns given by + *sequences*, so that the virtual event triggers whenever any of them + occurs. + If *virtual* is already defined, the new sequences are added to its + existing ones. + + .. method:: event_delete(virtual, *sequences) + + Remove each of *sequences* from those associated with the virtual event + *virtual*. + Sequences that are not currently associated with *virtual* are ignored. + If no *sequences* are given, all physical event sequences are removed, so + that *virtual* no longer triggers. + + .. method:: event_generate(sequence, **kw) + + Generate the event *sequence* on this widget and arrange for it to be + processed just as if it had come from the window system. + *sequence* must be a single event pattern, such as ``''`` or + ``'<>'``, not a concatenation of several. + Keyword arguments specify additional fields of the event, for example *x* + and *y* for the pointer position, or *when* to control when the event is + processed; refer to the Tk ``event`` manual page for the full list. + + .. method:: event_info(virtual=None) + + If *virtual* is omitted, return a tuple of all the virtual events that + are currently defined. + If *virtual* is given, return a tuple of the physical event sequences + currently associated with it, or an empty tuple if it is not defined. + + The methods with the ``after`` prefix schedule callbacks to run after a + delay or when the application is idle. + + .. method:: after(ms, func=None, *args, **kw) + + Schedule the callable *func* to be called after *ms* milliseconds, with + *args* and *kw* passed to it as positional and keyword arguments. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + If *func* is omitted, sleep for *ms* milliseconds instead, processing no + events during that time, and return ``None``. + + .. versionchanged:: 3.10 + *func* can now be any callable object, not only a function. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_cancel(id) + + Cancel a callback previously scheduled with :meth:`after` or + :meth:`after_idle`. + *id* must be an identifier returned by one of those methods; passing a + value that is not such an identifier raises :exc:`ValueError`. + If the callback has already run or been cancelled, this has no effect. + + .. versionchanged:: 3.7 + Passing ``None`` (or any false value) as *id* now raises + :exc:`ValueError`. + + + .. method:: after_idle(func, *args, **kw) + + Schedule the callable *func* to be called, with *args* and *kw* passed to + it, when the Tk main loop next becomes idle, that is, when it has no + other events to process. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_info(id=None) + + If *id* is omitted, return a tuple of the identifiers of all callbacks + currently scheduled with :meth:`after` and :meth:`after_idle` for this + interpreter. + + If *id* is given, it must identify a callback that has not yet run or + been cancelled, and the return value is a tuple ``(script, type)``, where + *script* refers to the function to be called and *type* is either + ``'idle'`` or ``'timer'``. + A :exc:`TclError` is raised if *id* does not exist. + + .. versionadded:: 3.13 + + + .. method:: mainloop(n=0) + + Enter the Tk event loop, which processes events until all windows are + destroyed. + This is normally called once, on the root window, to run the application. + + .. method:: quit() + + Quit the Tcl interpreter, causing :meth:`mainloop` to return. + + .. method:: update() + + Enter the event loop until all pending events, including idle callbacks, + have been processed. + This brings the display up to date and handles any events that are + already queued, then returns. + + .. method:: update_idletasks() + + Enter the event loop until all pending idle callbacks have been called. + This updates the display of windows, for example after geometry changes, + but does not process events caused by the user. + + .. method:: waitvar(name) + :no-typesetting: + + .. method:: wait_variable(name) + + Wait until the Tcl variable *name* is modified, continuing to process + events in the meantime so that the application stays responsive. + *name* is usually a :class:`Variable` instance, such as an + :class:`IntVar` or :class:`StringVar`. + + :meth:`waitvar` is an alias of :meth:`!wait_variable`. + + .. method:: wait_window(window=None) + + Wait until *window* is destroyed, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for the user to finish interacting with a + dialog box. + + .. method:: wait_visibility(window=None) + + Wait until the visibility state of *window* changes, for example when it + first appears on the screen, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for a newly created window to become + visible before acting on it. + + The methods with the ``focus_`` prefix manage the keyboard focus. + + .. method:: focus_set() + :no-typesetting: + + .. method:: focus() + + Direct the keyboard input focus for this widget's display to this widget. + If the application does not currently have the input focus on this + widget's display, the widget is remembered as the focus window for its + top level, and the focus will be redirected to it the next time the + window manager gives the focus to the top level. + :meth:`focus` is an alias of :meth:`!focus_set`, + except on the :class:`Canvas` and + :class:`ttk.Treeview ` widgets, + which provide their own :meth:`!focus` method. + + .. method:: focus_force() + + Direct the keyboard input focus to this widget even if the application + does not currently have the input focus for the widget's display. + This method should be used sparingly, if at all; normally an application + should wait for the window manager to give it the focus rather than + claiming it. + + .. method:: focus_get() + + Return the widget that currently has the keyboard focus in the + application, or ``None`` if no widget in the application has the focus. + Use :meth:`focus_displayof` to work correctly with several displays. + + .. method:: focus_displayof() + + Return the widget that currently has the keyboard focus on the display + where this widget is located, or ``None`` if no widget in the application + has the focus on that display. + + .. method:: focus_lastfor() + + Return the most recent widget to have had the keyboard focus among all + the widgets in the same top level as this widget; this is the widget that + will receive the focus the next time the window manager gives the focus + to the top level. + If no widget in that top level has ever had the focus, or if the most + recent focus widget has been deleted, the top level itself is returned. + + .. method:: tk_focusFollowsMouse() + + Reconfigure Tk to use an implicit focus model in which the focus is set + to a widget whenever the mouse pointer enters it. + This cannot easily be disabled once enabled. + + .. method:: tk_focusNext() + + Return the next widget after this one in the keyboard traversal order, or + ``None`` if there is none. + The traversal order goes first to the next child, then recursively to the + children of that child, and then to the next sibling higher in the + stacking order. + A widget is skipped if its ``takefocus`` option is set to ``0``. + This method is used in the default bindings for the :kbd:`Tab` key. + + .. method:: tk_focusPrev() + + Return the previous widget before this one in the keyboard traversal + order, or ``None`` if there is none. + See :meth:`tk_focusNext` for how the order is defined. + This method is used in the default bindings for the :kbd:`Shift-Tab` key. + + The methods with the ``grab_`` prefix set and query the input grab, which + directs all input events to a single widget. + + .. method:: grab_set() + + Set a local grab on this widget. + A grab confines pointer events to this widget and its descendants: while + the pointer is outside the widget's subtree, button presses and releases + and pointer motion are reported to the grab widget, and windows outside + the subtree become insensitive until the grab is released. + A local grab affects only the grabbing application. + Any grab previously set by this application on the widget's display is + automatically released. + Setting a grab is the usual way to make a dialog modal: while the grab is + in effect the user cannot interact with the other windows of the + application. + + .. method:: grab_set_global() + + Set a global grab on this widget. + A global grab is like the local grab set by :meth:`grab_set`, but it + locks out all other applications on the screen, so that only this + widget's subtree is sensitive to pointer events, and it also grabs the + keyboard. + Use with caution: it is easy to render a display unusable with a global + grab, since other applications stop receiving events until it is + released. + + .. method:: grab_release() + + Release the grab on this widget if there is one; otherwise do nothing. + + .. method:: grab_current() + + Return the widget that currently holds the grab in this application for + this widget's display, or ``None`` if there is no such widget. + + .. method:: grab_status() + + Return ``None`` if no grab is currently set on this widget, ``"local"`` + if a local grab is set, or ``"global"`` if a global grab is set. + + The methods with the ``selection_`` prefix retrieve and manage the X + selection. + + .. method:: selection_clear(**kw) + + Clear the X selection, so that no window owns it anymore. + The selection to clear is given by the keyword argument *selection*, an + atom name such as ``'PRIMARY'`` or ``'CLIPBOARD'``; it defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display on which to operate, and defaults to this widget. + + This is overridden by the :class:`Entry`, :class:`Listbox` and + :class:`Spinbox` widgets, + where :meth:`!selection_clear` clears the widget's own selection instead. + + .. method:: selection_get(**kw) + + Return the contents of the current X selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The keyword argument *type* specifies the form in which the data is to be + returned (the desired conversion target), an atom name such as + ``'STRING'`` or ``'FILE_NAME'``; it defaults to ``STRING``, except on + X11, where ``UTF8_STRING`` is tried first and ``STRING`` is used as a + fallback. + The *displayof* keyword argument names a widget that determines the + display from which to retrieve the selection, and defaults to this + widget. + + .. method:: selection_handle(command, **kw) + + Register *command* as a handler to supply the X selection owned by this + widget when another application requests it. + When the selection is retrieved, *command* is called with two arguments, + the starting character offset and the maximum number of characters to + return, and must return at most that many characters of the selection + starting at that offset; for very long selections it is called repeatedly + with increasing offsets. + The keyword argument *selection* names the selection (default + ``PRIMARY``) and the keyword argument *type* gives the form of the + selection that the handler supplies (such as ``'STRING'`` or + ``'FILE_NAME'``, default ``STRING``). + + .. method:: selection_own(**kw) + + Make this widget the owner of the X selection on its display. + The previous owner, if any, is notified that it has lost the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + + .. method:: selection_own_get(**kw) + + Return the widget in this application that owns the X selection on the + display containing this widget, or ``None`` if no widget in this + application owns the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display to query, and defaults to this widget. + + The methods with the ``clipboard_`` prefix manage the clipboard. + + .. method:: clipboard_append(string, **kw) + + Append *string* to the Tk clipboard and claim ownership of the clipboard + on this widget's display. + Before appending, the clipboard should be emptied with + :meth:`clipboard_clear`; all appends should be completed before returning + to the event loop so that the clipboard is updated atomically. + The keyword argument *type* specifies the form of the data, an atom name + such as ``'STRING'`` or ``'FILE_NAME'`` (default ``STRING``), and the + keyword argument *format* specifies the representation used to transmit + it (default ``STRING``). + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + The contents can be retrieved with :meth:`clipboard_get` or + :meth:`selection_get`. + + .. method:: clipboard_clear(**kw) + + Claim ownership of the clipboard on this widget's display and remove any + previous contents. + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + + .. method:: clipboard_get(**kw) + + Retrieve data from the clipboard on this widget's display. + The keyword argument *type* specifies the form in which the data is to be + returned, an atom name such as ``'STRING'`` or ``'FILE_NAME'``; it + defaults to ``STRING``, except on X11, where ``UTF8_STRING`` is tried + first and ``STRING`` is used as a fallback. + The *displayof* keyword argument names a widget that determines the + display, and defaults to the root window of the application. + This is equivalent to ``selection_get(selection='CLIPBOARD')``. + + The methods with the ``option_`` prefix query and modify the Tk option + database. + + .. method:: option_add(pattern, value, priority=None) + + Add an option to the Tk option database that associates *value* with + *pattern*. + *pattern* consists of names and/or classes separated by asterisks or + dots, in the usual X format. + *priority* is an integer between 0 and 100, or one of the symbolic names + ``'widgetDefault'`` (20), ``'startupFile'`` (40), ``'userDefault'`` (60), + or ``'interactive'`` (80); it defaults to ``interactive``. + + .. method:: option_clear() + + Clear the Tk option database. + Default options from the :envvar:`!RESOURCE_MANAGER` property or the + :file:`.Xdefaults` file are reloaded automatically the next time an + option is added to or removed from the database. + + .. method:: option_get(name, className) + + Return the value of the option matching this widget under *name* and + *className* from the Tk option database, or an empty string if there is + no matching entry. + When several entries match, the one with the highest priority is + returned, and among entries of equal priority the most recently added + one. + + .. method:: option_readfile(fileName, priority=None) + + Read the file named *fileName*, which should have the standard format for + an X resource database such as :file:`.Xdefaults`, and add all the + options it specifies to the Tk option database. + *priority* is interpreted as for :meth:`option_add` and defaults to + ``interactive``. + + .. method:: bell(displayof=0) + + Ring the bell on the display for this widget, using the display's current + bell-related settings, and reset the screen saver for the screen. + If *displayof* is given as a widget, the bell is rung on that widget's + display instead. + + .. method:: tk_setPalette(background, /) + tk_setPalette(*args, **kw) + + Set a new color scheme for all Tk widget elements. + Existing widgets are updated and the option database is changed so that + future widgets use the new colors. + A single color argument is taken as the normal background color, from + which a complete palette is computed. + Alternatively, the arguments may be given as keyword *name*/*value* pairs + naming individual options in the option database. + The recognized option names are ``activeBackground``, + ``activeForeground``, ``background``, ``disabledForeground``, + ``foreground``, ``highlightBackground``, ``highlightColor``, + ``insertBackground``, ``selectColor``, ``selectBackground``, + ``selectForeground``, and ``troughColor``; reasonable defaults are + computed for any that are not specified. + + .. method:: tk_bisque() + + Restore the application's colors to the light brown (bisque) color scheme + used in Tk 3.6 and earlier versions. + Provided for backward compatibility. + + .. method:: tk_strictMotif(boolean=None) + + Query or set whether Tk's look and feel should strictly adhere to Motif. + A true *boolean* value enables strict Motif compliance (for example, no + color change when the mouse passes over a slider). + Return the resulting setting. + + .. method:: tk_appname(name=None) + + Query or set the name used to communicate with this application through + the ``send`` command. + With no argument, return the current name; otherwise change it to *name* + and return the actual name set, which may have a suffix appended to keep + it unique among the applications on the display. + + .. versionadded:: next + + .. method:: tk_useinputmethods(boolean=None, *, displayof=0) + + Query or set whether Tk uses the X Input Methods (XIM) for filtering + events, and return the resulting state. + This is significant only on X11; if XIM support is not available it + always returns ``False``. + + .. versionadded:: next + + .. method:: tk_caret(*, x=None, y=None, height=None) + + Set or query the caret location for the widget's display. + The caret is the per-display insertion position used for global focus + indication (for accessibility) and for placing the input method + (XIM or IME) window. + With no argument, return the current location as a dictionary with the + keys ``'x'``, ``'y'`` and ``'height'``; otherwise update the given + coordinates. + + .. versionadded:: next + + .. method:: tk_scaling(number=None, *, displayof=0) + + Query or set the scaling factor used by Tk to convert between physical + units (such as points, inches or millimeters) and pixels, expressed as + the number of pixels per point (where a point is 1/72 inch). + With no argument, return the current factor; otherwise set it to the + floating-point *number*. + + .. versionadded:: next + + .. method:: tk_inactive(reset=False, *, displayof=0) + + Return the number of milliseconds since the last time the user interacted + with the system, or ``-1`` if the windowing system does not support this. + If *reset* is true, reset the inactivity timer to zero instead and return + ``None``. + + .. versionadded:: next + + The methods with the ``busy_`` prefix manage the busy state of a window, + which shows a busy cursor and ignores user input. + + .. method:: busy(**kw) + :no-typesetting: + + .. method:: busy_hold(**kw) + :no-typesetting: + + .. method:: tk_busy(**kw) + :no-typesetting: + + .. method:: tk_busy_hold(**kw) + + Make this widget appear busy. + A transparent window is placed in front of the widget, so that it and all + of its descendants in the widget hierarchy are blocked from pointer + events and display a busy cursor. + Normally :meth:`update` should be called immediately afterwards to ensure + that the hold operation is in effect before the application starts its + processing. + + The only supported configuration option is *cursor*, the cursor to be + displayed while the widget is busy; it may have any of the values + accepted by :meth:`!configure`. + + :meth:`busy_hold`, :meth:`busy` and :meth:`tk_busy` are aliases of + :meth:`!tk_busy_hold`. + + .. versionadded:: 3.13 + + + .. method:: busy_configure(cnf=None, **kw) + :no-typesetting: + + .. method:: busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_configure(cnf=None, **kw) + + Query or modify the configuration options of the busy window. + The widget must have been previously made busy by :meth:`tk_busy_hold`. + With no arguments, return a dictionary describing all of the available + options; if *cnf* is the name of an option, return a tuple describing + that one option. + Otherwise set the given options to the given values. + Options may have any of the values accepted by :meth:`tk_busy_hold`. + + The option database is referenced through the widget name or class. + For example, if a :class:`Frame` widget named ``frame`` is to be made + busy, the busy cursor can be specified for it by either of the calls:: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + + :meth:`busy_configure`, :meth:`busy_config` and :meth:`tk_busy_config` + are aliases of :meth:`!tk_busy_configure`. + + .. versionadded:: 3.13 + + + .. method:: busy_cget(option) + :no-typesetting: + + .. method:: tk_busy_cget(option) + + Return the current value of the busy configuration *option*. + The widget must have been previously made busy by :meth:`tk_busy_hold`, + and *option* may have any of the values accepted by that method. + + :meth:`busy_cget` is an alias of :meth:`!tk_busy_cget`. + + .. versionadded:: 3.13 + + + .. method:: busy_forget() + :no-typesetting: + + .. method:: tk_busy_forget() + + Make this widget no longer busy, releasing the resources (including the + transparent window) allocated when it was made busy. + User events will again be received by the widget. + These resources are also released when the widget is destroyed. + + :meth:`busy_forget` is an alias of :meth:`!tk_busy_forget`. + + .. versionadded:: 3.13 + + + .. method:: busy_status() + :no-typesetting: + + .. method:: tk_busy_status() + + Return ``True`` if the widget is currently busy, ``False`` otherwise. + + :meth:`busy_status` is an alias of :meth:`!tk_busy_status`. + + .. versionadded:: 3.13 + + + .. method:: busy_current(pattern=None) + :no-typesetting: + + .. method:: tk_busy_current(pattern=None) + + Return a list of widgets that are currently busy. + If *pattern* is given, only busy widgets whose path names match the + pattern are returned. + + :meth:`busy_current` is an alias of :meth:`!tk_busy_current`. + + .. versionadded:: 3.13 + + The methods with the ``winfo_`` prefix retrieve information about windows + managed by Tk. + + .. method:: winfo_atom(name, displayof=0) + + Return the integer identifier for the atom whose name is *name*, creating + a new atom if none exists. + If *displayof* is given, the atom is looked up on the display of that + window; otherwise it is looked up on the display of the application's + main window. + + .. method:: winfo_atomname(id, displayof=0) + + Return the textual name for the atom whose integer identifier is *id*. + This is the inverse of :meth:`winfo_atom`. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_cells() + + Return the number of cells in the colormap for the widget. + + .. method:: winfo_children() + + Return a list containing the widgets that are children of the widget, in + stacking order from lowest to highest. + Toplevel windows are returned as children of their logical parents. + + .. method:: winfo_class() + + Return the class name of the widget. + + .. method:: winfo_colormapfull() + + Return ``True`` if the colormap for the widget is known to be full, + ``False`` otherwise. + + .. method:: winfo_containing(rootX, rootY, displayof=0) + + Return the widget containing the point given by *rootX* and *rootY*, or + ``None`` if no window in this application contains the point. + The coordinates are in screen units in the coordinate system of the root + window. + If *displayof* is given, the coordinates refer to the screen containing + that window; otherwise they refer to the screen of the application's main + window. + + .. method:: winfo_depth() + + Return the depth of the widget, that is, the number of bits per pixel. + + .. method:: winfo_exists() + + Return ``1`` if the widget exists, ``0`` otherwise. + + .. method:: winfo_fpixels(number) + + Return a floating-point value giving the number of pixels in the widget + corresponding to the screen distance *number* (for example, + ``"2.0c"`` or ``"1i"``). + The result may be fractional; for a rounded integer value use + :meth:`winfo_pixels`. + + .. method:: winfo_geometry() + + Return the geometry of the widget, in the form ``widthxheight+x+y``. + All dimensions are in pixels. + An offset can be negative; see :meth:`~Wm.geometry`. + + .. method:: winfo_height() + + Return the height of the widget in pixels. + When a window is first created its height is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqheight`. + + .. method:: winfo_id() + + Return a low-level platform-specific identifier for the widget. + On Unix this is the X window identifier, and on Windows it is the window + handle. + + .. method:: winfo_interps(displayof=0) + + Return a tuple of the names of all Tcl interpreters currently registered + for a particular display. + If *displayof* is given, the return value refers to the display of that + window; otherwise it refers to the display of the application's main + window. + + .. method:: winfo_isdark() + + On macOS and Windows, return ``True`` if the widget is in "dark mode", + and ``False`` otherwise. + Always return ``False`` on X11. + + .. versionadded:: next + + Requires Tk 9.1 or newer. + + .. method:: winfo_ismapped() + + Return ``1`` if the widget is currently mapped, ``0`` otherwise. + + .. method:: winfo_manager() + + Return the name of the geometry manager currently responsible for the + widget, or an empty string if it is not managed by any geometry manager. + + .. method:: winfo_name() + + Return the widget's name within its parent, as opposed to its full path + name. + + .. method:: winfo_parent() + + Return the path name of the widget's parent, or an empty string if the + widget is the main window of the application. + + .. method:: winfo_pathname(id, displayof=0) + + Return the path name of the window whose identifier is *id*. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_pixels(number) + + Return the number of pixels in the widget corresponding to the screen + distance *number* (for example, ``"2.0c"`` or ``"1i"``). + The result is rounded to the nearest integer; for a fractional result use + :meth:`winfo_fpixels`. + + .. method:: winfo_pointerx() + + Return the pointer's *x* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_pointerxy() + + Return the pointer's coordinates as an ``(x, y)`` tuple, in pixels, + relative to the screen's root window (or virtual root, if one is in use). + Both coordinates are ``-1`` if the pointer is not on the same screen as + the widget. + + .. method:: winfo_pointery() + + Return the pointer's *y* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_reqheight() + + Return the widget's requested height in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_reqwidth() + + Return the widget's requested width in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_rgb(color) + + Return an ``(r, g, b)`` tuple of the red, green, and blue intensities, in + the range 0 to 65535, that correspond to *color* in the widget. + *color* may be specified in any of the forms acceptable for a color + option. + + .. method:: winfo_rootx() + + Return the *x* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_rooty() + + Return the *y* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_screen() + + Return the name of the screen associated with the widget, in the form + ``displayName.screenIndex``. + + .. method:: winfo_screencells() + + Return the number of cells in the default colormap for the widget's + screen. + + .. method:: winfo_screendepth() + + Return the depth of the root window of the widget's screen, that is, the + number of bits per pixel. + + .. method:: winfo_screenheight() + + Return the height of the widget's screen in pixels. + + .. method:: winfo_screenmmheight() + + Return the height of the widget's screen in millimeters. + + .. method:: winfo_screenmmwidth() + + Return the width of the widget's screen in millimeters. + + .. method:: winfo_screenvisual() + + Return the default visual class for the widget's screen, one of + ``"directcolor"``, ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, + ``"staticgray"``, or ``"truecolor"``. + + .. method:: winfo_screenwidth() + + Return the width of the widget's screen in pixels. + + .. method:: winfo_server() + + Return a string containing information about the server for the widget's + display. + The exact format of this string may vary from platform to platform. + + .. method:: winfo_toplevel() + + Return the top-of-hierarchy window containing the widget. + In standard Tk this is always a :class:`Toplevel` widget. + + .. method:: winfo_viewable() + + Return ``1`` if the widget and all of its ancestors up through the + nearest toplevel window are mapped, ``0`` otherwise. + + .. method:: winfo_visual() + + Return the visual class for the widget, one of ``"directcolor"``, + ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, ``"staticgray"``, + or ``"truecolor"``. + + .. method:: winfo_visualid() + + Return the X identifier for the visual for the widget. + + .. method:: winfo_visualsavailable(includeids=False) + + Return a list describing the visuals available for the widget's screen. + Each item consists of a visual class (see :meth:`winfo_visual`) followed + by an integer depth. + If *includeids* is true, the X identifier for the visual is also + included. + + .. method:: winfo_vrootheight() + + Return the height of the virtual root window associated with the widget + if there is one; otherwise return the height of the widget's screen. + + .. method:: winfo_vrootwidth() + + Return the width of the virtual root window associated with the widget if + there is one; otherwise return the width of the widget's screen. + + .. method:: winfo_vrootx() + + Return the *x* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_vrooty() + + Return the *y* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_width() + + Return the width of the widget in pixels. + When a window is first created its width is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqwidth`. + + .. method:: winfo_x() + + Return the *x* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: winfo_y() + + Return the *y* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: info_patchlevel() + + Return the Tcl/Tk patch level as a named tuple with the same five fields + as :data:`sys.version_info`: *major*, *minor*, *micro*, *releaselevel* + and *serial*. + *releaselevel* is ``'alpha'``, ``'beta'`` or ``'final'``. + Converting it to a string gives the version in the usual Tcl/Tk notation, + for example ``'9.0.3'`` for a final release or ``'9.1b2'`` for a + pre-release. + + .. versionadded:: 3.11 + + + +.. class:: Wm() + + The :class:`!Wm` mixin provides access to the window manager, allowing an + application to control such things as the title, geometry and icon of a + top-level window, the way it is resized, and how it responds to window + manager protocols. + It is mixed into :class:`Tk` and :class:`Toplevel`, so its methods are + available on every top-level window. + Each method has two equivalent spellings: a short name and a + ``wm_``-prefixed name (for example, :meth:`title` and :meth:`wm_title`). + + .. method:: wm_aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + :no-typesetting: + + .. method:: aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + + Constrain the aspect ratio (the ratio of width to height) of the window. + If all four arguments are given, the window manager keeps the ratio + between ``minNumer/minDenom`` and ``maxNumer/maxDenom``; passing empty + strings removes any existing restriction. + With no arguments, return a tuple of the four current values, or an empty + string if no aspect restriction is in effect. + :meth:`wm_aspect` is an alias of :meth:`!aspect`. + + .. method:: wm_attributes(*args, return_python_dict=False, **kwargs) + :no-typesetting: + + .. method:: attributes(*args, return_python_dict=False, **kwargs) + + Query or set platform-specific attributes of the window. + With no arguments, return the platform-specific flags and their values; + pass *return_python_dict* as true to get them as a dictionary. + A single option name such as ``'alpha'`` returns the value of that + option, and options are set using keyword arguments (``alpha=0.5``). + + The available attributes differ by platform. + All platforms support: + + *alpha* + The window's opacity, from ``0.0`` (fully transparent) to ``1.0`` + (opaque). + Where transparency is unsupported the value stays at ``1.0``. + + *appearance* + Whether the window is rendered in dark mode on Windows and macOS: + ``'auto'``, ``'light'`` or ``'dark'`` (this has no effect on X11). + + *fullscreen* + Whether the window takes up the entire screen and has no borders. + + *topmost* + Whether the window is displayed above all other windows. + + Windows additionally supports: + + *disabled* + Whether the window is in a disabled state. + + *toolwindow* + Whether the window uses the tool window style. + + *transparentcolor* + The color that is made fully transparent, or an empty string for none. + + macOS additionally supports: + + *class* + Whether the underlying Aqua window is an ``nswindow`` or an + ``nspanel``; this can only be set before the window is created. + + *modified* + The modification state shown by the window's close button and proxy + icon. + + *notify* + Whether the application's dock icon bounces to request attention. + + *stylemask* + The style mask of the underlying Aqua window, given as a list of bit + names such as ``titled`` or ``resizable``. + + *tabbingid* + The identifier of the tab group that the window belongs to. + + *tabbingmode* + Whether the window may be opened as a tab: ``'auto'``, ``'preferred'`` + or ``'disallowed'``. + + *titlepath* + The path of the file represented by the window's proxy icon. + + *transparent* + Whether the content area is transparent and the window shadow is + turned off. + + X11 additionally supports: + + *type* + The window type, or a list of types in order of preference, that the + window manager should use to interpret the window, such as + ``'dialog'`` or ``'splash'``. + + *zoomed* + Whether the window is maximized. + + .. note:: + + Tk 8.6 added the *type* attribute, and Tk 9.0 added the *appearance*, + *class*, *stylemask*, *tabbingid* and *tabbingmode* attributes. + + On X11 changes are applied asynchronously, so a queried value may not yet + reflect the most recent request. + :meth:`wm_attributes` is an alias of :meth:`!attributes`. + + .. versionchanged:: 3.13 + A single attribute may now be queried by name without the leading + ``-``, and attributes may be set using keyword arguments. + The *return_python_dict* parameter was added. + + .. deprecated:: 3.13 + Setting an attribute by passing the option name (with a leading + ``-``) and its value as two positional arguments, as in + ``w.attributes('-alpha', 0.5)``, is deprecated; use keyword arguments + instead. + + + .. method:: wm_client(name=None) + :no-typesetting: + + .. method:: client(name=None) + + Store *name*, which should be the name of the host on which the + application is running, in the window's ``WM_CLIENT_MACHINE`` property + for use by the window or session manager. + An empty string deletes the property. + With no argument, return the last name set, or an empty string. + :meth:`wm_client` is an alias of :meth:`!client`. + + .. method:: wm_colormapwindows(*wlist) + :no-typesetting: + + .. method:: colormapwindows(*wlist) + + Manipulate the ``WM_COLORMAP_WINDOWS`` property, which tells the window + manager about windows that have private colormaps. + If *wlist* is given, overwrite the property with those windows (their + order is a priority order for installing colormaps). + With no arguments, return the list of windows currently named in the + property. + :meth:`wm_colormapwindows` is an alias of :meth:`!colormapwindows`. + + .. method:: wm_command(value=None) + :no-typesetting: + + .. method:: command(value=None) + + Store *value* in the window's ``WM_COMMAND`` property for use by the + window or session manager; it should be a list giving the words of the + command used to invoke the application. + An empty string deletes the property. + With no argument, return the last value set, or an empty string. + :meth:`wm_command` is an alias of :meth:`!command`. + + .. method:: wm_deiconify() + :no-typesetting: + + .. method:: deiconify() + + Display the window in normal (non-iconified) form by mapping it. + If the window has never been mapped, this ensures it appears de-iconified + when it is first mapped. + On Windows the window is also raised and given the focus. + :meth:`wm_deiconify` is an alias of :meth:`!deiconify`. + + .. method:: wm_focusmodel(model=None) + :no-typesetting: + + .. method:: focusmodel(model=None) + + Set or query the focus model for the window. + *model* is either ``'active'`` (the window claims the input focus for + itself or its descendants, even when the focus is in another application) + or ``'passive'`` (the window relies on the window manager to give it the + focus). + With no argument, return the current model. + The default is ``'passive'``, which is what the :meth:`!focus` command + assumes. + :meth:`wm_focusmodel` is an alias of :meth:`!focusmodel`. + + .. method:: wm_forget(window) + :no-typesetting: + + .. method:: forget(window) + + Unmap *window* from the screen so that it is no longer managed by the + window manager. + A :class:`Toplevel` is then treated like a :class:`Frame`, although its + ``-menu`` configuration is remembered and the menu reappears if the + widget is managed again. + :meth:`wm_forget` is an alias of :meth:`!forget`. + + Not to be confused with :meth:`Pack.forget`. + + .. versionadded:: 3.3 + + .. method:: wm_frame() + :no-typesetting: + + .. method:: frame() + + Return the platform-specific window identifier for the outermost + decorative frame containing the window, if the window manager has + reparented it into such a frame; otherwise return the identifier of the + window itself. + :meth:`wm_frame` is an alias of :meth:`!frame`. + + .. method:: wm_geometry(newGeometry=None) + :no-typesetting: + + .. method:: geometry(newGeometry=None) + + Set or query the geometry of the window. + *newGeometry* has the form ``=widthxheight+x+y``, where any of ``=``, + ``widthxheight`` and the ``+x+y`` position may be omitted. + *width* and *height* are in pixels (or grid units for a gridded window); + a position preceded by ``+`` is measured from the left or top edge of the + screen and one preceded by ``-`` from the right or bottom edge. + An offset can be negative, as in ``'200x100+-9+-8'``, when the window + edge is positioned beyond the corresponding screen edge. + An empty string cancels any user-specified geometry, letting the window + revert to its natural size. + With no argument, return the current geometry as a string of the form + ``'200x200+10+10'``. + :meth:`wm_geometry` is an alias of :meth:`!geometry`. + + .. method:: wm_grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + :no-typesetting: + + .. method:: grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + + Manage the window as a gridded window and define the relationship between + grid units and pixels. + *baseWidth* and *baseHeight* are the numbers of grid units for the + window's internally requested size, and *widthInc* and *heightInc* are + the pixel sizes of a horizontal and vertical grid unit. + Empty strings turn off gridded management. + With no arguments, return a tuple of the four current values, or an empty + string if the window is not gridded. + :meth:`wm_grid` is an alias of :meth:`!grid`. + + Not to be confused with the grid geometry manager :meth:`Grid.grid`. + + .. method:: wm_group(pathName=None) + :no-typesetting: + + .. method:: group(pathName=None) + + Set or query the leader of a group of related windows. + *pathName* gives the path name of the group leader; the window manager + may, for example, unmap all windows in the group when the leader is + iconified. + An empty string removes the window from any group. + With no argument, return the path name of the current group leader, or an + empty string. + :meth:`wm_group` is an alias of :meth:`!group`. + + .. method:: wm_iconbadge(badge) + :no-typesetting: + + .. method:: iconbadge(badge) + + Set a badge for the window's icon, intended for display in the Dock + (macOS), taskbar (Windows) or app panel (X11). + *badge* may be a positive integer (for example a count of unread + messages) or an exclamation point to denote that attention is needed; + an empty string removes the badge. + On X11 the variable ``::tk::icons::base_icon(window)`` must be set to the + window's icon image for the badge to appear. + :meth:`wm_iconbadge` is an alias of :meth:`!iconbadge`. + + .. versionadded:: next + + Requires Tk 9.0 or newer. + + .. method:: wm_iconbitmap(bitmap=None, default=None) + :no-typesetting: + + .. method:: iconbitmap(bitmap=None, default=None) + + Set or query the bitmap used by the window manager for the window's icon. + *bitmap* names a bitmap in one of the standard forms accepted by Tk; an + empty string cancels the current icon bitmap. + With no argument, return the name of the current icon bitmap, or an empty + string. + On Windows the *default* argument names an icon (for example an ``.ico`` + file) applied to all top-level windows that have no icon of their own. + :meth:`wm_iconbitmap` is an alias of :meth:`!iconbitmap`. + + .. method:: wm_iconify() + :no-typesetting: + + .. method:: iconify() + + Iconify the window. + If the window has not yet been mapped for the first time, arrange for it + to appear in the iconified state when it is eventually mapped. + :meth:`wm_iconify` is an alias of :meth:`!iconify`. + + .. method:: wm_iconmask(bitmap=None) + :no-typesetting: + + .. method:: iconmask(bitmap=None) + + Set or query the bitmap used as a mask for the icon (see + :meth:`iconbitmap`). + Where the mask is zero no icon is displayed; where it is one, the + corresponding bits of the icon bitmap are shown. + An empty string cancels the current mask. + With no argument, return the name of the current icon mask, or an empty + string. + :meth:`wm_iconmask` is an alias of :meth:`!iconmask`. + + .. method:: wm_iconname(newName=None) + :no-typesetting: + + .. method:: iconname(newName=None) + + Set or query the name displayed by the window manager inside the window's + icon. + With no argument, return the current icon name, or an empty string if + none has been set (in which case the window manager normally displays the + window's title). + :meth:`wm_iconname` is an alias of :meth:`!iconname`. + + .. method:: wm_iconphoto(default=False, *images) + :no-typesetting: + + .. method:: iconphoto(default=False, *images) + + Set the titlebar icon for the window from one or more :class:`PhotoImage` + objects given in *images*. + Several images of different sizes (for example 16x16 and 32x32) may be + supplied so that the window manager can choose an appropriate one. + The image data is taken as a snapshot at the time of the call; later + changes to the images are not reflected. + If *default* is true, the icon is also applied to all top-level windows + created in the future. + On macOS only the first image is used. + :meth:`wm_iconphoto` is an alias of :meth:`!iconphoto`. + + .. versionadded:: 3.3 + + .. method:: wm_iconposition(x=None, y=None) + :no-typesetting: + + .. method:: iconposition(x=None, y=None) + + Set or query a hint to the window manager about where the window's icon + should be positioned. + Empty strings cancel an existing hint. + With no arguments, return a tuple of the two current values, or an empty + string if no hint is in effect. + :meth:`wm_iconposition` is an alias of :meth:`!iconposition`. + + .. method:: wm_iconwindow(pathName=None) + :no-typesetting: + + .. method:: iconwindow(pathName=None) + + Set or query the window used as the icon for the window. + When the window is iconified, *pathName* is mapped to serve as its icon + and unmapped again when it is de-iconified. + An empty string cancels the association. + With no argument, return the path name of the current icon window, or an + empty string. + Not all window managers support icon windows, and the concept is + meaningless on non-X11 platforms. + :meth:`wm_iconwindow` is an alias of :meth:`!iconwindow`. + + .. method:: wm_manage(widget) + :no-typesetting: + + .. method:: manage(widget) + + Make *widget* a stand-alone top-level window, decorated by the window + manager with a title bar and so on. + Only :class:`Frame`, :class:`LabelFrame` and :class:`Toplevel` widgets + may be used (the :mod:`tkinter.ttk` versions are **not** accepted); + passing any other widget type raises an error. + :meth:`wm_manage` is an alias of :meth:`!manage`. + + .. versionadded:: 3.3 + + .. method:: wm_maxsize(width=None, height=None) + :no-typesetting: + + .. method:: maxsize(width=None, height=None) + + Set or query the maximum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no larger than *width* and + *height*. + With no arguments, return a tuple of the current maximum width and + height. + The maximum size defaults to the size of the screen. + :meth:`wm_maxsize` is an alias of :meth:`!maxsize`. + + .. method:: wm_minsize(width=None, height=None) + :no-typesetting: + + .. method:: minsize(width=None, height=None) + + Set or query the minimum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no smaller than *width* and + *height*. + With no arguments, return a tuple of the current minimum width and + height. + The minimum size defaults to one pixel in each dimension. + :meth:`wm_minsize` is an alias of :meth:`!minsize`. + + .. method:: wm_overrideredirect(boolean=None) + :no-typesetting: + + .. method:: overrideredirect(boolean=None) + + Set or query the override-redirect flag for the window. + When this flag is set, the window is ignored by the window manager: it is + not reparented into a decorative frame and the user cannot manipulate it + through the usual window manager controls. + With no argument, return a boolean indicating whether the flag is set. + The flag is reliably honored only when the window is first mapped or + remapped from the withdrawn state. + :meth:`wm_overrideredirect` is an alias of :meth:`!overrideredirect`. + + .. method:: wm_positionfrom(who=None) + :no-typesetting: + + .. method:: positionfrom(who=None) + + Set or query the source of the window's current position. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + position was requested by the program or by the user; an empty string + cancels the current source. + With no argument, return the current source, or an empty string if none + has been set. + Tk automatically sets the source to ``'user'`` when :meth:`geometry` is + called, unless it has been set explicitly to ``'program'``. + :meth:`wm_positionfrom` is an alias of :meth:`!positionfrom`. + + .. method:: wm_protocol(name=None, func=None) + :no-typesetting: + + .. method:: protocol(name=None, func=None) + + Register *func* as the handler for the window manager protocol *name*, an + atom such as ``'WM_DELETE_WINDOW'``, ``'WM_SAVE_YOURSELF'`` or + ``'WM_TAKE_FOCUS'``; *func* is then called whenever the window manager + sends a message of that protocol. + Tk installs a default ``WM_DELETE_WINDOW`` handler that destroys the + window, which this method can replace. + If *func* is an empty string, the handler is removed. + With only *name*, return the name of its registered handler command, or + an empty string if none is set (the default ``WM_DELETE_WINDOW`` handler + is not reported); with no arguments, return a tuple of the protocols that + currently have handlers. + :meth:`wm_protocol` is an alias of :meth:`!protocol`. + + .. method:: wm_resizable(width=None, height=None) + :no-typesetting: + + .. method:: resizable(width=None, height=None) + + Control whether the user may interactively resize the window. + *width* and *height* are boolean values that determine whether the + window's width and height may be changed. + With no arguments, return a tuple of two ``0``/``1`` values indicating + whether each dimension is currently resizable. + By default a window is resizable in both dimensions. + :meth:`wm_resizable` is an alias of :meth:`!resizable`. + + .. method:: wm_sizefrom(who=None) + :no-typesetting: + + .. method:: sizefrom(who=None) + + Set or query the source of the window's current size. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + size was requested by the program or by the user; an empty string cancels + the current source. + With no argument, return the current source, or an empty string if none + has been set. + :meth:`wm_sizefrom` is an alias of :meth:`!sizefrom`. + + .. method:: wm_stackorder(relation=None, window=None) + :no-typesetting: + + .. method:: stackorder(relation=None, window=None) + + Query the stacking order of top-level windows. + With no arguments, return a list of the mapped top-level widgets in + stacking order, from lowest to highest, recursively including this + window's top-level children. + If *relation* is ``'isabove'`` or ``'isbelow'`` and *window* is another + top-level, return ``True`` if this window is respectively above or below + *window* in the stacking order, and ``False`` otherwise. + :meth:`wm_stackorder` is an alias of :meth:`!stackorder`. + + .. versionadded:: next + + .. method:: wm_state(newstate=None) + :no-typesetting: + + .. method:: state(newstate=None) + + Set or query the state of the window. + With no argument, return the current state: one of ``'normal'``, + ``'iconic'``, ``'withdrawn'``, ``'icon'`` or, on Windows and macOS only, + ``'zoomed'``. + ``'iconic'`` refers to a window that has been iconified, while ``'icon'`` + refers to a window serving as the icon for another window (see + :meth:`iconwindow`); the ``'icon'`` state cannot be set. + :meth:`wm_state` is an alias of :meth:`!state`. + + Not to be confused with :meth:`ttk.Widget.state + `. + + .. method:: wm_title(string=None) + :no-typesetting: + + .. method:: title(string=None) + + Set or query the title for the window, which the window manager should + display in the window's title bar. + With no argument, return the current title. + The title defaults to the window's name. + :meth:`wm_title` is an alias of :meth:`!title`. + + .. method:: wm_transient(master=None) + :no-typesetting: + + .. method:: transient(master=None) + + Mark the window as a transient window (such as a pull-down menu or + dialog) working on behalf of *master*, the path name of another top-level + window. + An empty string clears the transient status. + With no argument, return the path name of the current master, or an empty + string. + A transient window mirrors state changes in its master and may be + decorated differently by the window manager; it is an error to make a + window a transient of itself. + :meth:`wm_transient` is an alias of :meth:`!transient`. + + .. method:: wm_withdraw() + :no-typesetting: + + .. method:: withdraw() + + Withdraw the window from the screen, unmapping it and causing the window + manager to forget about it. + If the window has never been mapped, it is instead mapped in the + withdrawn state. + It is sometimes necessary to withdraw a window and then re-map it (for + example with :meth:`deiconify`) to make some window managers notice + changes to window attributes. + :meth:`wm_withdraw` is an alias of :meth:`!withdraw`. + + +.. class:: Pack() + + Geometry manager that arranges widgets by packing them against the sides of + their container. + The :class:`!Pack` mix-in is inherited by all widgets (through + :class:`Widget`) and provides the methods for managing a widget with the + *pack* geometry manager. + See also :ref:`pack-the-packer`. + + .. note:: + + :class:`Pack`, :class:`Place` and :class:`Grid` all define the short + method names :meth:`!forget`, :meth:`!info`, :meth:`!slaves`, + :meth:`!content` and :meth:`!propagate`. + On a widget the bare names resolve to the *pack* manager's versions, + since :class:`Pack` and :class:`Misc` precede :class:`Place` and + :class:`Grid` in the method resolution order, + whatever manager actually manages the widget; + and :meth:`!configure`/:meth:`!config` configure the widget's options, + not its geometry. + Use the explicit ``pack_*``, ``grid_*`` and ``place_*`` methods + (and ``pack``, ``grid``, ``place`` for geometry configuration) + to act on a specific geometry manager. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: pack_configure(cnf={}, **kw) + pack(cnf={}, **kw) + + Pack the widget inside its container, positioning it relative to the + siblings already packed there. + The supported options are: + + *side* + Which side of the container to pack the widget against: ``'top'`` (the + default), ``'bottom'``, ``'left'`` or ``'right'``. + + *fill* + Whether to stretch the widget to fill its parcel: ``'none'`` (the + default), ``'x'``, ``'y'`` or ``'both'``. + + *expand* + Whether the widget should expand to consume any extra space in its + container (a boolean, default false). + + *anchor* + Where to position the widget in its parcel when the parcel is larger + than the widget: an anchor such as ``'n'`` or ``'sw'`` (default + ``'center'``). + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *after* + Pack the widget after the given widget in the packing order, using the + same container. + + *before* + Pack the widget before the given widget in the packing order, using + the same container. + + *in_* + The container in which to pack the widget; it defaults to the parent + widget. + + :meth:`pack`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!pack_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: pack_forget() + + Unmap the widget and remove it from the packing order, forgetting its + packing options. + It can be packed again later with :meth:`pack_configure`. + :meth:`forget` is an alias of :meth:`!pack_forget`, + except on :class:`PanedWindow`, + :class:`ttk.Notebook ` and + :class:`ttk.PanedWindow `, + which provide their own :meth:`!forget` method. + + Not to be confused with :meth:`Wm.forget`. + + .. method:: info() + :no-typesetting: + + .. method:: pack_info() + + Return a dictionary of the widget's current packing options. + :meth:`info` is an alias of :meth:`!pack_info`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Same as :meth:`Misc.pack_propagate`, treating this widget as a container: + enable or disable geometry propagation. + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Same as :meth:`Misc.pack_slaves`: return the list of widgets packed in + this widget. + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`Misc.pack_content`. + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + +.. class:: Place() + + Geometry manager that places widgets at explicit positions and sizes within + their container. + The :class:`!Place` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: place_configure(cnf={}, **kw) + place(cnf={}, **kw) + + Place the widget inside its container at an absolute or relative + position. + The supported options are: + + *x*, *y* + The absolute horizontal and vertical position of the widget's anchor + point, as a screen distance (default ``0``). + + *relx*, *rely* + The horizontal and vertical position of the widget's anchor point as a + fraction of the container's width and height, where ``0.0`` is the + left or top edge and ``1.0`` is the right or bottom edge. + If both the absolute and the relative option are given, their values + are summed. + + *anchor* + Which point of the widget is placed at the given position: an anchor + such as ``'n'`` or ``'se'`` (default ``'nw'``). + + *width*, *height* + The absolute width and height of the widget, as a screen distance. + By default the widget's requested size is used. + + *relwidth*, *relheight* + The width and height of the widget as a fraction of the container's + width and height. + If both the absolute and the relative option are given, their values + are summed. + + *bordermode* + How the container's border affects placement: ``'inside'`` (the + default) measures the area inside the border, ``'outside'`` measures + the area including the border, and ``'ignore'`` uses the official X + area. + + *in_* + The container relative to which the widget is placed; it must be the + widget's parent or a descendant of the parent, and defaults to the + parent. + + :meth:`place`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!place_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: place_forget() + + Unmap the widget and remove it from the placement, forgetting its place + options. + + .. method:: info() + :no-typesetting: + + .. method:: place_info() + + Return a dictionary of the widget's current place options. + + .. method:: slaves() + :no-typesetting: + + .. method:: place_slaves() + + Same as :meth:`Misc.place_slaves`: return the list of widgets placed in + this widget. + + .. method:: content() + :no-typesetting: + + .. method:: place_content() + + Same as :meth:`Misc.place_content`. + + .. versionadded:: 3.15 + + +.. class:: Grid() + + Geometry manager that arranges widgets in a two-dimensional grid of rows and + columns within their container. + The :class:`!Grid` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: grid_configure(cnf={}, **kw) + grid(cnf={}, **kw) + + Position the widget in a cell of its container's grid. + + Not to be confused with :meth:`Wm.grid`. + + The supported options are: + + *row*, *column* + The row and column of the cell to place the widget in, counting from + ``0``. + *column* defaults to the column after the previous widget placed in + the same :meth:`!grid_configure` call (or ``0``), and *row* defaults + to the next empty row. + + *rowspan*, *columnspan* + The number of rows and columns the widget should span (default ``1``). + + *sticky* + How to position or stretch the widget when its cell is larger than the + widget: a string containing zero or more of the characters ``'n'``, + ``'s'``, ``'e'`` and ``'w'``, naming the cell sides the widget sticks + to. + Specifying both ``'n'`` and ``'s'`` (or ``'e'`` and ``'w'``) stretches + the widget to fill the height (or width) of the cell. + The default is ``''``, which centers the widget at its requested size. + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *in_* + The container in whose grid to place the widget; it defaults to the + parent widget. + + :meth:`grid`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!grid_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: grid_forget() + + Unmap the widget and remove it from the grid, forgetting its grid + options. + + .. method:: grid_remove() + + Unmap the widget and remove it from the grid, but remember its grid + options so that it is restored to the same cell if it is gridded again. + + .. method:: info() + :no-typesetting: + + .. method:: grid_info() + + Return a dictionary of the widget's current grid options. + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Same as :meth:`Misc.grid_bbox`. + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_columnconfigure`: query or set the options (such + as *weight*, *minsize*, *pad* and *uniform*) of a grid column. + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_rowconfigure`: query or set the options of a + grid row. + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: location(x, y) + :no-typesetting: + + .. method:: grid_location(x, y) + + Same as :meth:`Misc.grid_location`: return the ``(column, row)`` of the + cell that covers the pixel at *x*, *y*. + :meth:`location` is an alias of :meth:`!grid_location`. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Same as :meth:`Misc.grid_size`: return a ``(columns, rows)`` tuple giving + the size of the grid. + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: grid_propagate() + grid_propagate(flag) + + Same as :meth:`Misc.grid_propagate`. + + .. method:: slaves(row=None, column=None) + :no-typesetting: + + .. method:: grid_slaves(row=None, column=None) + + Same as :meth:`Misc.grid_slaves`: return the widgets managed in the grid, + optionally restricted to a *row* and/or *column*. + + .. method:: content(row=None, column=None) + :no-typesetting: + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`Misc.grid_content`. + + .. versionadded:: 3.15 + + +.. class:: XView() + + Mix-in providing the horizontal-scrolling interface shared by widgets such + as :class:`Entry`, :class:`Canvas`, :class:`Listbox`, :class:`Text` and + :class:`Spinbox`. + A widget's :meth:`xview` method is registered as the *command* of a + horizontal :class:`Scrollbar`. + + .. method:: xview(*args) + + Query or change the horizontal position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``xview`` widget command and + are usually generated by a scrollbar; :meth:`xview_moveto` and + :meth:`xview_scroll` provide a more convenient interface. + + .. method:: xview_moveto(fraction) + + Adjust the view so that *fraction* of the total width of the document is + off-screen to the left. + *fraction* is a number between 0 and 1. + + .. method:: xview_scroll(number, what) + + Shift the view left or right by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + left and a positive one scrolls right. + + +.. class:: YView() + + Mix-in providing the vertical-scrolling interface shared by widgets such as + :class:`Canvas`, :class:`Listbox` and :class:`Text`. + A widget's :meth:`yview` method is registered as the *command* of a vertical + :class:`Scrollbar`. + + .. method:: yview(*args) + + Query or change the vertical position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``yview`` widget command, + usually generated by a scrollbar; :meth:`yview_moveto` and + :meth:`yview_scroll` provide a more convenient interface. + + .. method:: yview_moveto(fraction) + + Adjust the view so that *fraction* of the total height of the document is + off-screen above the top. + *fraction* is a number between 0 and 1. + + .. method:: yview_scroll(number, what) + + Shift the view up or down by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + up and a positive one scrolls down. + + +.. class:: BaseWidget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for all widgets. + It inherits from :class:`Misc` and adds the machinery that creates the + underlying Tk widget; application code normally uses :class:`Widget` or a + concrete widget class rather than instantiating :class:`!BaseWidget` + directly. + + .. method:: destroy() + + Destroy this widget and all of its children, removing the corresponding + Tk widgets and deleting the associated Tcl commands. + + +.. class:: Widget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for the standard widgets. + It combines :class:`BaseWidget` with the geometry-manager mix-ins + :class:`Pack`, :class:`Place` and :class:`Grid`, so that every widget can be + managed by any of the three geometry managers. + The concrete widget classes (:class:`Button`, :class:`Label`, and so on) + derive from :class:`!Widget`. + + +Toplevel widgets +^^^^^^^^^^^^^^^^ + +.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) + + Construct a toplevel Tk widget, which is usually the main window of an + application, and initialize a Tcl interpreter for this widget. Each + instance has its own associated Tcl interpreter. + Inherits from :class:`Misc` and :class:`Wm`. + + To create a Tcl interpreter without initializing the Tk subsystem, use the + :func:`Tcl` factory function instead. + + The :class:`Tk` class is typically instantiated using all default values. + However, the following keyword arguments are currently recognized: + + *screenName* + When given (as a string), sets the :envvar:`DISPLAY` environment + variable. (X11 only) + *baseName* + Name of the profile file. By default, *baseName* is derived from the + program name (``sys.argv[0]``). + *className* + Name of the widget class. Used as a profile file and also as the name + with which Tcl is invoked (*argv0* in *interp*). + *useTk* + If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` + function sets this to ``False``. + *sync* + If ``True``, execute all X server commands synchronously, so that errors + are reported immediately. Can be used for debugging. (X11 only) + *use* + Specifies the *id* of the window in which to embed the application, + instead of it being created as an independent toplevel window. *id* must + be specified in the same way as the value for the -use option for + toplevel widgets (that is, it has a form like that returned by + :meth:`~Misc.winfo_id`). + + Note that on some platforms this will only work correctly if *id* refers + to a Tk frame or toplevel that has its -container option enabled. + + :class:`Tk` reads and interprets profile files, named + :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl + interpreter and calls :func:`exec` on the contents of + :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the + profile files is the :envvar:`HOME` environment variable or, if that + isn't defined, then :data:`os.curdir`. + + .. note:: + + On Windows, creating a Tcl interpreter (by instantiating :class:`Tk` or + calling :func:`Tcl`) sets the :envvar:`HOME` environment variable for + the process, if it is not already set, to ``%HOMEDRIVE%%HOMEPATH%`` (or + :envvar:`USERPROFILE`, or ``c:\``). This is done by Tcl and can affect + other code that reads :envvar:`HOME`. + + .. attribute:: tk + + The Tk application object created by instantiating :class:`Tk`. This + provides access to the Tcl interpreter. Each widget that is attached + the same instance of :class:`Tk` has the same value for its :attr:`tk` + attribute. + + .. attribute:: master + + The widget object that contains this widget. + For :class:`Tk`, the :attr:`!master` is :const:`None` because it is the + main window. + The terms *master* and *parent* are similar and sometimes used + interchangeably as argument names; however, calling + :meth:`~Misc.winfo_parent` returns a string of the widget name whereas + :attr:`!master` returns the object. + *parent*/*child* reflects the tree-like relationship while *master* (or + *container*)/*content* reflects the container structure. + + .. attribute:: children + + The immediate descendants of this widget as a :class:`dict` with the + child widget names as the keys and the child instance objects as the + values. + + .. method:: destroy() + + Destroy this and all descendant widgets and, for the main window, end the + connection to the underlying Tcl interpreter. + + .. method:: loadtk() + + Finish loading and initializing the Tk subsystem. + This is needed only when the interpreter was created without Tk (for + example through :func:`Tcl`); it is called automatically when *useTk* is + true. + + .. method:: readprofile(baseName, className) + + Read and source the user's profile files :file:`.{className}.tcl` and + :file:`.{baseName}.tcl` into the Tcl interpreter, and execute the + corresponding :file:`.{className}.py` and :file:`.{baseName}.py` files. + This is called during initialization; see the description of the + constructor above. + + .. method:: report_callback_exception(exc, val, tb) + + Report a callback exception. + This is called when an exception propagates out of a Tkinter callback; + *exc*, *val* and *tb* are the exception type, value and traceback as + returned by :func:`sys.exc_info`. + The default implementation prints a traceback to :data:`sys.stderr`. + It can be overridden to customize error handling, for example to display + the traceback in a dialog. + + +.. class:: Toplevel(master=None, cnf={}, **kw) + + A :class:`!Toplevel` widget is a top-level window, similar to a + :class:`Frame` except that its X parent is the root window of a screen + rather than its logical parent. + Its primary purpose is to serve as a container for dialog boxes and other + collections of widgets; its only visible features are its background and an + optional 3-D border. + Notable options include *menu*, which installs a :class:`Menu` as the + window's menubar. + Inherits from :class:`BaseWidget` and :class:`Wm`, so a toplevel is managed + by the window manager. + Refer to the Tk ``toplevel`` manual page for the full list of options. + + +Widget classes +^^^^^^^^^^^^^^ + +.. class:: Button(master=None, cnf={}, **kw) + + A :class:`!Button` widget displays a textual string, bitmap or image and + invokes a command when the user presses it (by clicking mouse button 1 over + the button or, when the button has focus, by pressing the space key). + Inherits from :class:`Widget`. + In addition to the standard widget options, a button accepts the options + documented in the Tk ``button`` manual page, such as *command* (the callback + invoked when the button is pressed), *textvariable*, *state* and *default*. + + .. method:: invoke() + + Invoke the command associated with the button, if there is one, and + return its result, or an empty string if no command is associated with + the button. + This is ignored if the button's state is ``disabled``. + + .. method:: flash() + + Flash the button by redisplaying it several times, alternating between + the active and normal colors. + At the end of the flash the button is left in the same normal or active + state as when the method was called. + This is ignored if the button's state is ``disabled``. + + +.. class:: Canvas(master=None, cnf={}, **kw) + + A :class:`!Canvas` widget implements structured graphics. + It displays any number of *items*, such as arcs, lines, ovals, polygons, + rectangles, text, bitmaps, images and embedded windows, which may be drawn, + moved, re-colored and bound to events. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``canvas`` manual page for the full list of widget and item + options. + + Each item has a unique integer *id*, assigned when it is created, and zero + or more string *tags*. + A tag is an arbitrary string that does not have the form of an integer; the + same tag may be shared by many items, which makes tags convenient for + grouping items. + The special tag ``'all'`` matches every item in the canvas, and + ``'current'`` matches the topmost item under the mouse pointer. + Most methods take a *tagOrId* argument that may be an integer id naming a + single item, or a tag naming zero or more items; as described in the Tk + ``canvas`` manual page, a tag may also be a logical expression of tags + combined with the operators ``&&``, ``||``, ``^``, ``!`` and parentheses. + When a method that operates on a single item is given a *tagOrId* matching + several items, it normally uses the lowest matching item in the display + list. + + The items are kept in a *display list* that determines drawing order: items + later in the list are drawn on top of earlier ones. + A newly created item is placed at the top of the list; the order can be + changed with :meth:`tag_raise` and :meth:`tag_lower`. + + .. method:: create_arc(*args, **kw) + create_bitmap(*args, **kw) + create_image(*args, **kw) + create_line(*args, **kw) + create_oval(*args, **kw) + create_polygon(*args, **kw) + create_rectangle(*args, **kw) + create_text(*args, **kw) + create_window(*args, **kw) + + Create a new item of the corresponding type and return its integer id. + Each method is called as ``create_TYPE(coord..., **options)``: the + leading positional arguments give the coordinates that define the item + (as separate numbers, as a single sequence of numbers, or as coordinate + pairs), and the keyword arguments set item-specific options. + Coordinates and screen distances may be given as numbers (interpreted as + pixels) or as strings with a unit suffix (``'m'``, ``'c'``, ``'i'`` or + ``'p'`` for millimetres, centimetres, inches or printer's points), but + are always stored and returned in pixels. + + The item types are: ``arc`` (an arc-shaped region that is a section of an + oval, defined by two diagonally opposite corners ``x1, y1, x2, y2`` of + the enclosing rectangle); ``bitmap`` (a two-color bitmap positioned at a + point ``x, y``); ``image`` (a Tk image positioned at a point ``x, y``); + ``line`` (a line or curve through the points ``x1, y1, ..., xn, yn``); + ``oval`` (a circle or ellipse inscribed in the rectangle + ``x1, y1, x2, y2``); ``polygon`` (a closed polygon through the points + ``x1, y1, ..., xn, yn``); ``rectangle`` (a rectangle with corners + ``x1, y1, x2, y2``); ``text`` (a string of text positioned at a point + ``x, y``); and ``window`` (a child widget embedded in the canvas at a + point ``x, y``, specified with the *window* option). + + Most item types accept a common set of *standard item options*, plus a + few options specific to each type. + Option names are passed as keyword arguments, without the leading + hyphen. + + The standard item options are: + + *fill* + The color used to fill the item's interior, or to draw a *line* + item or the characters of a *text* item. + An empty string (the default for all types except *line* and *text*) + leaves the item unfilled. + + *outline* + The color used to draw the item's outline. + An empty string draws no outline. + + *width* + The width of the outline, defaulting to ``1.0``. + Has no effect if *outline* is empty. + + *dash* + A dash pattern for the outline, given either as a sequence of + segment lengths in pixels or as a string of the characters + ``'.'``, ``','``, ``'-'``, ``'_'`` and space. + An empty pattern (the default) draws a solid outline. + + *dashoffset* + The starting offset in pixels into the *dash* pattern. + Ignored if there is no *dash* pattern. + + *stipple* + A bitmap used as a stipple pattern when filling the item. + Only well supported on X11. + + *outlinestipple* + A bitmap used as a stipple pattern when drawing the outline. + Has no effect if *outline* is empty. + + *offset*, *outlineoffset* + The offset of the fill and outline stipple patterns, given as + ``'x,y'`` or as a side such as ``'n'``, ``'se'`` or ``'center'``. + Stipple offsets are only supported on X11. + + *state* + Overrides the canvas state for this item; one of ``'normal'``, + ``'disabled'`` or ``'hidden'``. + + *tags* + A single tag or a sequence of tags to associate with the item, + replacing any existing tags. + + Many of these options have *active...* and *disabled...* variants + (such as *activefill*, *disabledfill*, *activewidth*, *disableddash*, + *activeoutline*, *disabledstipple*) that override the base option when + the item is the active item (under the mouse pointer) or is in the + disabled state. + + The following item types support additional options. + + For ``arc`` items: + + *start* + The start of the arc's angular range, in degrees measured + counter-clockwise from the 3-o'clock position. + + *extent* + The size of the angular range, in degrees counter-clockwise from + *start*. + + *style* + How the arc is drawn: ``'pieslice'`` (the default), ``'chord'`` or + ``'arc'``. + + For ``line`` items: + + *arrow* + Where to draw arrowheads: ``'none'`` (the default), ``'first'``, + ``'last'`` or ``'both'``. + + *arrowshape* + A sequence of three distances describing the shape of the + arrowheads. + + *capstyle* + How line ends are drawn: ``'butt'`` (the default), ``'projecting'`` + or ``'round'``. + + *joinstyle* + How line vertices are drawn: ``'round'`` (the default), ``'bevel'`` + or ``'miter'``. + + *smooth* + The smoothing method: a false value (the default) for no smoothing, + or ``'true'``/``'bezier'`` or ``'raw'`` to draw the line as a curve. + + *splinesteps* + The number of line segments approximating each spline when + *smooth* is enabled. + + For ``polygon`` items: + + *joinstyle*, *smooth*, *splinesteps* + As for ``line`` items, applied to the polygon's outline. + + For ``text`` items: + + *text* + The string to display; newline characters start new lines. + + *font* + The font used for the text. + + *justify* + How lines are justified: ``'left'`` (the default), ``'right'`` or + ``'center'``. + + *anchor* + How the text is positioned relative to its point, defaulting to + ``'center'``. + + *width* + The maximum line length; if non-zero, lines are wrapped at spaces. + + *angle* + How many degrees to rotate the text counter-clockwise about its + positioning point, from ``0.0`` to ``360.0`` (default ``0.0``). + + *underline* + The index of a character to underline, or ``-1`` for none. + + For ``bitmap`` items: + + *bitmap* + The bitmap to display. + + *anchor* + How the bitmap is positioned relative to its point. + + *background*, *foreground* + The colors used for the bitmap's ``0`` and ``1`` pixels; an empty + *background* makes the ``0`` pixels transparent. + Both have *active...* and *disabled...* variants, and *bitmap* has + *activebitmap* and *disabledbitmap* variants. + + For ``image`` items: + + *image* + The Tk image to display, previously created with the image + protocols. + + *anchor* + How the image is positioned relative to its point. + + Both options have *active...* and *disabled...* variants + (*activeimage*, *disabledimage*) used in the active and disabled + states. + + For ``window`` items: + + *window* + The widget to embed; it must be a child of the canvas or of one of + its ancestors, and may not be a top-level window. + + *anchor* + How the window is positioned relative to its point. + + *width*, *height* + The size to assign to the window; if zero (the default), the window + is given its requested size. + + ``oval`` and ``rectangle`` items have no type-specific options; they + use only the standard item options. + + .. note:: + + Tk 8.6 added the *angle* option and Tk 9.0 added the *underline* + option for ``text`` items. + + .. method:: coords(tagOrId) + coords(tagOrId, coordList, /) + coords(tagOrId, /, *coordList) + + Query or modify the coordinates of an item. + With only *tagOrId*, return a list of the floating-point coordinates of + the item given by *tagOrId* (the first matching item if it matches + several). + Given new coordinates, replace the coordinates of that item with them; + like the ``create_*`` methods, the coordinates may be given as separate + numbers, as a single sequence, or as coordinate pairs. + The returned coordinates are always in pixels, regardless of the units + used to specify them; for rectangles, ovals and arcs they are ordered + left, top, right, bottom. + + .. versionchanged:: 3.12 + The arguments are now flattened: the coordinates may be given as + separate arguments, as a single sequence, or grouped in pairs, like + the ``create_*`` methods. + + + .. method:: move(tagOrId, xAmount, yAmount, /) + + Move each of the items given by *tagOrId* in the canvas coordinate space + by adding *xAmount* to every x-coordinate and *yAmount* to every + y-coordinate of the item. + + .. method:: moveto(tagOrId, x='', y='') + + Move the items given by *tagOrId* so that the first coordinate pair (the + upper-left corner of the bounding box) of the lowest matching item is at + position (*x*, *y*). + *x* or *y* may be an empty string, in which case the corresponding + coordinate is unchanged. + All matching items keep their positions relative to each other. + + .. versionadded:: 3.8 + + + .. method:: scale(tagOrId, xOrigin, yOrigin, xScale, yScale, /) + + Rescale the coordinates of all items given by *tagOrId* in canvas + coordinate space. + Each x-coordinate is adjusted so that its distance from *xOrigin* changes + by a factor of *xScale*, and each y-coordinate so that its distance from + *yOrigin* changes by a factor of *yScale* (a factor of ``1.0`` leaves the + coordinate unchanged). + + .. method:: rotate(tagOrId, xOrigin, yOrigin, angle, /) + + Rotate the coordinates of all items given by *tagOrId* in canvas + coordinate space about the origin (*xOrigin*, *yOrigin*) by *angle* + degrees anticlockwise. + Negative values of *angle* rotate clockwise. + + .. versionadded:: next + + .. method:: delete(*tagOrIds) + + Delete each of the items given by the *tagOrIds* arguments. + + .. method:: dchars(tagOrId, first, /) + dchars(tagOrId, first, last, /) + + Delete from each of the items given by *tagOrId* the characters (for text + items) or coordinates (for line and polygon items) in the range from + *first* to *last* inclusive; *last* defaults to *first*. + Items that do not support indexing ignore this operation. + + .. method:: insert(tagOrId, beforeThis, string, /) + + Insert *string* into each of the items given by *tagOrId* just before the + character or coordinate whose index is *beforeThis*. + For line and polygon items *string* must be a valid sequence of + coordinates. + + .. method:: rchars(tagOrId, first, last, string, /) + + Replace the characters (for text items) or coordinates (for line and + polygon items) in the range from *first* to *last* inclusive of each of + the items given by *tagOrId* with *string*. + For line and polygon items *string* must be a valid sequence of + coordinates. + Items that do not support indexing ignore this operation. + + .. versionadded:: next + + .. method:: itemcget(tagOrId, option) + + Return the current value of the configuration option *option* for the + item given by *tagOrId* (the lowest matching item if it matches several). + This is like :meth:`~Misc.cget` but applies to an individual item. + + .. method:: itemconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(tagOrId, cnf=None, **kw) + + Query or modify the configuration options of the items given by + *tagOrId*. + This mirrors :meth:`~Misc.configure`, except that it applies to + individual items rather than to the canvas as a whole. + With no options, it returns a dictionary describing the current options + of the first matching item; otherwise it sets the given options on every + matching item. + The legal options are those accepted by the corresponding ``create_*`` + method. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: type(tagOrId) + + Return the type of the item given by *tagOrId* (the first matching item + if it matches several), such as ``'rectangle'`` or ``'text'``, or + ``None`` if *tagOrId* does not match any item. + + .. method:: gettags(tagOrId, /) + + Return a tuple of the tags associated with the item given by *tagOrId* + (the first matching item in display-list order if it matches several). + Return an empty tuple if no item matches or the item has no tags. + + .. method:: dtag(tagOrId, /) + dtag(tagOrId, tagToDelete, /) + + Remove the tag *tagToDelete* (which defaults to *tagOrId*) from each of + the items given by *tagOrId*. + Items that do not have that tag are unaffected. + + .. method:: addtag(newtag, searchSpec, /, *args) + + Add the tag *newtag* to each item selected by the search specification + *searchSpec* (and any further *args*). + *searchSpec* is one of ``'above'``, ``'all'``, ``'below'``, + ``'closest'``, ``'enclosed'``, ``'overlapping'`` or ``'withtag'``; the + ``addtag_*`` methods below are convenient wrappers that supply each of + these forms. + + .. method:: addtag_above(newtag, tagOrId) + + Add the tag *newtag* to the item just above (after) *tagOrId* in the + display list. + + .. method:: addtag_all(newtag) + + Add the tag *newtag* to all items in the canvas. + + .. method:: addtag_below(newtag, tagOrId) + + Add the tag *newtag* to the item just below (before) *tagOrId* in the + display list. + + .. method:: addtag_closest(newtag, x, y, halo=None, start=None) + + Add the tag *newtag* to the item closest to the point (*x*, *y*). + If *halo* is given, any item within that distance of the point is treated + as overlapping it. + If *start* is given (a tag or id), select the topmost closest item that + lies below *start* in the display list, which can be used to step through + all the closest items. + + .. method:: addtag_enclosed(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= *y2*. + + .. method:: addtag_overlapping(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item that overlaps or is enclosed within + the rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= + *y2*. + + .. method:: addtag_withtag(newtag, tagOrId) + + Add the tag *newtag* to every item given by *tagOrId*. + + .. method:: find(searchSpec, /, *args) + + Return a tuple of the ids of all items selected by the search + specification *searchSpec* (and any further *args*), in stacking order + with the lowest item first. + The search specification has any of the forms accepted by :meth:`addtag`. + The ``find_*`` methods below are more convenient wrappers around it. + + .. method:: find_above(tagOrId) + + Return a tuple containing the id of the item just above *tagOrId* in the + display list. + + .. method:: find_all() + + Return a tuple of the ids of all items in the canvas, in stacking order. + + .. method:: find_below(tagOrId) + + Return a tuple containing the id of the item just below *tagOrId* in the + display list. + + .. method:: find_closest(x, y, halo=None, start=None) + + Return a tuple containing the id of the item closest to the point (*x*, + *y*). + *halo* and *start* are interpreted as for :meth:`addtag_closest`. + + .. method:: find_enclosed(x1, y1, x2, y2) + + Return a tuple of the ids of all items completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_overlapping(x1, y1, x2, y2) + + Return a tuple of the ids of all items that overlap or are enclosed + within the rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_withtag(tagOrId) + + Return a tuple of the ids of all items given by *tagOrId*. + + .. method:: lift(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tkraise(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tag_raise(tagOrId, aboveThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just above the item given by *aboveThis*, or to the top of the display + list if *aboveThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items, whose stacking order is + controlled by :meth:`Misc.tkraise` and :meth:`Misc.lower` instead. + :meth:`lift` and :meth:`tkraise` are aliases of :meth:`!tag_raise`. + + .. method:: lower(tagOrId, belowThis=None, /) + :no-typesetting: + + .. method:: tag_lower(tagOrId, belowThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just below the item given by *belowThis*, or to the bottom of the display + list if *belowThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items. + :meth:`lower` is an alias of :meth:`!tag_lower`. + + .. note:: + + On a :class:`Canvas`, :meth:`tkraise`/:meth:`lift` and :meth:`lower` + restack canvas items, + shadowing the inherited :meth:`Misc.tkraise`/:meth:`Misc.lift` and + :meth:`Misc.lower` methods that restack the widget itself, + which are therefore not available. + + .. method:: tag_bind(tagOrId, sequence=None, func=None, add=None) + + Bind the callback *func* to the event *sequence* for all items given by + *tagOrId*, so that *func* is invoked whenever that event occurs for one + of the items. + This is like :meth:`Widget.bind ` but operates on canvas items + rather than on whole widgets; only mouse, keyboard and virtual events may + be bound. + Mouse events are directed to the current item and keyboard events to the + focus item (see :meth:`focus`). + If *add* is true the new binding is added to any existing bindings for + the same sequence, rather than replacing them. + Return the identifier of the bound function, which can be passed to + :meth:`tag_unbind`. + + .. method:: tag_unbind(tagOrId, sequence, funcid=None) + + Remove for all items given by *tagOrId* the binding for the event + *sequence*. + If *funcid* is given, only that callback (as returned by + :meth:`tag_bind`) is unbound and deregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: bbox(tagOrId, /, *tagOrIds) + + Return a 4-tuple ``(x1, y1, x2, y2)`` giving an approximate bounding box, + in pixels, that encloses all the items given by *tagOrId* and any further + *tagOrIds*. + The result may overestimate the true bounding box by a few pixels. + Return ``None`` if no item matches or the matching items have nothing to + display. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: canvasx(screenx, gridspacing=None) + + Given a window x-coordinate *screenx*, return the canvas x-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: canvasy(screeny, gridspacing=None) + + Given a window y-coordinate *screeny*, return the canvas y-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: focus() + focus(tagOrId, /) + + With *tagOrId*, set the keyboard focus for the canvas to the first item + given by *tagOrId* that supports the insertion cursor; the focus is left + unchanged if no such item exists. + If *tagOrId* is an empty string, reset the focus so that no item has it. + With no argument, return the id of the item that currently has the focus, + or an empty string if none does. + An item only displays the insertion cursor when both it is the focus item + and its canvas has the input focus. + + This shadows the inherited :meth:`!Misc.focus`; + use :meth:`~Misc.focus_set` to focus the widget itself. + + .. method:: icursor(tagOrId, index, /) + + Set the insertion cursor of the items given by *tagOrId* to just before + the character given by *index*. + Items that do not support an insertion cursor are unaffected. + The cursor is only displayed when the item has the focus, but its + position may be set at any time. + + .. method:: index(tagOrId, index, /) + + Return as an integer the numerical index within *tagOrId* corresponding + to *index*, which is a textual description of a position (for text items + an index into the characters, for line and polygon items an index into + the coordinates). + If *tagOrId* matches several items, the first one that supports indexing + is used. + + .. method:: select_adjust(tagOrId, index) + + Adjust the end of the selection in *tagOrId* nearest to *index* so that + it is at *index*, and make the other end the anchor point for future + :meth:`select_to` calls. + If the selection is not currently in *tagOrId*, this behaves like + :meth:`select_to`. + + .. method:: select_clear() + + Clear the selection if it is in this canvas; otherwise do nothing. + + .. method:: select_from(tagOrId, index) + + Set the selection anchor point to just before the character given by + *index* in the item given by *tagOrId*. + This does not change the selection itself; it sets the fixed end for + future :meth:`select_to` calls. + + .. method:: select_item() + + Return the id of the item that holds the selection, or ``None`` if the + selection is not in this canvas. + Unlike :meth:`find` and the ``find_*`` methods, this returns the id as a + string rather than an integer. + + .. method:: select_to(tagOrId, index) + + Set the selection to the characters of *tagOrId* between the selection + anchor point and *index*, inclusive of *index*. + The anchor point is the one set by the most recent :meth:`select_adjust` + or :meth:`select_from` call. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y, gain=10) + + Scroll the canvas by *gain* times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the canvas at high speed through its window. + + .. method:: postscript(cnf={}, **kw) + + Generate a PostScript (Encapsulated PostScript, version 3.0) + representation of part or all of the canvas. + If the *file* or *channel* option is given, the PostScript is written + there and an empty string is returned; otherwise it is returned as a + string. + By default only the area currently visible in the window is generated, so + it is usually necessary either to call :meth:`~Misc.update` first or to + use the *width* and *height* options. + Supported options include *colormap*, *colormode*, *file*, *fontmap*, + *height*, *pageanchor*, *pageheight*, *pagewidth*, *pagex*, *pagey*, + *rotate*, *width*, *x* and *y*. + + +.. class:: Checkbutton(master=None, cnf={}, **kw) + + A :class:`!Checkbutton` widget displays a textual string, bitmap or image + together with a square indicator, and toggles a boolean selection when + pressed. + It has all the behavior of a simple button and, in addition, can be + selected: when selected the indicator is drawn with a check mark and the + associated variable is set to the ``onvalue``, and when deselected the + indicator is drawn empty and the variable is set to the ``offvalue``. + Inherits from :class:`Widget`. + In addition to the standard widget options, a checkbutton accepts the + options documented in the Tk ``checkbutton`` manual page, such as + *variable*, *onvalue*, *offvalue* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the checkbutton with the + mouse: toggle the selection state of the button and invoke the associated + command, if there is one. + Return the result of the command, or an empty string if no command is + associated with the checkbutton. + This is ignored if the checkbutton's state is ``disabled``. + + .. method:: select() + + Select the checkbutton and set the associated variable to its + ``onvalue``. + + .. method:: deselect() + + Deselect the checkbutton and set the associated variable to its + ``offvalue``. + + .. method:: toggle() + + Toggle the selection state of the button, redisplaying it and modifying + its associated variable to reflect the new state. + + .. method:: flash() + + Flash the checkbutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the checkbutton is left in the same normal or + active state as when the method was called. + This is ignored if the checkbutton's state is ``disabled``. + + +.. class:: Entry(master=None, cnf={}, **kw) + + An :class:`!Entry` widget displays a single line of text and lets the user + edit it. + Inherits from :class:`Widget` and :class:`XView`; since entries can hold + strings too long to fit in the window, they support horizontal scrolling + through :meth:`~XView.xview`. + + In addition to the standard widget options, an entry accepts the options + documented in the Tk ``entry`` manual page. + Notable ones are *textvariable* (the name of a variable kept in sync with + the entry's contents), *show* (if set, each character is displayed as the + given character rather than its true value, useful for password entry), + *validate* and *validatecommand* (which together let a callback accept or + reject edits), and *state* (one of ``'normal'``, ``'disabled'`` or + ``'readonly'``). + + Many of the methods below take an *index* argument that selects a character + in the entry's string. + As described in the Tk ``entry`` manual page, *index* may be a number + (counting from 0), ``'insert'`` (the character just after the insertion + cursor), ``'end'`` (just after the last character), ``'anchor'`` (the + selection anchor point), ``'sel.first'`` and ``'sel.last'`` (the ends of the + selection), or ``@x`` (the character covering pixel x-coordinate *x* in the + window). + Out-of-range indices are rounded to the nearest legal value. + + .. method:: delete(first, last=None) + + Delete the characters from index *first* up to but not including index + *last*. + If *last* is omitted, only the single character at *first* is deleted. + + .. method:: get() + + Return the entry's current string. + + .. method:: insert(index, string) + + Insert *string* just before the character given by *index*. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*. + + .. method:: select_adjust(index) + :no-typesetting: + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by + *index*, and adjust that end to be at *index* (including but not going + beyond it); the other end becomes the anchor point for future + :meth:`selection_to` calls. + If there is no selection in the entry, a new one is created between + *index* and the most recent anchor point, inclusive. + :meth:`select_adjust` is an alias of :meth:`!selection_adjust`. + + .. method:: select_clear() + :no-typesetting: + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget the method has no effect. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on an :class:`Entry`. + + .. method:: select_from(index) + :no-typesetting: + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection. + :meth:`select_from` is an alias of :meth:`!selection_from`. + + .. method:: select_present() + :no-typesetting: + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the entry, ``False`` + otherwise. + :meth:`select_present` is an alias of :meth:`!selection_present`. + + .. method:: select_range(start, end) + :no-typesetting: + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + :meth:`select_range` is an alias of :meth:`!selection_range`. + + .. method:: select_to(index) + :no-typesetting: + + .. method:: selection_to(index) + + Set the selection between the anchor point and *index*: if *index* is + before the anchor point, the selection runs from *index* up to but not + including the anchor; if *index* is after it, from the anchor up to but + not including *index*; if they coincide, nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If there is no selection in the entry, a new one is created using the + most recent anchor point. + :meth:`select_to` is an alias of :meth:`!selection_to`. + + .. method:: scan_mark(x) + + Record *x* and the current view in the entry window, for use with later + :meth:`scan_dragto` calls. + Typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Compute the difference between *x* and the *x* given to the last + :meth:`scan_mark` call, and adjust the view left or right by 10 times + that difference. + Typically associated with mouse motion events, to produce the effect of + dragging the entry at high speed through the window. + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + +.. class:: Frame(master=None, cnf={}, **kw) + + A :class:`!Frame` widget is a simple container. + Its primary purpose is to act as a spacer or container for complex window + layouts; its only features are its background and an optional 3-D border to + make the frame appear raised or sunken. + Inherits from :class:`Widget`. + Refer to the Tk ``frame`` manual page for the full list of options. + + +.. class:: Label(master=None, cnf={}, **kw) + + A :class:`!Label` widget displays a non-interactive textual string, bitmap + or image. + The displayed text is set with the *text* option or linked to a variable + through *textvariable*, and an image can be shown using the *image* option. + Text must all be in a single font but may occupy multiple lines, and one + character may be underlined with the *underline* option. + Inherits from :class:`Widget`. + Refer to the Tk ``label`` manual page for the full list of options. + + +.. class:: LabelFrame(master=None, cnf={}, **kw) + + A :class:`!LabelFrame` widget is a container that has the features of a + :class:`Frame` plus the ability to display a label. + The label text is set with the *text* option and positioned with + *labelanchor*, or an arbitrary widget may be used as the label by giving it + as the *labelwidget* option. + Inherits from :class:`Widget`. + Refer to the Tk ``labelframe`` manual page for the full list of options. + + +.. class:: Listbox(master=None, cnf={}, **kw) + + A :class:`!Listbox` widget displays a list of single-line text items, one + per line, of which the user can select one or more. + The way the selection behaves is governed by the *selectmode* option, which + is one of ``browse`` (the default; at most one item, which may be dragged + with the mouse), ``single`` (at most one item), ``multiple`` (any number of + items, toggled individually), or ``extended`` (any number of items, + including discontiguous ranges, selected by clicking and dragging). + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``listbox`` manual page for the full list of options. + + Many of the methods take an *index* argument identifying a particular item. + As described in the Tk ``listbox`` manual page, *index* may be a numeric + index (counting from 0 at the top), ``'active'`` (the item with the location + cursor, set with :meth:`activate`), ``'anchor'`` (the selection anchor, set + with :meth:`selection_anchor`), ``'end'`` (the last item, or for + :meth:`index` and :meth:`insert` the position just after it), or ``@x,y`` + (the item covering pixel coordinates *x*, *y* in the listbox window). + Arguments named *first* and *last* are indices of the same forms. + + .. method:: insert(index, *elements) + + Insert the given *elements* as new items just before the item given by + *index*. + If *index* is ``'end'``, the new items are appended to the end of the + list. + + .. method:: delete(first, last=None) + + Delete the items in the range from *first* to *last* inclusive. + If *last* is omitted, it defaults to *first*, so that a single item is + deleted. + + .. method:: get(first, last=None) + + If *last* is omitted, return the contents of the item given by *first*, + or an empty string if *first* refers to a non-existent item. + If *last* is given, return a tuple of all the items in the range from + *first* to *last* inclusive. + + .. method:: size() + + Return the total number of items in the listbox. + + This shadows the inherited :meth:`!Misc.size`; + use :meth:`~Misc.grid_size` for the grid size. + + .. method:: index(index) + + Return the integer index value corresponding to *index*, or ``None`` if + *index* is out of range. + If *index* is ``'end'``, the result is a count of the number of items in + the listbox (not the index of the last item). + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` describing the bounding box, in + pixels relative to the widget, of the text of the item given by *index*. + Return ``None`` if no part of that item is visible on the screen, or if + *index* refers to a non-existent item; if the item is only partly + visible, the result still gives the full area of the item, including the + parts that are not visible. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: nearest(y) + + Given a y-coordinate within the listbox window, return the index of the + visible item nearest to that y-coordinate. + + .. method:: see(index) + + Adjust the view so that the item given by *index* is visible. + If the item is already visible the method has no effect; if it is near an + edge of the window the listbox scrolls just enough to bring it into view + at that edge, otherwise the listbox scrolls to center the item. + + .. method:: activate(index) + + Set the active item to the one given by *index*. + If *index* is outside the range of items, the closest item is activated + instead. + The active item is drawn as specified by the *activestyle* option when + the widget has the input focus, and its index may be retrieved with the + ``'active'`` index. + + .. method:: curselection() + + Return a tuple containing the numerical indices of all of the items that + are currently selected, or an empty tuple if no items are selected. + + .. method:: select_anchor(index) + :no-typesetting: + + .. method:: selection_anchor(index) + + Set the selection anchor to the item given by *index*. + If *index* refers to a non-existent item, the closest item is used. + The selection anchor is the end of the selection that is fixed while + dragging out a selection with the mouse, and may afterwards be referred + to with the ``'anchor'`` index. + :meth:`select_anchor` is an alias of :meth:`!selection_anchor`. + + .. method:: select_clear(first, last=None) + :no-typesetting: + + .. method:: selection_clear(first, last=None) + + Deselect any of the items in the range from *first* to *last* inclusive + that are selected. + The selection state of items outside this range is not changed. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Listbox`. + + .. method:: select_includes(index) + :no-typesetting: + + .. method:: selection_includes(index) + + Return ``True`` if the item given by *index* is currently selected, + ``False`` otherwise. + :meth:`select_includes` is an alias of :meth:`!selection_includes`. + + .. method:: select_set(first, last=None) + :no-typesetting: + + .. method:: selection_set(first, last=None) + + Select all of the items in the range from *first* to *last* inclusive, + without affecting the selection state of items outside that range. + :meth:`select_set` is an alias of :meth:`!selection_set`. + + .. method:: itemcget(index, option) + + Return the current value of the configuration option *option* for the + item given by *index*. + + .. method:: itemconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the item given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual item rather than to the listbox as a whole. + With no options, it returns a dictionary describing the current options + of the item; otherwise it sets the given options. + The supported item options are *background*, *foreground*, + *selectbackground* and *selectforeground*. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the listbox by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the list at high speed through the window. + + +.. class:: Menu(master=None, cnf={}, **kw) + + A :class:`!Menu` widget displays a column of entries, each of which may be a + command, a checkbutton, a radiobutton, a cascade (which posts an associated + submenu) or a separator. + Menus are used as the menubar of a toplevel window, as pulldown menus posted + from a cascade entry or menubutton, and as popup menus. + Inherits from :class:`Widget`. + + Many of the entry methods take an *index* argument that selects which entry + to operate on. + As described in the Tk ``menu`` manual page, *index* may be a numeric index + (counting from 0 at the top), ``'active'`` (the currently active entry), + ``'end'`` or ``'last'`` (the bottommost entry), ``'none'`` (no entry at all, + written ``{}`` in Tcl), ``@y`` (the entry covering pixel y-coordinate *y* in + the menu window), or a pattern matched against the labels of the entries + from the top down. + + .. method:: add(itemType, cnf={}, **kw) + + Add a new entry to the bottom of the menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'`` and determines the type of the new + entry; the remaining options configure it. + The :meth:`!add_command`, :meth:`!add_cascade`, :meth:`!add_checkbutton`, + :meth:`!add_radiobutton` and :meth:`!add_separator` convenience methods + call this method with the corresponding *itemType*. + + The entry is configured by the following options, although not every + option applies to every entry type (a separator accepts none of them): + + *label* + The text to display in the entry. + + *command* + The function to call when the entry is invoked (command, checkbutton + and radiobutton entries). + + *accelerator* + A string displayed at the right of the entry to advertise an + accelerator keystroke; it does not itself create the binding. + + *underline* + The index of a character in the label to underline for keyboard + traversal. + + *state* + One of ``'normal'``, ``'active'`` or ``'disabled'``. + + *image* + An image to display instead of, or together with, the text label. + + *compound* + Where to show the image relative to the text: ``'none'`` (the + default), ``'text'``, ``'image'``, ``'top'``, ``'bottom'``, ``'left'`` + or ``'right'``. + + *bitmap* + A bitmap to display instead of the text label. + + *font* + The font to use for the text. + + *background*, *foreground* + The entry's background and foreground colors in its normal state + (ignored on macOS). + + *activebackground*, *activeforeground* + The background and foreground colors used when the entry is active + (ignored on macOS). + + *columnbreak* + If true, the entry starts a new column instead of being placed below + the previous entry. + + *hidemargin* + If true, the standard margin around the entry is omitted, which is + useful when a menu is used as a palette. + + *menu* + The submenu posted by a cascade entry; it must be a child of this + menu. + + *variable* + The variable associated with a checkbutton or radiobutton entry. + + *onvalue*, *offvalue* + The values stored in *variable* when a checkbutton entry is selected + or cleared. + + *value* + The value stored in *variable* when a radiobutton entry is selected. + + *indicatoron* + Whether to display the indicator of a checkbutton or radiobutton + entry. + + *selectcolor* + The color of the indicator of a checkbutton or radiobutton entry when + it is selected. + + *selectimage* + The image displayed when a checkbutton or radiobutton entry is + selected and *image* is also given. + + .. method:: add_cascade(cnf={}, **kw) + + Add a new cascade entry to the bottom of the menu. + A cascade entry has an associated submenu, given by its *menu* option, + which must be a child of this menu; posting the entry posts the submenu + next to it. + + .. method:: add_checkbutton(cnf={}, **kw) + + Add a new checkbutton entry to the bottom of the menu. + When invoked, a checkbutton entry toggles between its *onvalue* and + *offvalue*, storing the result in its associated *variable*, and displays + an indicator showing whether it is selected. + + .. method:: add_command(cnf={}, **kw) + + Add a new command entry to the bottom of the menu. + A command entry behaves much like a button: when it is invoked, the + callback given by its *command* option is called. + + .. method:: add_radiobutton(cnf={}, **kw) + + Add a new radiobutton entry to the bottom of the menu. + Radiobutton entries sharing the same *variable* form a group of which + only one may be selected at a time; selecting an entry stores its *value* + in the variable. + + .. method:: add_separator(cnf={}, **kw) + + Add a separator to the bottom of the menu. + A separator is displayed as a horizontal dividing line and cannot be + activated or invoked. + + .. method:: insert(index, itemType, cnf={}, **kw) + + Same as :meth:`add`, except that the new entry is inserted just before + the entry given by *index* instead of being appended to the end of the + menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'``. + The :meth:`!insert_command`, :meth:`!insert_cascade`, + :meth:`!insert_checkbutton`, :meth:`!insert_radiobutton` and + :meth:`!insert_separator` convenience methods call this method with the + corresponding *itemType*. + + .. method:: insert_cascade(index, cnf={}, **kw) + + Insert a new cascade entry before the entry given by *index* (see + :meth:`add_cascade`). + + .. method:: insert_checkbutton(index, cnf={}, **kw) + + Insert a new checkbutton entry before the entry given by *index* (see + :meth:`add_checkbutton`). + + .. method:: insert_command(index, cnf={}, **kw) + + Insert a new command entry before the entry given by *index* (see + :meth:`add_command`). + + .. method:: insert_radiobutton(index, cnf={}, **kw) + + Insert a new radiobutton entry before the entry given by *index* (see + :meth:`add_radiobutton`). + + .. method:: insert_separator(index, cnf={}, **kw) + + Insert a separator before the entry given by *index* (see + :meth:`add_separator`). + + .. method:: delete(index1, index2=None) + + Delete all of the menu entries between *index1* and *index2* inclusive. + If *index2* is omitted, it defaults to *index1*, so that a single entry + is deleted. + Attempts to delete a tear-off entry are ignored; remove it by changing + the *tearoff* option instead. + + .. method:: entrycget(index, option) + + Return the current value of the configuration option *option* for the + entry given by *index*. + + .. method:: entryconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: entryconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the entry given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual entry rather than to the menu as a whole. + With no options, it returns a dictionary describing the current options + of the entry; otherwise it sets the given options. + The supported options are those accepted by :meth:`add` for the entry's + type. + :meth:`entryconfig` is an alias of :meth:`!entryconfigure`. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, or ``None`` if + *index* selects no entry. + + .. method:: type(index) + + Return the type of the entry given by *index*: one of ``'command'``, + ``'cascade'``, ``'checkbutton'``, ``'radiobutton'``, ``'separator'`` or + ``'tearoff'`` (for the tear-off entry). + + .. method:: activate(index) + + Make the entry given by *index* the active entry, redisplaying it with + its active colors, and deactivate any previously active entry. + If *index* selects no entry, or the selected entry is disabled, the menu + ends up with no active entry. + + .. method:: invoke(index) + + Invoke the action of the entry given by *index*, as if it had been + clicked. + Nothing happens if the entry is disabled. + If the entry has a *command* associated with it, the result of that + command is returned; otherwise the result is an empty string. + + .. method:: post(x, y) + + Display the menu on the screen at the root-window coordinates *x* and + *y*, adjusting them if necessary so that the whole menu is visible. + If the *postcommand* option has been specified, it is evaluated before + the menu is posted. + + .. method:: postcascade(index) + + Post the submenu associated with the cascade entry given by *index*, + unposting any previously posted submenu. + This has no effect if *index* does not name a cascade entry or if the + menu itself is not posted. + + .. versionadded:: next + + .. method:: tk_popup(x, y, entry='') + + Post the menu as a popup at the root-window coordinates *x* and *y*. + If *entry* is given, the menu is positioned so that this entry is + displayed under the pointer. + + .. method:: unpost() + + Unmap the menu so that it is no longer displayed, also unposting any + posted lower-level cascaded submenu. + This has no effect on Windows and macOS, which manage the unposting of + menus themselves. + + .. method:: xposition(index) + + Return the x-coordinate, within the menu window, of the leftmost pixel of + the entry given by *index*. + + .. versionadded:: 3.3 + + .. method:: yposition(index) + + Return the y-coordinate, within the menu window, of the topmost pixel of + the entry given by *index*. + + +.. class:: Menubutton(master=None, cnf={}, **kw) + + A :class:`!Menubutton` widget displays a textual string, bitmap or image and + posts an associated :class:`Menu`, given by its *menu* option, when the user + presses it. + Like a :class:`Label` it can show *text*, a *textvariable*, or an *image*, + and the *direction* option controls where the menu appears relative to the + button. + Inherits from :class:`Widget`. + Refer to the Tk ``menubutton`` manual page for the full list of options. + + +.. class:: Message(master=None, cnf={}, **kw) + + A :class:`!Message` widget displays a non-interactive textual string, given + by the *text* option or linked to a variable through *textvariable*. + Unlike a :class:`Label`, it breaks the string into multiple lines in order + to produce a given aspect ratio, choosing line breaks at word boundaries, + and it can justify the text left, centered or right. + Inherits from :class:`Widget`. + Refer to the Tk ``message`` manual page for the full list of options. + + +.. class:: OptionMenu(master, variable, value, *values, **kwargs) + + A helper subclass of :class:`Menubutton` that displays a pop-up menu of + mutually exclusive choices. + *variable* is a :class:`Variable` kept in sync with the selection, *value* + is the initial choice, and *values* are the remaining menu entries. + The keyword argument *command* may be given a callback that is invoked with + the selected value, and the keyword argument *name* sets the Tk widget name. + Other keyword arguments are passed to the underlying :class:`Menubutton` + and may override its default appearance. + + .. method:: destroy() + + Destroy the widget, also cleaning up the associated pop-up menu. + + .. versionchanged:: 3.14 + Added support for the *name* keyword argument. + + .. versionchanged:: next + Other :class:`Menubutton` options can now be passed as keyword arguments. + + + +.. class:: PanedWindow(master=None, cnf={}, **kw) + + A :class:`!PanedWindow` is a geometry-manager widget that arranges any + number of child *panes* in a row (when *orient* is ``'horizontal'``) or a + column (when *orient* is ``'vertical'``). + Each pane holds one widget, and each pair of adjacent panes is separated by + a movable *sash* that the user can drag with the mouse to resize the widgets + on either side of it. + Inherits from :class:`Widget`. + + The *orient* option selects the layout direction, *sashwidth* sets the width + of each sash and *sashrelief* its relief. + When *showhandle* is true a small handle is drawn on each sash that the user + can grab to drag it. + Refer to the Tk ``panedwindow`` manual page for the full list of options. + + .. method:: add(child, **kw) + + Add *child* to the panedwindow as a new pane, placed after any existing + panes. + The keyword arguments specify per-pane management options for *child*; + they may be any of the options accepted by :meth:`paneconfigure`. + + .. method:: forget(child) + :no-typesetting: + + .. method:: remove(child) + + Remove the pane containing *child* from the panedwindow. + All geometry management options for *child* are forgotten. + :meth:`forget` is an alias of :meth:`!remove`. + This shadows the inherited geometry-manager :meth:`!forget`; + use :meth:`~Pack.pack_forget`, :meth:`~Grid.grid_forget` or + :meth:`~Place.place_forget` to remove the widget itself from its manager. + + .. method:: panes() + + Return a tuple of the widgets managed by the panedwindow, one per pane, + in order. + + .. method:: panecget(child, option) + + Return the current value of the management option *option* for the pane + containing *child*. + *option* may be any value allowed by :meth:`paneconfigure`. + + .. method:: paneconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: paneconfigure(tagOrId, cnf=None, **kw) + + Query or modify the management options of the pane containing the widget + *tagOrId*. + With no options, it returns a dictionary describing all of the available + options for the pane; given a single option name as a string, it returns + a description of that one option; otherwise it sets the given options. + The supported options include *after* and *before* (insert the pane after + or before another managed window), *height* and *width* (the outer + dimensions of the window, including any border), *minsize* (the minimum + size in the paned dimension), *padx* and *pady* (extra space to leave on + each side of the window), *sticky* (position or stretch the window within + an oversized pane, using a string of the characters ``n``, ``s``, ``e`` + and ``w``), *hide* (hide the pane while keeping it in the list of panes) + and *stretch* (how extra space is allocated to the pane: one of + ``'always'``, ``'first'``, ``'last'``, ``'middle'`` or ``'never'``). + :meth:`paneconfig` is an alias of :meth:`!paneconfigure`. + + .. method:: identify(x, y) + + Identify the panedwindow component underneath the point given by *x* and + *y*, in window coordinates. + If the point is over a sash or a sash handle, the result is a two-element + tuple containing the index of the sash or handle and a word indicating + whether it is over a sash or a handle, such as ``(0, 'sash')`` or + ``(2, 'handle')``. + If the point is over any other part of the panedwindow, the result is an + empty string. + + .. method:: sash(*args) + + Query or change the position of the sashes in the panedwindow. + This is a thin wrapper around the Tk ``sash`` subcommand; the convenience + methods :meth:`sash_coord`, :meth:`sash_mark` and :meth:`sash_place` + should normally be used instead. + + .. method:: sash_coord(index) + + Return the current x and y coordinate pair for the sash given by *index*, + which must be an integer between 0 and one less than the number of panes + in the panedwindow. + The coordinates returned are those of the top left corner of the region + containing the sash. + + .. method:: sash_mark(index) + + Record the current mouse position for the sash given by *index*, for use + together with later sash-drag operations to move the sash. + + .. method:: sash_place(index, x, y) + + Place the sash given by *index* at the coordinates *x* and *y*. + + .. method:: proxy(*args) + + Query or change the position of the sash proxy, the "ghost" sash shown + while a sash is being dragged with non-opaque resizing. + This is a thin wrapper around the Tk ``proxy`` subcommand; the + convenience methods :meth:`proxy_coord`, :meth:`proxy_forget` and + :meth:`proxy_place` should normally be used instead. + + .. method:: proxy_coord() + + Return a tuple containing the x and y coordinates of the most recent + proxy location. + + .. method:: proxy_forget() + + Remove the proxy from the display. + + .. method:: proxy_place(x, y) + + Place the proxy at the coordinates *x* and *y*. + + +.. class:: Radiobutton(master=None, cnf={}, **kw) + + A :class:`!Radiobutton` widget displays a textual string, bitmap or image + together with a diamond or circular indicator, and selects one choice out of + several. + It has all the behavior of a simple button and, in addition, can be + selected: typically several radiobuttons share a single *variable*, and + selecting one sets that variable to the radiobutton's *value*; each + radiobutton also monitors the variable and automatically selects or + deselects itself when the variable changes. + Inherits from :class:`Widget`. + In addition to the standard widget options, a radiobutton accepts the + options documented in the Tk ``radiobutton`` manual page, such as + *variable*, *value* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the radiobutton with the + mouse: select the button and invoke the associated command, if there is + one. + Return the result of the command, or an empty string if no command is + associated with the radiobutton. + This is ignored if the radiobutton's state is ``disabled``. + + .. method:: select() + + Select the radiobutton and set the associated variable to the value + corresponding to this widget. + + .. method:: deselect() + + Deselect the radiobutton and set the associated variable to an empty + string. + If this radiobutton was not currently selected, this has no effect. + + .. method:: flash() + + Flash the radiobutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the radiobutton is left in the same normal or + active state as when the method was called. + This is ignored if the radiobutton's state is ``disabled``. + + +.. class:: Scale(master=None, cnf={}, **kw) + + A :class:`!Scale` widget lets the user select a numerical value by moving a + slider along a trough. + It can be oriented vertically or horizontally and can optionally display a + label and the current value. + Inherits from :class:`Widget`. + + In addition to the standard widget options, a scale accepts the options + documented in the Tk ``scale`` manual page, such as *from_*, *to*, + *resolution*, *orient*, *tickinterval*, *variable* and *command*. + As elsewhere in :mod:`!tkinter`, the leading ``-`` of the Tk option name is + dropped; *from* is spelled ``from_`` because :keyword:`from` is a Python + keyword. + + With a non-integer *resolution*, see :ref:`numeric values and the locale + `. + + .. method:: get() + + Return the current value of the scale. + The result is an integer if the scale's *resolution* yields whole + numbers, and a float otherwise. + + .. method:: set(value) + + Set the scale to *value*, moving the slider accordingly. + This has no effect if the scale is disabled. + + .. method:: coords(value=None) + + Return a tuple ``(x, y)`` giving the pixel coordinates, relative to the + widget, of the point on the centerline of the trough that corresponds to + *value*. + If *value* is omitted, the scale's current value is used. + + .. method:: identify(x, y) + + Return a string describing the part of the scale at the pixel coordinates + *x*, *y*: ``'slider'``, ``'trough1'`` (the part of the trough above or to + the left of the slider), ``'trough2'`` (below or to the right of the + slider), or an empty string if the point is not over any of these + elements. + + +.. class:: Scrollbar(master=None, cnf={}, **kw) + + A :class:`!Scrollbar` widget displays a slider and two arrows that let the + user scroll an associated widget, such as a :class:`Listbox`, :class:`Text`, + :class:`Canvas` or :class:`Entry`. + It is connected to the scrolled widget by setting that widget's + *xscrollcommand* or *yscrollcommand* option to the scrollbar's :meth:`set` + method, and the scrollbar's *command* option to the scrolled widget's + :meth:`~XView.xview` or :meth:`~YView.yview` method. + Inherits from :class:`Widget`. + + .. method:: get() + + Return the current scrollbar settings as a tuple ``(first, last)`` of two + fractions between 0 and 1, describing the portion of the document that is + currently visible, as last passed to :meth:`set`. + + .. method:: set(first, last) + + Set the scrollbar. + *first* and *last* are fractions between 0 and 1 giving the positions of + the start and end of the visible portion of the associated document. + This method is normally registered as the scrolled widget's + *xscrollcommand* or *yscrollcommand* and called by that widget. + + .. method:: activate(index=None) + + Mark the element *index* (one of ``'arrow1'``, ``'slider'`` or + ``'arrow2'``) as active, displaying it according to the + *activebackground* and *activerelief* options. + If *index* is omitted, return the name of the currently active element, + or ``None`` if no element is active. + + .. versionchanged:: 3.5 + The *index* argument is now optional. + + .. method:: delta(deltax, deltay) + + Return a float indicating the fractional change in the scrollbar setting + that corresponds to moving the slider by *deltax* pixels horizontally + (for horizontal scrollbars) or *deltay* pixels vertically (for vertical + scrollbars). + + .. method:: fraction(x, y) + + Return a float between 0 and 1 indicating where the point at pixel + coordinates *x*, *y* lies in the trough: 0 corresponds to the top or left + of the trough and 1 to the bottom or right. + + .. method:: identify(x, y) + + Return the name of the element under the pixel coordinates *x*, *y* (such + as ``'arrow1'``), or an empty string if the point does not lie in any + element of the scrollbar. + + +.. class:: Spinbox(master=None, cnf={}, **kw) + + A :class:`!Spinbox` widget is an :class:`Entry`-like widget with a pair of + up/down arrow buttons that let the user step through a range of values in + addition to editing the value directly. + The set of values may be a numeric range given by the *from_*, *to* and + *increment* options, or an explicit list of strings given by the *values* + option (which takes precedence over the range). + Each time an arrow is invoked the *command* callback, if any, is called; the + *wrap* option controls whether stepping past either end of the range wraps + around to the other end; the *format* option specifies how numeric values + are formatted; and the *validate* option enables validation of the entered + text. + Inherits from :class:`Widget` and :class:`XView`. + + With a non-integer *increment*, see :ref:`numeric values and the locale + `. + + Many of the methods take an *index* argument identifying a character in the + spinbox's string. + As described in the Tk ``spinbox`` manual page, *index* may be a numeric + index (counting from 0), ``'anchor'`` (the selection anchor point), + ``'end'`` (just after the last character), ``'insert'`` (the character just + after the insertion cursor), ``'sel.first'`` or ``'sel.last'`` (the ends of + the selection), or ``@x`` (the character covering pixel x-coordinate *x* in + the window). + + .. method:: get() + + Return the spinbox's string. + + .. method:: insert(index, s) + + Insert the characters of the string *s* just before the character given + by *index*. + + .. method:: delete(first, last=None) + + Delete one or more characters of the spinbox. + *first* is the index of the first character to delete, and *last* is the + index of the character just after the last one to delete. + If *last* is omitted, a single character at *first* is deleted. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, as a string. + + .. method:: bbox(index) + + Return a tuple of four integers ``(x, y, width, height)`` describing the + bounding box of the character given by *index*. + *x* and *y* are the pixel coordinates of the upper-left corner of the + character relative to the widget, and *width* and *height* are its size + in pixels. + The bounding box may refer to a region outside the visible area of the + window. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: identify(x, y) + + Return the name of the window element at the pixel coordinates *x*, *y*: + one of ``'buttondown'``, ``'buttonup'``, ``'entry'`` or ``'none'``. + + .. method:: invoke(element) + + Invoke the spin button given by *element*, either ``'buttonup'`` or + ``'buttondown'``, triggering the action associated with it. + + .. method:: scan(*args) + + A thin wrapper around the Tk ``scan`` widget subcommand, used to + implement fast dragging of the view: ``scan('mark', x)`` records *x* and + the current view, and ``scan('dragto', x)`` adjusts the view relative to + that mark. + The :meth:`scan_mark` and :meth:`scan_dragto` methods wrap the two forms. + + .. method:: scan_mark(x) + + Record *x* and the current view in the spinbox window, for use with a + later :meth:`scan_dragto` call. + This is typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Adjust the view by 10 times the difference between *x* and the *x* passed + to the last :meth:`scan_mark` call. + This is typically associated with mouse motion events, producing the + effect of dragging the spinbox at high speed through the window. + + .. method:: selection(*args) + + A thin wrapper around the Tk ``selection`` widget subcommand, used to + adjust the selection within the spinbox. + It has several forms depending on the first argument, such as + ``selection('adjust', index)``, ``selection('clear')``, + ``selection('element', ?elem?)``, ``selection('from', index)``, + ``selection('present')``, ``selection('range', start, end)`` and + ``selection('to', index)``. + The :meth:`selection_adjust`, :meth:`selection_clear`, + :meth:`selection_element`, :meth:`selection_from`, + :meth:`selection_present`, :meth:`selection_range` and + :meth:`selection_to` methods wrap these forms. + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by *index* + and adjust that end of the selection to be at *index* (including but not + going beyond *index*). + The other end becomes the anchor point for future :meth:`selection_to` + calls. + If the selection is not currently in the spinbox, a new selection is + created to include the characters between *index* and the most recent + anchor point, inclusive. + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget, the method has no effect. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Spinbox`. + + .. method:: selection_element(element=None) + + Set or get the currently selected element. + If *element* (one of ``'buttonup'``, ``'buttondown'`` or ``'none'``) is + given, that spin button is selected and displayed depressed; otherwise + the name of the currently selected element is returned. + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection itself. + + .. versionadded:: 3.8 + + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the spinbox, + ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + + .. versionadded:: 3.8 + + + .. method:: selection_to(index) + + Set the selection between *index* and the anchor point. + If *index* is before the anchor point, the selection runs from *index* up + to but not including the anchor point; if it is after, the selection runs + from the anchor point up to but not including *index*; if it is the same, + nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If the selection is not in this widget, a new selection is created using + the most recent anchor point. + + .. versionadded:: 3.8 + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + + +.. class:: Text(master=None, cnf={}, **kw) + + A :class:`!Text` widget displays and edits multi-line text. + Portions of the text may be styled with **tags**, particular positions may + be annotated with floating **marks**, and arbitrary images and other widgets + may be embedded in the text. + The widget also provides an unlimited undo/redo mechanism and supports peer + widgets that share the same underlying data. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``text`` manual page for the full list of options. + + Most of the methods take one or more *index* arguments that identify a + position within the text. + As described in the Tk ``text`` manual page, an index is a string consisting + of a base, optionally followed by one or more modifiers. + The base may be ``'line.char'`` (line *line*, character *char*, where lines + are counted from 1 and characters within a line from 0; ``'line.end'`` + refers to the newline ending the line), ``'end'`` (the position just after + the last newline), the name of a mark, ``'tag.first'`` or ``'tag.last'`` + (the first character tagged with *tag*, or the position just after the last + such character), the name of an embedded image or window, or ``@x,y`` (the + character covering pixel coordinates *x*, *y* in the widget). + A modifier such as ``'+5 chars'``, ``'-3 lines'``, ``'linestart'``, + ``'lineend'``, ``'wordstart'`` or ``'wordend'`` adjusts the index relative + to its base; several modifiers may be combined and are applied from left to + right, for example ``'insert wordstart - 1 c'``. + + .. method:: insert(index, chars, *args) + + Insert the string *chars* just before the character at *index* (if + *index* is ``'end'``, just before the final newline). + By default the new text inherits any tags present on both sides of the + insertion point. + If *args* is given, it consists of alternating *tagList*, *chars* values: + the preceding *chars* receives exactly the tags listed (a tag list may be + a single tag name or a sequence of names), overriding the surrounding + tags. + + .. method:: delete(index1, index2=None) + + Delete the range of characters from *index1* up to but not including + *index2*. + If *index2* is omitted, the single character at *index1* is deleted. + The widget always keeps a newline as its last character, so a deletion + that would remove it is adjusted accordingly. + + .. method:: replace(index1, index2, chars, *args) + + Replace the range of characters from *index1* up to but not including + *index2* with *chars*. + This is equivalent to a :meth:`delete` followed by an :meth:`insert` at + *index1*; *args* is interpreted as for :meth:`insert`. + + .. versionadded:: 3.3 + + .. method:: get(index1, index2=None) + + Return the text from *index1* up to but not including *index2* as a + string. + If *index2* is omitted, return the single character at *index1*. + Embedded images and windows are omitted from the result. + + .. method:: index(index) + + Return the position corresponding to *index* in the canonical + ``'line.char'`` form. + + .. method:: compare(index1, op, index2) + + Compare the positions of *index1* and *index2* using the relational + operator *op*, which must be one of ``'<'``, ``'<='``, ``'=='``, + ``'>='``, ``'>'`` or ``'!='``, and return the boolean result. + + .. method:: count(index1, index2, *options, return_ints=False) + + Count the number of items of the requested kinds between *index1* and + *index2*; the count is negative if *index1* is after *index2*. + Each of *options* names a kind of item to count: ``'chars'``, + ``'displaychars'``, ``'displayindices'``, ``'displaylines'``, + ``'indices'``, ``'lines'``, ``'xpixels'`` or ``'ypixels'`` (the default, + used when no option is given, is ``'indices'``). + The pseudo-option ``'update'`` forces any out-of-date layout information + to be recalculated before the following options are evaluated. + When *return_ints* is true and a single counting option is given, return + a plain integer; otherwise return a tuple with one integer per counting + option (or ``None`` if the result is empty). + + .. versionadded:: 3.3 + + .. versionchanged:: 3.13 + Added the *return_ints* parameter. + + + .. method:: see(index) + + Adjust the view so that the character given by *index* is visible. + If it is already visible the method has no effect; if it is a short + distance out of view the widget scrolls just enough to bring it to the + nearest edge, otherwise it scrolls to center *index* in the window. + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` giving the bounding box, in + pixels, of the visible part of the character at *index*, or ``None`` if + that character is not visible on the screen. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: dlineinfo(index) + + Return a tuple ``(x, y, width, height, baseline)`` describing the display + line that contains *index*: the first four values give the bounding box + of the line in pixels and *baseline* gives the offset of the baseline + measured down from the top of the area. + Return ``None`` if that display line is not visible on the screen. + + .. method:: mark_set(markName, index) + + Set the mark named *markName* to the position just before the character + at *index*, creating the mark if it does not already exist. + A mark created this way has right gravity by default. + + .. method:: mark_unset(*markNames) + + Remove each of the marks named in *markNames*. + The special ``insert`` and ``current`` marks may not be removed. + + .. method:: mark_names() + + Return a tuple of the names of all marks currently set in the widget. + + .. method:: mark_gravity(markName, direction=None) + + If *direction* is omitted, return the gravity of mark *markName*, either + ``'left'`` or ``'right'``. + Otherwise set its gravity to *direction*. + The gravity determines on which side of the mark text inserted at the + mark's position appears: a mark with right gravity (the default) stays to + the right of such text. + + .. method:: mark_next(index) + + Return the name of the first mark at or after *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just after that + mark. + + .. method:: mark_previous(index) + + Return the name of the last mark at or before *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just before that + mark. + + .. method:: tag_add(tagName, index1, *args) + + Add the tag *tagName* to the range of characters from *index1* up to but + not including the next index in *args*. + Further pairs of indices may follow in *args* to tag additional ranges; a + trailing single index tags just the character at that index. + + .. method:: tag_remove(tagName, index1, index2=None) + + Remove the tag *tagName* from the characters from *index1* up to but not + including *index2* (or from the single character at *index1* if *index2* + is omitted). + The tag itself continues to exist even if no characters carry it. + + .. method:: tag_delete(*tagNames) + + Delete each of the tags named in *tagNames*, removing them from all + characters and discarding their options and bindings. + + .. method:: tag_config(tagName, cnf=None, **kw) + :no-typesetting: + + .. method:: tag_configure(tagName, cnf=None, **kw) + + Query or modify the configuration options of the tag *tagName*. + This mirrors :meth:`~Misc.configure`, except that it applies to a tag + rather than to the widget as a whole: with no options it returns a + dictionary describing the current options, otherwise it sets the given + options. + Defining a tag this way also gives it a priority higher than any existing + tag. + + The supported tag options, all controlling the appearance of the tagged + text, are: + + *font* + The font to use for the text. + + *foreground* + The color to use for the text. + + *background* + The color to use for the area behind the text. + + *fgstipple*, *bgstipple* + Bitmaps used to stipple the foreground (text) and the background; + only well supported on X11. + + *borderwidth* + The width of the border drawn around the text according to *relief* + (default ``0``). + + *relief* + The 3-D appearance of the text's border: ``'flat'`` (the default), + ``'raised'``, ``'sunken'``, ``'ridge'``, ``'groove'`` or ``'solid'``. + + *offset* + How far the text is raised above (or, if negative, lowered below) the + baseline, for superscripts and subscripts. + + *underline* + Whether to underline the text. + + *underlinefg* + The color of the underline; it defaults to the text color. + + *overstrike* + Whether to draw a line through the middle of the text. + + *overstrikefg* + The color of the overstrike line; it defaults to the text color. + + *elide* + Whether the text is elided (hidden). + + *justify* + How to justify the first character of a display line: ``'left'`` (the + default), ``'right'`` or ``'center'``. + + *wrap* + How to wrap lines that are too long: ``'char'``, ``'word'`` or + ``'none'``. + + *lmargin1*, *lmargin2* + The indentation, in pixels, of the first display line of a logical + line and of the remaining display lines. + + *lmargincolor* + The color of the left margin area. + + *rmargin* + The right-hand margin, in pixels. + + *rmargincolor* + The color of the right margin area. + + *spacing1*, *spacing2*, *spacing3* + Extra space, in pixels, above the first display line of a logical + line, between its display lines, and below its last display line. + + *tabs* + The set of tab stops, in the same form as the widget's *tabs* option. + + *tabstyle* + How tab stops are interpreted: ``'tabular'`` or ``'wordprocessor'``. + + *selectbackground*, *selectforeground* + The background and foreground colors used for the text while it is + selected. + + .. note:: + + Tk 8.6 added the *lmargincolor*, *overstrikefg*, *rmargincolor*, + *selectbackground*, *selectforeground* and *underlinefg* options. + + :meth:`tag_config` is an alias of :meth:`!tag_configure`. + + .. method:: tag_cget(tagName, option) + + Return the current value of the configuration option *option* for the tag + *tagName*. + + .. method:: tag_names(index=None) + + If *index* is omitted, return a tuple of the names of all tags defined in + the widget; otherwise return only the names of the tags applied to the + character at *index*. + The names are ordered from lowest to highest priority. + + .. method:: tag_ranges(tagName) + + Return a tuple of indices describing all ranges of text tagged with + *tagName*. + The result alternates start and end indices, so that elements ``2*i`` and + ``2*i+1`` bound the *i*-th range. + + .. method:: tag_nextrange(tagName, index1, index2=None) + + Search forward from *index1* (up to *index2* if given) for the first + range of characters tagged with *tagName*, and return a two-element tuple + of its start and end indices, or an empty tuple if there is no such + range. + + .. method:: tag_prevrange(tagName, index1, index2=None) + + Search backward from *index1* (down to *index2* if given) for the nearest + preceding range of characters tagged with *tagName*, and return a + two-element tuple of its start and end indices, or an empty tuple if + there is no such range. + + .. method:: tag_raise(tagName, aboveThis=None) + + Raise the priority of tag *tagName* so that it is just above the priority + of *aboveThis*, or to the highest priority of all tags if *aboveThis* is + omitted. + When the display options of overlapping tags conflict, the + higher-priority tag wins. + + .. method:: tag_lower(tagName, belowThis=None) + + Lower the priority of tag *tagName* so that it is just below the priority + of *belowThis*, or to the lowest priority of all tags if *belowThis* is + omitted. + + .. method:: tag_bind(tagName, sequence, func, add=None) + + Bind the event *sequence* on characters tagged with *tagName* to the + callback *func*, so that *func* is invoked when that event occurs over + such a character. + If *add* is true the binding is added alongside any existing bindings for + *sequence*, otherwise it replaces them. + Works like :meth:`~Misc.bind` and returns the identifier of the new + binding. + + .. method:: tag_unbind(tagName, sequence, funcid=None) + + Remove the bindings of the event *sequence* on characters tagged with + *tagName*. + If *funcid* is given, only that binding (as returned by :meth:`tag_bind`) + is removed and its callback is unregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: image_create(index, cnf={}, **kw) + + Embed an image at *index* and return the name assigned to this image + instance, which may then be used as an index or passed to the other + ``image_*`` methods. + The options, given in *cnf* and *kw*, include *image* (the Tk image to + display), *name* (a base name for the instance), *align*, *padx* and + *pady*. + + .. method:: image_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded image at *index*. + + .. method:: image_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded image at + *index*, like :meth:`~Misc.configure` but applied to that image. + + .. method:: image_names() + + Return a tuple of the names of all images embedded in the widget. + + .. note:: + + This shadows the inherited :meth:`Misc.image_names`, + which returns the names of all images in the Tcl interpreter; + that method is not available on a :class:`Text`. + + .. method:: window_create(index, cnf={}, **kw) + + Embed a window (any widget) at *index*. + The options, given in *cnf* and *kw*, include *window* (the widget to + embed), *create* (a callback that creates the widget on demand), *align*, + *stretch*, *padx* and *pady*. + The embedded widget must be a descendant of the text widget's parent. + + .. method:: window_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded window at *index*. + + .. method:: window_config(index, cnf=None, **kw) + :no-typesetting: + + .. method:: window_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded window at + *index*, like :meth:`~Misc.configure` but applied to that window. + + :meth:`window_config` is an alias of :meth:`!window_configure`. + + .. method:: window_names() + + Return a tuple of the names of all windows embedded in the widget. + + .. method:: edit(*args) + + Low-level wrapper around the Tk ``edit`` widget command that controls the + undo/redo mechanism and the modified flag; *args* is the ``edit`` + subcommand and its arguments. + The :meth:`!edit_\*` methods below are thin wrappers around it and are + usually more convenient. + + .. method:: edit_modified(arg=None) + + If *arg* is omitted, return the current state of the modified flag as + ``0`` or ``1``; the flag is set automatically whenever the text is + inserted or deleted. + Otherwise set the flag to the boolean *arg*. + + .. method:: edit_canundo() + + Return ``True`` if there is an edit action on the undo stack that can be + undone, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_canredo() + + Return ``True`` if there is an edit action on the redo stack that can be + reapplied, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_undo() + + Undo the most recent edit action, that is, all the inserts and deletes + recorded on the undo stack since the previous separator, and move it to + the redo stack. + Raises :exc:`TclError` if the undo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_redo() + + Reapply the most recently undone edit action, provided no further edits + have been made since, and move it back to the undo stack. + Raises :exc:`TclError` if the redo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_reset() + + Clear the undo and redo stacks. + + .. method:: edit_separator() + + Push a separator onto the undo stack, marking a boundary between edit + actions for undo and redo. + Has no effect unless the *undo* option is true. + Separators are inserted automatically when the *autoseparators* option is + true. + + .. method:: search(pattern, index, stopindex=None, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, *, nolinestop=None, strictlimits=None) + + Search for *pattern* starting at *index* and return the index of the + first character of the first match, or an empty string if there is no + match. + Searching stops at *stopindex* if given; otherwise it wraps around the + ends of the text until the starting position is reached again. + The following boolean keyword flags control the search: *forwards* or + *backwards* select the direction (forward is the default); *exact* (the + default) or *regexp* select literal or regular-expression matching; + *nocase* makes the match case-insensitive; *elide* causes hidden text to + be searched as well; *nolinestop* (regexp only) lets ``.`` and ``[^`` + match newlines; and *strictlimits* requires the whole match to lie within + *index* and *stopindex*. + If *count* is a :class:`Variable`, the number of index positions in the + match is stored in it. + + .. versionchanged:: 3.15 + Added the *nolinestop* and *strictlimits* parameters. + + + .. method:: search_all(pattern, index, stopindex=None, *, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, nolinestop=None, overlap=None, strictlimits=None) + + Like :meth:`search`, but find every match in the searched range and + return a tuple of the starting indices of all matches (empty if there are + none). + By default overlapping matches are not reported; passing a true *overlap* + returns every match that is not wholly contained in another. + If *count* is a :class:`Variable`, it receives a list with one element + per match. + + .. versionadded:: 3.15 + + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the widget by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events, producing the effect of + dragging the text at high speed through the window. + + .. method:: debug(boolean=None) + + If *boolean* is omitted, return whether internal consistency checks of + the B-tree data structure are enabled. + Otherwise enable or disable them. + The setting is shared by all text widgets and may noticeably slow down + widgets holding large amounts of text. + + .. method:: dump(index1, index2=None, command=None, **kw) + + Return the contents of the widget from *index1* up to but not including + *index2* (or just the segment at *index1* if *index2* is omitted), + including text and information about marks, tags, images and windows. + The result is a list of ``(key, value, index)`` triples, where *key* is + one of ``'text'``, ``'mark'``, ``'tagon'``, ``'tagoff'``, ``'image'`` or + ``'window'``. + By default all kinds are reported; passing any of the keyword arguments + *all*, *text*, *mark*, *tag*, *image* or *window* as true restricts the + dump to the selected kinds. + If *command* is given, it is called once per triple with the three values + as arguments and nothing is returned. + + .. method:: peer_create(newPathName, cnf={}, **kw) + + Create a peer text widget with the path name *newPathName* that shares + this widget's underlying data (text, marks, tags, images and the undo + stack). + Changes made through any peer are reflected in all of them. + By default the peer covers the same lines as this widget; standard text + options, including *startline* and *endline*, may be given to override + this. + + .. versionadded:: 3.3 + + .. method:: peer_names() + + Return a tuple of the path names of this widget's peers, not including + the widget itself. + + .. versionadded:: 3.3 + + .. method:: sync(command=None) + + Control the synchronization of the displayed view with the underlying + text, which may lag behind when line heights have not yet been computed + (for example, for lines that have never been displayed). + If *command* is omitted, bring the line metrics up to date immediately by + forcing computation of any outdated line heights, and return once they + are current. + Otherwise schedule *command* to be called, with no arguments, exactly + once as soon as all line heights are up to date; if there are no pending + calculations, it is called immediately. + + .. versionadded:: next + + .. method:: pendingsync() + + Return ``True`` if the line height calculations are not up to date, and + ``False`` otherwise. + The ``<>`` virtual event fires whenever this state + changes, with the *detail* field set to the new value. + + .. versionadded:: next + + .. method:: yview_pickplace(*what) + + Adjust the view so that the location given by *what* is visible. + This is an obsolete equivalent of :meth:`see`, which should be used + instead. + + +Variable classes +^^^^^^^^^^^^^^^^ + +.. class:: Variable(master=None, value=None, name=None) + + The base class for the Tk variable wrappers. + A Tk variable is a value stored in the Tcl interpreter that can be linked to + widgets through their *variable* or *textvariable* options (see + :ref:`coupling-widget-variables`), so that changes propagate both ways: + updating the variable updates every widget bound to it, and a user editing + such a widget updates the variable. + + *master* is the widget whose Tcl interpreter owns the variable; if omitted, + the default root window is used. + *value* is the initial value; if omitted, a type-specific default is used. + *name* is the name of the variable in the Tcl interpreter; if omitted, a + unique name of the form ``'PY_VARnum'`` is generated. + If *name* matches an existing variable and *value* is omitted, the existing + value is retained. + + In most cases you should use one of the typed subclasses below -- + :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` or + :class:`BooleanVar` -- rather than :class:`!Variable` directly. + + .. note:: + + When a :class:`!Variable` is garbage collected, its Tcl variable is unset. + Keep a reference to it for as long as a widget is linked to it, for example + by storing it as an attribute rather than in a local variable. + Otherwise Tk recreates the Tcl variable to keep the widget working, but it + is never unset again, leaking one Tcl variable per dropped wrapper. + + .. versionchanged:: 3.10 + Two variables now compare equal (``==``) only when they have the same + name, are of the same class, and belong to the same Tcl interpreter. + + .. method:: get() + + Return the current value of the variable. + For the base class the value is returned as a string; the typed + subclasses convert it to the appropriate Python type. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + .. method:: trace_add(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'array'``, ``'read'``, ``'write'`` or + ``'unset'``, or a list or tuple of such strings. + + When triggered, *callback* is called with three arguments: the name of + the Tcl variable, an index (or an empty string if the variable is not an + element of an array), and the *mode* that triggered the call. + + Return the internal name of the registered callback, which can be passed + to :meth:`trace_remove`. + + .. versionadded:: 3.6 + + .. method:: trace_remove(mode, cbname) + + Remove a trace callback from the variable. + *mode* must match the *mode* that was passed to :meth:`trace_add`, and + *cbname* is the callback name returned by :meth:`trace_add`. + + .. versionadded:: 3.6 + + .. method:: trace_info() + + Return a list of ``(modes, cbname)`` pairs describing all traces + currently set on the variable, where *modes* is a tuple of mode strings + and *cbname* is the internal callback name. + + .. versionadded:: 3.6 + + .. method:: trace(mode, callback) + :no-typesetting: + + .. method:: trace_variable(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'r'``, ``'w'`` or ``'u'``, for read, write + or unset. + Return the internal name of the registered callback. + :meth:`trace` is an alias of :meth:`!trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_add` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + .. method:: trace_vdelete(mode, cbname) + + Remove the trace callback named *cbname* registered for *mode* with + :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_remove` instead. This method wraps a Tcl feature + that was removed in Tcl 9.0. + + .. method:: trace_vinfo() + + Return a list of ``(mode, cbname)`` pairs for all traces set on the + variable with :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_info` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + +.. class:: StringVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a string. + The default value is ``''``. + + .. method:: get() + + Return the value of the variable as a :class:`str`. + + +.. class:: IntVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds an integer. + The default value is ``0``. + + .. method:: get() + + Return the value of the variable as an :class:`int`. + + +.. class:: DoubleVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a float. + The default value is ``0.0``. + + .. method:: get() + + Return the value of the variable as a :class:`float`. + + .. _tkinter-numeric-locale: + + .. note:: + + A floating-point value is always parsed with a period (``.``) as the + decimal separator, but :class:`Spinbox`, :class:`Scale` and + :class:`ttk.Spinbox ` format it according to the + ``LC_NUMERIC`` locale. Under a locale that uses a comma they produce a + value that :meth:`get` cannot read, raising :exc:`TclError`. Set + ``LC_NUMERIC`` to a locale that uses a period (such as ``'C'``) to avoid + this. + + +.. class:: BooleanVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a boolean. + The default value is ``False``. + + .. method:: get() + + Return the value of the variable as a :class:`bool`. + Raise a :exc:`ValueError` if the value cannot be interpreted as a + boolean. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*, converting it to a boolean. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + +Image classes +^^^^^^^^^^^^^ + +.. class:: Image(imgtype, name=None, cnf={}, master=None, **kw) + + Base class for Tk images. + *imgtype* is the Tk image type, one of ``'photo'`` or ``'bitmap'``. + An image is a named object that can be displayed by widgets through their + *image* option; deleting all references to the :class:`!Image` object + deletes the underlying Tk image. + Usually you create a :class:`PhotoImage` or :class:`BitmapImage` rather than + an :class:`!Image` directly. + + The image's configuration options are given by *cnf* and *kw* and may be + queried and changed later with the mapping protocol (using ``image[key]``) + or with the :meth:`configure` method. + + .. method:: config(**kw) + :no-typesetting: + + .. method:: configure(**kw) + + Modify one or more configuration options of the image. + The valid options depend on the image type; see :class:`PhotoImage` and + :class:`BitmapImage`. + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: height() + + Return the height of the image, in pixels. + + .. method:: width() + + Return the width of the image, in pixels. + + .. method:: type() + + Return the type of the image, that is the value of *imgtype* with which + it was created (for example ``'photo'`` or ``'bitmap'``). + + +.. class:: PhotoImage(name=None, cnf={}, master=None, **kw) + + A full-color image (the Tk ``photo`` image type), stored internally with a + varying degree of transparency per pixel. + It can read and write GIF, PPM/PGM and (in Tk 8.6 and later) PNG files, read + SVG files (in Tk 9.0 and later), and be drawn in widgets. + Inherits from :class:`Image`. + + The configuration options include *data* (the image contents as a string), + *file* (the name of a file to read the contents from), *format* (the name of + the file format handler), *width* and *height* (the size of the image, used + when building it up piece by piece), *gamma* and *palette*. + + .. method:: blank() + + Blank the image; that is, set the entire image to have no data, so that + it is displayed as transparent and the background of whatever window it + is displayed in shows through. + + .. method:: redither() + + Recalculate the dithered image in each window where it is displayed. + This is useful when the image data was supplied in pieces, in which case + the dithered image may not be exactly correct. + + .. versionadded:: next + + .. method:: cget(option) + + Return the current value of the configuration option *option*. + + .. method:: copy(*, from_coords=None, zoom=None, subsample=None) + + Return a new :class:`PhotoImage` with a copy of this image. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + ``(x1, y1)`` and ``(x2, y2)`` specify diagonally opposite corners of the + rectangle. + If *x2* and *y2* are not specified, they default to the bottom-right + corner of the source image. + The pixels copied include the left and top edges of the rectangle but not + the bottom or right edges. + If *from_coords* is not given, the whole source image is copied. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + .. versionchanged:: 3.13 + Added the *from_coords*, *zoom* and *subsample* parameters. + + + .. method:: copy_replace(sourceImage, *, from_coords=None, to=None, \ + shrink=False, zoom=None, subsample=None, \ + compositingrule=None) + + Copy a region from *sourceImage* (which must be a :class:`PhotoImage`) + into this image, possibly with pixel zooming and/or subsampling. + If no options are specified, the whole of *sourceImage* is copied into + this image, starting at coordinates ``(0, 0)``. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + *to* specifies a rectangular sub-region of the destination image to be + affected. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If *x2* and *y2* are not specified, they default to ``(x1, y1)`` plus the + size of the source region (after subsampling and zooming, if specified). + If *x2* and *y2* are specified, the source region is replicated if + necessary to fill the destination region in a tiled fashion. + + If *shrink* is true, the size of the destination image is reduced, if + necessary, so that the region being copied into is at the bottom-right + corner of the image. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + *compositingrule* specifies how transparent pixels in the source image + are combined with the destination image. + With ``'overlay'`` (the default), the old contents of the destination + image remain visible, as if the source image were printed on a piece of + transparent film and placed over the top of the destination. + With ``'set'``, the old contents of the destination image are discarded + and the source image is used as-is. + + .. versionadded:: 3.13 + + + .. method:: data(format=None, *, from_coords=None, background=None, \ + grayscale=False, metadata=None) + + Return the image data. + + *format* specifies the name of the image file format handler to use. + If it is not given, the data is returned as a tuple (one element per row) + of strings containing space-separated (one element per pixel/column) + colors in ``#RRGGBB`` format. + + *from_coords* specifies a rectangular region of the image to be returned. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region, including ``(x1, y1)`` and excluding ``(x2, y2)``. + If *from_coords* is not given, the whole image is returned. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: get(x, y, *, withalpha=False) + + Return the color of the pixel at coordinates (*x*, *y*) as an + ``(r, g, b)`` tuple of three integers between 0 and 255, representing the + red, green and blue components respectively. + If *withalpha* is true, the returned tuple has a fourth element giving + the alpha (opacity) value of the pixel. + + .. versionchanged:: next + Added the *withalpha* parameter, which requires Tcl/Tk 9.0 or newer. + + .. method:: put(data, to=None, *, format=None, metadata=None) + + Set pixels of the image to the colors given in *data*, which must be a + string or a nested sequence of horizontal rows of pixel colors (for + example ``"{red green} {blue yellow}"``). + + *to* specifies the coordinates of the region of the image into which the + data are copied. + It must be a tuple or a list of 2 or 4 integers ``(x1, y1)`` or + ``(x1, y1, x2, y2)`` giving the top-left corner, and optionally the + bottom-right corner, of the region. + The default position is ``(0, 0)``. + + *format* specifies the format of the image *data*, so that only image + file format handlers whose names begin with it are tried. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: next + Added the *format* and *metadata* parameters. + + .. method:: read(filename, format=None, *, from_coords=None, to=None, \ + shrink=False, metadata=None) + + Read image data from the file named *filename* into the image. + + *format* specifies the format of the image data in the file. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + *from_coords* specifies a rectangular sub-region of the image file data + to be copied to the destination image. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image in the file. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole of the image in the file is + read. + + *to* specifies the coordinates of the top-left corner of the region of + the image into which the data are read. + The default is ``(0, 0)``. + + If *shrink* is true, the size of the image is reduced, if necessary, so + that the region into which the file data are read is at the bottom-right + corner of the image. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: subsample(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` based on this image but using only every + *x*-th pixel in the X direction and every *y*-th pixel in the Y + direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + .. method:: transparency_get(x, y) + + Return ``True`` if the pixel at coordinates (*x*, *y*) is fully + transparent, ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: transparency_set(x, y, boolean) + + Make the pixel at coordinates (*x*, *y*) fully transparent if *boolean* + is true, fully opaque otherwise. + + .. versionadded:: 3.8 + + + .. method:: write(filename, format=None, from_coords=None, *, \ + background=None, grayscale=False, metadata=None) + + Write image data from the image to the file named *filename*. + + *format* specifies the name of the image file format handler to use. + If it is not given, the format is guessed from the file extension. + + *from_coords* specifies a rectangular region of the image to be written. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole image is written. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: 3.13 + Added the *background* and *grayscale* parameters. + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: zoom(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` with this image magnified by a factor of + *x* in the X direction and *y* in the Y direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + +.. class:: BitmapImage(name=None, cnf={}, master=None, **kw) + + A two-color image (the Tk ``bitmap`` image type) created from an X11 bitmap. + Each pixel displays a foreground color, a background color, or nothing + (producing a transparent effect). + Inherits from :class:`Image`. + + The configuration options are *data* or *file* (the source bitmap, given as + a string in X11 bitmap format or as the name of a file in that format), + *maskdata* or *maskfile* (the mask bitmap, in the same forms), and + *foreground* and *background* (the two colors). + For pixels where the mask is zero the image displays nothing; for other + pixels it displays the foreground color where the source is one and the + background color where the source is zero. + If *background* is set to an empty string, the background pixels are + transparent. + + :class:`!BitmapImage` has no methods of its own beyond those inherited from + :class:`Image`. + + +Other classes +^^^^^^^^^^^^^ + +.. class:: Event() + + A container for the attributes of an event passed to a callback bound with + :meth:`Misc.bind`. + An :class:`!Event` instance has the following attributes, each corresponding + to a field of the underlying Tk event; depending on the event type, some + attributes may be set to the string ``'??'`` to indicate that they are not + meaningful. + See :ref:`bindings-and-events`. + + .. attribute:: serial + + The serial number of the event. + + .. attribute:: num + + The mouse button that was pressed or released (for button events). + + .. attribute:: focus + + Whether the window has the focus (for ``Enter`` and ``Leave`` events). + + .. attribute:: height + width + + The new height and width of the window (for ``Configure`` and ``Expose`` + events). + + .. attribute:: keycode + + The keycode of the key that was pressed or released. + + .. attribute:: state + + The state of the event, as a number (for most events) or a string (for + ``Visibility`` events). + + .. attribute:: time + + The timestamp of the event, in milliseconds. + + .. attribute:: x + y + + The pointer position relative to the widget, in pixels. + + .. attribute:: x_root + y_root + + The pointer position relative to the top-left corner of the screen, in + pixels. + + .. attribute:: char + + The character typed, as a string (for key events). + + .. attribute:: send_event + + ``True`` if the event was sent by another application. + + .. attribute:: keysym + + The symbolic name of the key that was pressed or released. + + .. attribute:: keysym_num + + The numeric value of :attr:`keysym`. + + .. attribute:: type + + The :class:`EventType` of the event. + + .. attribute:: widget + + The widget on which the event occurred. + + .. attribute:: delta + + The amount the mouse wheel was rotated (for ``MouseWheel`` events). + + .. attribute:: user_data + + The data string of a virtual event, as passed to the *data* option of + :meth:`Misc.event_generate`. + It is ``'??'`` for non-virtual events. + + .. versionadded:: 3.15 + + .. attribute:: detail + + A fixed detail string for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut`` + and ``ConfigureRequest`` events (see the Tcl/Tk documentation). + It is ``'??'`` for other events. + + .. versionadded:: 3.15 + + +.. class:: EventType(*values) + + An :class:`enum.StrEnum` enumerating the Tk event types, used as the value + of :attr:`Event.type`. + Its members include, among others, ``KeyPress``, ``KeyRelease``, + ``ButtonPress``, ``ButtonRelease``, ``Motion``, ``Enter``, ``Leave``, + ``FocusIn``, ``FocusOut``, ``Configure``, ``Map``, ``Unmap``, ``Expose``, + ``Destroy`` and ``MouseWheel``. + + .. versionadded:: 3.6 + + + +.. class:: CallWrapper(func, subst, widget) + + Internal helper that wraps a Python callback so that it can be invoked from + Tcl. + *func* is the Python function, *subst* is an optional function that + pre-processes the Tcl arguments, and *widget* is the widget used for error + reporting. + Instances are created automatically by :meth:`Misc.register`; this class is + not normally used directly. + + +Module-level functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) + + The :func:`Tcl` function is a factory function which creates an object much + like that created by the :class:`Tk` class, except that it does not + initialize the Tk subsystem. + This is most often useful when driving the Tcl interpreter in an environment + where one doesn't want to create extraneous toplevel windows, or where one + cannot (such as Unix/Linux systems without an X server). + An object created by the :func:`Tcl` object can have a Toplevel window + created (and the Tk subsystem initialized) by calling its :meth:`~Tk.loadtk` + method. + +.. function:: NoDefaultRoot() + + Inhibit the creation of an implicit default root window. + Afterwards :mod:`!tkinter` no longer creates a shared default root + automatically, and operations that rely on one --- such as constructing a + widget without an explicit *master* --- raise a :exc:`RuntimeError`. + Call this early in larger applications to make the root window explicit. + +.. function:: mainloop(n=0) + + Run the Tk main event loop on the default root window until all windows are + destroyed. + Equivalent to calling :meth:`Misc.mainloop` on the default root. + +.. function:: getboolean(s) + + Convert the Tcl boolean string *s* (one of ``'1'``, ``'true'``, ``'yes'``, + ``'on'`` and similar, or their false counterparts) to a Python + :class:`bool`. + Raise :exc:`TclError` for an invalid value. + +.. function:: getdouble(s) + + Convert *s* to a floating-point number. + This is the built-in :class:`float`. + +.. function:: getint(s) + + Convert *s* to an integer. + This is the built-in :class:`int`. + +.. function:: image_names() + + Return the names of all existing images in the default root's interpreter. + +.. function:: image_types() + + Return the available image types (such as ``'photo'`` and ``'bitmap'``) in + the default root's interpreter. + + +.. _tkinter-file-handlers: + +File handlers +^^^^^^^^^^^^^ + +Tk allows you to register and unregister a callback function which will be +called from the Tk mainloop when I/O is possible on a file descriptor. +Only one handler may be registered per file descriptor. Example code:: + + import tkinter + widget = tkinter.Tk() + mask = tkinter.READABLE | tkinter.WRITABLE + widget.tk.createfilehandler(file, mask, callback) + ... + widget.tk.deletefilehandler(file) + +This feature is not available on Windows. + +Since you don't know how many bytes are available for reading, you may not +want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` +:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, +since these will insist on reading a predefined number of bytes. +For sockets, the :meth:`~socket.socket.recv` or +:meth:`~socket.socket.recvfrom` methods will work fine; for other files, +use raw reads or ``os.read(file.fileno(), maxbytecount)``. + + +.. method:: Widget.tk.createfilehandler(file, mask, func) + + Registers the file handler callback function *func*. The *file* argument + may either be an object with a :meth:`~io.IOBase.fileno` method (such as + a file or socket object), or an integer file descriptor. The *mask* + argument is an ORed combination of any of the three constants below. + The callback is called as follows:: + + callback(file, mask) + + +.. method:: Widget.tk.deletefilehandler(file) + + Unregisters a file handler. + + +.. data:: READABLE + WRITABLE + EXCEPTION + + Constants used in the *mask* arguments. + + +Constants +^^^^^^^^^ + +The following symbolic constants are available in both the :mod:`!tkinter` +and :mod:`!tkinter.constants` namespaces. + +.. data:: TRUE + YES + ON + + Truthy values, all equal to the integer ``1``. + +.. data:: FALSE + NO + OFF + + Falsy values, all equal to the integer ``0``. + +.. data:: N + S + E + W + NE + NW + SE + SW + NS + EW + NSEW + CENTER + + Compass directions (``'n'``, ``'s'``, ``'e'``, ``'w'`` and the diagonals and + edges) plus ``CENTER`` (``'center'``), used as values for the *anchor* and + *sticky* options and by methods such as :meth:`Misc.grid_anchor`. + +.. data:: LEFT + RIGHT + TOP + BOTTOM + + Sides for the *side* option of the packer (see :meth:`Pack.pack_configure`). + +.. data:: X + Y + BOTH + NONE + + Values for the *fill* option of the packer: ``'x'``, ``'y'``, ``'both'`` or + ``'none'``. + +.. data:: RAISED + SUNKEN + FLAT + RIDGE + GROOVE + SOLID + + Values for the *relief* option, which controls a widget's 3-D border. + +.. data:: HORIZONTAL + VERTICAL + + Values for the *orient* option of widgets such as :class:`Scale`, + :class:`Scrollbar` and :class:`PanedWindow`. + +.. data:: CHAR + WORD + + Values for the *wrap* option of the :class:`Text` widget, selecting line + wrapping on character or word boundaries. + +.. data:: BASELINE + + The text-alignment value ``'baseline'``. + +.. data:: INSIDE + OUTSIDE + + Values for the *bordermode* option of the placer (see + :meth:`Place.place_configure`). + +.. data:: INSERT + CURRENT + END + ANCHOR + SEL + SEL_FIRST + SEL_LAST + + Symbolic indices used by the :class:`Text`, :class:`Entry`, :class:`Listbox` + and :class:`Canvas` widgets, such as ``'insert'`` (the insertion cursor), + ``'current'``, ``'end'``, ``'anchor'`` and the bounds of the selection + (``'sel.first'`` and ``'sel.last'``). + +.. data:: ALL + + The special tag ``'all'``, which matches every item of a :class:`Canvas` or + every character of a :class:`Text` (for example ``canvas.delete(ALL)``). + +.. data:: NORMAL + DISABLED + ACTIVE + HIDDEN + + Values for the *state* option of various widgets and items. + +.. data:: CASCADE + CHECKBUTTON + COMMAND + RADIOBUTTON + SEPARATOR + + Menu entry types, used as the *itemType* argument of :meth:`Menu.add` and + :meth:`Menu.insert`. + +.. data:: SINGLE + BROWSE + MULTIPLE + EXTENDED + + Values for the *selectmode* option of the :class:`Listbox` widget. + +.. data:: PIESLICE + CHORD + ARC + + Values for the *style* option of :class:`Canvas` arc items. + +.. data:: BUTT + PROJECTING + ROUND + BEVEL + MITER + + Values for the *capstyle* (``'butt'``, ``'projecting'``, ``'round'``) and + *joinstyle* (``'round'``, ``'bevel'``, ``'miter'``) options of + :class:`Canvas` line items. + +.. data:: FIRST + LAST + + Values for the *arrow* option of :class:`Canvas` line items, indicating + which ends have arrowheads. + +.. data:: MOVETO + SCROLL + + The first argument passed by a :class:`Scrollbar` to the :meth:`XView.xview` + or :meth:`YView.yview` method of the scrolled widget. + +.. data:: UNITS + PAGES + + Values for the *what* argument of :meth:`XView.xview_scroll` and + :meth:`YView.yview_scroll`. + +.. data:: UNDERLINE + NUMERIC + DOTBOX + + Other option values: ``'underline'``, ``'numeric'`` and ``'dotbox'``. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 763e24929d74b51..30aef8748edb72b 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,25 +1,24 @@ -:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +:mod:`!tkinter.scrolledtext` --- Scrolled text widget ===================================================== .. module:: tkinter.scrolledtext - :platform: Tk :synopsis: Text widget with a vertical scroll bar. -.. sectionauthor:: Fred L. Drake, Jr. - **Source code:** :source:`Lib/tkinter/scrolledtext.py` -------------- -The :mod:`tkinter.scrolledtext` module provides a class of the same name which +The :mod:`!tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. +The text widget and scrollbar are packed together in a :class:`~tkinter.Frame`, +and the methods of the :class:`~tkinter.Pack`, :class:`~tkinter.Grid` and +:class:`~tkinter.Place` geometry managers are acquired from the +:class:`~tkinter.Frame` object. +This allows the :class:`ScrolledText` widget to be used directly to achieve +most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 628e9f945ac3658..13b374e23558fc3 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -4,23 +4,23 @@ .. module:: tkinter.ttk :synopsis: Tk themed widget set -.. sectionauthor:: Guilherme Polo - **Source code:** :source:`Lib/tkinter/ttk.py` .. index:: single: ttk -------------- -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, +The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, introduced in Tk 8.5. It provides additional benefits including anti-aliased font rendering under X11 and window transparency (requiring a composition window manager on X11). -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, +The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. +.. versionadded:: 3.1 + .. seealso:: @@ -40,18 +40,18 @@ To override the basic Tk widgets, the import should follow the Tk import:: from tkinter import * from tkinter.ttk import * -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, +That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to automatically replace the Tk widgets. -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. +This has the direct benefit of using the new widgets which gives a better look +and feel across platforms; however, the replacement widgets are not completely +compatible. +The main difference is that widget options such as "fg", "bg" and others +related to widget styling are no longer present in Ttk widgets. +Instead, use the :class:`ttk.Style {%- endblock -%} {% block body %}

{{ docstitle|e }}

@@ -21,63 +50,76 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Documentation sections:{% endtrans %}

- - -
- - - - - - - - - - - - - -
+
+ + +
+ +

{% trans %}Other resources:{% endtrans %}

+
+ + +

{% trans %}Indices, glossary, and search:{% endtrans %}

- - -
- - - - - - -
+
+ + +

{% trans %}Project information:{% endtrans %}

- - -
- - - - - - - -
+ {% endblock %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 5020c428741febe..2539edb51ba3a09 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,7 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default since Python 3.13. +Since Python 3.13, a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9ab003d5cd3dd5d..4f7da5253f78bc7 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -420,7 +420,7 @@ of the class:: 'Buddy' As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. +effects involving :term:`mutable` objects such as lists and dictionaries. For example, the *tricks* list in the following code should not be used as a class variable because just a single list would be shared by all *Dog* instances:: @@ -929,6 +929,25 @@ Examples:: >>> list(data[i] for i in range(len(data)-1, -1, -1)) ['f', 'l', 'o', 'g'] + >>> x = [[1,2,3], [], [4, 5]] + >>> g = (*i for i in x) + >>> list(g) + [1, 2, 3, 4, 5] + +In most cases, generator expressions must be wrapped in parentheses. As a +special case, however, when provided as the sole argument to a function (as in +the examples involving ``sum``, ``set``, ``max``, and ``list`` above), the +generator expression does not need to be wrapped in an additional set of +parentheses. That is to say, the following two pieces of code are semantically +equivalent:: + + >>> f(x for x in y) + >>> f((x for x in y)) + +as are the following:: + + >>> f(*x for x in y) + >>> f((*x for x in y)) .. rubric:: Footnotes diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 5ec8789f98c7016..8bac8df4368c00a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -155,8 +155,8 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. +arguments. In chapter :ref:`tut-structures`, we will discuss :func:`list` in more +detail. .. _tut-break: @@ -441,7 +441,7 @@ Several other key features of this statement: ``False`` and ``None`` are compared by identity. - Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: + to prevent them from being interpreted as capture variables:: from enum import Enum class Color(Enum): @@ -1039,31 +1039,28 @@ blank, visually separating the summary from the rest of the description. The following lines should be one or more paragraphs describing the object's calling conventions, its side effects, etc. -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). +The Python parser strips indentation from multi-line string literals when they +serve as module, class, or function docstrings. Here is an example of a multi-line docstring:: >>> def my_function(): ... """Do nothing, but document it. ... - ... No, really, it doesn't do anything. + ... No, really, it doesn't do anything: + ... + ... >>> my_function() + ... >>> ... """ ... pass ... >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything: + + >>> my_function() + >>> .. _tut-annotations: @@ -1071,7 +1068,6 @@ Here is an example of a multi-line docstring:: Function Annotations -------------------- -.. sectionauthor:: Zachary Ware .. index:: pair: function; annotations single: ->; function annotations @@ -1105,12 +1101,11 @@ value annotated:: Intermezzo: Coding Style ======================== -.. sectionauthor:: Georg Brandl .. index:: pair: coding; style Now that you are about to write longer, more complex pieces of Python, it is a good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. +concisely, *formatted*) in different styles; some are more readable than others. Making it easy for others to read your code is always a good idea, and adopting a nice coding style helps tremendously for that. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 7e02e74177c4570..276e31a3056f0ee 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -15,20 +15,20 @@ More on Lists The :ref:`list ` data type has some more methods. Here are all of the methods of list objects: -.. method:: list.append(x) +.. method:: list.append(value, /) :noindex: Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. -.. method:: list.extend(iterable) +.. method:: list.extend(iterable, /) :noindex: Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. -.. method:: list.insert(i, x) +.. method:: list.insert(index, value, /) :noindex: Insert an item at a given position. The first argument is the index of the @@ -36,14 +36,14 @@ of the methods of list objects: the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. -.. method:: list.remove(x) +.. method:: list.remove(value, /) :noindex: - Remove the first item from the list whose value is equal to *x*. It raises a + Remove the first item from the list whose value is equal to *value*. It raises a :exc:`ValueError` if there is no such item. -.. method:: list.pop([i]) +.. method:: list.pop(index=-1, /) :noindex: Remove the item at the given position in the list, and return it. If no index @@ -58,10 +58,10 @@ of the methods of list objects: Remove all items from the list. Similar to ``del a[:]``. -.. method:: list.index(x[, start[, end]]) +.. method:: list.index(value[, start[, stop]]) :noindex: - Return zero-based index of the first occurrence of *x* in the list. + Return zero-based index of the first occurrence of *value* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -70,10 +70,10 @@ of the methods of list objects: sequence rather than the *start* argument. -.. method:: list.count(x) +.. method:: list.count(value, /) :noindex: - Return the number of times *x* appears in the list. + Return the number of times *value* appears in the list. .. method:: list.sort(*, key=None, reverse=False) @@ -136,9 +136,6 @@ comparison. Using Lists as Stacks --------------------- -.. sectionauthor:: Ka-Ping Yee - - The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the @@ -166,8 +163,6 @@ top of the stack, use :meth:`~list.pop` without an explicit index. For example: Using Lists as Queues --------------------- -.. sectionauthor:: Ka-Ping Yee - It is also possible to use a list as a queue, where the first element added is the first element retrieved ("first-in, first-out"); however, lists are not efficient for this purpose. While appends and pops from the end of list are @@ -333,6 +328,47 @@ The :func:`zip` function would do a great job for this use case:: See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. +Unpacking in Lists and List Comprehensions +------------------------------------------ + +The section on :ref:`tut-unpacking-arguments` describes the use of ``*`` to +"unpack" the elements of an iterable object, providing each one separately as +an argument to a function. Unpacking can also be used in other contexts, for +example, when creating lists. When specifying elements of a list, prefixing an +expression by a ``*`` will unpack the result of that expression, adding each of +its elements to the list we're creating:: + + >>> x = [1, 2, 3] + >>> [0, *x, 4, 5, 6] + [0, 1, 2, 3, 4, 5, 6] + +This only works if the expression following the ``*`` evaluates to an iterable +object; trying to unpack a non-iterable object will raise an exception:: + + >>> x = 1 + >>> [0, *x, 2, 3, 4] + Traceback (most recent call last): + File "", line 1, in + [0, *x, 2, 3, 4] + TypeError: Value after * must be an iterable, not int + +Unpacking can also be used in list comprehensions, as a way to build a new list +representing the concatenation of an arbitrary number of iterables:: + + >>> x = [[1, 2, 3], [4, 5, 6], [], [7], [8, 9]] + >>> [*element for element in x] + [1, 2, 3, 4, 5, 6, 7, 8, 9] + +Note that the effect is that each element from ``x`` is unpacked. This works +for arbitrary iterable objects, not just lists:: + + >>> x = [[1, 2, 3], 'cat', {'spam': 'eggs'}] + >>> [*element for element in x] + [1, 2, 3, 'c', 'a', 't', 'spam'] + +But if the objects in ``x`` are not iterable, this expression would again raise +an exception. + .. _tut-del: The :keyword:`!del` statement @@ -394,7 +430,10 @@ A tuple consists of a number of values separated by commas, for instance:: >>> v = ([1, 2, 3], [3, 2, 1]) >>> v ([1, 2, 3], [3, 2, 1]) - + >>> # they support unpacking just like lists: + >>> x = [1, 2, 3] + >>> 0, *x, 4 + (0, 1, 2, 3, 4) As you see, on output tuples are always enclosed in parentheses, so that nested tuples are interpreted correctly; they may be input with or without surrounding @@ -454,6 +493,9 @@ Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an empty dictionary, a data structure that we discuss in the next section. +Because sets are unordered, iterating over them or printing them can +produce the elements in a different order than you expect. + Here is a brief demonstration:: >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} @@ -480,12 +522,16 @@ Here is a brief demonstration:: {'r', 'd', 'b', 'm', 'z', 'l'} Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: +are also supported, including comprehensions with unpacking:: >>> a = {x for x in 'abracadabra' if x not in 'abc'} >>> a {'r', 'd'} + >>> fruits = [{'apple', 'avocado', 'apricot'}, {'banana', 'blueberry'}] + >>> {*fruit for fruit in fruits} + {'blueberry', 'banana', 'avocado', 'apple', 'apricot'} + .. _tut-dictionaries: @@ -563,6 +609,18 @@ arbitrary key and value expressions:: >>> {x: x**2 for x in (2, 4, 6)} {2: 4, 4: 16, 6: 36} +And dictionary unpacking (via ``**``) can be used to merge multiple +dictionaries:: + + >>> odds = {i: i**2 for i in (1, 3, 5)} + >>> evens = {i: i**2 for i in (2, 4, 6)} + >>> {**odds, **evens} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36} + + >>> all_values = [odds, evens, {0: 0}] + >>> {**i for i in all_values} + {1: 1, 3: 9, 5: 25, 2: 4, 4: 16, 6: 36, 0: 0} + When the keys are simple strings, it is sometimes easier to specify pairs using keyword arguments:: diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 1c20fa2f0b6ae58..3c6edf2c4793abd 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -121,9 +121,9 @@ A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. Handlers only handle exceptions that occur in the corresponding *try clause*, not in other handlers of the same :keyword:`!try` statement. An *except clause* -may name multiple exceptions as a parenthesized tuple, for example:: +may name multiple exceptions, for example:: - ... except (RuntimeError, TypeError, NameError): + ... except RuntimeError, TypeError, NameError: ... pass A class in an :keyword:`except` clause matches exceptions which are instances of the @@ -549,9 +549,9 @@ caught like any other exception. :: >>> try: ... f() ... except Exception as e: - ... print(f'caught {type(e)}: e') + ... print(f'caught {type(e)}: {e}') ... - caught : e + caught : there were problems (2 sub-exceptions) >>> By using ``except*`` instead of ``except``, we can selectively diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index dfe2d1d3a8378fa..37e23ba1cd0c812 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -9,10 +9,6 @@ Floating-Point Arithmetic: Issues and Limitations ************************************************** -.. sectionauthor:: Tim Peters -.. sectionauthor:: Raymond Hettinger - - Floating-point numbers are represented in computer hardware as base 2 (binary) fractions. For example, the **decimal** fraction ``0.625`` has value 6/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.101`` diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index cd52607142485e6..45a2ca21801437f 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.15 + python3.16 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -34,13 +34,13 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. +substitution and code completion on most systems. Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use +typing a word in on the Python prompt, then pressing Left arrow (or :kbd:`Control-b`). +If the cursor moves, you have command line editing; see Appendix +:ref:`tut-interacting` for an introduction to the keys. +If nothing appears to happen, or if a sequence like ``^[[D`` or ``^B`` appears, +command line editing isn't available; you'll only be able to use backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.15 - Python 3.15 (default, May 7 2025, 15:46:04) + $ python3.16 + Python 3.16 (default, May 7 2026, 19:03:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index fb491149793cf79..7778e37a9adaa95 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -49,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. @@ -184,11 +184,11 @@ If you don't want characters prefaced by ``\`` to be interpreted as special characters, you can use *raw strings* by adding an ``r`` before the first quote:: - >>> print('C:\some\name') # here \n means newline! - C:\some + >>> print('C:\this\name') # here \t means tab, \n means newline + C: his ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + >>> print(r'C:\this\name') # note the r before the quote + C:\this\name There is one subtle aspect to raw strings: a raw string may not end in an odd number of ``\`` characters; see diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 342c1a00193959f..dec2008add1bf50 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python315' + 'C:\\Python316' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 678b71c9274c1c6..cf3a44850b0b8f2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -279,14 +278,14 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python315/lib/weakref.py", line 46, in __getitem__ + File "C:/python316/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f362e1943b666f7..6b6b8a768e3f58d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +Note that the activated virtual environment does not alter the ``PYTHONPATH`` variable in any way. +This may lead to unexpected results if the path includes references to code which is incompatible with +the Python version the virtual environment is using. The best practice is to ``unset PYTHONPATH`` +in bash or the equivalent for the shell you are using. + To deactivate a virtual environment, type:: deactivate diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index 359cf80a7b2ecf8..aae8f29b0077627 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -68,6 +68,6 @@ already contain the solution for your problem. .. rubric:: Footnotes -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, +.. [#] "Cheese Shop" is a Monty Python sketch: a customer enters a cheese shop, but whatever cheese he asks for, the clerk says it's missing. diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 45345d045ddfd99..60a135693181412 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index aff165191b76e8a..677fbbae3f4219a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -49,9 +49,9 @@ additional methods of invocation: appropriately named script from that directory. * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by - newlines. Leading whitespace is significant in Python statements! -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. + newlines. +* When called with ``-m module-name``, the given module is located using the standard + import mechanism and executed as a script. In non-interactive mode, the entire input is parsed before it is executed. @@ -78,8 +78,8 @@ source. .. option:: -m - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. + Locate the module using the standard import mechanism and execute its contents + as the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension (``.py``). The module name should be a valid absolute Python module name, but @@ -390,7 +390,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. + https://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. @@ -654,13 +654,17 @@ Miscellaneous options .. versionadded:: 3.13 - * :samp:`-X presite={package.module}` specifies a module that should be - imported before the :mod:`site` module is executed and before the + * :samp:`-X presite={module}` or :samp:`-X presite={module:func}` specifies + an entry point that should be executed before the :mod:`site` module is + executed and before the :mod:`__main__` module exists. Therefore, the imported module isn't :mod:`__main__`. This can be used to execute code early during Python initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. + .. versionchanged:: 3.15 + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 * :samp:`-X gil={0,1}` forces the GIL to be disabled or enabled, @@ -687,6 +691,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X pathconfig_warnings={0,1}` if true (``1``) then + :ref:`sys-path-init` is allowed to log warnings into stderr. + If false (``0``) suppress these warnings. Set to true by default. + See also :envvar:`PYTHON_PATHCONFIG_WARNINGS`. + + .. versionadded:: 3.15 + * :samp:`-X tlbc={0,1}` enables (1, the default) or disables (0) thread-local bytecode in builds configured with :option:`--disable-gil`. When disabled, this also disables the specializing interpreter. See also @@ -694,6 +705,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X lazy_imports={all,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, and ``normal`` (the default) + respects the ``lazy`` keyword in source code. + See also :envvar:`PYTHON_LAZY_IMPORTS`. + + .. versionadded:: 3.15 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -941,8 +959,9 @@ conflict. .. envvar:: PYTHONNOUSERSITE - If this is set, Python won't add the :data:`user site-packages directory - ` to :data:`sys.path`. + This is equivalent to the :option:`-s` option. If this is set, Python won't + add the :data:`user site-packages directory ` to + :data:`sys.path`. .. seealso:: @@ -956,6 +975,9 @@ conflict. and :ref:`installation paths ` for ``python -m pip install --user``. + To disable the user site-packages, see :envvar:`PYTHONNOUSERSITE` or the :option:`-s` + option. + .. seealso:: :pep:`370` -- Per user site-packages directory @@ -1066,6 +1088,13 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. note:: + + In the :term:`free-threaded ` build, the ``malloc``, + ``malloc_debug``, ``pymalloc``, and ``pymalloc_debug`` values are not + supported. Only ``default``, ``debug``, ``mimalloc``, and + ``mimalloc_debug`` are accepted. + .. versionadded:: 3.6 .. versionchanged:: 3.7 @@ -1075,18 +1104,48 @@ conflict. .. envvar:: PYTHONMALLOCSTATS If set to a non-empty string, Python will print statistics of the - :ref:`pymalloc memory allocator ` every time a new pymalloc object - arena is created, and on shutdown. + :ref:`pymalloc memory allocator ` or the + :ref:`mimalloc memory allocator ` (whichever is in use) + every time a new object arena is created, and on shutdown. This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable is used to force the :c:func:`malloc` allocator of the C library, or if - Python is configured without ``pymalloc`` support. + Python is configured without both ``pymalloc`` and ``mimalloc`` support. .. versionchanged:: 3.6 This variable can now also be used on Python compiled in release mode. It now has no effect if set to an empty string. +.. envvar:: PYTHON_PYMALLOC_HUGEPAGES + + If set to a non-zero integer, enable huge page support for + :ref:`pymalloc ` arenas. Set to ``0`` or unset to disable. + Python must be compiled with :option:`--with-pymalloc-hugepages` for this + variable to have any effect. + + When enabled, arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages if + huge pages are not available. + + .. warning:: + + On Linux, if the huge-page pool is exhausted, page faults — including + copy-on-write faults triggered by :func:`os.fork` — deliver ``SIGBUS`` + and kill the process. Only enable this in environments where the + huge-page pool is properly sized and fork-safety is not a concern. + + On Windows you need a special privilege. See the + `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + .. versionadded:: 3.15 + + .. envvar:: PYTHONLEGACYWINDOWSFSENCODING If set to a non-empty string, the default :term:`filesystem encoding and @@ -1094,9 +1153,6 @@ conflict. 'replace', respectively. Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. - This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding`. - .. availability:: Windows. .. versionadded:: 3.6 @@ -1282,6 +1338,13 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_COMPLETER + + If this variable is set to any value, PyREPL will use :mod:`rlcompleter` to + implement tab completion, instead of the default one which uses colors. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a @@ -1321,6 +1384,14 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_PATHCONFIG_WARNINGS + + If true (``1``) then :ref:`sys-path-init` is allowed to log warnings into + stderr. If false (``0``) suppress these warnings. Set to true by default. + See also :option:`-X pathconfig_warnings<-X>`. + + .. versionadded:: 3.15 + .. envvar:: PYTHON_JIT On builds where experimental just-in-time compilation is available, this @@ -1339,6 +1410,16 @@ conflict. .. versionadded:: 3.14 +.. envvar:: PYTHON_LAZY_IMPORTS + + Controls lazy import behavior. Accepts two values: ``all`` makes all + imports lazy by default, and ``normal`` (the default) respects the + ``lazy`` keyword in source code. + + See also the :option:`-X lazy_imports <-X>` command-line option. + + .. versionadded:: 3.15 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -1376,4 +1457,7 @@ Debug-mode variables Needs Python configured with the :option:`--with-pydebug` build option. + .. versionchanged:: 3.15 + Accept also ``module:func`` entry point format. + .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index cdadbe514174995..22a10db976c4fc6 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -112,7 +112,6 @@ Dependencies to build optional modules are: .. [1] If *libmpdec* is not available, the :mod:`decimal` module will use a pure-Python implementation. - See :option:`--with-system-libmpdec` for details. .. [2] See :option:`--with-readline` for choosing the backend for the :mod:`readline` module. .. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. @@ -322,6 +321,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution", + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: 3.15 + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. @@ -397,7 +420,7 @@ General Options :no-typesetting: Enables support for running Python without the :term:`global interpreter - lock` (GIL): free threading build. + lock` (GIL): :term:`free-threaded build`. Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. @@ -439,6 +462,26 @@ General Options ``pkg-config`` options. +.. option:: --disable-epoll + + Build without ``epoll``, meaning that :py:func:`select.epoll` will not be + present even if the system provides an + :manpage:`epoll_create ` function. + This may be used on systems where :manpage:`!epoll_create` or + :manpage:`epoll_create1 ` is available + but incompatible with Linux semantics. + + .. versionadded:: 3.15 + +.. option:: --with-build-details-suffix=[yes|SUFFIX] + + Rename ``build-details.json`` to permit multiple co-located Python + installs. If a custom ``SUFFIX`` is supplied it is used verbatim, + otherwise one will be generated from the ``MULTIARCH`` tag with + ``-free-threading`` and ``-debug``, as appropriate. + + .. versionadded:: 3.16 + C compiler options ------------------ @@ -518,11 +561,6 @@ Options for third-party dependencies C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, overriding ``pkg-config``. - .. note:: - - These environment variables have no effect unless - :option:`--with-system-libmpdec` is specified. - .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -745,11 +783,41 @@ also be used to improve performance. .. versionadded:: 3.14 +.. option:: --without-frame-pointers + + Disable frame pointers, which are enabled by default (see :pep:`831`). + + By default, the build appends flags to generate frame or backchain + pointers to ``BASECFLAGS``: + + - ``-fno-omit-frame-pointer`` and/or ``-mno-omit-leaf-frame-pointer`` + are added when the compiler supports them. + - ``-marm`` and/or ``-mno-thumb`` is added on 32-bit ARM when supported, + - on s390x platforms, when supported, ``-mbackchain`` is added *instead*. + of the above frame pointer flags. + - on ppc64le platforms, no compiler flags is needed since the power ABI + requires that compilers maintain a back chain by default. + + Frame pointers enable profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) to walk the C call stack + without DWARF metadata. The flags propagate to third-party C + extensions through :mod:`sysconfig`. On compilers that do not + understand them, the build silently skips them. + + Downstream packagers and authors of native libraries built with + custom build systems should set the same flags so the unwind chain + stays unbroken across all native frames. + + .. versionadded:: 3.15 + .. option:: --without-mimalloc Disable the fast :ref:`mimalloc ` allocator (enabled by default). + This option cannot be used together with :option:`--disable-gil` + because the :term:`free-threaded ` build requires mimalloc. + See also :envvar:`PYTHONMALLOC` environment variable. .. option:: --without-pymalloc @@ -759,6 +827,36 @@ also be used to improve performance. See also :envvar:`PYTHONMALLOC` environment variable. +.. option:: --with-pymalloc-hugepages + + Enable huge page support for :ref:`pymalloc ` arenas (disabled by + default). When enabled, the arena size on 64-bit platforms is increased to + 2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + + Even when compiled with this option, huge pages are **not** used at runtime + unless the :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable is set + to ``1``. This opt-in is required because huge pages + + * carry risks on Linux: if the huge-page pool is exhausted, page faults + (including copy-on-write faults after :func:`os.fork`) deliver ``SIGBUS`` + and kill the process. + + * need a special privilege on Windows. See the `Windows documentation for large pages + `_ + for details. Python will fail on startup if the required privilege + `SeLockMemoryPrivilege + `_ + is not held by the user. + + The configure script checks that the platform supports ``MAP_HUGETLB`` + and emits a warning if it is not available. + + On Windows, use the ``--pymalloc-hugepages`` flag with ``build.bat`` or + set the ``UsePymallocHugepages`` MSBuild property. + + .. versionadded:: 3.15 + .. option:: --without-doc-strings Disable static documentation strings to reduce the memory footprint (enabled @@ -838,9 +936,11 @@ See also the :ref:`Python Development Mode ` and the :option:`--with-trace-refs` configure option. .. versionchanged:: 3.8 - Release builds and debug builds are now ABI compatible: defining the + Release builds are now ABI compatible with debug builds: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option). + :option:`--with-trace-refs` option). However, debug builds still expose + more symbols than release builds and code built against a debug build is not + necessarily compatible with a release build. Debug options @@ -942,6 +1042,21 @@ Linker options .. versionadded:: 3.10 +.. option:: --enable-static-libpython-for-interpreter + + Do not link the Python interpreter binary (``python3``) against the + shared Python library; instead, statically link the interpreter + against ``libpython`` as if ``--enable-shared`` had not been used, + but continue to build the shared ``libpython`` (for use by other + programs). + + This option does nothing if ``--enable-shared`` is not used. + + The default (when ``-enable-shared`` is used) is to link the Python + interpreter against the built shared library. + + .. versionadded:: 3.15 + Libraries options ----------------- @@ -955,29 +1070,6 @@ Libraries options Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. option:: --with-system-libmpdec - - Build the ``_decimal`` extension module using an installed ``mpdecimal`` - library, see the :mod:`decimal` module (default is yes). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.13 - Default to using the installed ``mpdecimal`` library. - - .. versionchanged:: 3.15 - - A bundled copy of the library will no longer be selected - implicitly if an installed ``mpdecimal`` library is not found. - In Python 3.15 only, it can still be selected explicitly using - ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. - - .. deprecated-removed:: 3.13 3.16 - A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.16. - - .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. - .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -1504,6 +1596,12 @@ Compiler flags .. versionadded:: 3.7 +.. envvar:: CFLAGS_CEVAL + + Flags used to compile ``Python/ceval.c``. + + .. versionadded:: 3.14.5 + .. envvar:: CCSHARED Compiler flags used to build a shared library. @@ -1605,6 +1703,9 @@ Linker flags value to be able to build extension modules using the directories specified in the environment variables. + Please consider using ``EXE_LDFLAGS`` if the supplied linker flags are + executable specific, e.g. GCC's ``-pie`` flag. + .. envvar:: LIBS Linker flags to pass libraries to the linker when linking the Python @@ -1640,6 +1741,30 @@ Linker flags .. versionadded:: 3.8 +.. envvar:: EXE_LDFLAGS + + Linker flags used for building executable targets such as the + interpreter. If supplied, :envvar:`PY_CORE_EXE_LDFLAGS` + will be used in replacement of :envvar:`PY_CORE_LDFLAGS`. + + .. versionadded:: 3.15 + +.. envvar:: CONFIGURE_EXE_LDFLAGS + + Value of :envvar:`EXE_LDFLAGS` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.15 + +.. envvar:: PY_CORE_EXE_LDFLAGS + + Linker flags used for building the interpreter and + executable targets. + + Default: ``$(PY_CORE_LDFLAGS)`` + + .. versionadded:: 3.15 + .. rubric:: Footnotes diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 2fd6eace2b5dda2..6cf945de5b3f3b4 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -5,9 +5,6 @@ Using Python on macOS ********************* -.. sectionauthor:: Bob Savage -.. sectionauthor:: Ned Deily - This document aims to give an overview of macOS-specific behavior you should know about to get started with Python on Mac computers. Python on a Mac running macOS is very similar to Python on other Unix-derived platforms, diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index a9950ef75254976..829bdfe8b1121fd 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -6,9 +6,6 @@ Using Python on Unix platforms ******************************** -.. sectionauthor:: Shriphani Palakodety - - Getting and installing the latest version of Python =================================================== diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e6619b73bd2c26c..5b715d4614dad8f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,14 +4,14 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* Using Python on Windows ************************* -.. sectionauthor:: Steve Dower - This document aims to give an overview of Windows-specific behaviour you should know about when using Python on Microsoft Windows. @@ -124,9 +124,8 @@ is also an unambiguous ``pymanager`` command. Scripted installs that are intending to use Python install manager should consider using ``pymanager``, due to the lower chance of encountering a conflict with existing installs. The only difference between the two commands is when running without any arguments: -``py`` will install and launch your default interpreter, while ``pymanager`` -will display help (``pymanager exec ...`` provides equivalent behaviour to -``py ...``). +``py`` will launch your default interpreter, while ``pymanager`` will display +help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` @@ -185,12 +184,11 @@ that virtual environment. In this scenario, the ``python`` command was likely already overridden and none of these checks occurred. However, this behaviour ensures that the ``py`` command can be used interchangeably. -When you launch either ``python`` or ``py`` but do not have any runtimes -installed, and the requested version is the default, it will be installed -automatically and then launched. Otherwise, the requested version will be -installed if automatic installation is configured (most likely by setting -``PYTHON_MANAGER_AUTOMATIC_INSTALL`` to ``true``), or if the ``py exec`` or -``pymanager exec`` forms of the command were used. +When no runtimes are installed, any launch command will try to install the +requested version and launch it. However, after any version is installed, only +the ``py exec ...`` and ``pymanager exec ...`` commands will install if the +requested version is absent. Other forms of commands will display an error and +direct you to use ``py install`` first. Command help @@ -287,14 +285,32 @@ work. Passing ``--dry-run`` will generate output and logs, but will not modify any installs. +Passing ``--refresh`` will update all registrations for installed runtimes. This +will recreate Start menu shortcuts, registry keys, and global aliases (such as +``python3.14.exe`` or for any installed scripts). These are automatically +refreshed on installation of any runtime, but may need to be manually refreshed +after installing packages. + In addition to the above options, the ``--target`` option will extract the -runtime to the specified directory instead of doing a normal install. This is -useful for embedding runtimes into larger applications. +runtime to the specified directory instead of doing a normal install. +This is useful for embedding runtimes into larger applications. +Unlike a normal install, ``py`` will not be aware of the extracted runtime, +and no Start menu or other shortcuts will be created. +To launch the runtime, directly execute the main executable (typically +``python.exe``) in the target directory. .. code:: $> py install ... [-t=|--target=] +The ``py exec`` command will install the requested runtime if it is not already +present. This is controlled by the ``automatic_install`` configuration +(:envvar:`PYTHON_MANAGER_AUTOMATIC_INSTALL`), and is enabled by default. +If no runtimes are available at all, all launch commands will do an automatic +install if the configuration setting allows. This is to ensure a good experience +for new users, but should not generally be relied on rather than using the +``py exec`` command or explicit install commands. + .. _pymanager-offline: @@ -376,10 +392,13 @@ overridden installs may resolve settings differently. A global configuration file may be configured by an administrator, and would be read first. The user configuration file is stored at -:file:`%AppData%\\Python\\pymanager.json` (by default) and is read next, +:file:`%AppData%\\Python\\pymanager.json` +(note that this location is under ``Roaming``, not ``Local``) and is read next, overwriting any settings from earlier files. An additional configuration file may be specified as the ``PYTHON_MANAGER_CONFIG`` environment variable or the ``--config`` command line option (but not both). +These locations may be modified by administrative customization options listed +later. The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative @@ -417,9 +436,11 @@ customization. By default, :file:`%TEMP%`. * - ``automatic_install`` - - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` - - True to allow automatic installs when specifying a particular runtime - to launch. + - .. envvar:: PYTHON_MANAGER_AUTOMATIC_INSTALL + - True to allow automatic installs when using ``py exec`` to launch (or + ``py`` when no runtimes are installed yet). + Other commands will not automatically install, regardless of this + setting. By default, true. * - ``include_unmanaged`` @@ -434,6 +455,12 @@ customization. Python runtimes, or false to prevent it. By default, true. + * - ``shebang_templates`` + - (none) + - Mapping from shebang line template to alternative command, such as + ``py -V:`` or a substitute string. + See :ref:`pymanager-shebang` for more details. + * - ``log_level`` - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` - Set the default level of output (0-50). @@ -452,6 +479,14 @@ customization. - ``PYTHON_MANAGER_SOURCE_URL`` - Override the index feed to obtain new installs from. + * - ``install.enable_entrypoints`` + - (none) + - True to generate global commands for installed packages (such as + ``pip.exe``). These are defined by the packages themselves. + If set to false, only the Python interpreter has global commands created. + By default, true. You should run ``py install --refresh`` after changing + this setting. + * - ``list.format`` - ``PYTHON_MANAGER_LIST_FORMAT`` - Specify the default format used by the ``py list`` command. @@ -465,8 +500,8 @@ customization. * - ``global_dir`` - (none) - - Specify the directory where global commands (such as ``python3.14.exe``) - are stored. + - Specify the directory where global commands (such as ``python3.14.exe`` + and ``pip.exe``) are stored. This directory should be added to your :envvar:`PATH` to make the commands available from your terminal. @@ -476,6 +511,7 @@ customization. This directory is a temporary cache, and can be cleaned up from time to time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -538,17 +574,38 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. +Since version 26.3 of the Python install manager, custom shebang templates may +be added to your configuration file. Add the ``shebang_templates`` object with +one member for each template (the string to match) and the command to use when +the template is matched. Most commands should be ``py -V:`` (or ``pyw``) to +launch one of your installed runtimes. The ``py -3.`` form is also +allowed, as is a plain ``py`` to launch the default. No other arguments are +supported. + +.. code:: json5 + + { + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/my_custom_python": "py -V:MyCustomPython/3" + } + } + +If the substitute command is not ``py`` or ``pyw``, it will be written back into +the shebang and regular handling continues. If launching arbitrary executables +is permitted, then providing a full path will allow you to redirect from Python +to any executable. The template should match either the entire line (ignoring +leading and trailing whitespace), or up to the first space in the shebang line. + + .. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -725,6 +782,14 @@ directory containing the configuration file that specified them. (e.g. ``"pep514,start"``). Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + * - ``install.hard_link_entrypoints`` + - True to use hard links for global shortcuts to save disk space. If false, + each shortcut executable is copied instead. After changing this setting, + you must run ``py install --refresh --force`` to update existing + commands. + By default, true. Disabling this may be necessary for troubleshooting or + systems that have issues with file links. + * - ``pep514_root`` - Registry location to read and write PEP 514 entries into. By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. @@ -743,6 +808,14 @@ directory containing the configuration file that specified them. - True to suppress visible warnings when a shebang launches an application other than a Python runtime. + * - ``source_settings`` + - A mapping from source URL to settings specific to that index. + When multiple configuration files include this section, URL settings are + added or overwritten, but individual settings are not merged. + These settings are currently only for :ref:`index signatures + `. + + .. _install-freethreaded-windows: Installing free-threaded binaries @@ -764,6 +837,101 @@ installed, then ``python`` will launch this one. Otherwise, you will need to use ``py -V:3.14t ...`` or, if you have added the global aliases directory to your :envvar:`PATH` environment variable, the ``python3.14t.exe`` commands. + +.. _pymanager-index-signatures: + +Index signatures +---------------- + +.. versionadded:: 26.2 + +Index files may be signed to detect tampering. A signature is a catalog file +at the same URL as the index with ``.cat`` added to the filename. The catalog +file should contain the hash of its matching index file, and should be signed +with a valid Authenticode signature. This allows standard tooling (on Windows) +to generate a signature, and any certificate may be used as long as the client +operating system already trusts its certification authority (root CA). + +Index signatures are only downloaded and checked when the local configuration's +``source_settings`` section includes the index URL and ``requires_signature`` is +true, or the index JSON contains ``requires_signature`` set to true. When the +setting exists in local configuration, even when false, settings in the index +are ignored. + +As well as requiring a valid signature, the ``required_root_subject`` and +``required_publisher_subject`` settings can further restrict acceptable +signatures based on the certificate Subject fields. Any attribute specified in +the configuration must match the attribute in the certificate (additional +attributes in the certificate are ignored). Typical attributes are ``CN=`` for +the common name, ``O=`` for the organizational unit, and ``C=`` for the +publisher's country. + +Finally, the ``required_publisher_eku`` setting allows requiring that a specific +Enhanced Key Usage (EKU) has been assigned to the publisher certificate. For +example, the EKU ``1.3.6.1.5.5.7.3.3`` indicates that the certificate was +intended for code signing (as opposed to server or client authentication). +In combination with a specific root CA, this provides another mechanism to +verify a legitimate signature. + +This is an example ``source_settings`` section from a configuration file. In +this case, the publisher of the feed is uniquely identified by the combination +of the Microsoft Identity Verification root and the EKU assigned by that root. +The signature for this case would be found at +``https://www.python.org/ftp/python/index-windows.json.cat``. + +.. code:: json5 + + { + "source_settings": { + "https://www.python.org/ftp/python/index-windows.json": { + "requires_signature": true, + "required_root_subject": "CN=Microsoft Identity Verification Root Certificate Authority 2020", + "required_publisher_subject": "CN=Python Software Foundation", + "required_publisher_eku": "1.3.6.1.4.1.311.97.608394634.79987812.305991749.578777327" + } + } + } + +The same settings could be specified in the ``index.json`` file instead. In this +case, the root and EKU are omitted, meaning that the signature must be valid and +have a specific common name in the publisher's certificate, but no other checks +are used. + +.. code:: json5 + + { + "requires_signature": true, + "required_publisher_subject": "CN=Python Software Foundation", + "versions": [ + // ... + ] + } + +When settings from inside a feed are used, the user is notified and the settings +are shown in the log file or verbose output. It is recommended to copy these +settings into a local configuration file for feeds that will be used frequently, +so that unauthorised modifications to the feed cannot disable verification. + +It is not possible to override the location of the signature file in the feed or +through a configuration file. Administrators can provide their own +``source_settings`` in a mandatory configuration file (see +:ref:`pymanager-admin-config`). + +If signature validation fails, you will be notified and prompted to continue. +When interactive confirmation is not allowed (for example, because ``--yes`` was +specified), it will always abort. To use a feed with invalid configuration in +this scenario, you must provide a configuration file that disables signature +checking for that feed. + +.. code:: json5 + + "source_settings": { + "https://www.example.com/feed-with-invalid-signature.json": { + "requires_signature": false + } + } + + .. _pymanager-troubleshoot: Troubleshooting @@ -771,7 +939,7 @@ Troubleshooting If your Python install manager does not seem to be working correctly, please work through these tests and fixes to see if it helps. If not, please report an -issue at `our bug tracker `_, +issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). @@ -800,6 +968,12 @@ default). * - - Check that the ``py`` and ``pymanager`` commands work. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "command not found" error when I type it in my terminal. - Did you :ref:`install the Python install manager `? @@ -810,6 +984,12 @@ default). The "Python (default windowed)" and "Python install manager" commands may also need refreshing. + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + * - ``py`` gives me a "can't open file" error when I type commands in my terminal. - This usually means you have the legacy launcher installed and @@ -840,7 +1020,7 @@ default). - Prerelease and experimental installs that are not managed by the Python install manager may be chosen ahead of stable releases. Configure your default tag or uninstall the prerelease runtime - and reinstall using ``py install``. + and reinstall it using ``py install``. * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` - Click Start, open "Manage app execution aliases", and check that your @@ -852,13 +1032,48 @@ default). * - - The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, - or alternatively the ``python -m pip install --force pip`` command - will recreate the executables and show you the path to - add to :envvar:`PATH`. - These scripts are separated for each runtime, and so you may need to - add multiple paths. + We recommend using the ``python -m pip`` command instead. + Running ``py install --refresh`` and ensuring that the global shortcuts + directory is on :envvar:`PATH` (it will be shown in the command output if + it is not) should make commands such as ``pip`` (and other installed + packages) available. + + * - I installed a package with ``pip`` but its command is not found. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - New packages do not automatically have global shortcuts created by the + Python install manager. Similarly, uninstalled packages do not have their + shortcuts removed. + Run ``py install --refresh`` to update the global shortcuts for newly + installed packages. + + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + + * - I have installed the Python install manager multiple times. + - It is possible to install from the Store or WinGet, from the MSIX on + the Python website, and from the MSI, all at once. + They are all compatible and will share configuration and runtimes. + + * - + - See the earlier :ref:`pymanager-advancedinstall` section for ways to + uninstall the install manager other than the typical Installed Apps + (Add and Remove Programs) settings page. + * - My old ``py.ini`` settings no longer work. + - The new Python install manager no longer supports this configuration file + or its settings, and so it will be ignored. + See :ref:`pymanager-config` for information about configuration settings. .. _windows-embeddable: @@ -876,7 +1091,7 @@ To install an embedded distribution, we recommend using ``py install`` with the .. code:: - $> py install 3.14-embed --target=runtime + $> py install 3.14-embed --target= When extracted, the embedded distribution is (almost) fully isolated from the user's system, including environment variables, system registry settings, and diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce4aa0..43ab19037d26273 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 3430ac8668e280c..4b92ba82c991e44 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -745,7 +745,7 @@ The contextlib module The new :mod:`contextlib` module provides some functions and a decorator that are useful for writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will be the method's return @@ -1277,12 +1277,12 @@ complete list of changes, or look through the SVN logs for all the details. with the new ':keyword:`with`' statement. See section :ref:`contextlibmod` for more about this module. -* New module: The :mod:`cProfile` module is a C implementation of the existing - :mod:`profile` module that has much lower overhead. The module's interface is - the same as :mod:`profile`: you run ``cProfile.run('main()')`` to profile a +* New module: The :mod:`!cProfile` module is a C implementation of the existing + :mod:`!profile` module that has much lower overhead. The module's interface is + the same as :mod:`!profile`: you run ``cProfile.run('main()')`` to profile a function, can save profile data to a file, etc. It's not yet known if the Hotshot profiler, which is also written in C but doesn't match the - :mod:`profile` module's interface, will continue to be maintained in future + :mod:`!profile` module's interface, will continue to be maintained in future versions of Python. (Contributed by Armin Rigo.) Also, the :mod:`pstats` module for analyzing the data measured by the profiler @@ -1313,10 +1313,10 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') * The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching @@ -2169,9 +2169,9 @@ Changes to Python's build process and to the C API include: * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:macro:`Py_LOCAL` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index f5e3a47037c65fa..e1f72b6c4a6c92c 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -431,7 +431,7 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`~contextlib.contextmanager`, and lets you write +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will @@ -1557,8 +1557,9 @@ Some smaller changes made to the core Python language are: .. Revision 57619 -* Properties now have three attributes, :attr:`getter`, :attr:`setter` - and :attr:`deleter`, that are decorators providing useful shortcuts +* Properties now have three decorators, :deco:`~property.getter`, + :deco:`~property.setter` and :deco:`~property.deleter`, that are + decorators providing useful shortcuts for adding a getter, setter or deleter function to an existing property. You would use them like this:: @@ -2822,10 +2823,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 7296296d144803d..de90c37a38d38ff 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1267,7 +1267,7 @@ changes, or look through the Subversion logs for all the details. uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` +* New class decorator: :deco:`~functools.total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`~object.__eq__` method and one of :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index d8251185fa7f4b6..390b82d5a1fdc1b 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,7 +402,7 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of +The :attr:`!codeobject.co_lnotab` attribute of :ref:`code objects ` is deprecated and will be removed in 3.12. Code that needs to convert from offset to line number should use the new @@ -996,7 +996,7 @@ dataclasses __slots__ ~~~~~~~~~ -Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +Added ``slots`` parameter in :deco:`dataclasses.dataclass` decorator. (Contributed by Yurii Karabas in :issue:`42269`) Keyword-only fields @@ -1446,8 +1446,8 @@ as a positional-only argument. types ----- -Reintroduce the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set +Reintroduce the :class:`types.EllipsisType`, :class:`types.NoneType` +and :class:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) @@ -1485,9 +1485,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`~typing.runtime_checkable`. Previously, these checks +are decorated with :deco:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`!runtime_checkable` decorator +subclasses with the :deco:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1622,7 +1622,7 @@ Deprecated compatibility. Specifically, :meth:`!find_loader`/:meth:`!find_module` (superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` + ``importlib.abc.Loader.load_module`` (superseded by :meth:`~importlib.abc.Loader.exec_module`), :meth:`!module_repr` (which the import system takes care of for you), the ``__package__`` attribute @@ -1652,7 +1652,7 @@ Deprecated preference for :meth:`~zipimport.zipimporter.exec_module`. (Contributed by Brett Cannon in :issue:`26131`.) -* The use of :meth:`~importlib.abc.Loader.load_module` by the import +* The use of ``importlib.abc.Loader.load_module`` by the import system now triggers an :exc:`ImportWarning` as :meth:`~importlib.abc.Loader.exec_module` is preferred. (Contributed by Brett Cannon in :issue:`26131`.) @@ -1928,7 +1928,7 @@ Changes in the Python API (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley in :issue:`42392`.) -* The :data:`types.FunctionType` constructor now inherits the current builtins +* The :class:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index a095d887352127a..7b3475882063635 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -388,7 +388,7 @@ Kumar Srinivasan and Graham Bleaney.) PEP 681: Data class transforms ------------------------------ -:data:`~typing.dataclass_transform` may be used to +:deco:`~typing.dataclass_transform` may be used to decorate a class, metaclass, or a function that is itself a decorator. The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class, @@ -686,8 +686,8 @@ enum * Added the :func:`~enum.member` and :func:`~enum.nonmember` decorators, to ensure the decorated object is/is not converted to an enum member. -* Added the :func:`~enum.property` decorator, - which works like :func:`property` except for enums. +* Added the :deco:`~enum.property` decorator, + which works like :deco:`property` except for enums. Use this instead of :func:`types.DynamicClassAttribute`. * Added the :func:`~enum.global_enum` enum decorator, @@ -740,7 +740,7 @@ fractions functools --------- -* :func:`functools.singledispatch` now supports :class:`types.UnionType` +* :deco:`functools.singledispatch` now supports :class:`types.UnionType` and :class:`typing.Union` as annotations to the dispatch argument.:: >>> from functools import singledispatch @@ -1164,7 +1164,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. type checker errors related to highly dynamic class, such as mocks. (Contributed by Shantanu Jain in :gh:`91154`.) -* The :func:`typing.final` decorator now sets the ``__final__`` attributed on +* The :deco:`typing.final` decorator now sets the ``__final__`` attributed on the decorated object. (Contributed by Jelle Zijlstra in :gh:`90500`.) @@ -1197,7 +1197,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. :data:`~typing.ClassVar` annotations. (Contributed by Gregory Beauregard in :gh:`90711`.) -* :func:`typing.no_type_check` no longer modifies external classes and functions. +* :deco:`typing.no_type_check` no longer modifies external classes and functions. It also now correctly marks classmethods as not to be type checked. (Contributed by Nikita Sobolev in :gh:`90729`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8badfe9a6b49b95..9e48b3ab7393c87 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -146,7 +146,7 @@ New typing features: * :ref:`PEP 692 `, using :class:`~typing.TypedDict` to annotate :term:`**kwargs ` -* :ref:`PEP 698 `, :func:`typing.override` decorator +* :ref:`PEP 698 `, :deco:`typing.override` decorator Important deprecations, removals or restrictions: @@ -511,7 +511,7 @@ See :pep:`692` for more details. PEP 698: Override Decorator for Static Typing --------------------------------------------- -A new decorator :func:`typing.override` has been added to the :mod:`typing` +A new decorator :deco:`typing.override` has been added to the :mod:`typing` module. It indicates to type checkers that the method is intended to override a method in a superclass. This allows type checkers to catch mistakes where a method that is intended to override something in a base class @@ -1021,7 +1021,7 @@ typing ``__orig_bases__`` attribute. (Contributed by Adrian Garcia Badaracco in :gh:`103699`.) -* Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. +* Add ``frozen_default`` parameter to :deco:`typing.dataclass_transform`. (Contributed by Erik De Bonte in :gh:`99957`.) unicodedata @@ -1337,7 +1337,7 @@ Deprecated it was :exc:`ImportWarning`). (Contributed by Brett Cannon in :gh:`65961`.) -* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a +* Setting :attr:`~module.__package__` or ``__cached__`` on a module is deprecated, and will cease to be set or taken into consideration by the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.) @@ -1347,7 +1347,7 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in +* Accessing :attr:`!codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.15. @@ -1367,6 +1367,8 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: @@ -1747,7 +1749,7 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Donghee Na in :gh:`99127`.) -* The undocumented locking behavior of :func:`~functools.cached_property` +* The undocumented locking behavior of :deco:`~functools.cached_property` is removed, because it locked across all instances of the class, leading to high lock contention. This means that a cached property getter function could now run more than once for a single instance, if two threads race. For most simple diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index dc105156f33080d..d8845fe921ededa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1861,6 +1861,7 @@ New Deprecations * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` format specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation. + Scheduled to be removed in Python 3.18. (Contributed by Serhiy Storchaka in :gh:`89902`.) * :mod:`dis`: @@ -1957,14 +1958,14 @@ New Deprecations * :mod:`sys`: - * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + * Deprecate the :func:`!_enablelegacywindowsfsencoding` function, to be removed in Python 3.16. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. (Contributed by Inada Naoki in :gh:`73427`.) * :mod:`tarfile`: - * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + * Deprecate the undocumented and unused :attr:`!TarInfo.tarfile` attribute, to be removed in Python 3.16. (Contributed in :gh:`115256`.) @@ -1993,7 +1994,7 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`!typing.no_type_check_decorator` decorator function, + * Deprecate the :deco:`!typing.no_type_check_decorator` decorator function, to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -2024,10 +2025,14 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + .. include:: ../deprecations/pending-removal-in-3.19.rst .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2337,7 +2342,7 @@ Limited C API Changes * :c:func:`PySys_AuditTuple` * :c:func:`PyType_GetModuleByDef` - (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) + (Contributed by Victor Stinner in :gh:`85283` and :gh:`116936`.) * Python built with :option:`--with-trace-refs` (tracing references) now supports the :ref:`Limited API `. @@ -2594,7 +2599,7 @@ Build Changes .. _mimalloc library: https://github.com/microsoft/mimalloc/ -* The :file:`configure` option :option:`--with-system-libmpdec` +* The :file:`configure` option :option:`!--with-system-libmpdec` now defaults to ``yes``. The bundled copy of ``libmpdec`` will be removed in Python 3.16. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9459b73bcb502fa..2fa7267292d9940 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -454,7 +454,7 @@ on x86-64 and AArch64 architectures. However, a future release of GCC is expected to support this as well. This feature is opt-in for now. Enabling profile-guided optimization is highly -recommendeded when using the new interpreter as it is the only configuration +recommended when using the new interpreter as it is the only configuration that has been tested and validated for improved performance. For further information, see :option:`--with-tail-call-interp`. @@ -897,8 +897,7 @@ Command line and environment (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) * The command-line option :option:`-c` now automatically dedents its code - argument before execution. The auto-dedentation behavior mirrors - :func:`textwrap.dedent`. + argument before execution. (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) * :option:`!-J` is no longer a reserved flag for Jython_, @@ -954,10 +953,24 @@ when a module is imported) will still emit the syntax warning. (Contributed by Irit Katriel in :gh:`130080`.) +.. _incremental-garbage-collection: .. _whatsnew314-incremental-gc: -Incremental garbage collection ------------------------------- +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** The cycle garbage collector is now incremental. This means that maximum pause times are reduced @@ -2204,7 +2217,18 @@ difflib gc -- -* The new :ref:`incremental garbage collector ` +* **From Python 3.14.5 onwards:** + + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. + +* **Previously in Python 3.14.0-3.14.4:** + + The new :ref:`incremental garbage collector ` means that maximum pause times are reduced by an order of magnitude or more for larger heaps. @@ -2265,6 +2289,15 @@ pdb (Contributed by Tian Gao in :gh:`124533`.) +textwrap +-------- + +* Optimize the :func:`~textwrap.dedent` function, improving performance by + an average of 2.4x, with larger improvements for bigger inputs, + and fix a bug with incomplete normalization of blank lines with whitespace + characters other than space and tab. + + uuid ---- @@ -2595,12 +2628,12 @@ New deprecations and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` + * :class:`!asyncio.AbstractEventLoopPolicy` + * :class:`!asyncio.DefaultEventLoopPolicy` + * :class:`!asyncio.WindowsSelectorEventLoopPolicy` + * :class:`!asyncio.WindowsProactorEventLoopPolicy` + * :func:`!asyncio.get_event_loop_policy` + * :func:`!asyncio.set_event_loop_policy` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with the *loop_factory* argument to use the desired event loop implementation. @@ -2685,7 +2718,7 @@ New deprecations (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) * :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + Deprecate :meth:`!symtable.Class.get_methods` due to the lack of interest, scheduled for removal in Python 3.16. (Contributed by Bénédikt Tran in :gh:`119698`.) @@ -2709,10 +2742,14 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + .. include:: ../deprecations/pending-removal-in-3.19.rst .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst @@ -3437,3 +3474,17 @@ Changes in the C API functions on Python 3.13 and older. .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + + +Notable changes in 3.14.5 +========================= + +gc +-- + +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 31594a2e70bd4c7..aad4758297c30d1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -65,127 +65,506 @@ Summary -- Release highlights .. PEP-sized items next. +* :pep:`810`: :ref:`Explicit lazy imports for faster startup times + ` +* :pep:`814`: :ref:`Add frozendict built-in type + ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python - profiling tools ` + profiling tools ` +* :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler + ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` +* :pep:`798`: :ref:`Unpacking in comprehensions + ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` +* :pep:`829`: :ref:`Package startup configuration files ` +* :pep:`728`: :ref:`TypedDict with typed extra items ` +* :pep:`747`: :ref:`Annotating type forms with TypeForm + ` +* :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object - ` + ` +* :pep:`803`, :pep:`820 <820>`, :pep:`793 <793>`: + :ref:`Stable ABI for free-threaded builds ` and + related C API +* :pep:`788`: :ref:`Protection against finalization in the C API ` +* :ref:`The JIT compiler has been significantly upgraded ` +* :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter + ` * :ref:`Improved error messages ` +* :ref:`More color ` New features ============ +.. _whatsnew315-lazy-imports: + +:pep:`810`: Explicit lazy imports +--------------------------------- + +Large Python applications often suffer from slow startup times. A +significant contributor to this problem is the import system: when a module +is imported, Python must locate the file, read it from disk, compile it to +bytecode, and execute all top-level code. For applications with deep +dependency trees, this process can take seconds, even when most of the +imported code is never actually used during a particular run. + +Developers have worked around this by moving imports inside functions, using +:mod:`importlib` to load modules on demand, or restructuring code to avoid +unnecessary dependencies. These approaches work but make code harder to read +and maintain, scatter import statements throughout the codebase, and require +discipline to apply consistently. + +Python now provides a cleaner solution through explicit :keyword:`lazy` +imports using the new ``lazy`` soft keyword. When you mark an import as +lazy, Python defers the actual module loading until the imported name is +first used. This gives you the organizational benefits of declaring all +imports at the top of the file while only paying the loading cost for +modules you actually use. + +The ``lazy`` keyword works with both ``import`` and ``from ... import`` +statements. When you write ``lazy import heavy_module``, Python does not +immediately load the module. Instead, it creates a lightweight proxy object. +The actual module loading happens transparently when you first access the +name: + +.. code-block:: python + + lazy import json + lazy from pathlib import Path + + print("Starting up...") # json and pathlib not loaded yet + + data = json.loads('{"key": "value"}') # json loads here + p = Path(".") # pathlib loads here + +This mechanism is particularly useful for applications that import many +modules at the top level but may only use a subset of them in any given run. +The deferred loading reduces startup latency without requiring code +restructuring or conditional imports scattered throughout the codebase. + +In the case where loading a lazily imported module fails (for example, if +the module does not exist), Python raises the exception at the point of +first use rather than at import time. The associated traceback includes both +the location where the name was accessed and the original import statement, +making it straightforward to diagnose and debug the failure. + +For cases where you want to enable lazy loading globally without modifying +source code, Python provides the :option:`-X lazy_imports <-X>` command-line +option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both +accept two values: ``all`` makes all imports lazy by default, and ``normal`` +(the default) respects the ``lazy`` keyword in source code. The +:func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` functions allow +changing and querying this mode at runtime. + +For more selective control, :func:`sys.set_lazy_imports_filter` accepts a +callable that determines whether a specific module should be loaded lazily. +The filter receives three arguments: the importing module's name (or +``None``), the imported module's name, and the fromlist (or ``None`` for +regular imports). It should return ``True`` to allow the import to be lazy, +or ``False`` to force eager loading. This allows patterns like making only +your own application's modules lazy while keeping third-party dependencies +eager: + +.. code-block:: python + + import sys + + def myapp_filter(importing, imported, fromlist): + return imported.startswith("myapp.") + sys.set_lazy_imports_filter(myapp_filter) + sys.set_lazy_imports("all") + + import myapp.slow_module # lazy (matches filter) + import json # eager (does not match filter) + +The proxy type itself is available as :class:`types.LazyImportType` for code +that needs to detect lazy imports programmatically. + +There are some restrictions on where the ``lazy`` keyword can be used. Lazy +imports are only permitted at module scope; using ``lazy`` inside a +function, class body, or ``try``/``except``/``finally`` block raises a +:exc:`SyntaxError`. Neither star imports nor future imports can be lazy +(``lazy from module import *`` and ``lazy from __future__ import ...`` both +raise :exc:`SyntaxError`). + +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + +.. seealso:: :pep:`810` for the full specification and rationale. + +(Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) + + +.. _whatsnew315-frozendict: + +:pep:`814`: Add frozendict built-in type +---------------------------------------- + +A new :term:`immutable` type, :class:`frozendict`, is added to the :mod:`builtins` module. +It does not allow modification after creation. A :class:`!frozendict` is not a subclass of ``dict``; +it inherits directly from ``object``. A :class:`!frozendict` is :term:`hashable` +as long as all of its keys and values are hashable. A :class:`!frozendict` preserves +insertion order, but comparison does not take order into account. + +For example:: + + >>> a = frozendict(x=1, y=2) + >>> a + frozendict({'x': 1, 'y': 2}) + >>> a['z'] = 3 + Traceback (most recent call last): + File "", line 1, in + a['z'] = 3 + ~^^^^^ + TypeError: 'frozendict' object does not support item assignment + >>> b = frozendict(y=2, x=1) + >>> hash(a) == hash(b) + True + >>> a == b + True + +The following standard library modules have been updated to accept +:class:`!frozendict`: :mod:`copy`, :mod:`decimal`, :mod:`json`, :mod:`marshal`, +:mod:`plistlib` (only for serialization), :mod:`pickle`, :mod:`pprint` and +:mod:`xml.etree.ElementTree`. + +:func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and +:func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*. + +Code checking for :class:`dict` type using ``isinstance(arg, dict)`` can be +updated to ``isinstance(arg, (dict, frozendict))`` to accept also the +:class:`!frozendict` type, or to ``isinstance(arg, collections.abc.Mapping)`` +to accept also other mapping types such as :class:`~types.MappingProxyType`. + +.. seealso:: :pep:`814` for the full specification and rationale. + +(Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) + + +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + +.. seealso:: :pep:`661` for further details. + + +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`profiling.tracing`: deterministic function-call tracing (relocated from + ``cProfile``). +* :mod:`profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The ``cProfile`` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + .. _whatsnew315-sampling-profiler: -:pep:`799`: High frequency statistical sampling profiler --------------------------------------------------------- +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon profiler logo + :align: center + :width: 200px -A new statistical sampling profiler has been added to the new :mod:`!profiling` module as -:mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of +A new statistical sampling profiler (Tachyon) has been added as +:mod:`profiling.sampling`. This profiler enables low-overhead performance analysis of running Python processes without requiring code modification or process restart. -Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +Unlike deterministic profilers (such as :mod:`profiling.tracing`) that instrument every function call, the sampling profiler periodically captures stack traces from running processes. This approach provides virtually zero overhead while achieving sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging -performance issues in production environments. +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. Key features include: * **Zero-overhead profiling**: Attach to any running Python process without - affecting its performance -* **No code modification required**: Profile existing applications without restart -* **Real-time statistics**: Monitor sampling quality during data collection -* **Multiple output formats**: Generate both detailed statistics and flamegraph data -* **Thread-aware profiling**: Option to profile all threads or just the main thread + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. + +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. + +* **Flexible target modes**: + + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + * Capture a one-shot snapshot of a running process (``dump``) - print a + traceback-style stack of every thread (or all asyncio tasks with + ``--async-aware``). Useful for investigating hung processes. + +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: + + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + * **Exception handling time** (``--mode exception``): Captures samples only from threads with + an active exception. Use this to analyze exception handling overhead. -Profile process 1234 for 10 seconds with default settings: +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. -.. code-block:: shell +* **Multiple output formats**: Choose the visualization that best fits your workflow: + + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flame graphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flame graph using D3.js. + Opens directly in your browser for immediate visual analysis. Flame graphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with `Firefox Profiler + `__. Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. - python -m profiling.sampling 1234 +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. -Profile with custom interval and duration, save to file: +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. -.. code-block:: shell +* **Opcode-level profiling**: Gather bytecode opcode information for instruction-level + profiling (``--opcodes``). Shows which bytecode instructions are executing, including + specializations from the adaptive interpreter. - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 +See :mod:`profiling.sampling` for the complete documentation, including all +available output formats, profiling modes, and configuration options. -Generate collapsed stacks for flamegraph: +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) + + +.. _whatsnew315-frame-pointers: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. -.. code-block:: shell +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. - python -m profiling.sampling --collapsed 1234 +.. important:: + + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) + +.. seealso:: :pep:`831` for further details. + + +.. _whatsnew315-unpacking-in-comprehensions: + +:pep:`798`: Unpacking in comprehensions +--------------------------------------- + +List, set, and dictionary comprehensions, as well as generator expressions, now +support unpacking with ``*`` and ``**``. This extends the unpacking syntax +from :pep:`448` to comprehensions, providing a new syntax for combining an +arbitrary number of iterables or dictionaries into a single flat structure. +This new syntax is a direct alternative to nested comprehensions, +:func:`itertools.chain`, and :meth:`itertools.chain.from_iterable`. For +example:: + + >>> lists = [[1, 2], [3, 4], [5]] + >>> [*L for L in lists] # equivalent to [x for L in lists for x in L] + [1, 2, 3, 4, 5] + + >>> sets = [{1, 2}, {2, 3}, {3, 4}] + >>> {*s for s in sets} # equivalent to {x for s in sets for x in s} + {1, 2, 3, 4} + + >>> dicts = [{'a': 1}, {'b': 2}, {'a': 3}] + >>> {**d for d in dicts} # equivalent to {k: v for d in dicts for k,v in d.items()} + {'a': 3, 'b': 2} + +Generator expressions can similarly use unpacking to yield values from multiple +iterables:: + + >>> gen = (*L for L in lists) # equivalent to (x for L in lists for x in L) + >>> list(gen) + [1, 2, 3, 4, 5] + +This change also extends to asynchronous generator expressions, such that, for +example, ``(*a async for a in agen())`` is equivalent to ``(x async for a in +agen() for x in a)``. + +.. seealso:: :pep:`798` for further details. + +(Contributed by Adam Hartz in :gh:`143055`.) + +.. _whatsnew315-startup-files: + +:pep:`829`: Package startup configuration files +----------------------------------------------- + +Loaded by the :mod:`site` module when ``-S`` is not given, :ref:`.pth files ` +can contain lines that both extend :data:`sys.path` and execute arbitrary code +when the line starts with ``import`` (followed by a space or tab). The latter +functionality can be problematic, since it is difficult to know exactly what +gets executed when Python starts up. -Profile all threads and sort by total time: +As a step towards improving the ability to audit pre-start executable code, +Python 3.15 introduces :ref:`.start files ` which contain entry point +specifications of the form ``pkg.mod:callable`` where ``pkg.mod`` is the +import path to the given callable. When Python starts up, the callable is +located and called with no arguments. -.. code-block:: shell +``import`` lines in :file:`.pth` files are silently deprecated. When a +matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files +are ignored. There is no change to :data:`sys.path` extension lines in +:file:`.pth` files. - python -m profiling.sampling -a --sort-tottime 1234 +The :mod:`site` module also provides :class:`site.StartupState` to batch +startup processing for multiple site directories, ensuring all static path +extensions are applied before any startup code is executed. :func:`site.main` +uses an instance of this class implicitly to batch process all startup +configuration files during normal interpreter startup. Callers needing the +same batching behavior can build a :class:`~site.StartupState` directly and +drive it with :meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, and +:meth:`~site.StartupState.addsitepackages`, then call +:meth:`~site.StartupState.process` once at the end of the batch. -The profiler generates statistical estimates of where time is spent: +(Contributed by Barry Warsaw in :gh:`148641` and :gh:`150228`.) -.. code-block:: text - Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001 - Captured 498841 samples in 5.00 seconds - Sample rate: 99768.04 samples/sec - Error rate: 0.72% - Profile Stats: - nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) - 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) - 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) - 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) - 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) - 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) - 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) - 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) - 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) - 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) - 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) - 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) - 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) - 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) - 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) - 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) +.. _whatsnew315-abi3t: - Legend: - nsamples: Direct/Cumulative samples (direct executing / on call stack) - sample%: Percentage of total samples this function was directly executing - tottime: Estimated total time spent directly in this function - cumul%: Percentage of total samples when this function was on the call stack - cumtime: Estimated cumulative time (including time in called functions) - filename:lineno(function): Function location and name +:pep:`803` -- Stable ABI for free-threaded builds +------------------------------------------------- - Summary of Interesting Functions: +C extensions that target the :ref:`Stable ABI ` can now be +compiled for the new *Stable ABI for Free-Threaded Builds* (also known +as ``abi3t``), which makes them compatible with +:term:`free-threaded builds ` of CPython. +This usually requires some non-trivial changes to the source code; +specifically: - Functions with Highest Direct/Cumulative Ratio (Hot Spots): - 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) - 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) - 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) +- Switching to API introduced in :pep:`697` (Python 3.12), such as + negative :c:member:`~PyType_Spec.basicsize` and + :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject` + part of the instance struct; and +- Switching from a ``PyInit_`` function to a new export hook, + :c:func:`PyModExport_* `, introduced for this + purpose in :pep:`793`, with a new :c:type:`PySlot` structure + introduced in :pep:`820`. - Functions with Highest Call Frequency (Indirect Calls): - 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) - 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) - 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) +Note that Stable ABI does not offer all the functionality that CPython +has to offer. +Extensions that cannot switch to ``abi3t`` should continue to build for +the existing Stable ABI (``abi3``) and the version-specific ABI for +free-threading (``cp315t``) separately. + +Stable ABI for Free-Threaded Builds should typically +be selected in a build tool (such as, for example, Setuptools, meson-python, +scikit-build-core, or Maturin). +At the time of writing, these tools did **not** support ``abi3t``. +If this is the case for your tool, compile for ``cp315t`` separately. +If not using a build tool -- or when writing such a tool -- you can select +``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed +in :ref:`abi3-compiling`. - Functions with Highest Call Magnification (Cumulative/Direct): - 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) - 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) +A practical :ref:`migration guide ` for switching to +``abi3t`` is available. -The profiler automatically identifies performance bottlenecks through statistical -analysis, highlighting functions with high CPU usage and call frequency patterns. +.. seealso:: :pep:`803` for further details. -This capability is particularly valuable for debugging performance issues in -production systems where traditional profiling approaches would be too intrusive. - .. seealso:: :pep:`799` for further details. +.. _whatsnew315-c-api-interpreter-finalization: -(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.) +:pep:`788`: Protecting the C API from interpreter finalization +-------------------------------------------------------------- + +In the C API, :term:`interpreter finalization ` can be +problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks +and other spurious issues. Additionally, it has historically been impossible +to safely check whether an interpreter is alive before using it, leading to crashes +when a thread concurrently deletes an interpreter while another thread is +trying to attach to it. + +There are now several new suites of APIs to circumvent these problems: + +* :ref:`Interpreter guards `, which prevent an interpreter + from finalizing. +* :ref:`Interpreter views `, which allow thread-safe access + to an interpreter that may be concurrently finalizing or deleted. +* :ref:`New APIs ` to automatically attach and detach + thread states that come with built-in protection against finalization. + +In addition, APIs in the ``PyGILState`` family (most notably +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`) have been +:term:`soft deprecated`. There is **no** plan to remove them, and existing +code will continue to work, but there will be no new ``PyGILState`` APIs +in future versions of Python. + +.. seealso:: :pep:`788` for further details. + +(Contributed by Peter Bierma in :gh:`149101`.) .. _whatsnew315-improved-error-messages: @@ -220,13 +599,76 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ - AttributeError: 'Container' object has no attribute 'area'. Did you mean: 'inner.area'? + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ + AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? + +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) Other language changes @@ -259,32 +701,38 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) + .. _whatsnew315-color-interpreter-help: -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? +* The interpreter help (such as ``python --help``) is now in color. + This can be controlled by :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148766`.) - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + .. _whatsnew315-color-exceptions: * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) + .. _whatsnew315-more-color: + +* More color in + :ref:`argparse `, + :ref:`ast `, + :ref:`calendar `, + :ref:`difflib `, + :ref:`http.server `, + :ref:`pickletools `, + :ref:`PyREPL tab completion `, + :ref:`python --help `, + :ref:`sqlite3 `, + :ref:`timeit `, + :ref:`tokenize `, + :ref:`unraisable exceptions ` and + :term:`stdlib` (ast, compileall, doctest, gzip, inspect, json.tool, pdb, + profiling.sampling, random, regrtest, sqlite3, timeit, tokenize, trace, + unittest, uuid, zipapp, zipfile) CLI help. + * The :meth:`~object.__repr__` of :class:`ImportError` and :class:`ModuleNotFoundError` now shows "name" and "path" as ``name=`` and ``path=`` if they were given as keyword arguments at construction time. @@ -316,9 +764,7 @@ Other language changes and compression. Common code patterns which can be optimized with :func:`~bytearray.take_bytes` are listed below. - (Contributed by Cody Maloney in :gh:`139871`.) - - .. list-table:: Suggested Optimizing Refactors + .. list-table:: Suggested optimizing refactors :header-rows: 1 * - Description @@ -361,7 +807,7 @@ Other language changes n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] - assert data == b'abc' + assert data == b'abc\n' assert buffer == bytearray(b'def') - .. code:: python @@ -387,13 +833,59 @@ Other language changes buffer.resize(n) data = buffer.take_bytes() + (Contributed by Cody Maloney in :gh:`139871`.) + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, - and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass - the module name. It is needed to unambiguous :ref:`filter ` + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` syntax warnings by module name. (Contributed by Serhiy Storchaka in :gh:`135801`.) +* Allowed defining the *__dict__* and *__weakref__* :ref:`__slots__ ` + for any class. + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* Allowed defining any :ref:`__slots__ ` for a class derived from + :class:`tuple` (including classes created by :func:`collections.namedtuple`). + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* The :class:`slice` type now supports subscription, + making it a :term:`generic type`. + (Contributed by James Hilton-Balfe in :gh:`128335`.) + +* The class :class:`memoryview` now supports the :c:expr:`float complex` and + :c:expr:`double complex` C types: formatting characters ``'Zf'`` and ``'Zd'`` + respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Allow the *count* argument of :meth:`bytes.replace` to be a keyword. + (Contributed by Stan Ulbrych in :gh:`147856`.) + +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + + +Default interactive shell +========================= + +.. _whatsnew315-pyrepl-completion: + +* Tab completions are now colored by object kind, based on + :pypi:`fancycompleter`. + Set :envvar:`PYTHON_BASIC_COMPLETER` to fall back to :mod:`rlcompleter`. + Color can also be controlled by :ref:`environment variables + `. + (Contributed by Antonio Cuni and Pablo Galindo in :gh:`130472`.) + New modules =========== @@ -412,21 +904,158 @@ Improved modules argparse -------- +* The :class:`~argparse.BooleanOptionalAction` action now supports single-dash + long options and alternate prefix characters. + (Contributed by Serhiy Storchaka in :gh:`138525`.) + * Changed the *suggest_on_error* parameter of :class:`argparse.ArgumentParser` to default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) + .. _whatsnew315-color-argparse: + +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. + (Contributed by Savannah Ostrowski in :gh:`142390`.) + +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + + +array +----- + +* Support the :c:expr:`float complex` and :c:expr:`double complex` C types: + formatting characters ``'Zf'`` and ``'Zd'`` respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Support half-floats (16-bit IEEE 754 binary interchange format): formatting + character ``'e'``. + (Contributed by Sergey B Kirpichev in :gh:`146238`.) + +* The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` + to support type codes longer than 1 character (``Zf`` and ``Zd``). + (Contributed by Victor Stinner in :gh:`148675`.) + + +ast +--- + + .. _whatsnew315-color-ast: + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + + +base64 +------ + +* Added the *pad* parameter in :func:`~base64.z85encode`. + (Contributed by Hauke Dämpfling in :gh:`143103`.) + +* Added the *padded* parameter in + :func:`~base64.b32encode`, :func:`~base64.b32decode`, + :func:`~base64.b32hexencode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64encode`, :func:`~base64.b64decode`, + :func:`~base64.urlsafe_b64encode`, and :func:`~base64.urlsafe_b64decode`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~base64.b16encode`, + :func:`~base64.b32encode`, :func:`~base64.b32hexencode`, + :func:`~base64.b64encode`, :func:`~base64.b85encode`, and + :func:`~base64.z85encode`. + (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) + +* Added the *ignorechars* parameter in :func:`~base64.b16decode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.urlsafe_b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +binascii +-------- + +* Added functions for Base32 encoding: + + - :func:`~binascii.b2a_base32` and :func:`~binascii.a2b_base32` + + (Contributed by James Seo in :gh:`146192`.) + +* Added functions for Ascii85, Base85, and Z85 encoding: + + - :func:`~binascii.b2a_ascii85` and :func:`~binascii.a2b_ascii85` + - :func:`~binascii.b2a_base85` and :func:`~binascii.a2b_base85` + + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Added the *padded* parameter in + :func:`~binascii.b2a_base32`, :func:`~binascii.a2b_base32`, + :func:`~binascii.b2a_base64`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`. + (Contributed by Serhiy Storchaka in :gh:`143214`.) + +* Added the *alphabet* parameter in :func:`~binascii.b2a_base64` and + :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`145980`.) + +* Added the *ignorechars* parameter in :func:`~binascii.a2b_hex`, + :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + calendar -------- -* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support - dark mode and have been migrated to the HTML5 standard for improved accessibility. - (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + .. _whatsnew315-color-calendar: + +* :mod:`calendar`'s :ref:`command-line ` text output has more + color. This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148352`.) * The :mod:`calendar`'s :ref:`command-line ` HTML output now accepts the year-month option: ``python -m calendar -t html 2009 06``. (Contributed by Pål Grønås Drange in :gh:`140212`.) +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + collections ----------- @@ -436,25 +1065,6 @@ collections between :class:`~collections.Counter` objects. (Contributed by Raymond Hettinger in :gh:`138682`.) -collections.abc ---------------- - -* :class:`collections.abc.ByteString` has been removed from - ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been - deprecated since Python 3.12, and is scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from collections.abc import ByteString`` - * ``import collections.abc; collections.abc.ByteString``. - - ``DeprecationWarning``\ s were already emitted if - :class:`collections.abc.ByteString` was subclassed or used as the second - argument to :func:`isinstance` or :func:`issubclass`, but warnings were not - previously emitted if it was merely imported or accessed from the - :mod:`!collections.abc` module. - concurrent.futures ------------------ @@ -466,6 +1076,25 @@ concurrent.futures (Contributed by Jonathan Berg in :gh:`139486`.) +contextlib +---------- + +* Added support for arbitrary descriptors :meth:`!__enter__`, + :meth:`!__exit__`, :meth:`!__aenter__`, and :meth:`!__aexit__` in + :class:`~contextlib.ExitStack` and :class:`contextlib.AsyncExitStack`, for + consistency with the :keyword:`with` and :keyword:`async with` statements. + (Contributed by Serhiy Storchaka in :gh:`144386`.) + +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + + dataclasses ----------- @@ -477,11 +1106,10 @@ dbm --- * Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` - which allow to recover unused free space previously occupied by deleted entries. + to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) - difflib ------- @@ -498,6 +1126,25 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + functools --------- @@ -505,6 +1152,21 @@ functools callables. (Contributed by Serhiy Storchaka in :gh:`140873`.) +* :func:`~functools.singledispatchmethod` now dispatches on the second argument + if it wraps a regular method and is called as a class attribute. + (Contributed by Bartosz Sławecki in :gh:`143535`.) + + +gc +-- + +* Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later and Python 3.15. + hashlib ------- @@ -527,11 +1189,45 @@ http.client (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) -http.cookies ------------- +http.server +----------- + + .. _whatsnew315-color-http.server: -* Allow '``"``' double quotes in cookie values. - (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +* The logging of :mod:`~http.server.BaseHTTPRequestHandler`, + as used by the :ref:`command-line interface `, + is colored by default. + This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`146292`.) + +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + +* Add a new ``extra_response_headers`` keyword argument to + :class:`~http.server.SimpleHTTPRequestHandler` to support custom headers in + HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + +* Add a ``-H/--header`` option to the :program:`python -m http.server` + command-line interface to support custom headers in HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + + +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata()`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) inspect @@ -542,6 +1238,18 @@ inspect (Contributed by Serhiy Storchaka in :gh:`132686`.) +json +---- + +* Add the *array_hook* parameter to :func:`~json.load` and + :func:`~json.loads` functions: + allow a callback for JSON literal array types to customize Python lists in + the resulting decoded object. Passing combined :class:`frozendict` to + *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a + deeply nested immutable Python structure representing the JSON data. + (Contributed by Joao S. O. Bueno in :gh:`146440`.) + + locale ------ @@ -550,6 +1258,9 @@ locale but included in the language code. (Contributed by Serhiy Storchaka in :gh:`137729`.) +* Undeprecate the :func:`locale.getdefaultlocale` function. + (Contributed by Victor Stinner in :gh:`130796`.) + math ---- @@ -564,8 +1275,11 @@ math mimetypes --------- -* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -580,6 +1294,11 @@ mmap not be duplicated. (Contributed by Serhiy Storchaka in :gh:`78502`.) +* Added the :meth:`mmap.mmap.set_name` method + to annotate an anonymous memory mapping + if Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + (Contributed by Donghee Na in :gh:`142419`.) + os -- @@ -588,6 +1307,10 @@ os glibc versions 2.28 and later. (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) os.path ------- @@ -602,6 +1325,59 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + +pickle +------ + +* Add support for pickling private methods and nested classes. + (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) + + +pickletools +----------- + + .. _whatsnew315-color-pickletools: + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + +re +-- + +* :func:`re.prefixmatch` and a corresponding :meth:`re.Pattern.prefixmatch` + have been added as alternate, more explicit names for the existing + and now :term:`soft deprecated` + :func:`re.match` and :meth:`re.Pattern.match` APIs. These are intended + to be used to alleviate confusion around what *match* means by following the + Zen of Python's *"Explicit is better than implicit"* mantra. Most other + language regular expression libraries use an API named *match* to mean what + Python has always called *search*. + (Contributed by Gregory P. Smith in :gh:`86519`.) + + resource -------- @@ -617,6 +1393,17 @@ shelve * Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) +* Add support for custom serialization and deserialization functions + in the :mod:`shelve` module. + (Contributed by Furkan Onder in :gh:`99631`.) + + +shutil +------ + +* :func:`shutil.copyfile` now also raises :exc:`~shutil.SpecialFileError` for + sockets and device files. + (Contributed by Savannah Ostrowski in :gh:`142693`.) socket @@ -631,42 +1418,44 @@ sqlite3 * The :ref:`command-line interface ` has several new features: - * SQL keyword completion on . - (Contributed by Long Tan in :gh:`133393`.) + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) + + .. _whatsnew315-color-sqlite3: - * Prompts, error messages, and help text are now colored. - This is enabled by default, see :ref:`using-on-controlling-color` for - details. - (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) - * Table, index, trigger, view, column, function, and schema completion on . - (Contributed by Long Tan in :gh:`136101`.) + * Table, index, trigger, view, column, function, and schema completion on . + (Contributed by Long Tan in :gh:`136101`.) ssl --- * Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module - supports "External PSKs" in TLSv1.3, as described in RFC 9258. + supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. (Contributed by Will Childs-Klein in :gh:`133624`.) * Added new methods for managing groups used for SSL key agreement - * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing - key agreement, extending the previous - :meth:`ssl.SSLContext.set_ecdh_curve` method. - This new API provides the ability to list multiple groups and - supports fixed-field and post-quantum groups in addition to ECDH - curves. This method can also be used to control what key shares - are sent in the TLS handshake. - * :meth:`ssl.SSLSocket.group` returns the group selected for doing key - agreement on the current connection after the TLS handshake completes. - This call requires OpenSSL 3.2 or later. - * :meth:`ssl.SSLContext.get_groups` returns a list of all available key - agreement groups compatible with the minimum and maximum TLS versions - currently set in the context. This call requires OpenSSL 3.5 or later. - - (Contributed by Ron Frederick in :gh:`136306`.) + * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing + key agreement, extending the previous + :meth:`ssl.SSLContext.set_ecdh_curve` method. + This new API provides the ability to list multiple groups and + supports fixed-field and post-quantum groups in addition to ECDH + curves. This method can also be used to control what key shares + are sent in the TLS handshake. + * :meth:`ssl.SSLSocket.group` returns the group selected for doing key + agreement on the current connection after the TLS handshake completes. + This call requires OpenSSL 3.2 or later. + * :meth:`ssl.SSLContext.get_groups` returns a list of all available key + agreement groups compatible with the minimum and maximum TLS versions + currently set in the context. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`136306`.) * Added a new method :meth:`ssl.SSLContext.set_ciphersuites` for setting TLS 1.3 ciphers. For TLS 1.2 or earlier, :meth:`ssl.SSLContext.set_ciphers` should @@ -690,7 +1479,30 @@ ssl selected for the server to complete the TLS handshake on the current connection. This call requires OpenSSL 3.5 or later. - (Contributed by Ron Frederick in :gh:`138252`.) + (Contributed by Ron Frederick in :gh:`138252`.) + + +subprocess +---------- + +* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the + platform supports it, an efficient event-driven mechanism is used to wait for + process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`. + - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``. + - Windows keeps using ``WaitForSingleObject`` (unchanged). + + If none of these mechanisms are available, the function falls back to the + traditional busy loop (non-blocking call and short sleeps). + (Contributed by Giampaolo Rodola in :gh:`83069`.) + + +symtable +-------- + +* Add :meth:`symtable.Function.get_cells` and :meth:`symtable.Symbol.is_cell` methods. + (Contributed by Yashp002 in :gh:`143504`.) sys @@ -700,6 +1512,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -721,29 +1546,188 @@ tarfile (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` and :cve:`2025-4435`.) * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` - now replace slashes by backslashes in symlink targets on Windows to prevent + now replace slashes with backslashes in symlink targets on Windows to prevent creation of corrupted links. (Contributed by Christoph Walcher in :gh:`57911`.) +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) + + timeit ------ + .. _whatsnew315-color-timeit: + +* The output of the :mod:`timeit` command-line interface is colored by default. + This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`146609`.) + * The command-line interface now colorizes error tracebacks by default. This can be controlled with :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +* Make the target time of :meth:`timeit.Timer.autorange` configurable + and add ``--target-time`` option to the command-line interface. + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) + + +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`.) + +* Added new methods :meth:`!pack_content`, :meth:`!place_content` and + :meth:`!grid_content` which use Tk commands with new names (introduced + in Tk 8.6) instead of :meth:`!*_slaves` methods which use Tk commands + with outdated names. + (Contributed by Serhiy Storchaka in :gh:`143754`.) + +* Added :class:`!Event` attributes :attr:`!user_data` for Tk virtual events + and :attr:`!detail` for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut``, + and ``ConfigureRequest`` events. + (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) + + +tokenize +-------- + + .. _whatsnew315-color-tokenize: + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + +.. _whatsnew315-tomllib-1-1-0: + +tomllib +------- + +* The :mod:`tomllib` module now supports TOML 1.1.0. + This is a backwards compatible update, meaning that all valid TOML 1.0.0 + documents are parsed the same way. + + The changes, according to the `official TOML changelog`_, are: + + - Allow newlines and trailing commas in inline tables. + + Previously an inline table had to be on a single line and couldn't end + with a trailing comma. This is now relaxed so that the following is valid: + + .. code-block:: toml + + tbl = { + key = "a string", + moar-tbl = { + key = 1, + }, + } + + - Add ``\xHH`` notation to basic strings for codepoints under 255, + and the ``\e`` escape for the escape character: + + .. code-block:: toml + + null = "null byte: \x00; letter a: \x61" + csi = "\e[" + + - Seconds in datetime and time values are now optional. + The following are now valid: + + .. code-block:: toml + + dt = 2010-02-03 14:15 + t = 14:15 + + (Contributed by Taneli Hukkinen in :gh:`142956`.) + +.. _official TOML changelog: https://github.com/toml-lang/toml/blob/main/CHANGELOG.md + types ------- +----- * Expose the write-through :func:`locals` proxy type - as :data:`types.FrameLocalsProxyType`. + as :class:`types.FrameLocalsProxyType`. This represents the type of the :attr:`frame.f_locals` attribute, as described in :pep:`667`. +typing +------ + +.. _whatsnew315-typeform: + +* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating + values that are themselves type expressions. + ``TypeForm[T]`` means "a type form object describing ``T`` (or a type + assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, + which allows explicit annotation of type-form values without changing + behavior. + + This helps libraries that accept user-provided type expressions + (for example ``int``, ``str | None``, :class:`~typing.TypedDict` + classes, or ``list[int]``) expose precise signatures: + + .. code-block:: python + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + (Contributed by Jelle Zijlstra in :gh:`145033`.) + +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred at runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + + unicodedata ----------- @@ -754,14 +1738,58 @@ unicodedata `Unicode Standard Annex #31 `_ identifier. (Contributed by Stan Ulbrych in :gh:`129117`.) +* Add the :func:`~unicodedata.iter_graphemes` + function to iterate over grapheme clusters according to rules defined in + `Unicode Standard Annex #29, "Unicode Text Segmentation" + `_. + Add :func:`~unicodedata.grapheme_cluster_break`, + :func:`~unicodedata.indic_conjunct_break` and + :func:`~unicodedata.extended_pictographic` functions to get the properties + of the character which are related to the above algorithm. + (Contributed by Serhiy Storchaka and Guillaume Sanchez in :gh:`74902`.) + +* Add :func:`~unicodedata.block` function to return the `Unicode block + `_ + assigned to a character. + (Contributed by Stan Ulbrych in :gh:`66802`.) + unittest -------- -* :func:`unittest.TestCase.assertLogs` will now accept a formatter +* :meth:`unittest.TestCase.assertLogs` will now accept a formatter to control how messages are formatted. (Contributed by Garry Cairns in :gh:`134567`.) +* :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex. + Nested context managers are now supported. + (Contributed by Serhiy Storchaka in :gh:`143231`.) + + +urllib.parse +------------ + +* Add the *missing_as_none* parameter to :func:`~urllib.parse.urlsplit`, + :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. + Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and + :func:`~urllib.parse.urlunparse` functions. + This allows distinguishing between empty and undefined URI components + and preserving empty components. + (Contributed by Serhiy Storchaka in :gh:`67041`.) + + +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using a ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + warnings -------- @@ -775,15 +1803,49 @@ warnings (Contributed by Serhiy Storchaka in :gh:`135801`.) -venv +wave ---- -* On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. - This means purelib and platlib of virtual environments no longer share the - same ``lib`` directory on platforms where :data:`sys.platlibdir` is not - equal to ``lib``. - (Contributed by Rui Xi in :gh:`133951`.) +* Added support for IEEE floating-point WAVE audio + (``WAVE_FORMAT_IEEE_FLOAT``) in :mod:`wave`. + +* Added :meth:`wave.Wave_read.getformat`, :meth:`wave.Wave_write.getformat`, + and :meth:`wave.Wave_write.setformat` for explicit frame format handling. + +* :meth:`wave.Wave_write.setparams` accepts both 7-item tuples including + ``format`` and 6-item tuples for backwards compatibility (defaulting to + ``WAVE_FORMAT_PCM``). + +* ``WAVE_FORMAT_IEEE_FLOAT`` output now includes a ``fact`` chunk, + as required for non-PCM WAVE formats. + +(Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) + + +webbrowser +---------- + +* On macOS, the new :class:`!webbrowser.MacOS` class opens URLs via + :program:`/usr/bin/open` instead of constructing and executing AppleScript + via :program:`osascript`. The default browser is detected from the + LaunchServices preferences file using :mod:`plistlib`, with + :class:`!com.apple.Safari` as the fallback on fresh installations. + For non-HTTP(S) URLs, :program:`open -b ` is used to route the + URL through a browser rather than the OS file handler, preventing + file injection attacks. + (Contributed by Jeff Lyon in :gh:`137586`.) + + +xml +--- + +* Add the :func:`xml.is_valid_name` function to check + whether a string can be used as an element or attribute name in XML. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + +* Add the :func:`xml.is_valid_text` function, which allows to check + whether a string can be used in the XML document. + (Contributed by Serhiy Storchaka in :gh:`139489`.) xml.parsers.expat @@ -819,6 +1881,29 @@ zlib Optimizations ============= +* ``mimalloc`` is now used as the default allocator for + raw memory allocations such as via :c:func:`PyMem_RawMalloc` + for better performance on :term:`free-threaded builds `. + (Contributed by Kumar Aditya in :gh:`144914`.) + + +base64 & binascii +----------------- + +* CPython's underlying base64 implementation now encodes 2x faster and decodes 3x + faster thanks to simple CPU pipelining optimizations. + (Contributed by Gregory P. Smith and Serhiy Storchaka in :gh:`143262`.) + +* Implementation for Ascii85, Base85, and Z85 encoding has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster and consumes + two orders of magnitude less memory. + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Implementation for Base32 has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster. + (Contributed by James Seo in :gh:`146192`.) + + csv --- @@ -826,8 +1911,134 @@ csv (Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.) +.. _whatsnew315-jit: + +Upgraded JIT compiler +--------------------- + +Results from the `pyperformance `__ +benchmark suite report +`8-9% `__ +geometric mean performance improvement for the JIT over the standard CPython +interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 +macOS, the JIT has a +`12-13% `__ +speedup over the :ref:`tail calling interpreter ` +with all optimizations enabled. The speedups for JIT +builds versus no JIT builds range from roughly 15% slowdown to over +100% speedup (ignoring the ``unpack_sequence`` microbenchmark) on +x86-64 Linux and AArch64 macOS systems. + +.. attention:: + These results are not yet final. + +The major upgrades to the JIT are: + +* LLVM 21 build-time dependency +* New tracing frontend +* Basic register allocation in the JIT +* More JIT optimizations +* GDB and GNU ``backtrace()`` unwinding support +* Better machine code generation + +.. rubric:: LLVM 21 build-time dependency + +The JIT compiler now uses LLVM 21 for build-time stencil generation. As +always, LLVM is only needed when building CPython with the JIT enabled; +end users running Python do not need LLVM installed. Instructions for +installing LLVM can be found in the `JIT compiler documentation +`__ +for all supported platforms. +(Contributed by Savannah Ostrowski in :gh:`140973`.) + +.. rubric:: A new tracing frontend + +The JIT compiler now supports significantly more bytecode operations and +control flow than in Python 3.14, enabling speedups on a wider variety of +code. For example, simple Python object creation is now understood by the +3.15 JIT compiler. Overloaded operations and generators are also partially +supported. This was made possible by an overhauled JIT tracing frontend +that records actual execution paths through code, rather than estimating +them as the previous implementation did. +(Contributed by Ken Jin in :gh:`139109`. Support for Windows added by +Mark Shannon in :gh:`141703`.) + +.. rubric:: Basic register allocation in the JIT + +A basic form of register allocation has been added to the JIT compiler's +optimizer. This allows the JIT compiler to avoid certain stack operations +altogether and instead operate on registers. This allows the JIT to produce +more efficient traces by avoiding reads and writes to memory. +(Contributed by Mark Shannon in :gh:`135379`.) + +.. rubric:: More JIT optimizations + +More `constant-propagation `__ +is now performed. This means when the JIT compiler detects that certain user +code results in constants, the code can be simplified by the JIT. +(Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.) + +:term:`Reference count`\ s are avoided whenever it is safe to do so. This generally +reduces the cost of most operations in Python. +(Contributed by Ken Jin, Donghee Na, Zheao Li, Hai Zhu, Savannah Ostrowski, +Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) + +By tracking unique references to objects, the JIT optimizer can now eliminate +reference count updates and perform in-place operations on ints and floats. +(Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) + +The JIT optimizer now supports significantly more operations than in 3.14. +(Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) + +.. rubric:: GDB and GNU ``backtrace()`` unwinding support + +The JIT compiler now publishes unwind information for generated machine code to the +GDB interface on supported Linux ELF platforms. When libgcc frame +registration is available, the same unwind information is also registered for +GNU ``backtrace()`` stack walkers. This allows native debuggers, crash +handlers, and diagnostic tools using these mechanisms to unwind through JIT +frames instead of stopping at generated code. +(Contributed by Diego Russo and Pablo Galindo Salgado in :gh:`146071` and +:gh:`149104`.) + +.. rubric:: Better machine code generation + +The JIT compiler's machine code generator now produces better machine code +for x86-64 and AArch64 macOS and Linux targets. In general, users should +experience lower memory usage for generated machine code and more efficient +machine code versus 3.14. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. +Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. +Additional optimizations for AArch64 contributed by Mark Shannon and +Diego Russo in :gh:`140683` and :gh:`142305`.) + +.. rubric:: Maintainability + +The JIT optimizer's operations have been simplified. +This was made possible by a refactoring of JIT data structures. +(Contributed by Zhongtian Zheng in :gh:`148211` and Hai Zhu in :gh:`143421`.) + + Removed -======= +======== + +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + ctypes ------ @@ -836,6 +2047,21 @@ ctypes which has been deprecated since Python 3.13. (Contributed by Bénédikt Tran in :gh:`133866`.) +* Change the :py:attr:`~ctypes._SimpleCData._type_` of + :class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and + :class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, + ``Zd`` and ``Zg`` for compatibility with numpy. + (Contributed by Victor Stinner in :gh:`148675`.) + + +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + glob ---- @@ -860,12 +2086,16 @@ importlib.resources * Removed deprecated ``package`` parameter from :func:`importlib.resources.files` function. - (Contributed by Semyon Moroz in :gh:`138044`) + (Contributed by Semyon Moroz in :gh:`138044`.) pathlib ------- +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + * Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Nikita Sobolev in :gh:`133875`.) @@ -902,9 +2132,21 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed deprecated in :pep:`626` since Python 3.12 + :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. + * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes (for example, ``Point = NamedTuple("Point", x=int, y=int)``) is no longer supported. @@ -917,31 +2159,7 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) -* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises - a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles - type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly inferred in runtime before. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* :class:`typing.ByteString` has been removed from ``typing.__all__``. - :class:`!typing.ByteString` has been deprecated since Python 3.9, and is - scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from typing import ByteString`` - * ``import typing; typing.ByteString``. - - ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` - was subclassed or used as the second argument to :func:`isinstance` or - :func:`issubclass`, but warnings were not previously emitted if it was merely - imported or accessed from the :mod:`!typing` module. - -* Deprecated :func:`!typing.no_type_check_decorator` has been removed. +* Deprecated :deco:`!typing.no_type_check_decorator` has been removed. (Contributed by Nikita Sobolev in :gh:`133601`.) @@ -968,6 +2186,22 @@ Deprecated New deprecations ---------------- +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + +* :mod:`base64`: + + * Accepting the ``+`` and ``/`` characters with an alternative alphabet in + :func:`~base64.b64decode` and :func:`~base64.urlsafe_b64decode` is now + deprecated. + In future Python versions they will be errors in the strict mode and + discarded in the non-strict mode. + (Contributed by Serhiy Storchaka in :gh:`125346`.) + * CLI: * Deprecate :option:`-b` and :option:`!-bb` command-line options @@ -978,6 +2212,21 @@ New deprecations (Contributed by Nikita Sobolev in :gh:`136355`.) +* :mod:`collections.abc` + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. + + * :mod:`hashlib`: * In hash function constructors such as :func:`~hashlib.new` or the @@ -992,16 +2241,90 @@ New deprecations (Contributed by Bénédikt Tran in :gh:`134978`.) + +* :mod:`http.cookies`: + + * :meth:`Morsel.js_output ` and + :meth:`BaseCookie.js_output ` are + deprecated and will be removed in Python 3.19. Use + :meth:`Morsel.output ` or + :meth:`BaseCookie.output ` instead. + (Contributed by kishorhange111 in :gh:`148849`.) + + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + + +* :mod:`re`: + + * :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) + + +* :mod:`struct`: + + * Calling ``Struct.__new__()`` without a required argument is now + deprecated and will be removed in Python 3.20. Calling the + :meth:`~object.__init__` method on an initialized :class:`~struct.Struct` + object is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + + * Using ``'F'`` and ``'D'`` type codes now are :term:`soft deprecated` + in favor of two-letter forms ``'Zf'`` and ``'Zd'``. + (Contributed by Sergey B Kirpichev in :gh:`121249`.) + + +* :mod:`typing`: + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was + merely imported or accessed from the :mod:`!typing` module. + + +* :mod:`webbrowser`: + + * :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS` and scheduled for removal in Python 3.17. + (Contributed by Jeff Lyon in :gh:`137586`.) + * ``__version__`` - * The ``__version__`` attribute has been deprecated in these standard library - modules and will be removed in Python 3.20. - Use :py:data:`sys.version_info` instead. + * The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. - :mod:`argparse` - :mod:`csv` + - :mod:`ctypes` - :mod:`!ctypes.macholib` - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` - :mod:`imaplib` - :mod:`ipaddress` - :mod:`json` @@ -1012,8 +2335,13 @@ New deprecations - :mod:`re` - :mod:`socketserver` - :mod:`tabnanny` + - :mod:`tarfile` - :mod:`tkinter.font` - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` - :mod:`zlib` (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) @@ -1024,12 +2352,18 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + .. include:: ../deprecations/pending-removal-in-3.19.rst .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst +.. include:: ../deprecations/soft-deprecations.rst + C API changes ============= @@ -1037,6 +2371,25 @@ C API changes New features ------------ +* Add :c:func:`PyArg_ParseArray` and :c:func:`PyArg_ParseArrayAndKeywords` + functions to parse arguments of functions using the :c:macro:`METH_FASTCALL` + calling convention. + (Contributed by Victor Stinner in :gh:`144175`.) + +* Add the following functions for the new :class:`frozendict` type: + + * :c:func:`PyAnyDict_Check` + * :c:func:`PyAnyDict_CheckExact` + * :c:func:`PyFrozenDict_Check` + * :c:func:`PyFrozenDict_CheckExact` + * :c:func:`PyFrozenDict_New` + + (Contributed by Victor Stinner in :gh:`141510`.) + +* Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C + API. + (Contributed by Victor Stinner in :gh:`146063`.) + * Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` functions as replacements for :c:func:`PySys_GetObject`. @@ -1047,11 +2400,11 @@ New features (Contributed by Petr Viktorin in :gh:`131510`.) * Add API for checking an extension module's ABI compatibility: - :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, :c:macro:`PyABIInfo_VAR` - and :c:data:`Py_mod_abi`. + :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, + and :c:macro:`PyABIInfo_VAR`. (Contributed by Petr Viktorin in :gh:`137210`.) -.. _whatsnew315-pep782: +.. _whatsnew315-pybyteswriter: * Implement :pep:`782`, the :ref:`PyBytesWriter API `. Add functions: @@ -1071,31 +2424,107 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* :c:type:`PyCriticalSection` and related functions are added to the Stable + ABI. + + (Contributed in :gh:`149227`.) + +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a new module export hook, + :c:func:`PyModExport_* `. + + (Contributed by Petr Viktorin in :pep:`793` and :gh:`140550`.) + +* Add functions that are guaranteed to be safe for use in + :c:member:`~PyTypeObject.tp_traverse` handlers: + :c:func:`PyObject_GetTypeData_DuringGC`, + :c:func:`PyObject_GetItemData_DuringGC`, + :c:func:`PyType_GetModuleState_DuringGC`, + :c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, + :c:func:`PyType_GetBaseByToken_DuringGC`, + :c:func:`PyType_GetModule_DuringGC`, + :c:func:`PyType_GetModuleByToken_DuringGC`. + (Contributed by Petr Viktorin in :gh:`145925`.) + +* Add :c:func:`PyObject_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + +* Implement :pep:`820`: ``PySlot`` -- Unified slot system for the C API. + See :ref:`capi-slots` for documentation. + + This adds: + + * The :c:type:`PySlot` struct; + * the :c:func:`PyType_FromSlots` function; + * new slot IDs: :c:macro:`Py_slot_end`, :c:macro:`Py_slot_invalid`; + :c:macro:`Py_slot_subslots`, :c:macro:`Py_tp_slots`, + :c:macro:`Py_mod_slots`; + :c:macro:`Py_tp_name`, :c:macro:`Py_tp_basicsize`, + :c:macro:`Py_tp_extra_basicsize`, :c:macro:`Py_tp_itemsize`, + :c:macro:`Py_tp_flags`, :c:macro:`Py_tp_metaclass`, + :c:macro:`Py_tp_module`; + * convenience macros: :c:macro:`PySlot_DATA`, :c:macro:`PySlot_FUNC`, + :c:macro:`PySlot_SIZE`, :c:macro:`PySlot_INT64`, :c:macro:`PySlot_UINT64`, + :c:macro:`PySlot_STATIC_DATA`, :c:macro:`PySlot_END`, + :c:macro:`PySlot_PTR`, :c:macro:`PySlot_PTR_STATIC`. + + The :c:func:`PyModule_FromSlotsAndSpec` function and + ``PyModExport`` :ref:`module export hook ` also + use the new :c:type:`!PySlot` struct. + + These following functions are :term:`soft deprecated`: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + * :c:func:`PyType_FromMetaclass` + * :c:func:`PyModule_FromDefAndSpec` + * :c:func:`PyModule_FromDefAndSpec2` + * :c:func:`PyModule_ExecDef` + + + The slots :c:macro:`Py_tp_bases` and :c:macro:`Py_tp_base` are now + equivalent: they can be set either to a single type or a tuple of types. + The :c:macro:`Py_tp_bases` slot is preferred; the other is ignored if both + are specified. + + (Contributed by Petr Viktorin in :gh:`149044`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add :c:func:`PyUnstable_SetImmortal` C-API function to mark objects as :term:`immortal`. + (Contributed by Kumar Aditya in :gh:`143300`.) + +* Restore private provisional ``_Py_InitializeMain()`` function removed in + Python 3.14. + (Contributed by Victor Stinner in :gh:`142417`.) + +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) Changed C APIs -------------- * If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` or :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flag is set then :c:macro:`Py_TPFLAGS_HAVE_GC` must be set too. - (Contributed by Sergey Miryanov in :gh:`134786`) - - -Porting to Python 3.15 ----------------------- + (Contributed by Sergey Miryanov in :gh:`134786`.) -* Private functions promoted to public C APIs: - - The |pythoncapi_compat_project| can be used to get most of these new - functions on Python 3.14 and older. +* :c:macro:`PyDateTime_IMPORT` is now thread safe. Code that directly checks ``PyDateTimeAPI`` + for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. + (Contributed by Kumar Aditya in :gh:`141563`.) Removed C APIs @@ -1106,12 +2535,12 @@ Removed C APIs * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: - Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Decode` instead; note that some codecs (for example, "base64") may return a type other than :class:`str`, such as :class:`bytes`. * :c:func:`!PyUnicode_AsEncodedObject`: Use :c:func:`PyCodec_Encode` instead. * :c:func:`!PyUnicode_AsEncodedUnicode`: - Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Encode` instead; note that some codecs (for example, "base64") may return a type other than :class:`bytes`, such as :class:`str`. (Contributed by Stan Ulbrych in :gh:`133612`.) @@ -1171,6 +2600,16 @@ on Python 3.13 and older. Deprecated C APIs ----------------- +* Deprecate :pep:`456` support for providing an external definition + of the string hashing scheme. Removal is scheduled for Python 3.19. + + Previously, embedders could define :c:macro:`Py_HASH_ALGORITHM` to be + ``Py_HASH_EXTERNAL`` to indicate that the hashing scheme was provided + externally but this feature was undocumented, untested and most likely + unused. + + (Contributed by Bénédikt Tran in :gh:`141226`.) + * For unsigned integer formats in :c:func:`PyArg_ParseTuple`, accepting Python integers with value that is larger than the maximal value for the C type or less than the minimal value for the corresponding @@ -1182,6 +2621,13 @@ Deprecated C APIs use the :c:type:`PyBytesWriter` API instead. (Contributed by Victor Stinner in :gh:`129813`.) +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + * Deprecate :c:member:`~PyComplexObject.cval` field of the :c:type:`PyComplexObject` type. Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` @@ -1202,6 +2648,24 @@ Deprecated C APIs use the C11 standard ```` :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`141004`.) +* The following macros are :term:`soft deprecated`: + + - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. + - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. + - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, + :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, + :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: + Use C99 types/limits. + - :c:macro:`Py_UNICODE_SIZE`: Use ``sizeof(wchar_t)`` directly. + - :c:macro:`Py_VA_COPY`: Use ``va_copy`` directly. + + The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, + is :term:`soft deprecated` instead. + + (Contributed by Petr Viktorin in :gh:`146175`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) @@ -1214,10 +2678,58 @@ Build changes ============= * Removed implicit fallback to the bundled copy of the ``libmpdec`` library. - Now this should be explicitly enabled with :option:`--with-system-libmpdec` + Now this should be explicitly enabled with :option:`!--with-system-libmpdec` set to ``no`` or with :option:`!--without-system-libmpdec`. (Contributed by Sergey B Kirpichev in :gh:`115119`.) +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + +* The new configure option :option:`--with-pymalloc-hugepages` enables huge + page support for :ref:`pymalloc ` arenas. When enabled, arena size + increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + On Windows, use ``build.bat --pymalloc-hugepages``. + At runtime, huge pages must be explicitly enabled by setting the + :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable to ``1``. + +* Annotating anonymous mmap usage is now supported if Linux kernel supports + :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + Annotations are visible in ``/proc//maps`` if the kernel supports the feature + and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + (Contributed by Donghee Na in :gh:`141770`.) + +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + + Authors of C extensions and native libraries built with custom build + systems should ensure the unwind chain is intact. + This is usually done by adding ``-fno-omit-frame-pointer`` and + similar flags to ``CFLAGS``. See :option:`--without-frame-pointers` + documentation for the specific flags Python uses. + + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + +.. _whatsnew315-windows-tail-calling-interpreter: + +* 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new + :ref:`tail-calling interpreter `. + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries + to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18, + which the official Windows 64-bit binaries on python.org__ now use. + (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. + Special thanks to Steve Dower, and the MSVC team including Hulon Jenkins.) + + __ https://www.python.org/downloads/windows/ + Porting to Python 3.15 ====================== @@ -1225,7 +2737,7 @@ Porting to Python 3.15 This section lists previously described changes and other bugfixes that may require changes to your code. -* :class:`sqlite3.Connection` APIs has been cleaned up. +* :class:`sqlite3.Connection` APIs have been cleaned up. * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` @@ -1242,11 +2754,32 @@ that may require changes to your code. :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. (Contributed by Serhiy Storchaka in :gh:`137044`.) -* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the +* :meth:`mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. -* Resource warning is now emitted for unclosed +* A resource warning is now emitted for an unclosed :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. Use its :meth:`!close` method or the :func:`contextlib.closing` context manager to close it. (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) + +* If a short option and a single-dash long option are passed to + :meth:`argparse.ArgumentParser.add_argument`, *dest* is now inferred from + the single-dash long option. For example, in ``add_argument('-f', '-foo')``, + *dest* is now ``'foo'`` instead of ``'f'``. + Pass an explicit *dest* argument to preserve the old behavior. + (Contributed by Serhiy Storchaka in :gh:`138697`.) + +* Padding of input no longer required in :func:`base64.urlsafe_b64decode`. + Pass a new argument ``padded=True`` or use :func:`base64.b64decode` + with argument ``altchars=b'-_'`` (this works with older Python versions) + to make padding required. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Since :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex, your tests may start leaking + some warnings that were previously masked. + Use warning filters to silence them or additional :meth:`!assertWarns*` + to catch and check them. + (Contributed by Serhiy Storchaka in :gh:`143231`.) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst new file mode 100644 index 000000000000000..e8c75d2f571061a --- /dev/null +++ b/Doc/whatsnew/3.16.rst @@ -0,0 +1,667 @@ + +**************************** + What's new in Python 3.16 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.16, compared to 3.15. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.16 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary --- release highlights +============================== + +.. This section singles out the most important changes in Python 3.16. + Brevity is key. + + +.. PEP-sized items next. + + + +New features +============ + + + +Other language changes +====================== + + + +New modules +=========== + +* None yet. + + +Improved modules +================ + +curses +------ + +* Add support for multiple terminals to the :mod:`curses` module: + the new functions :func:`curses.newterm`, :func:`curses.set_term` + and :func:`curses.new_prescr`, + the corresponding :ref:`screen ` object, + and the :meth:`window.use() ` method. + (Contributed by Serhiy Storchaka in :gh:`90092`.) + +* The :mod:`curses` character-cell window methods now accept a full character + cell --- a spacing character optionally followed by combining characters --- + in addition to a single integer or byte character. This affects + :meth:`~curses.window.addch`, :meth:`~curses.window.bkgd`, + :meth:`~curses.window.bkgdset`, :meth:`~curses.window.border`, + :meth:`~curses.window.box`, :meth:`~curses.window.echochar`, + :meth:`~curses.window.hline`, :meth:`~curses.window.insch` and + :meth:`~curses.window.vline`. + Also add the wide-character read methods :meth:`~curses.window.get_wstr` and + :meth:`~curses.window.in_wstr`, the counterparts of + :meth:`~curses.window.getstr` and :meth:`~curses.window.instr` that return a + :class:`str` rather than :class:`bytes`, + and the module functions :func:`curses.erasewchar`, :func:`curses.killwchar` + and :func:`curses.wunctrl`, the wide-character counterparts of + :func:`curses.erasechar`, :func:`curses.killchar` and :func:`curses.unctrl`. + On a narrow (non-ncursesw) build the character cell holds a single character + without combining marks, representable as one byte in the window's encoding, + and :meth:`~curses.window.in_wstr` returns its decoded text. + (Contributed by Serhiy Storchaka in :gh:`151757`.) + +* The wide-character :mod:`curses` functions and methods + :meth:`~curses.window.get_wch`, :meth:`~curses.window.get_wstr`, + :func:`curses.unget_wch`, :func:`curses.erasewchar`, + :func:`curses.killwchar` and :func:`curses.wunctrl` now also work when Python + is not built against a wide-character-aware curses library, on an 8-bit + locale, where each character is a single byte in the relevant encoding. + :func:`curses.ungetch` now also accepts a one-character string, like + :func:`curses.unget_wch`; on a wide-character build it can be any character + (previously a multibyte character raised :exc:`OverflowError`). + (Contributed by Serhiy Storchaka in :gh:`152470`.) + +* Add :func:`curses.nofilter`, which undoes the effect of :func:`curses.filter`. + (Contributed by Serhiy Storchaka in :gh:`151744`.) + +* Add the :mod:`curses` functions :func:`curses.alloc_pair`, + :func:`curses.find_pair`, :func:`curses.free_pair` and + :func:`curses.reset_color_pairs` for dynamic color-pair management, + available when built against a wide-character ncurses with extended-color + support. + (Contributed by Serhiy Storchaka in :gh:`151774`.) + +* Add :mod:`curses` functions and window methods that report state which could + previously only be set, such as :meth:`curses.window.is_keypad`, + :meth:`curses.window.getparent` and :func:`curses.is_cbreak`, + available when built against an ncurses with ``NCURSES_EXT_FUNCS``. + (Contributed by Serhiy Storchaka in :gh:`151776`.) + +* Add the :mod:`curses` window methods :meth:`~curses.window.attr_get`, + :meth:`~curses.window.attr_set`, :meth:`~curses.window.attr_on`, + :meth:`~curses.window.attr_off` and :meth:`~curses.window.color_set`, which + pass the color pair as a separate argument instead of packing it into the + attribute value, and the corresponding ``WA_*`` attribute constants. + (Contributed by Serhiy Storchaka in :gh:`152219`.) + +* Add the :class:`curses.complexchar` type, representing a styled + character cell (its text, attributes and color pair), and the window + methods :meth:`~curses.window.in_wch` and :meth:`~curses.window.getbkgrnd` + that return one --- the counterparts of + :meth:`~curses.window.inch` and :meth:`~curses.window.getbkgd`. The + character-cell methods, such as :meth:`~curses.window.addch` and + :meth:`~curses.window.border`, now also accept a + :class:`~curses.complexchar`. These work whether or not Python was built + against a wide-character-aware curses library; on a narrow build a cell holds a + single character representable as one byte in the window's encoding (so only + 8-bit locales are supported). + (Contributed by Serhiy Storchaka in :gh:`152233`.) + +* Add the :class:`curses.complexstr` type, an immutable run of styled cells + (the string counterpart of :class:`~curses.complexchar`), and the window + method :meth:`~curses.window.in_wchstr` that returns one. The string-cell + methods :meth:`~curses.window.addstr`, :meth:`~curses.window.addnstr`, + :meth:`~curses.window.insstr` and :meth:`~curses.window.insnstr` now also + accept a :class:`~curses.complexstr`. Like :class:`~curses.complexchar`, it + works whether or not Python was built against a wide-character-aware curses + library. + (Contributed by Serhiy Storchaka in :gh:`152233`.) + +* Add the :mod:`curses` window method :meth:`~curses.window.dupwin`, which + returns a new window that is an independent duplicate of an existing one. + (Contributed by Serhiy Storchaka in :gh:`152258`.) + +* Add the :mod:`curses` functions :func:`~curses.scr_dump`, + :func:`~curses.scr_restore`, :func:`~curses.scr_init` and + :func:`~curses.scr_set`, which dump the whole screen to a file and restore it. + (Contributed by Serhiy Storchaka in :gh:`152260`.) + +* Add the soft-label-key functions to the :mod:`curses` module, which manage a + row of labels along the bottom line of the screen: + :func:`~curses.slk_init`, :func:`~curses.slk_set`, :func:`~curses.slk_label`, + :func:`~curses.slk_refresh`, :func:`~curses.slk_noutrefresh`, + :func:`~curses.slk_clear`, :func:`~curses.slk_restore`, + :func:`~curses.slk_touch`, the attribute functions + :func:`~curses.slk_attron`, :func:`~curses.slk_attroff`, + :func:`~curses.slk_attrset`, :func:`~curses.slk_attr`, + :func:`~curses.slk_attr_on`, :func:`~curses.slk_attr_off`, + :func:`~curses.slk_attr_set`, and :func:`~curses.slk_color`. + (Contributed by Serhiy Storchaka in :gh:`152263`.) + +* Add the :func:`curses.term_attrs` function, which returns the supported + video attributes as :ref:`WA_* ` values, the + counterpart of :func:`curses.termattrs`. + (Contributed by Serhiy Storchaka in :gh:`152332`.) + +* Add the :mod:`curses` key-management functions :func:`~curses.define_key`, + :func:`~curses.key_defined` and :func:`~curses.keyok`, available when built + against an ncurses with ``NCURSES_EXT_FUNCS``. + (Contributed by Serhiy Storchaka in :gh:`152334`.) + +* Add the :func:`curses.has_mouse` function and the + :meth:`curses.window.mouse_trafo` method, completing the :mod:`curses` + mouse interface. + (Contributed by Serhiy Storchaka in :gh:`152325`.) + +* :class:`curses.textpad.Textbox` now supports entering and reading back the + full Unicode range, including combining characters, when curses is built with + wide-character support. + (Contributed by Serhiy Storchaka in :gh:`133031`.) + +gzip +---- + +* :func:`gzip.open` now accepts an optional argument ``mtime`` + which is passed on to the constructor of the :class:`~gzip.GzipFile` class. + (Contributed by Marin Misur in :gh:`91372`.) + +io +-- + +* Add :meth:`io.BytesIO.peek` method to read without advancing position. + (Contributed by Marcel Martin in :gh:`90533`.) + + +imaplib +------- + +* Add the :meth:`~imaplib.IMAP4.id` method, + a wrapper for the ``ID`` command (:rfc:`2971`). + (Contributed by Serhiy Storchaka in :gh:`98092`.) + +* Add the :meth:`~imaplib.IMAP4.move` method, + a wrapper for the ``MOVE`` command (:rfc:`6851`). + (Contributed by Serhiy Storchaka in :gh:`77508`.) + +* Add the :meth:`~imaplib.IMAP4.login_plain` method, which authenticates + using the ``PLAIN`` SASL mechanism (:rfc:`4616`) and supports non-ASCII + user names and passwords. + (Contributed by Przemysław Buczkowski and Serhiy Storchaka in :gh:`89869`.) + + +logging +------- + +* :class:`~logging.handlers.TimedRotatingFileHandler` now uses the creation + time instead of the last modification time of an existing log file as + the basis for the first rotation after handler creation, if supported by + the OS and file system. + This allows it to be used in short-running programs that start and end + before the rotation interval expires. + (Contributed by Iván Márton and Serhiy Storchaka in :gh:`84649`.) + + +lzma +---- + +* Add support of new BCJ filters ARM64 and RISC-V via + :const:`!lzma.FILTER_ARM64` and :const:`!lzma.FILTER_RISCV`. Note that the + new filters will work only if the runtime library supports them. ARM64 filter + requires ``lzma`` 5.4.0 or newer while RISC-V requires 5.6.0 or newer. + (Contributed by Chien Wong in :gh:`115988`.) + + +math +---- + +* Added trigonometric functions that work in units of half turns, rather than + radians. The new functions :func:`math.acospi`, :func:`math.asinpi`, + :func:`math.atanpi`, and :func:`math.atan2pi` return half-turn angles. The + new functions :func:`math.cospi`, :func:`math.sinpi`, and :func:`math.tanpi` + take half-turn angle arguments. These functions are recommended by IEEE + 754-2019 and standardized in C23. + (Contributed by Jeff Epler in :gh:`150534`.) + + +os +-- + +* Add :func:`os.pidfd_getfd` for duplicating a file descriptor from another + process via a pidfd. Available on Linux 5.6+. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.) + + +re +-- + +* :mod:`re` now supports set operations and nested sets in character classes, + as described in `Unicode Technical Standard #18 + `__: set difference (``[A--B]``), + intersection (``[A&&B]``) and union (``[A||B]``), where an operand may be a + nested set written in square brackets. For example, ``[a-z--[aeiou]]`` + matches an ASCII lowercase consonant. + (Contributed by Serhiy Storchaka in :gh:`152100`.) + +* Regular expressions now support Unicode property escapes ``\p{...}`` and + ``\P{...}``, which match a character by a Unicode property -- for example + ``\p{Lu}`` (an uppercase letter), ``\p{Cased}`` or ``\p{ASCII}``. See + :ref:`the regular expression syntax ` for the supported + properties. + (Contributed by Serhiy Storchaka in :gh:`95555`.) + + +shlex +----- + +* Add keyword-only parameter *force* to :func:`shlex.quote` to force quoting + a string, even if it is already safe for a shell without being quoted. + (Contributed by Jay Berry in :gh:`148846`.) + + +tkinter +------- + +* Added many :class:`tkinter.ttk.Treeview` methods wrapping the enhanced + ``ttk::treeview`` widget commands from Tk 9.1, such as + :meth:`~tkinter.ttk.Treeview.sort`, :meth:`~tkinter.ttk.Treeview.search`, + :meth:`~tkinter.ttk.Treeview.expand`, :meth:`~tkinter.ttk.Treeview.collapse`, + :meth:`~tkinter.ttk.Treeview.hide`, :meth:`~tkinter.ttk.Treeview.unhide`, and + methods for cell focus, selection and tagging. The + :meth:`~tkinter.ttk.Treeview.expand` and :meth:`~tkinter.ttk.Treeview.collapse` + methods (without recursion) also work on Tk older than 9.1. + (Contributed by Serhiy Storchaka in :gh:`151910`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.edit_canundo` + and :meth:`~tkinter.Text.edit_canredo` which return whether an undo or redo + is possible. + (Contributed by Serhiy Storchaka in :gh:`151674`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.sync` and + :meth:`~tkinter.Text.pendingsync` which control and report the + synchronization of the displayed view with the underlying text. + (Contributed by Serhiy Storchaka in :gh:`151675`.) + +* Added the :meth:`ttk.Style.theme_styles + ` method which returns the list of styles + defined in a theme. + (Contributed by Serhiy Storchaka in :gh:`151920`.) + +* Added new :class:`!tkinter.Canvas` methods :meth:`~tkinter.Canvas.rchars` + which replaces the text or coordinates of canvas items, and + :meth:`~tkinter.Canvas.rotate` which rotates the coordinates of canvas items. + (Contributed by Serhiy Storchaka in :gh:`151876`.) + +* Added a :meth:`!validate` method to the :class:`!tkinter.Entry` and + :class:`!tkinter.Spinbox` widgets, which forces an evaluation of the + validation command. + (Contributed by Serhiy Storchaka in :gh:`151878`.) + +* Added the :meth:`tkinter.Menu.postcascade` method, and the + :meth:`~tkinter.Misc.tk_scaling` and :meth:`~tkinter.Misc.tk_inactive` + methods which respectively query or set the display scaling factor and + report the user idle time. + (Contributed by Serhiy Storchaka in :gh:`151881`.) + +* Added new window-management methods :meth:`~tkinter.Misc.winfo_isdark` + (dark mode detection), :meth:`~tkinter.Wm.wm_iconbadge` (application icon + badge) and :meth:`~tkinter.Wm.wm_stackorder` (toplevel stacking order). + (Contributed by Serhiy Storchaka in :gh:`151874`.) + +* Added the :meth:`~tkinter.Misc.tk_appname`, + :meth:`~tkinter.Misc.tk_useinputmethods` and :meth:`~tkinter.Misc.tk_caret` + methods, exposing the application send name, the X Input Methods state and + the input method caret location. + (Contributed by Serhiy Storchaka in :gh:`151886`.) + +* Added support for more options in :class:`!tkinter.PhotoImage` methods: the + *format* parameter of :meth:`~tkinter.PhotoImage.put`, the *metadata* + parameter of :meth:`~tkinter.PhotoImage.put`, :meth:`~tkinter.PhotoImage.read`, + :meth:`~tkinter.PhotoImage.write` and :meth:`~tkinter.PhotoImage.data`, and + the *withalpha* parameter of :meth:`~tkinter.PhotoImage.get`. + (Contributed by Serhiy Storchaka in :gh:`151890`.) + +* Added the :meth:`~tkinter.PhotoImage.redither` method which recalculates the + dithered image when its data was supplied in pieces. + (Contributed by Serhiy Storchaka in :gh:`151888`.) + +* :class:`tkinter.OptionMenu` now accepts arbitrary :class:`!tkinter.Menubutton` + options as keyword arguments, which can also override its default appearance. + (Contributed by Serhiy Storchaka in :gh:`101284`.) + +* The :mod:`tkinter.simpledialog` dialogs were modernized to match the look + and feel of the native Tk dialogs. + :class:`!tkinter.simpledialog.SimpleDialog` and the + :func:`~tkinter.simpledialog.askinteger`, + :func:`~tkinter.simpledialog.askfloat` and + :func:`~tkinter.simpledialog.askstring` dialogs are now built from the themed + :mod:`tkinter.ttk` widgets instead of the classic :mod:`tkinter` widgets; + the :class:`!tkinter.simpledialog.Dialog` base class still defaults to the + classic widgets for compatibility. Both :class:`!Dialog` and + :class:`!SimpleDialog` gained a *use_ttk* parameter that selects between the + classic Tk widgets and the themed ttk widgets. :class:`!SimpleDialog` also + gained *bitmap* and + *detail* parameters, draws the standard icons with themed images in the + ttk version, and accepts mappings of button options as *buttons* entries. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +* The :class:`!tkinter.filedialog.FileDialog` dialog and its + :class:`!tkinter.filedialog.LoadFileDialog` and + :class:`!tkinter.filedialog.SaveFileDialog` subclasses are now built from the + themed :mod:`tkinter.ttk` widgets by default instead of the classic + :mod:`tkinter` widgets, and gained a *use_ttk* parameter to select between + them. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +xml +--- + +* Add support for multiple multi-byte encodings in the :mod:`XML parser + `: "cp932", "cp949", "cp950", "Big5", "EUC-JP", + "GB2312", "GBK", "johab", and "Shift_JIS". + Add partial support (only BMP characters) for multi-byte encodings + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", "Shift_JIS-2004", + "Shift_JISX0213", "utf-8-sig" and non-standard aliases like "UTF8" + (without hyphen). + The parser now raises :exc:`ValueError` for known unsupported + multi-byte encodings such as "ISO-2022-JP" or "raw-unicode-escape" + instead of failing later, when encountering non-ASCII data. + (Contributed by Serhiy Storchaka in :gh:`62259`.) + +zipfile +------- + +* Add :meth:`ZipFile.remove() ` to remove a member + from an archive's central directory, and + :meth:`ZipFile.repack() ` to reclaim the space used + by the local file entries of removed members. + (Contributed by Danny Lin in :gh:`51067`.) + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +re +-- + +* Character class escapes (``\d``, ``\D``, ``\s``, ``\S``, ``\w`` and ``\W``) + outside a character set, and character sets containing a single such escape + (such as ``[\d]`` or ``[^\s]``), are now compiled to a single ``CATEGORY`` + opcode instead of being wrapped in an ``IN`` block. This speeds up matching + of patterns such as ``\d+`` and reduces the size of the compiled byte code. + (Contributed by Serhiy Storchaka in :gh:`152033` and Pieter Eendebak in + :gh:`152056`.) + +module_name +----------- + +* TODO + + + +Removed +======= + +annotationlib +------------- + +* The :meth:`!annotationlib.ForwardRef._evaluate` method + which has been deprecated since Python 3.14. + Use :meth:`annotationlib.ForwardRef.evaluate` + or :func:`typing.evaluate_forward_ref` instead. + +array +----- + +* The ``'u'`` format code (:c:type:`wchar_t`) which has been deprecated in + documentation since Python 3.3 and at runtime since Python 3.13. + Use ``'w'`` format code instead (:c:type:`Py_UCS4`, always 4 bytes). + +asyncio +------- + +* The :func:`!asyncio.iscoroutinefunction` + which has been deprecated since Python 3.14. + Use :func:`inspect.iscoroutinefunction` instead. + +* The event loop policy system, including the + :class:`!asyncio.AbstractEventLoopPolicy`, + :class:`!asyncio.DefaultEventLoopPolicy`, + :class:`!asyncio.WindowsSelectorEventLoopPolicy` and + :class:`!asyncio.WindowsProactorEventLoopPolicy` classes and the + :func:`!asyncio.get_event_loop_policy` and + :func:`!asyncio.set_event_loop_policy` functions, + which have been deprecated since Python 3.14. + Use :func:`asyncio.run` or :class:`asyncio.Runner` with a custom + *loop_factory* instead. + +functools +--------- + +* Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +logging +------- + +* Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. + +mimetypes +--------- + +* Valid extensions start with a '.' or are empty for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions now raise a :exc:`ValueError`. + +shutil +------ + +* The :exc:`!ExecError` exception which has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4. + (Contributed by Stan Ulbrych in :gh:`149567`.) + +symtable +-------- + +* The :meth:`!symtable.Class.get_methods` method + which has been deprecated since Python 3.14. + +sys +--- + +* The :func:`!_enablelegacywindowsfsencoding` function + which has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Stan Ulbrych in :gh:`149595`.) + +sysconfig +--------- + +* The :func:`!sysconfig.expand_makefile_vars` function + which has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + (Contributed by Stan Ulbrych in :gh:`149499`.) + +tarfile +------- + +* The undocumented and unused :attr:`!tarfile.TarFile.tarfile` attribute + has been deprecated since Python 3.13. + +.. Add removals above alphabetically, not here at the end. + + +Deprecated +========== + +New deprecations +---------------- + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, deprecated since Python 3.9, are no longer + imported by ``from ast import *`` and issue a deprecation warning on + use. The classes are slated for removal in Python 3.21. These types are not + generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` objects, deprecated since Python + 3.9, now issues a deprecation warning on use. This property is slated for + removal in 3.21. Use ``ast.Tuple.elts`` instead. + +* :mod:`struct`: + + * Soft-deprecated since Python 3.15, using ``'F'`` and ``'D'`` type codes are now + deprecated. These codes will be removed in Python 3.21. Use instead + two-letter forms ``'Zf'`` and ``'Zd'``. + (Contributed by Sergey B Kirpichev in :gh:`121249`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +Porting to Python 3.16 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +* In :mod:`tkinter`, the *name* parameter of the + :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and + :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of + :meth:`!setvar` are now required. Calling these methods without + them, which formerly defaulted to ``'PY_VAR'`` and ``'1'``, now raises + :exc:`TypeError`. + (Contributed by Serhiy Storchaka in :gh:`152587`.) + + +Build changes +============= + +* Remove the bundled copy of the libmpdec_ decimal library from the CPython source tree + to simplify maintenance and updates. The :mod:`decimal` module will now + unconditionally use the system's libmpdec decimal library. Also remove the + now unused :option:`!--with-system-libmpdec` :program:`configure` flag. + This change has no impact on binary releases of Python, which have been + built against a separate copy of libmpdec for the past several releases. + + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + + .. _libmpdec: https://www.bytereef.org/mpdecimal/ + +* Add a :option:`--with-build-details-suffix` configure flag to allow + Linux distributions that co-install multiple versions of Python in the + same tree to avoid ``build-details.json`` clashes. + + (Contributed by Stefano Rivera in :gh:`131372`.) + + +C API changes +============= + +New features +------------ + +* TODO + +Porting to Python 3.16 +---------------------- + +* TODO + +Deprecated C APIs +----------------- + +* :c:func:`PyGen_New`, :c:func:`PyGen_NewWithQualName`, :c:func:`PyCoro_New`, + and :c:func:`PyAsyncGen_New` are deprecated. + They are scheduled for removal in 3.18. + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.19.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.20.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +Removed C APIs +-------------- diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 47c4d9acbc870e5..c61eb659ccc0f62 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -312,7 +312,7 @@ cluttering source directories, the *pyc* files are now collected in a Aside from the filenames and target directories, the new scheme has a few aspects that are visible to the programmer: -* Imported modules now have a :attr:`~module.__cached__` attribute which stores +* Imported modules now have a ``__cached__`` attribute which stores the name of the actual file that was imported: >>> import collections @@ -756,7 +756,7 @@ functools --------- * The :mod:`functools` module includes a new decorator for caching function - calls. :func:`functools.lru_cache` can save repeated queries to an external + calls. :deco:`functools.lru_cache` can save repeated queries to an external resource whenever the results are expected to be the same. For example, adding a caching decorator to a database query function can save @@ -789,7 +789,7 @@ functools `_\, :issue:`10586`, and :issue:`10593`.) -* The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute +* The :deco:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to be introspected. It also copies :attr:`~function.__annotations__` if defined. And now it also gracefully skips over missing attributes such as @@ -805,11 +805,11 @@ functools :issue:`8814`.) * To help write classes with rich comparison methods, a new decorator - :func:`functools.total_ordering` will use existing equality and inequality + :deco:`functools.total_ordering` will use existing equality and inequality methods to fill in the remaining methods. For example, supplying *__eq__* and *__lt__* will enable - :func:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: + :deco:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: @total_ordering class Student: @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by @@ -1097,11 +1097,11 @@ logarithm of the gamma function: abc --- -The :mod:`abc` module now supports :func:`~abc.abstractclassmethod` and -:func:`~abc.abstractstaticmethod`. +The :mod:`abc` module now supports :deco:`~abc.abstractclassmethod` and +:deco:`~abc.abstractstaticmethod`. These tools make it possible to define an :term:`abstract base class` that -requires a particular :func:`classmethod` or :func:`staticmethod` to be +requires a particular :deco:`classmethod` or :deco:`staticmethod` to be implemented:: class Temperature(metaclass=abc.ABCMeta): @@ -1155,7 +1155,7 @@ self-reference by displaying "..." in the recursive part of the representation string. To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new -decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to +decorator, :deco:`~reprlib.recursive_repr`, for detecting recursive calls to :meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): @@ -1241,7 +1241,7 @@ There is a new and slightly mind-blowing tool :term:`context manager` that does double duty as a function decorator. As a convenience, this new functionality is used by -:func:`~contextlib.contextmanager` so that no extra effort is needed to support +:deco:`~contextlib.contextmanager` so that no extra effort is needed to support both roles. The basic idea is that both context managers and function decorators can be used @@ -1253,7 +1253,7 @@ write a pre-action or post-action wrapper that can be used in either role. For example, it is sometimes useful to wrap functions or groups of statements with a logger that can track the time of entry and time of exit. Rather than writing both a function decorator and a context manager for the task, the -:func:`~contextlib.contextmanager` provides both capabilities in a single +:deco:`~contextlib.contextmanager` provides both capabilities in a single definition:: from contextlib import contextmanager diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 1bb79bce2c3e972..02fd264e53e1b06 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -919,12 +919,12 @@ abstract methods. The recommended approach to declaring abstract descriptors is now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. -* :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. +* :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. (Contributed by Darren Dale in :issue:`11610`.) @@ -1390,7 +1390,7 @@ ftplib functools --------- -The :func:`functools.lru_cache` decorator now accepts a ``typed`` keyword +The :deco:`functools.lru_cache` decorator now accepts a ``typed`` keyword argument (that defaults to ``False`` to ensure that it caches values of different types that compare equal in separate cache slots. (Contributed by Raymond Hettinger in :issue:`13227`.) @@ -2244,12 +2244,12 @@ Deprecated Python modules, functions and methods * The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. + * :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. * :mod:`importlib` package: diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 59afd6520c418f5..0067491c569cc05 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2,7 +2,7 @@ What's New In Python 3.4 **************************** -:Author: R. David Murray (Editor) +:Author: \R. David Murray (Editor) .. Rules for maintenance: @@ -858,7 +858,7 @@ to behave like normal instance methods when included in a class definition. .. _whatsnew-singledispatch: -The new :func:`~functools.singledispatch` decorator brings support for +The new :deco:`~functools.singledispatch` decorator brings support for single-dispatch generic functions to the Python standard library. Where object oriented programming focuses on grouping multiple operations on a common set of data into a class, a generic function focuses on grouping @@ -870,7 +870,7 @@ multiple implementations of an operation that allows it to work with :pep:`443` -- Single-dispatch generic functions PEP written and implemented by Łukasz Langa. -:func:`~functools.total_ordering` now supports a return value of +:deco:`~functools.total_ordering` now supports a return value of :data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) @@ -1029,7 +1029,7 @@ information for modules, classes and functions. (Contributed by Claudiu Popa and Nick Coghlan in :issue:`18626`.) :func:`~inspect.unwrap` makes it easy to unravel wrapper function chains -created by :func:`functools.wraps` (and any other API that sets the +created by :deco:`functools.wraps` (and any other API that sets the ``__wrapped__`` attribute on a wrapper function). (Contributed by Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`.) @@ -2086,10 +2086,10 @@ Deprecations in the Python API :meth:`!importlib.abc.PathEntryFinder.find_loader` and :meth:`!find_module` are replaced by :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the :samp:`{xxx}Loader` ABC - ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, - :meth:`!importlib.abc.InspectLoader.load_module`, - :meth:`!importlib.abc.FileLoader.load_module`, - :meth:`!importlib.abc.SourceLoader.load_module`) should no longer be + ``load_module`` methods (``importlib.abc.Loader.load_module``, + ``importlib.abc.InspectLoader.load_module``, + ``importlib.abc.FileLoader.load_module``, + ``importlib.abc.SourceLoader.load_module``) should no longer be implemented, instead loaders should implement an ``exec_module`` method (:meth:`importlib.abc.Loader.exec_module`, @@ -2317,7 +2317,7 @@ Changes in the Python API wish to continue to ignore syntax or decoding issues, catch all three exceptions now. -* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly +* :func:`functools.update_wrapper` and :deco:`functools.wraps` now correctly set the ``__wrapped__`` attribute to the function being wrapped, even if that function also had its ``__wrapped__`` attribute set. This means ``__wrapped__`` attributes now correctly link a stack of decorated diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 6009dd8a71eea52..24a2ffbefd0cfba 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -121,7 +121,7 @@ Significant improvements in the standard library: :ref:`better and significantly faster way ` of directory traversal. -* :func:`functools.lru_cache` has been mostly +* :deco:`functools.lru_cache` has been mostly :ref:`reimplemented in C `, yielding much better performance. @@ -1148,7 +1148,7 @@ functools .. _whatsnew-lrucache: -Most of the :func:`~functools.lru_cache` machinery is now implemented in C, making +Most of the :deco:`~functools.lru_cache` machinery is now implemented in C, making it significantly faster. (Contributed by Matt Joiner, Alexey Kachayev, and Serhiy Storchaka in :issue:`14373`.) @@ -2158,7 +2158,7 @@ improvement in some benchmarks. Objects from the :mod:`random` module now use 50% less memory on 64-bit builds. (Contributed by Serhiy Storchaka in :issue:`23488`.) -The :func:`property` getter calls are up to 25% faster. +The :deco:`property` getter calls are up to 25% faster. (Contributed by Joe Jevnik in :issue:`23910`.) Instantiation of :class:`fractions.Fraction` is now up to 30% faster. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b1e3269239d6297..b0382dd8215bc00 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -517,7 +517,7 @@ Applications that do not use str to represent paths should use :func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call -:func:`sys._enablelegacywindowsfsencoding`. +:func:`!sys._enablelegacywindowsfsencoding`. See :pep:`529` for more information and discussion of code modifications that may be required. @@ -2006,10 +2006,10 @@ deprecated. importlib ~~~~~~~~~ -The :meth:`importlib.machinery.SourceFileLoader.load_module` and -:meth:`importlib.machinery.SourcelessFileLoader.load_module` methods +The ``importlib.machinery.SourceFileLoader.load_module`` and +``importlib.machinery.SourcelessFileLoader.load_module`` methods are now deprecated. They were the only remaining implementations of -:meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not +``importlib.abc.Loader.load_module`` in :mod:`importlib` that had not been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. @@ -2173,7 +2173,7 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects +* The format of the :attr:`!codeobject.co_lnotab` attribute of code objects changed to support a negative line number delta. By default, Python does not emit bytecode with a negative line number delta. Functions using :attr:`frame.f_lineno`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9ac3bf53833f93e..3af3e6ec9cac08f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -571,7 +571,7 @@ decimal operations to work with the correct context in asynchronous code. dataclasses ----------- -The new :func:`~dataclasses.dataclass` decorator provides a way to declare +The new :deco:`~dataclasses.dataclass` decorator provides a way to declare *data classes*. A data class describes its attributes using class variable annotations. Its constructor and other magic methods, such as :meth:`~object.__repr__`, :meth:`~object.__eq__`, and @@ -604,7 +604,7 @@ new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a :meth:`!get_resource_reader` function which returns -a :class:`importlib.abc.ResourceReader` instance to support this +a :class:`!importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. Contributed by Barry Warsaw and Brett Cannon in :issue:`32248`. @@ -830,7 +830,7 @@ The new :func:`~contextlib.nullcontext` is a simpler and faster no-op context manager than :class:`~contextlib.ExitStack`. (Contributed by Jesse-Bakker in :issue:`10049`.) -The new :func:`~contextlib.asynccontextmanager`, +The new :deco:`~contextlib.asynccontextmanager`, :class:`~contextlib.AbstractAsyncContextManager`, and :class:`~contextlib.AsyncExitStack` have been added to complement their synchronous counterparts. (Contributed @@ -841,7 +841,7 @@ and by Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- -The :mod:`cProfile` command line now accepts ``-m module_name`` as an +The :mod:`!cProfile` command line now accepts ``-m module_name`` as an alternative to script path. (Contributed by Sanyam Khurana in :issue:`21862`.) @@ -921,7 +921,7 @@ return :const:`False` instead and are deprecated. functools --------- -:func:`functools.singledispatch` now supports registering implementations +:deco:`functools.singledispatch` now supports registering implementations using type annotations. (Contributed by Łukasz Langa in :issue:`32227`.) @@ -1043,7 +1043,7 @@ window are shown and hidden in the Options menu. importlib --------- -The :class:`importlib.abc.ResourceReader` ABC was introduced to +The :class:`!importlib.abc.ResourceReader` ABC was introduced to support the loading of resources from packages. See also :ref:`whatsnew37_importlib_resources`. (Contributed by Barry Warsaw, Brett Cannon in :issue:`32248`.) @@ -2032,7 +2032,7 @@ both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in -favour of :class:`importlib.abc.ResourceReader`. +favour of :class:`!importlib.abc.ResourceReader`. locale diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 545a17aecab8ee3..42d98ff94ec0d1b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -208,14 +207,15 @@ subdirectories). Debug build uses the same ABI as release build ----------------------------------------------- -Python now uses the same ABI whether it's built in release or debug mode. On -Unix, when Python is built in debug mode, it is now possible to load C -extensions built in release mode and C extensions built using the stable ABI. +The ABI of Python :ref:`debug builds ` is now compatible with +Python release builds. On Unix, when Python is built in debug mode, it is now +possible to load C extensions built in release mode and C extensions built +using the stable ABI. The inverse is not true, as debug builds expose +additional symbols not available in release builds. -Release builds and :ref:`debug builds ` are now ABI compatible: defining the -``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which -introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which -adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` +Defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, +which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, +which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` environment variable, can be set using the new :option:`./configure --with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) @@ -259,15 +259,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' @@ -763,7 +764,7 @@ these are the inverse of each class's ``isocalendar`` method. functools --------- -:func:`functools.lru_cache` can now be used as a straight decorator rather +:deco:`functools.lru_cache` can now be used as a straight decorator rather than as a function returning a decorator. So both of these are now supported:: @lru_cache @@ -776,7 +777,7 @@ than as a function returning a decorator. So both of these are now supported:: (Contributed by Raymond Hettinger in :issue:`36772`.) -Added a new :func:`functools.cached_property` decorator, for computed properties +Added a new :deco:`functools.cached_property` decorator, for computed properties cached for the life of the instance. :: import functools @@ -892,7 +893,7 @@ inspect The :func:`inspect.getdoc` function can now find docstrings for ``__slots__`` if that attribute is a :class:`dict` where the values are docstrings. This provides documentation options similar to what we already have -for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: +for :deco:`property`, :deco:`classmethod`, and :deco:`staticmethod`:: class AudioClip: __slots__ = {'bit_rate': 'expressed in kilohertz to one decimal place', @@ -1332,14 +1333,14 @@ The :mod:`typing` module incorporates several new features: ... * "Final" variables, functions, methods and classes. See :pep:`591`, - :class:`typing.Final` and :func:`typing.final`. + :class:`typing.Final` and :deco:`typing.final`. The final qualifier instructs a static type checker to restrict subclassing, overriding, or reassignment:: pi: Final[float] = 3.1415926536 * Protocol definitions. See :pep:`544`, :class:`typing.Protocol` and - :func:`typing.runtime_checkable`. Simple ABCs like + :deco:`typing.runtime_checkable`. Simple ABCs like :class:`typing.SupportsInt` are now ``Protocol`` subclasses. * New protocol class :class:`typing.SupportsIndex`. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 40d4a27bff9fee5..49a52b7504bc951 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 38194db670b8390..420876a9b3ecd00 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.16.rst 3.15.rst 3.14.rst 3.13.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 7ae00c6f005e7e3..ac1b4a64f38c75a 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -121,9 +121,9 @@ simple_stmts[asdl_stmt_seq*]: simple_stmt[stmt_ty] (memo): | assignment | &"type" type_alias + | &('import' | 'from' | "lazy") import_stmt | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt - | &('import' | 'from') import_stmt | &'raise' raise_stmt | &'pass' pass_stmt | &'del' del_stmt @@ -216,7 +216,7 @@ assert_stmt[stmt_ty]: | invalid_assert_stmt | 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } -import_stmt[stmt_ty]: +import_stmt[stmt_ty](memo): | invalid_import | import_name | import_from @@ -224,13 +224,16 @@ import_stmt[stmt_ty]: # Import statements # ----------------- -import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } +import_name[stmt_ty]: + | lazy="lazy"? 'import' a=dotted_as_names { _PyAST_Import(a, lazy ? 1 : 0, EXTRA) } + # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: - | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } - | 'from' a=('.' | '...')+ 'import' b=import_from_targets { - _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } + | invalid_import_from + | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { + _PyPegen_checked_from_import(p, a, b, c, lazy, EXTRA) } + | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets { + _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) } import_from_targets[asdl_alias_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' @@ -552,10 +555,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -563,6 +568,7 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } @@ -709,12 +715,16 @@ expressions[expr_ty]: | expression expression[expr_ty] (memo): + | invalid_if_expression | invalid_expression | invalid_legacy_expression - | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + | if_expression | disjunction | lambdef +if_expression[expr_ty]: + | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } + yield_expr[expr_ty]: | 'yield' 'from' a=expression { _PyAST_YieldFrom(a, EXTRA) } | 'yield' a=[star_expressions] { _PyAST_Yield(a, EXTRA) } @@ -731,10 +741,16 @@ star_expression[expr_ty] (memo): star_named_expressions[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression+ [','] { a } +star_named_expressions_sequence[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression_sequence+ [','] { a } + star_named_expression[expr_ty]: | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | named_expression +star_named_expression_sequence[expr_ty]: + | invalid_starred_expression_unpacking_sequence + | star_named_expression + assignment_expression[expr_ty]: | a=NAME ':=' ~ b=expression { CHECK_VERSION(expr_ty, 8, "Assignment expressions are", @@ -804,6 +820,7 @@ is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, I bitwise_or[expr_ty]: | a=bitwise_or '|' b=bitwise_xor { _PyAST_BinOp(a, BitOr, b, EXTRA) } + | invalid_bitwise_or | bitwise_xor bitwise_xor[expr_ty]: @@ -812,12 +829,12 @@ bitwise_xor[expr_ty]: bitwise_and[expr_ty]: | a=bitwise_and '&' b=shift_expr { _PyAST_BinOp(a, BitAnd, b, EXTRA) } + | invalid_bitwise_and | shift_expr shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } - | invalid_arithmetic | sum # Arithmetic operators @@ -826,6 +843,7 @@ shift_expr[expr_ty]: sum[expr_ty]: | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) } | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) } + | invalid_arithmetic | term term[expr_ty]: @@ -834,7 +852,6 @@ term[expr_ty]: | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } - | invalid_factor | factor factor[expr_ty] (memo): @@ -842,6 +859,7 @@ factor[expr_ty] (memo): | '-' a=factor { _PyAST_UnaryOp(USub, a, EXTRA) } | '~' a=factor { _PyAST_UnaryOp(Invert, a, EXTRA) } | power + | invalid_factor power[expr_ty]: | a=await_primary '**' b=factor { _PyAST_BinOp(a, Pow, b, EXTRA) } @@ -882,9 +900,9 @@ atom[expr_ty]: | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } | &(STRING|FSTRING_START|TSTRING_START) strings | NUMBER - | &'(' (tuple | group | genexp) - | &'[' (list | listcomp) - | &'{' (dict | set | dictcomp | setcomp) + | &'(' (genexp | tuple | group) + | &'[' (listcomp | list) + | &'{' (dictcomp | setcomp | dict | set) | '...' { _PyAST_Constant(Py_Ellipsis, NULL, EXTRA) } group[expr_ty]: @@ -998,13 +1016,13 @@ strings[expr_ty] (memo): | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: - | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } + | '[' a=[star_named_expressions_sequence] ']' { _PyAST_List(a, Load, EXTRA) } tuple[expr_ty]: - | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { + | '(' a=[y=star_named_expression_sequence ',' z=[star_named_expressions_sequence] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { _PyAST_Tuple(a, Load, EXTRA) } -set[expr_ty]: '{' a=star_named_expressions '}' { _PyAST_Set(a, EXTRA) } +set[expr_ty]: '{' a=star_named_expressions_sequence '}' { _PyAST_Set(a, EXTRA) } # Dicts # ----- @@ -1040,20 +1058,20 @@ for_if_clause[comprehension_ty]: | invalid_for_target listcomp[expr_ty]: - | '[' a=named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } + | '[' a=star_named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } | invalid_comprehension setcomp[expr_ty]: - | '{' a=named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } + | '{' a=star_named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } | invalid_comprehension genexp[expr_ty]: - | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } + | '(' a=( assignment_expression | expression !':=' | starred_expression ) b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } | invalid_comprehension dictcomp[expr_ty]: | '{' a=kvpair b=for_if_clauses '}' { _PyAST_DictComp(a->key, a->value, b, EXTRA) } - | invalid_dict_comprehension + | '{' '**' a=expression b=for_if_clauses '}' { _PyAST_DictComp(a, NULL, b, EXTRA) } # FUNCTION CALL ARGUMENTS # ======================= @@ -1251,8 +1269,7 @@ invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid { - _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } + _PyPegen_raise_error_for_missing_comma(p, a, b) } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } | a=disjunction 'if' b=disjunction 'else' !expression { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") } @@ -1263,6 +1280,12 @@ invalid_expression: | a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") } +invalid_if_expression: + | disjunction 'if' b=disjunction 'else' a='*' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot unpack only part of a conditional expression") } + | disjunction 'if' b=disjunction 'else' a='**' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking on only part of a conditional expression") } + invalid_named_expression(memo): | a=expression ':=' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( @@ -1327,16 +1350,15 @@ invalid_assert_stmt: invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } invalid_comprehension: - | ('[' | '(' | '{') a=starred_expression for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } + | '[' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in list comprehension") } + | '(' a='**' b=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in generator expression") } | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty), "did you forget parentheses around the comprehension target?") } | ('[' | '{') a=star_named_expression b=',' for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") } -invalid_dict_comprehension: - | '{' a='**' bitwise_or for_if_clauses '}' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | a="/" ',' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "at least one parameter must precede /") } @@ -1424,6 +1446,11 @@ invalid_import_from_as_name: RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } +invalid_import_from: + | 'from' ('.' | '...')* dotted_name a="lazy" 'import' import_from_targets { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + "use 'lazy from ... ' instead of 'from ... lazy import'") } + invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } @@ -1431,6 +1458,8 @@ invalid_import_from_targets: RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_with_stmt: + | ['async'] 'with' ','.(expression ['as' star_target])+ trailing=',' ':' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(trailing, "the last 'with' item has a trailing comma") } | ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } invalid_with_stmt_indent: @@ -1531,19 +1560,32 @@ invalid_class_def_raw: RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } invalid_double_starred_kvpairs: - | ','.double_starred_kvpair+ ',' invalid_kvpair - | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | invalid_kvpair_unpacking [','] + | ','.double_starred_kvpair+ ',' (invalid_kvpair | invalid_kvpair_unpacking) | expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } +invalid_kvpair_unpacking: + | a='**' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid double starred expression. Did you forget to wrap the conditional expression in parentheses?") } + | a='*' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary key") } + | a='**' b=bitwise_or ':' expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary key") } + | expression ':' a='*' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' b=bitwise_or { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot use dict unpacking in a dictionary value") } invalid_kvpair: | a=expression !(':') { RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | expression ':' a='**' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use dict unpacking in a dictionary value") } | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } invalid_starred_expression_unpacking: + | a='*' b=if_expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid starred expression. Did you forget to wrap the conditional expression in parentheses?") } | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_starred_expression_unpacking_sequence: + | a='**' bitwise_or { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use dict unpacking here") } + | invalid_starred_expression_unpacking invalid_starred_expression: | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") } - invalid_fstring_replacement_field: | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } @@ -1604,3 +1646,17 @@ invalid_type_params: RAISE_SYNTAX_ERROR_STARTING_FROM( token, "Type parameter list cannot be empty")} + +invalid_bitwise_and: + | a=bitwise_and b='&' c='&' { + _PyPegen_tokens_are_adjacent(b, c) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'and' or '&' instead of '&&'?") + : NULL + } + +invalid_bitwise_or: + | a=bitwise_or b='|' c='|' { + _PyPegen_tokens_are_adjacent(b, c) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'or' or '|' instead of '||'?") + : NULL + } diff --git a/Include/Python.h b/Include/Python.h index 78083bbf31db751..337119c15fe8b6f 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -22,12 +23,13 @@ #include // INT_MAX #include // HUGE_VAL #include // va_list +#include // memcpy() #include // wchar_t #ifdef HAVE_SYS_TYPES_H # include // ssize_t #endif -// , , and headers are no longer used +// , and headers are no longer used // by Python, but kept for the backward compatibility of existing third party C // extensions. They are not included by limited C API version 3.11 and newer. // @@ -37,7 +39,6 @@ # include // errno # include // FILE* # include // getenv() -# include // memcpy() #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000 # include // tolower() @@ -46,17 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) -# error "Py_LIMITED_API is not currently supported in the free-threaded build" -# endif - -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -71,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -83,7 +79,8 @@ __pragma(warning(disable: 4201)) #include "object.h" #include "refcount.h" #include "objimpl.h" -#include "typeslots.h" +#include "slots.h" +#include "slots_generated.h" #include "pyhash.h" #include "cpython/pydebug.h" #include "bytearrayobject.h" @@ -121,6 +118,7 @@ __pragma(warning(disable: 4201)) #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "cpython/sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index ffd19ccd3500fac..7490ece52e5220e 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -6,7 +6,7 @@ /* Like PyObject_CallMethod(), but expect a _Py_Identifier* as the method name. */ -PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_CallMethodId( PyObject *obj, _Py_Identifier *name, const char *format, ...); diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index ca8109e3248a8d8..5b66fa1040d738f 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -23,6 +23,9 @@ _PyEval_RequestCodeExtraIndex(freefunc f) { PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); +PyAPI_FUNC(int) _PyEval_UnpackIndices(PyObject *, PyObject *, + Py_ssize_t, + Py_ssize_t *, Py_ssize_t *); // Trampoline API @@ -35,7 +38,7 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name); PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 84456a709a6abe7..1be47e42ed62bfb 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -34,9 +34,13 @@ typedef struct { char *entries[1]; } _PyCodeArray; +#define _PyCode_DEF_UNIQUE_ID() \ + Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ + #define _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ _PyCodeArray *co_tlbc; #else +#define _PyCode_DEF_UNIQUE_ID() #define _PyCode_DEF_THREAD_LOCAL_BYTECODE() #endif @@ -101,7 +105,7 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ struct _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ - Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \ + _PyCode_DEF_UNIQUE_ID() \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 4fc46fefb93a24a..bcba32da412f326 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -2,15 +2,6 @@ # error "this header file must not be included directly" #endif -// Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// // Straightforward per-object locking could introduce deadlocks that were not // present when running with the GIL. Threads may hold locks for multiple // objects simultaneously because Python operations can nest. If threads were @@ -43,52 +34,19 @@ // `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) // critical section by reacquiring the associated lock or locks. See // `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - -typedef struct PyCriticalSection PyCriticalSection; -typedef struct PyCriticalSection2 PyCriticalSection2; - -PyAPI_FUNC(void) -PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); PyAPI_FUNC(void) PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); -PyAPI_FUNC(void) -PyCriticalSection_End(PyCriticalSection *c); - -PyAPI_FUNC(void) -PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); - PyAPI_FUNC(void) PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); -PyAPI_FUNC(void) -PyCriticalSection2_End(PyCriticalSection2 *c); - #ifndef Py_GIL_DISABLED +#undef Py_BEGIN_CRITICAL_SECTION +#undef Py_END_CRITICAL_SECTION +#undef Py_BEGIN_CRITICAL_SECTION2 +#undef Py_END_CRITICAL_SECTION2 + # define Py_BEGIN_CRITICAL_SECTION(op) \ { # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ @@ -101,54 +59,17 @@ PyCriticalSection2_End(PyCriticalSection2 *c); { # define Py_END_CRITICAL_SECTION2() \ } -#else /* !Py_GIL_DISABLED */ - -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection { - // Tagged pointer to an outer active critical section (or 0). - uintptr_t _cs_prev; - - // Mutex used to protect critical section - PyMutex *_cs_mutex; -}; - -// A critical section protected by two mutexes. Use -// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection2 { - PyCriticalSection _cs_base; - PyMutex *_cs_mutex2; -}; - -# define Py_BEGIN_CRITICAL_SECTION(op) \ - { \ - PyCriticalSection _py_cs; \ - PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +#else /* !Py_GIL_DISABLED */ # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ { \ PyCriticalSection _py_cs; \ PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_END_CRITICAL_SECTION() \ - PyCriticalSection_End(&_py_cs); \ - } - -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ - { \ - PyCriticalSection2 _py_cs2; \ - PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) - # define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ { \ PyCriticalSection2 _py_cs2; \ PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) -# define Py_END_CRITICAL_SECTION2() \ - PyCriticalSection2_End(&_py_cs2); \ - } - -#endif +#endif /* !Py_GIL_DISABLED */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fca1a9..5e7811416aba63f 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -32,6 +32,16 @@ typedef struct { PyDictValues *ma_values; } PyDictObject; +// frozendict +PyAPI_DATA(PyTypeObject) PyFrozenDict_Type; +#define PyFrozenDict_Check(op) PyObject_TypeCheck((op), &PyFrozenDict_Type) +#define PyFrozenDict_CheckExact(op) Py_IS_TYPE((op), &PyFrozenDict_Type) + +#define PyAnyDict_CheckExact(ob) \ + (PyDict_CheckExact(ob) || PyFrozenDict_CheckExact(ob)) +#define PyAnyDict_Check(ob) \ + (PyDict_Check(ob) || PyFrozenDict_Check(ob)) + PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); // PyDict_GetItemStringRef() can be used instead @@ -39,20 +49,10 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); mp = _Py_CAST(PyDictObject*, op); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&mp->ma_used); @@ -103,3 +103,6 @@ PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id); // Mark given dictionary as "watched" (callback will be called if it is modified) PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict); PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict); + +// Create a frozendict. Create an empty dictionary if iterable is NULL. +PyAPI_FUNC(PyObject*) PyFrozenDict_New(PyObject *iterable); diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c24..e14facd77edec53 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -16,8 +16,8 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; #define PyGen_Check(op) PyObject_TypeCheck((op), &PyGen_Type) #define PyGen_CheckExact(op) Py_IS_TYPE((op), &PyGen_Type) -PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); -PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -29,7 +29,7 @@ typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) -PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); @@ -40,7 +40,7 @@ typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); #define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cce2a2..149a20af8b9cbb2 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 1c979d91a40850b..1ccc496c63ac780 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -149,6 +149,7 @@ typedef struct PyConfig { int dump_refs; wchar_t *dump_refs_file; int malloc_stats; + int pymalloc_hugepages; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; wchar_t *pycache_prefix; @@ -190,6 +191,7 @@ typedef struct PyConfig { int enable_gil; int tlbc_enabled; #endif + int lazy_imports; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 4b6f97a5e475d65..998ebe6891577e8 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -133,50 +133,16 @@ _PyLong_CompactValue(const PyLongObject *op) assert(PyType_HasFeature(op->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS)); assert(PyUnstable_Long_IsCompact(op)); sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); + if (sign == 0) { + // gh-147988: Make sure that the digit is zero. + // It helps detecting the usage of uninitialized digits. + assert(op->long_value.ob_digit[0] == 0); + } return sign * (Py_ssize_t)op->long_value.ob_digit[0]; } #define PyUnstable_Long_CompactValue _PyLong_CompactValue - -/* --- Import/Export API -------------------------------------------------- */ - -typedef struct PyLongLayout { - uint8_t bits_per_digit; - uint8_t digit_size; - int8_t digits_order; - int8_t digit_endianness; -} PyLongLayout; - -PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); - -typedef struct PyLongExport { - int64_t value; - uint8_t negative; - Py_ssize_t ndigits; - const void *digits; - // Member used internally, must not be used for other purpose. - Py_uintptr_t _reserved; -} PyLongExport; - -PyAPI_FUNC(int) PyLong_Export( - PyObject *obj, - PyLongExport *export_long); -PyAPI_FUNC(void) PyLong_FreeExport( - PyLongExport *export_long); - - -/* --- PyLongWriter API --------------------------------------------------- */ - -typedef struct PyLongWriter PyLongWriter; - -PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( - int negative, - Py_ssize_t ndigits, - void **digits); -PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); -PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); - #ifdef __cplusplus } #endif diff --git a/Include/cpython/marshal.h b/Include/cpython/marshal.h index 6c1f7f96b6a2e81..159459fcaec3d93 100644 --- a/Include/cpython/marshal.h +++ b/Include/cpython/marshal.h @@ -6,7 +6,7 @@ PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#define Py_MARSHAL_VERSION 5 +#define Py_MARSHAL_VERSION 6 PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); diff --git a/Include/cpython/modsupport.h b/Include/cpython/modsupport.h index 6134442106474fd..cfeee6e8ab3414d 100644 --- a/Include/cpython/modsupport.h +++ b/Include/cpython/modsupport.h @@ -2,6 +2,19 @@ # error "this header file must not be included directly" #endif +PyAPI_FUNC(int) PyArg_ParseArray( + PyObject *const *args, + Py_ssize_t nargs, + const char *format, + ...); +PyAPI_FUNC(int) PyArg_ParseArrayAndKeywords( + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames, + const char *format, + const char * const *kwlist, + ...); + // A data structure that can be used to run initialization code once in a // thread-safe manner. The C++11 equivalent is std::call_once. typedef struct { @@ -26,13 +39,7 @@ PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); #ifdef Py_BUILD_CORE -// Internal; defined here to avoid explicitly including pycore_modsupport.h -#define _Py_INTERNAL_ABI_SLOT \ - {Py_mod_abi, (void*) &(PyABIInfo) { \ - .abiinfo_major_version = 1, \ - .abiinfo_minor_version = 0, \ - .flags = PyABIInfo_INTERNAL, \ - .build_version = PY_VERSION_HEX, \ - .abi_version = PY_VERSION_HEX }} \ - /////////////////////////////////////////////////////// +// For internal use in stdlib. Needs C99 compound literals. +// Defined here to avoid every stdlib module including pycore_modsupport.h +#define _Py_ABI_SLOT {Py_mod_abi, (void*) &(PyABIInfo) _PyABIInfo_DEFAULT} #endif diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 5094c8c23ae32bb..c93271f6ca95f5b 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -24,9 +24,10 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ #define PY_MONITORING_EVENT_RAISE 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 @@ -34,6 +35,9 @@ extern "C" { #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d64298232e705c8..326254c335b4895 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,8 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + /* Below here all fields are internal to the VM */ + /* bitset of which type-watchers care about this type */ unsigned char tp_watched; @@ -239,6 +241,11 @@ struct _typeobject { * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). */ uint16_t tp_versions_used; + + /* Virtual iterator next function. + * This function must escape to any code that can result in + * the GC being run, such as Py_DECREF. */ + _Py_iteritemfunc _tp_iteritem; }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) @@ -295,13 +302,15 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyObject_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyObject_Dump -PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); -PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); @@ -387,10 +396,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyObject_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ @@ -432,14 +442,13 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count); - /* For backwards compatibility with the old trashcan mechanism */ #define Py_TRASHCAN_BEGIN(op, dealloc) #define Py_TRASHCAN_END PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); +PyAPI_FUNC(void *) PyObject_GetItemData_DuringGC(PyObject *obj); PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj); @@ -491,3 +500,84 @@ PyAPI_FUNC(int) PyUnstable_TryIncRef(PyObject *); PyAPI_FUNC(void) PyUnstable_EnableTryIncRef(PyObject *); PyAPI_FUNC(int) PyUnstable_Object_IsUniquelyReferenced(PyObject *); + +PyAPI_FUNC(int) PyUnstable_SetImmortal(PyObject *op); + +#if defined(Py_GIL_DISABLED) +PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); + +static inline uintptr_t +_Py_ThreadId(void) +{ + uintptr_t tid; +#if defined(_MSC_VER) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(_MSC_VER) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(_MSC_VER) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__MINGW32__) && defined(_M_X64) + tid = __readgsqword(48); +#elif defined(__MINGW32__) && defined(_M_IX86) + tid = __readfsdword(24); +#elif defined(__MINGW32__) && defined(_M_ARM64) + tid = __getReg(18); +#elif defined(__i386__) + __asm__("{movl %%gs:0, %0|mov %0, dword ptr gs:[0]}" : "=r" (tid)); // 32-bit always uses GS +#elif defined(__MACH__) && defined(__x86_64__) + __asm__("{movq %%gs:0, %0|mov %0, qword ptr gs:[0]}" : "=r" (tid)); // x86_64 macOSX uses GS +#elif defined(__x86_64__) + __asm__("{movq %%fs:0, %0|mov %0, qword ptr fs:[0]}" : "=r" (tid)); // x86_64 Linux, BSD uses FS +#elif defined(__arm__) && __ARM_ARCH >= 7 + __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); +#elif defined(__aarch64__) && defined(__APPLE__) + __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); +#elif defined(__aarch64__) + __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); +#elif defined(__powerpc64__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r13 is reserved for use as system thread ID by the Power 64-bit ABI. + register uintptr_t tp __asm__ ("r13"); + __asm__("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__powerpc__) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // r2 is reserved for use as system thread ID by the Power 32-bit ABI. + register uintptr_t tp __asm__ ("r2"); + __asm__ ("" : "=r" (tp)); + tid = tp; + #endif +#elif defined(__s390__) && defined(__GNUC__) + // Both GCC and Clang have supported __builtin_thread_pointer + // for s390 from long time ago. + tid = (uintptr_t)__builtin_thread_pointer(); +#elif defined(__riscv) + #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) + tid = (uintptr_t)__builtin_thread_pointer(); + #else + // tp is Thread Pointer provided by the RISC-V ABI. + __asm__ ("mv %0, tp" : "=r" (tid)); + #endif +#else + // Fallback to a portable implementation if we do not have a faster + // platform-specific implementation. + tid = _Py_GetThreadLocal_Addr(); +#endif + return tid; +} + +static inline Py_ALWAYS_INLINE int +_Py_IsOwnedByCurrentThread(PyObject *ob) +{ +#ifdef _Py_THREAD_SANITIZER + return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); +#else + return ob->ob_tid == _Py_ThreadId(); +#endif +} +#endif diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad665..e85b360c986668c 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -72,8 +72,8 @@ // def _Py_atomic_load_ptr_acquire(obj): // return obj # acquire // -// def _Py_atomic_store_ptr_release(obj): -// return obj # release +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release // // def _Py_atomic_fence_seq_cst(): // # sequential consistency @@ -523,12 +523,18 @@ _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value); + static inline void _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline void _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); @@ -591,6 +597,17 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- +// Compilers don't really support "consume" semantics, so we fake it. Use +// "acquire" with TSan to support false positives. Use "relaxed" otherwise, +// because CPUs on all platforms we support respect address dependencies without +// extra barriers. +// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +#if defined(_Py_THREAD_SANITIZER) +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire +#else +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed +#endif + #if SIZEOF_LONG == 8 # define _Py_atomic_load_ulong(p) \ _Py_atomic_load_uint64((uint64_t *)p) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b6..253b35082aafcd2 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -572,10 +572,18 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf6..3b3c5f7017e9575 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -971,12 +971,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) *(volatile unsigned short *)obj = value; } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - *(volatile unsigned int *)obj = value; -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1066,6 +1060,32 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int8_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_int8_release" +#endif +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(volatile unsigned int *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5ff..faef303da70314c 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -459,7 +459,7 @@ static inline uint16_t _Py_atomic_load_uint16(const uint16_t *obj) { _Py_USING_STD; - return atomic_load((const _Atomic(uint32_t)*)obj); + return atomic_load((const _Atomic(uint16_t)*)obj); } static inline uint32_t @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,22 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_int8_release(int8_t *obj, int8_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(int8_t)*)obj, value, + memory_order_release); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d9297..be2e3b641c25cb6 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 86ce6e6f79824a7..e46dfe59ec46304 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -25,6 +25,9 @@ PyAPI_FUNC(PyStatus) Py_PreInitializeFromArgs( PyAPI_FUNC(PyStatus) Py_InitializeFromConfig( const PyConfig *config); +// Python 3.8 provisional API (PEP 587) +PyAPI_FUNC(PyStatus) _Py_InitializeMain(void); + PyAPI_FUNC(int) Py_RunMain(void); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c53abe43ebe65cb..f367146e262bfe8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -105,7 +105,7 @@ struct _ts { # define _PyThreadState_WHENCE_INIT 1 # define _PyThreadState_WHENCE_FINI 2 # define _PyThreadState_WHENCE_THREADING 3 -# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_C_API 4 # define _PyThreadState_WHENCE_EXEC 5 # define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif @@ -113,6 +113,9 @@ struct _ts { /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). @@ -132,6 +135,16 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + /* Pointer to the base frame (bottommost sentinel frame). + Used by profilers to validate complete stack unwinding. + Points to the embedded base_frame in _PyThreadStateImpl. + The frame is embedded there rather than here because _PyInterpreterFrame + is defined in internal headers that cannot be exposed in the public API. */ + struct _PyInterpreterFrame *base_frame; + + struct _PyInterpreterFrame *last_profiled_frame; + uintptr_t last_profiled_frame_seq; + Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; PyObject *c_profileobj; @@ -186,6 +199,7 @@ struct _ts { _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; + _PyStackChunk *datastack_cached_chunk; /* XXX signal handlers should also be here */ /* The following fields are here to avoid allocation during init. @@ -226,6 +240,20 @@ struct _ts { // structure and all share the same per-interpreter structure). PyStats *pystats; #endif + + struct { + /* Number of nested PyThreadState_Ensure() calls on this thread state */ + Py_ssize_t counter; + + /* Should this thread state be deleted upon calling + PyThreadState_Release() (with the counter at 1)? + + This is only true for thread states created by PyThreadState_Ensure() */ + int delete_on_release; + + /* The interpreter guard owned by PyThreadState_EnsureFromView(), if any. */ + PyInterpreterGuard *owned_guard; + } ensure; }; /* other API */ @@ -306,3 +334,8 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization( + PyInterpreterState *interp, + int allow_specialization); +PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled( + PyInterpreterState *interp); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 1c94603c08b9b60..69659c48a3bf882 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -29,7 +29,7 @@ # error "this header file must not be included directly" #endif -#define PYSTATS_MAX_UOP_ID 1024 +#define PYSTATS_MAX_UOP_ID 2000 #define SPECIALIZATION_FAILURE_KINDS 60 @@ -142,7 +142,9 @@ typedef struct _optimization_stats { uint64_t recursive_call; uint64_t low_confidence; uint64_t unknown_callee; + uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; @@ -161,6 +163,7 @@ typedef struct _optimization_stats { uint64_t jit_code_size; uint64_t jit_trampoline_size; uint64_t jit_data_size; + uint64_t jit_got_size; uint64_t jit_padding_size; uint64_t jit_freed_memory_size; uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h new file mode 100644 index 000000000000000..e621d6abbfed8aa --- /dev/null +++ b/Include/cpython/sentinelobject.h @@ -0,0 +1,26 @@ +/* Sentinel object interface */ + +#ifndef Py_LIMITED_API +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name, + const char *repr); + +#ifdef __cplusplus +} +#endif +#endif /* !_Py_SENTINELOBJECT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h index 4c3ea1facebc4ed..137206eff15b33f 100644 --- a/Include/cpython/sliceobject.h +++ b/Include/cpython/sliceobject.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_SLICEOBJECT_H +#ifndef _Py_CPYTHON_SLICEOBJECT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h index 328fbe86143b024..83a1abcd6f3b347 100644 --- a/Include/cpython/structseq.h +++ b/Include/cpython/structseq.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_STRUCTSEQ_H +#ifndef _Py_CPYTHON_STRUCTSEQ_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f29..7f42730f1b09194 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,11 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c9ca6..ea91f4158eb3929 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else @@ -497,7 +496,7 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteUCS4( PyUnicodeWriter *writer, - Py_UCS4 *str, + const Py_UCS4 *str, Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteStr( @@ -779,4 +778,4 @@ static inline int Py_UNICODE_ISALNUM(Py_UCS4 ch) { // Return an interned Unicode object for an Identifier; may fail if there is no // memory. -PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); +Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); diff --git a/Include/critical_section.h b/Include/critical_section.h index 3b37615a8b17e2a..732bfab7ecf2349 100644 --- a/Include/critical_section.h +++ b/Include/critical_section.h @@ -4,6 +4,91 @@ extern "C" { #endif +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +// NOTE: the contents of this struct are private and their meaning may +// change betweeen Python releases without a deprecation period. +typedef struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + struct PyMutex *_cs_mutex; +} PyCriticalSection; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +typedef struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + struct PyMutex *_cs_mutex2; +} PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +// These are definitions for the stable ABI. For GIL-ful builds they're +// conditionally redefined as no-ops in cpython/critical_section.h. + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + #ifndef Py_LIMITED_API # define Py_CPYTHON_CRITICAL_SECTION_H # include "cpython/critical_section.h" diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e5acc..66e6c6e3ac35755 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -196,8 +196,23 @@ typedef struct { /* Define global variable for the C API and a macro for setting it. */ static PyDateTime_CAPI *PyDateTimeAPI = NULL; -#define PyDateTime_IMPORT \ - PyDateTimeAPI = (PyDateTime_CAPI *)PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0) +static inline PyDateTime_CAPI * +_PyDateTime_IMPORT(void) { + PyDateTime_CAPI *val = (PyDateTime_CAPI *)_Py_atomic_load_ptr(&PyDateTimeAPI); + if (val == NULL) { + PyDateTime_CAPI *capi = (PyDateTime_CAPI *)PyCapsule_Import( + PyDateTime_CAPSULE_NAME, 0); + if (capi != NULL) { + /* if the compare exchange fails then in that case + another thread would have initialized it */ + _Py_atomic_compare_exchange_ptr(&PyDateTimeAPI, &val, (void *)capi); + return capi; + } + } + return val; +} + +#define PyDateTime_IMPORT _PyDateTime_IMPORT() /* Macro for access to the UTC singleton */ #define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC @@ -263,5 +278,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/descrobject.h b/Include/descrobject.h index fd66d17b497a31f..340de4e0e1e6ffc 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -80,10 +80,14 @@ struct PyMemberDef { #define _Py_T_NONE 20 // Deprecated. Value is always None. /* Flags */ -#define Py_READONLY 1 -#define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that -#define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. -#define Py_RELATIVE_OFFSET 8 +#define Py_READONLY (1 << 0) +#define Py_AUDIT_READ (1 << 1) // Added in 3.10, harmless no-op before that +#define _Py_WRITE_RESTRICTED (1 << 2) // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET (1 << 3) + +#ifndef Py_LIMITED_API +# define _Py_AFTER_ITEMS (1 << 4) // For internal use. +#endif PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab699e75..0384e3131dcdb52 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 4d4def9bf8983e2..5290319dd762c55 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -461,6 +461,7 @@ int RunningOnValgrind(void); #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) +extern "C++" { /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. Instead of doing @@ -476,6 +477,8 @@ int RunningOnValgrind(void); _Py_ANNOTATE_IGNORE_READS_END(); return res; } +} + /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ namespace { \ diff --git a/Include/exports.h b/Include/exports.h index 62feb09ed2b4336..18692283005e596 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -35,6 +35,12 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + /* module init functions outside the core must be exported */ + #if defined(_PyEXPORTS_CORE) + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL + #else + #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) + #endif #else /* * If we only ever used gcc >= 5, we could use __has_attribute(visibility) @@ -52,22 +58,19 @@ #define Py_EXPORTED_SYMBOL #define Py_LOCAL_SYMBOL #endif + #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #endif /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE -# define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) -# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL -# else /* __CYGWIN__ */ +# if !defined(__CYGWIN__) # define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -77,13 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE - /* module init functions outside the core must be exported */ -# if defined(__cplusplus) -# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL -# else /* __cplusplus */ -# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL -# endif /* __cplusplus */ -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ @@ -96,13 +93,17 @@ #endif #ifndef _PyINIT_FUNC_DECLSPEC # if defined(__cplusplus) -# define _PyINIT_FUNC_DECLSPEC extern "C" Py_EXPORTED_SYMBOL +# define _PyINIT_FUNC_DECLSPEC extern "C" _PyINIT_EXPORTED_SYMBOL # else /* __cplusplus */ -# define _PyINIT_FUNC_DECLSPEC Py_EXPORTED_SYMBOL +# define _PyINIT_FUNC_DECLSPEC _PyINIT_EXPORTED_SYMBOL # endif /* __cplusplus */ #endif -#define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* -#define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* +#ifndef PyMODINIT_FUNC + #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* +#endif +#ifndef PyMODEXPORT_FUNC + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PySlot* +#endif #endif /* Py_EXPORTS_H */ diff --git a/Include/import.h b/Include/import.h index d91ebe96ca868d1..c062e46bff46bf7 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,6 +88,19 @@ PyAPI_FUNC(int) PyImport_AppendInittab( PyObject* (*initfunc)(void) ); +typedef enum { + PyImport_LAZY_NORMAL, + PyImport_LAZY_ALL +} PyImport_LazyImportsMode; + +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyImport_SetLazyImportsMode(PyImport_LazyImportsMode mode); +PyAPI_FUNC(int) PyImport_SetLazyImportsFilter(PyObject *filter); + +PyAPI_FUNC(PyImport_LazyImportsMode) PyImport_GetLazyImportsMode(void); +PyAPI_FUNC(PyObject *) PyImport_GetLazyImportsFilter(void); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H # include "cpython/import.h" diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 19e932241743141..286e7bf668312db 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -608,8 +608,8 @@ struct mi_heap_s { #if (MI_DEBUG) // use our own assertion to print without memory allocation -mi_decl_noreturn mi_decl_cold mi_decl_throw -void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func); +mi_decl_noreturn mi_decl_cold +void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func) mi_decl_throw; #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) #else #define mi_assert(x) diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 30809e097002ddf..67c6fa7c0c4ed5b 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -16,10 +16,11 @@ _PyIndex_Check(PyObject *obj) return (tp_as_number != NULL && tp_as_number->nb_index != NULL); } -PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); -PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -extern int _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); /* === Sequence protocol ================================================ */ @@ -59,6 +60,14 @@ PyAPI_FUNC(int) _Py_convert_optional_to_non_negative_ssize_t(PyObject *, void *) // Export for 'math' shared extension. PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); +typedef struct { + PyObject *object; + PySendResult kind; +} PySendResultPair; + +// Same as PyIter_Send but returns a struct for MSVC tailcall support +PyAPI_FUNC(PySendResultPair) _PyIter_Send(PyObject *iter, PyObject *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index 60367202bab6370..b47398669bbe513 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -329,12 +329,14 @@ struct _stmt { struct { asdl_alias_seq *names; + int is_lazy; } Import; struct { identifier module; asdl_alias_seq *names; int level; + int is_lazy; } ImportFrom; struct { @@ -764,11 +766,12 @@ stmt_ty _PyAST_TryStar(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, end_col_offset, PyArena *arena); stmt_ty _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); -stmt_ty _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Import(asdl_alias_seq * names, int is_lazy, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); + int is_lazy, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Global(asdl_identifier_seq * names, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); stmt_ty _PyAST_Nonlocal(asdl_identifier_seq * names, int lineno, int diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d6b2d..32c12fb5875e8ea 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; @@ -205,6 +206,7 @@ struct ast_state { PyObject *id; PyObject *ifs; PyObject *is_async; + PyObject *is_lazy; PyObject *items; PyObject *iter; PyObject *key; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 71066f1bd9f19b7..38dd82f6fc8a140 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -12,6 +12,7 @@ extern "C" { #include #include #include "pycore_structs.h" // _Py_BackoffCounter +#include "pycore_interp_structs.h" // _PyOptimizationConfig /* 16-bit countdown counters using exponential backoff. @@ -22,33 +23,48 @@ extern "C" { Another use is for the Tier 2 optimizer to decide when to create a new Tier 2 trace (executor). Again, exponential backoff is used. - The 16-bit counter is structured as a 12-bit unsigned 'value' - and a 4-bit 'backoff' field. When resetting the counter, the + The 16-bit counter is structured as a 13-bit unsigned 'value' + and a 3-bit 'backoff' field. When resetting the counter, the backoff field is incremented (until it reaches a limit) and the - value is set to a bit mask representing the value 2**backoff - 1. - The maximum backoff is 12 (the number of bits in the value). + value is set to a bit mask representing some prime value - 1. + New values and backoffs for each backoff are calculated once + at compile time and saved to value_and_backoff_next table. + The maximum backoff is 6, since 7 is an UNREACHABLE_BACKOFF. There is an exceptional value which must not be updated, 0xFFFF. */ -#define BACKOFF_BITS 4 -#define MAX_BACKOFF 12 -#define UNREACHABLE_BACKOFF 15 - -static inline bool -is_unreachable_backoff_counter(_Py_BackoffCounter counter) -{ - return counter.value_and_backoff == UNREACHABLE_BACKOFF; -} +#define BACKOFF_BITS 3 +#define BACKOFF_MASK 7 +#define MAX_BACKOFF 6 +#define UNREACHABLE_BACKOFF 7 +#define MAX_VALUE 0x1FFF + +#define MAKE_VALUE_AND_BACKOFF(value, backoff) \ + ((uint16_t)(((value) << BACKOFF_BITS) | (backoff))) + +// For previous backoff b we use value x such that +// x + 1 is near to 2**(2*b+1) and x + 1 is prime. +static const uint16_t value_and_backoff_next[] = { + MAKE_VALUE_AND_BACKOFF(1, 1), + MAKE_VALUE_AND_BACKOFF(6, 2), + MAKE_VALUE_AND_BACKOFF(30, 3), + MAKE_VALUE_AND_BACKOFF(126, 4), + MAKE_VALUE_AND_BACKOFF(508, 5), + MAKE_VALUE_AND_BACKOFF(2052, 6), + // We use the same backoff counter for all backoffs >= MAX_BACKOFF. + MAKE_VALUE_AND_BACKOFF(8190, 6), + MAKE_VALUE_AND_BACKOFF(8190, 6), +}; static inline _Py_BackoffCounter make_backoff_counter(uint16_t value, uint16_t backoff) { - assert(backoff <= 15); - assert(value <= 0xFFF); - _Py_BackoffCounter result; - result.value_and_backoff = (value << BACKOFF_BITS) | backoff; - return result; + assert(backoff <= UNREACHABLE_BACKOFF); + assert(value <= MAX_VALUE); + return ((_Py_BackoffCounter){ + .value_and_backoff = MAKE_VALUE_AND_BACKOFF(value, backoff) + }); } static inline _Py_BackoffCounter @@ -62,14 +78,11 @@ forge_backoff_counter(uint16_t counter) static inline _Py_BackoffCounter restart_backoff_counter(_Py_BackoffCounter counter) { - assert(!is_unreachable_backoff_counter(counter)); - int backoff = counter.value_and_backoff & 15; - if (backoff < MAX_BACKOFF) { - return make_backoff_counter((1 << (backoff + 1)) - 1, backoff + 1); - } - else { - return make_backoff_counter((1 << MAX_BACKOFF) - 1, MAX_BACKOFF); - } + uint16_t backoff = counter.value_and_backoff & BACKOFF_MASK; + assert(backoff <= MAX_BACKOFF); + return ((_Py_BackoffCounter){ + .value_and_backoff = value_and_backoff_next[backoff] + }); } static inline _Py_BackoffCounter @@ -113,12 +126,27 @@ trigger_backoff_counter(void) // as we always end up tracing the loop iteration's // exhaustion iteration. Which aborts our current tracer. #define JUMP_BACKWARD_INITIAL_VALUE 4000 -#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +#define JUMP_BACKWARD_INITIAL_BACKOFF 6 +static inline _Py_BackoffCounter +initial_jump_backoff_counter(_PyOptimizationConfig *opt_config) +{ + return make_backoff_counter( + opt_config->jump_backward_initial_value, + opt_config->jump_backward_initial_backoff); +} + +// This needs to be around 2-4x of JUMP_BACKWARD_INITIAL_VALUE +// The reasoning is that we always want loop traces to form and inline +// functions before functions themselves warm up and link to them instead +// of inlining. +#define RESUME_INITIAL_VALUE 8190 +#define RESUME_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_jump_backoff_counter(void) +initial_resume_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(JUMP_BACKWARD_INITIAL_VALUE, - JUMP_BACKWARD_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->resume_initial_value, + opt_config->resume_initial_backoff); } /* Initial exit temperature. @@ -126,13 +154,14 @@ initial_jump_backoff_counter(void) * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ #define SIDE_EXIT_INITIAL_VALUE 4000 -#define SIDE_EXIT_INITIAL_BACKOFF 12 +#define SIDE_EXIT_INITIAL_BACKOFF 6 static inline _Py_BackoffCounter -initial_temperature_backoff_counter(void) +initial_temperature_backoff_counter(_PyOptimizationConfig *opt_config) { - return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE, - SIDE_EXIT_INITIAL_BACKOFF); + return make_backoff_counter( + opt_config->side_exit_initial_value, + opt_config->side_exit_initial_backoff); } /* Unreachable backoff counter. */ diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665859f..322c1e93344ba30 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 8e8fa696ee0350b..27a7a46152f57b8 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -14,6 +14,11 @@ extern PyObject* _PyBytes_FormatEx( PyObject *args, int use_bytearray); +/* Concatenate two bytes objects. Used as the sq_concat slot and by the + * specializing interpreter. Unlike PyBytes_Concat(), this returns a new + * reference rather than modifying its first argument in place. */ +extern PyObject* _PyBytes_Concat(PyObject *a, PyObject *b); + extern PyObject* _PyBytes_FromHex( PyObject *string, int use_bytearray); @@ -57,9 +62,11 @@ _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, // // Export for 'array' shared extension. PyAPI_FUNC(void) -_PyBytes_Repeat(char* dest, Py_ssize_t len_dest, +_PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n); + /* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation for a bytes object of length n should request PyBytesObject_SIZE + n bytes. diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 506da06f7087d2e..a9db8860e91c06c 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,14 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} +extern PyObject *_PyObject_VectorcallPrepend( + PyThreadState *tstate, + PyObject *callable, + PyObject *arg, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); /* === Vectorcall protocol (PEP 590) ============================= */ @@ -191,7 +166,8 @@ _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); -extern void _PyStack_UnpackDict_Free( +// Exported for external JIT support +PyAPI_FUNC(void) _PyStack_UnpackDict_Free( PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index cef01e80514f4b1..d0d45a2343654fa 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -53,7 +53,7 @@ _PyCell_GetStackRef(PyCellObject *cell) { PyObject *value; #ifdef Py_GIL_DISABLED - value = _Py_atomic_load_ptr(&cell->ob_ref); + value = _PyObject_CAST(_Py_atomic_load_ptr(&cell->ob_ref)); if (value == NULL) { return PyStackRef_NULL; } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 33b9fd053f70cb8..19b5473271fffb5 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -33,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); @@ -96,7 +94,7 @@ typedef struct { void* (*init_state)(void); // Callback to register every trampoline being created void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); // Callback to free the trampoline state int (*free_state)(void* state); } _PyPerf_Callbacks; @@ -105,12 +103,15 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; +extern void _PyPerfJit_WriteNamedCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); #endif static inline PyObject* @@ -124,18 +125,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitTrampoline( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; @@ -214,10 +208,17 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { +static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Possible overflow if stack pointer is beyond the soft limit. + // _Py_CheckRecursiveCall will check for corner cases and + // report an error if there is an overflow. +#if _Py_STACK_GROWS_DOWN return here_addr < _tstate->c_stack_soft_limit; +#else + return here_addr > _tstate->c_stack_soft_limit; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -231,7 +232,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where)); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -245,12 +246,10 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); - return here_addr <= _tstate->c_stack_soft_limit; -} +// Export for test_peg_generator +PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( + PyThreadState *tstate, + int margin_count); static inline void _Py_LeaveRecursiveCall(void) { } @@ -273,6 +272,9 @@ PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); and asynchronous exception */ PyAPI_FUNC(int) _Py_HandlePending(PyThreadState *tstate); +/* Raise exception set by PyThreadState_SetAsyncExc, if any */ +PyAPI_FUNC(int) _PyEval_RaiseAsyncExc(PyThreadState *tstate); + extern PyObject * _PyEval_GetFrameLocals(void); typedef PyObject *(*conversion_func)(PyObject *); @@ -295,13 +297,25 @@ PyAPI_FUNC(int) _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *, PyObject* exc PyAPI_FUNC(void) _PyEval_FormatAwaitableError(PyThreadState *tstate, PyTypeObject *type, int oparg); PyAPI_FUNC(void) _PyEval_FormatExcCheckArg(PyThreadState *tstate, PyObject *exc, const char *format_str, PyObject *obj); PyAPI_FUNC(void) _PyEval_FormatExcUnbound(PyThreadState *tstate, PyCodeObject *co, int oparg); -PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs); +PyAPI_FUNC(void) _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject *kwargs, PyObject *dupkey); PyAPI_FUNC(PyObject *) _PyEval_ImportFrom(PyThreadState *, PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame *, PyObject *, PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyEval_LazyImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, + int lazy); +PyAPI_FUNC(PyObject *) _PyEval_LazyImportFrom( + PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *v, PyObject *name); +PyAPI_FUNC(PyObject *) _PyEval_ImportName( + PyThreadState *tstate, PyObject *builtins, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); +PyObject * _PyEval_ImportNameWithImport( + PyThreadState *tstate, PyObject *import_func, PyObject *globals, + PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level); PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -314,6 +328,8 @@ PyAPI_FUNC(PyObject *) _PyEval_GetAwaitable(PyObject *iterable, int oparg); PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject *name); PyAPI_FUNC(int) _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); +PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from); +PyAPI_FUNC(int) _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyObject* ns); /* * Indicate whether a special method of given 'oparg' can use the (improved) @@ -363,8 +379,6 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit) void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit); -PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value); - #ifndef Py_SUPPORTS_REMOTE_DEBUG #if defined(__APPLE__) #include @@ -394,6 +408,85 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject*) +_Py_VectorCallInstrumentation_StackRefSteal( + _PyStackRef callable, + _PyStackRef* arguments, + int total_args, + _PyStackRef kwnames, + bool call_instrumentation, + _PyInterpreterFrame* frame, + _Py_CODEUNIT* this_instr, + PyThreadState* tstate); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRef( + _PyStackRef callable, + PyCFunctionFast cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRef( + _PyStackRef callable, + PyCFunctionFastWithKeywords cfunc, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRef( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + +PyAPI_FUNC(_PyStackRef) +_Py_LoadAttr_StackRefSteal( + PyThreadState *tstate, _PyStackRef owner, + PyObject *name, _PyStackRef *self_or_null); + +// Like PyMapping_GetOptionalItem, but returns the PyObject* instead of taking +// it as an out parameter. This helps MSVC's escape analysis when used with +// tail calling. +PyAPI_FUNC(PyObject*) _PyMapping_GetOptionalItem2(PyObject* obj, PyObject* key, int* err); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9748e036bf2874e..5b1fddbe15b98b8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -141,6 +141,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyGetIterCache; + +#define INLINE_CACHE_ENTRIES_GET_ITER CACHE_ENTRIES(_PyGetIterCache) + typedef struct { _Py_BackoffCounter counter; } _PySendCache; @@ -160,6 +166,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_CONTAINS_OP CACHE_ENTRIES(_PyContainsOpCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyCallFunctionExCache; + +#define INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX CACHE_ENTRIES(_PyCallFunctionExCache) + /* "Locals plus" for a code object is the set of locals + cell vars + * free vars. This relates to variable names as well as offsets into * the "fast locals" storage array of execution frames. The compiler @@ -286,17 +298,7 @@ extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); extern void _PyCode_Clear_Executors(PyCodeObject *code); -#ifdef Py_GIL_DISABLED -// gh-115999 tracks progress on addressing this. -#define ENABLE_SPECIALIZATION 0 -// Use this to enable specialization families once they are thread-safe. All -// uses will be replaced with ENABLE_SPECIALIZATION once all families are -// thread-safe. -#define ENABLE_SPECIALIZATION_FT 1 -#else #define ENABLE_SPECIALIZATION 1 -#define ENABLE_SPECIALIZATION_FT ENABLE_SPECIALIZATION -#endif /* Specialization functions, these are exported only for other re-generated * interpreters to call */ @@ -311,8 +313,8 @@ PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins _Py_CODEUNIT *instr, PyObject *name); PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, @@ -326,6 +328,9 @@ PyAPI_FUNC(void) _Py_Specialize_Send(_PyStackRef receiver, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_Specialize_ToBool(_PyStackRef value, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); +PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr); +PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr); // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and @@ -498,6 +503,18 @@ typedef struct { int oparg; binaryopguardfunc guard; binaryopactionfunc action; + /* Static type of the result, or NULL if unknown. Used by the tier 2 + optimizer to propagate type information through _BINARY_OP_EXTEND. */ + PyTypeObject *result_type; + /* Nonzero iff `action` always returns a freshly allocated object (not + aliased to either operand). Used by the tier 2 optimizer to enable + inplace follow-up ops. */ + int result_unique; + /* Expected types of the left and right operands. Used by the tier 2 + optimizer to eliminate _GUARD_BINARY_OP_EXTEND when the operand + types are already known. NULL means unknown/don't eliminate. */ + PyTypeObject *lhs_type; + PyTypeObject *rhs_type; } _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ @@ -522,7 +539,8 @@ typedef struct { PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); @@ -556,7 +574,7 @@ _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) // Return a pointer to the thread-local bytecode for the current thread, // creating it if necessary. -extern _Py_CODEUNIT *_PyCode_GetTLBC(PyCodeObject *co); +PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co); // Reserve an index for the current thread into thread-local bytecode // arrays diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 52dca1362592d69..bfa10eadf735731 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name); in Python 3.5+? */ -extern PyObject* _PyCodec_LookupTextEncoding( +PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding( const char *encoding, const char *alternate_command); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 527141b54d0dca2..911cc1f10f15131 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -131,6 +131,7 @@ int _PyCompile_PushFBlock(struct _PyCompiler *c, _Py_SourceLocation loc, void _PyCompile_PopFBlock(struct _PyCompiler *c, enum _PyCompile_FBlockType t, _PyJumpTargetLabel block_label); _PyCompile_FBlockInfo *_PyCompile_TopFBlock(struct _PyCompiler *c); +bool _PyCompile_InExceptionHandler(struct _PyCompiler *c); int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index c77ef7910c09aa6..a833f790a621b1d 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -55,5 +55,8 @@ struct _pycontexttokenobject { // Export for '_testcapi' shared extension PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); +PyAPI_FUNC(int) _PyContext_Enter(PyThreadState *ts, PyObject *octx); +PyAPI_FUNC(int) _PyContext_Exit(PyThreadState *ts, PyObject *octx); + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 2601de40737e85c..51d99d74ca159f6 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -32,7 +32,7 @@ extern "C" { const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \ PyCriticalSection _cs; \ if (_should_lock_cs) { \ - _PyCriticalSection_Begin(&_cs, _orig_seq); \ + PyCriticalSection_Begin(&_cs, _orig_seq); \ } # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \ @@ -50,6 +50,15 @@ extern "C" { // Asserts that the mutex for the given object is locked. The mutex must // be held by the top-most critical section otherwise there's the // possibility that the mutex would be swalled out in some code paths. +// +// NOTE: We use Py_REFCNT(op) != 1 instead of +// !PyUnstable_Object_IsUniquelyReferenced(op) because the free threading +// GC can change an object's ob_tid (it overwrites ob_tid and later +// restores it from the mimalloc segment). This means +// PyUnstable_Object_IsUniquelyReferenced() may spuriously return false +// after a GC collection, even though the thread may still have exclusive +// access to the object. The refcount check is a looser but still catches +// most misuses. #ifdef Py_DEBUG # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \ @@ -77,10 +86,10 @@ _PyCriticalSection_Resume(PyThreadState *tstate); // (private) slow path for locking the mutex PyAPI_FUNC(void) -_PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m); +_PyCriticalSection_BeginSlow(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m); PyAPI_FUNC(void) -_PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, +_PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, int is_m1_locked); PyAPI_FUNC(void) @@ -95,34 +104,30 @@ _PyCriticalSection_IsActive(uintptr_t tag) } static inline void -_PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +_PyCriticalSection_BeginMutex(PyThreadState *tstate, PyCriticalSection *c, PyMutex *m) { if (PyMutex_LockFast(m)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_mutex = m; c->_cs_prev = tstate->critical_section; tstate->critical_section = (uintptr_t)c; } else { - _PyCriticalSection_BeginSlow(c, m); + _PyCriticalSection_BeginSlow(tstate, c, m); } } -#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void -_PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) +_PyCriticalSection_Begin(PyThreadState *tstate, PyCriticalSection *c, PyObject *op) { - _PyCriticalSection_BeginMutex(c, &op->ob_mutex); + _PyCriticalSection_BeginMutex(tstate, c, &op->ob_mutex); } -#define PyCriticalSection_Begin _PyCriticalSection_Begin // Removes the top-most critical section from the thread's stack of critical // sections. If the new top-most critical section is inactive, then it is // resumed. static inline void -_PyCriticalSection_Pop(PyCriticalSection *c) +_PyCriticalSection_Pop(PyThreadState *tstate, PyCriticalSection *c) { - PyThreadState *tstate = _PyThreadState_GET(); uintptr_t prev = c->_cs_prev; tstate->critical_section = prev; @@ -132,7 +137,7 @@ _PyCriticalSection_Pop(PyCriticalSection *c) } static inline void -_PyCriticalSection_End(PyCriticalSection *c) +_PyCriticalSection_End(PyThreadState *tstate, PyCriticalSection *c) { // If the mutex is NULL, we used the fast path in // _PyCriticalSection_BeginSlow for locks already held in the top-most @@ -141,18 +146,17 @@ _PyCriticalSection_End(PyCriticalSection *c) return; } PyMutex_Unlock(c->_cs_mutex); - _PyCriticalSection_Pop(c); + _PyCriticalSection_Pop(tstate, c); } -#define PyCriticalSection_End _PyCriticalSection_End static inline void -_PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +_PyCriticalSection2_BeginMutex(PyThreadState *tstate, PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) { if (m1 == m2) { // If the two mutex arguments are the same, treat this as a critical // section with a single mutex. c->_cs_mutex2 = NULL; - _PyCriticalSection_BeginMutex(&c->_cs_base, m1); + _PyCriticalSection_BeginMutex(tstate, &c->_cs_base, m1); return; } @@ -167,7 +171,6 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) if (PyMutex_LockFast(m1)) { if (PyMutex_LockFast(m2)) { - PyThreadState *tstate = _PyThreadState_GET(); c->_cs_base._cs_mutex = m1; c->_cs_mutex2 = m2; c->_cs_base._cs_prev = tstate->critical_section; @@ -176,29 +179,26 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) tstate->critical_section = p; } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 1); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 1); } } else { - _PyCriticalSection2_BeginSlow(c, m1, m2, 0); + _PyCriticalSection2_BeginSlow(tstate, c, m1, m2, 0); } } -#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void -_PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) +_PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject *a, PyObject *b) { - _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex); + _PyCriticalSection2_BeginMutex(tstate, c, &a->ob_mutex, &b->ob_mutex); } -#define PyCriticalSection2_Begin _PyCriticalSection2_Begin static inline void -_PyCriticalSection2_End(PyCriticalSection2 *c) +_PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { - // if mutex1 is NULL, we used the fast path in - // _PyCriticalSection_BeginSlow for mutexes that are already held, - // which should only happen when mutex1 and mutex2 were the same mutex, - // and mutex2 should also be NULL. + // if mutex1 is NULL, we used the fast path in either + // _PyCriticalSection_BeginSlow or _PyCriticalSection2_BeginSlow for mutexes + // that are already held, and mutex2 should also be NULL. if (c->_cs_base._cs_mutex == NULL) { assert(c->_cs_mutex2 == NULL); return; @@ -207,9 +207,8 @@ _PyCriticalSection2_End(PyCriticalSection2 *c) PyMutex_Unlock(c->_cs_mutex2); } PyMutex_Unlock(c->_cs_base._cs_mutex); - _PyCriticalSection_Pop(&c->_cs_base); + _PyCriticalSection_Pop(tstate, &c->_cs_base); } -#define PyCriticalSection2_End _PyCriticalSection2_End static inline void _PyCriticalSection_AssertHeld(PyMutex *mutex) @@ -251,6 +250,45 @@ _PyCriticalSection_AssertHeldObj(PyObject *op) #endif } + +#undef Py_BEGIN_CRITICAL_SECTION +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_Begin(_cs_tstate, &_py_cs, _PyObject_CAST(op)) + +#undef Py_BEGIN_CRITICAL_SECTION_MUTEX +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection_BeginMutex(_cs_tstate, &_py_cs, mutex) + +#undef Py_END_CRITICAL_SECTION +# define Py_END_CRITICAL_SECTION() \ + _PyCriticalSection_End(_cs_tstate, &_py_cs); \ + } + +#undef Py_BEGIN_CRITICAL_SECTION2 +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_Begin(_cs_tstate, &_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +#undef Py_BEGIN_CRITICAL_SECTION2_MUTEX +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyThreadState *_cs_tstate = _PyThreadState_GET(); \ + _PyCriticalSection2_BeginMutex(_cs_tstate, &_py_cs2, m1, m2) + +#undef Py_END_CRITICAL_SECTION2 +# define Py_END_CRITICAL_SECTION2() \ + _PyCriticalSection2_End(_cs_tstate, &_py_cs2); \ + } + #endif /* Py_GIL_DISABLED */ #ifdef __cplusplus diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 81faffac1941719..bed966681fa1f0c 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -265,6 +265,12 @@ typedef struct { // heap types PyObject *PyExc_NotShareableError; } exceptions; + + // Cached references to pickle.dumps/loads (per-interpreter). + struct { + PyObject *dumps; + PyObject *loads; + } pickle; } _PyXI_state_t; #define _PyXI_GET_GLOBAL_STATE(interp) (&(interp)->runtime->xi) diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 8e7cd16acffa48a..6e1eb573c8c2c83 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -102,12 +102,24 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t base_frame; + uint64_t last_profiled_frame; + uint64_t last_profiled_frame_seq; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; + uint64_t current_exception; + uint64_t exc_state; } thread_state; + // Exception stack item offset + struct { + uint64_t exc_value; + } err_stackitem; + // InterpreterFrame offset; struct _interpreter_frame { uint64_t size; @@ -147,8 +159,16 @@ typedef struct _Py_DebugOffsets { uint64_t tp_name; uint64_t tp_repr; uint64_t tp_flags; + uint64_t tp_basicsize; + uint64_t tp_dictoffset; } type_object; + // PyHeapTypeObject offset; + struct _heap_type_object { + uint64_t size; + uint64_t ht_cached_keys; + } heap_type_object; + // PyTuple object offset; struct _tuple_object { uint64_t size; @@ -204,12 +224,16 @@ typedef struct _Py_DebugOffsets { uint64_t state; uint64_t length; uint64_t asciiobject_size; + uint64_t compactunicodeobject_size; } unicode_object; // GC runtime state offset; struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; + uint64_t generation_stats_size; + uint64_t generation_stats; } gc; // Generator object offset; @@ -269,10 +293,20 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .base_frame = offsetof(PyThreadState, base_frame), \ + .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ + .last_profiled_frame_seq = offsetof(PyThreadState, last_profiled_frame_seq), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ + .current_exception = offsetof(PyThreadState, current_exception), \ + .exc_state = offsetof(PyThreadState, exc_state), \ + }, \ + .err_stackitem = { \ + .exc_value = offsetof(_PyErr_StackItem, exc_value), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ @@ -306,6 +340,12 @@ typedef struct _Py_DebugOffsets { .tp_name = offsetof(PyTypeObject, tp_name), \ .tp_repr = offsetof(PyTypeObject, tp_repr), \ .tp_flags = offsetof(PyTypeObject, tp_flags), \ + .tp_basicsize = offsetof(PyTypeObject, tp_basicsize), \ + .tp_dictoffset = offsetof(PyTypeObject, tp_dictoffset), \ + }, \ + .heap_type_object = { \ + .size = sizeof(PyHeapTypeObject), \ + .ht_cached_keys = offsetof(PyHeapTypeObject, ht_cached_keys), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ @@ -347,10 +387,14 @@ typedef struct _Py_DebugOffsets { .state = offsetof(PyUnicodeObject, _base._base.state), \ .length = offsetof(PyUnicodeObject, _base._base.length), \ .asciiobject_size = sizeof(PyASCIIObject), \ + .compactunicodeobject_size = sizeof(PyCompactUnicodeObject), \ }, \ .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ + .generation_stats_size = sizeof(struct gc_stats), \ + .generation_stats = offsetof(struct _gc_runtime_state, generation_stats), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_descrobject.h b/Include/internal/pycore_descrobject.h index 3cec59a68a3d2b2..6143f82176a1f2f 100644 --- a/Include/internal/pycore_descrobject.h +++ b/Include/internal/pycore_descrobject.h @@ -22,6 +22,8 @@ typedef propertyobject _PyPropertyObject; extern PyTypeObject _PyMethodWrapper_Type; +extern void *_PyMember_GetOffset(PyObject *, PyMemberDef *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index b8fe360321d14b4..7032b61d7654b89 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -31,23 +31,23 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, Py_hash_t hash); extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); +extern void _PyDict_ClearKeysVersionLockHeld(PyObject *mp); extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); extern int _PyDict_HasOnlyStringKeys(PyObject *mp); +PyAPI_FUNC(PyObject *) _PyDict_Subscript(PyObject *self, PyObject *key); +PyAPI_FUNC(PyObject *) _PyDict_SubscriptKnownHash(PyObject *self, PyObject *key, Py_hash_t hash); +PyAPI_FUNC(int) _PyDict_StoreSubscript(PyObject *self, PyObject *key, PyObject *value); + // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); @@ -60,7 +60,7 @@ extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); of a key wins, if override is 2, a KeyError with conflicting key as argument is raised. */ -PyAPI_FUNC(int) _PyDict_MergeEx(PyObject *mp, PyObject *other, int override); +PyAPI_FUNC(int) _PyDict_MergeUniq(PyObject *mp, PyObject *other, PyObject **dupkey); extern void _PyDict_DebugMallocStats(FILE *out); @@ -90,11 +90,18 @@ typedef struct { } PyDictUnicodeEntry; extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); +extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); +/* Implementations of the `|` and `|=` operators for dict, used by the + * specializing interpreter. */ +extern PyObject *_PyDict_Or(PyObject *self, PyObject *other); +extern PyObject *_PyDict_IOr(PyObject *self, PyObject *other); + /* Gets a version number unique to the current state of the keys of dict, if possible. - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDictKeys_GetVersionForCurrentState( PyInterpreterState *interp, PyDictKeysObject *dictkeys); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -104,8 +111,9 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( * * The caller must hold the per-object lock on dict. * - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDict_GetKeysVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDict_GetKeysVersionForCurrentState( PyInterpreterState *interp, PyDictObject *dict); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); @@ -114,15 +122,18 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. + * Exported for external JIT support */ -extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +PyAPI_FUNC(Py_ssize_t) _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); -extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); -extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +// Exported for external JIT support +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndexAndValue(PyDictObject *, PyObject *, PyObject **); +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndex(PyDictObject *, PyObject *); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); /* Look up a string key in an all unicode dict keys, assign the keys object a version, and * store it in version. @@ -131,9 +142,11 @@ extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject * strings. * * Returns DKIX_EMPTY if the key is not present. + * + * Exported for external JIT support */ -extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); -extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); @@ -142,13 +155,15 @@ extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +PyAPI_FUNC(int) _PyDict_SetItem_Take2_KnownHash(PyDictObject *op, PyObject *key, PyObject *value, Py_hash_t hash); +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash_LockHeld(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); -extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +PyAPI_FUNC(int) _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); PyAPI_FUNC(int) _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); @@ -164,6 +179,9 @@ extern void _PyDict_Clear_LockHeld(PyObject *op); PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); #endif +// Export for '_elementtree' shared extension +PyAPI_FUNC(PyObject*) _PyDict_CopyAsDict(PyObject *op); + #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ #define DKIX_ERROR (-3) @@ -222,6 +240,17 @@ struct _dictkeysobject { see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; +struct _instancekeysobject { + PyTypeObject* dsk_owning_type; + struct _dictkeysobject dsk_keys; +}; + +static inline struct _instancekeysobject *_PyDictKeys_AsSharedKeys(struct _dictkeysobject *keys) +{ + assert(keys->dk_kind == DICT_KEYS_SPLIT); + return _Py_CONTAINER_OF(keys, struct _instancekeysobject, dsk_keys); +} + /* This must be no more than 250, for the prefix size to fit in one byte. */ #define SHARED_KEYS_MAX_SIZE 30 #define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6 @@ -270,6 +299,13 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_UNIQUE_ID_SHIFT (32) #define DICT_UNIQUE_ID_MAX ((UINT64_C(1) << (64 - DICT_UNIQUE_ID_SHIFT)) - 1) +/* The first three dict watcher IDs are reserved for CPython, + * so we don't need to check that they haven't been used */ +#define BUILTINS_WATCHER_ID 0 +#define GLOBALS_WATCHER_ID 1 +#define MODULE_WATCHER_ID 2 +#define FIRST_AVAILABLE_WATCHER 3 + PyAPI_FUNC(void) _PyDict_SendEvent(int watcher_bits, @@ -279,14 +315,13 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline void -_PyDict_NotifyEvent(PyInterpreterState *interp, - PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); @@ -318,6 +353,10 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) values->size = size+1; } +// Exported for external JIT support +PyAPI_FUNC(void) +_PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix); + static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { @@ -362,13 +401,13 @@ PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); static inline Py_ssize_t _PyDict_UniqueId(PyDictObject *mp) { - return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); + return (Py_ssize_t)(FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) >> DICT_UNIQUE_ID_SHIFT); } static inline void _Py_INCREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_INCREF_OBJECT(op, id); } @@ -376,7 +415,7 @@ _Py_INCREF_DICT(PyObject *op) static inline void _Py_DECREF_DICT(PyObject *op) { - assert(PyDict_Check(op)); + assert(PyAnyDict_Check(op)); Py_ssize_t id = _PyDict_UniqueId((PyDictObject *)op); _Py_THREAD_DECREF_OBJECT(op, id); } @@ -406,6 +445,15 @@ _Py_DECREF_BUILTINS(PyObject *op) } #endif +/* frozendict */ +typedef struct { + PyDictObject ob_base; + Py_hash_t ma_hash; +} PyFrozenDictObject; + +#define _PyFrozenDictObject_CAST(op) \ + (assert(PyFrozenDict_Check(op)), _Py_CAST(PyFrozenDictObject*, (op))) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index 16916f1a8eb16cc..e37c53a64f4a723 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,9 +27,6 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index 78cd657e6ae5aee..9ddd70d39ed0d51 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -42,6 +42,7 @@ struct faulthandler_user_signal { int chain; _Py_sighandler_t previous; PyInterpreterState *interp; + Py_ssize_t max_threads; }; #endif /* FAULTHANDLER_USER */ @@ -57,6 +58,7 @@ struct _faulthandler_runtime_state { void *exc_handler; #endif int c_stack; + Py_ssize_t max_threads; } fatal_error; struct { @@ -68,6 +70,7 @@ struct _faulthandler_runtime_state { int exit; char *header; size_t header_len; + Py_ssize_t max_threads; /* The main thread always holds this lock. It is only released when faulthandler_thread() is interrupted before this thread exits, or at Python exit. */ diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index 317f984188bad83..62501cdaf44f072 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -12,7 +12,6 @@ extern "C" { /* runtime lifecycle */ -extern void _PyFloat_InitState(PyInterpreterState *); extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); extern void _PyFloat_FiniType(PyInterpreterState *); @@ -42,6 +41,15 @@ extern double _Py_parse_inf_or_nan(const char *p, char **endptr); extern int _Py_convert_int_to_double(PyObject **v, double *dbl); +/* Should match endianness of the platform in most (all?) cases. */ + +#ifdef DOUBLE_IS_BIG_ENDIAN_IEEE754 +# define _PY_FLOAT_BIG_ENDIAN 1 +# define _PY_FLOAT_LITTLE_ENDIAN 0 +#else +# define _PY_FLOAT_BIG_ENDIAN 0 +# define _PY_FLOAT_LITTLE_ENDIAN 1 +#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 5043260d2fd99f2..fc3ae6b369c54c1 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -27,7 +27,7 @@ int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject struct _PyCfgBuilder* _PyCfg_FromInstructionSequence(_PyInstructionSequence *seq); int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyInstructionSequence *seq); int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, - int code_flags, int *stackdepth, int *nlocalsplus, + int *stackdepth, int *nlocalsplus, _PyInstructionSequence *seq); PyCodeObject * diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c410e9e2083407..3c9ab99c34ebc64 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -38,23 +38,23 @@ struct _frame { PyObject *_f_frame_data[1]; }; -extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); +// Exported for external JIT support +PyAPI_FUNC(PyFrameObject *) _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ typedef enum _framestate { - FRAME_CREATED = -3, - FRAME_SUSPENDED = -2, - FRAME_SUSPENDED_YIELD_FROM = -1, - FRAME_EXECUTING = 0, - FRAME_COMPLETED = 1, - FRAME_CLEARED = 4 + FRAME_CREATED = 0, + FRAME_SUSPENDED = 1, + FRAME_SUSPENDED_YIELD_FROM = 2, + FRAME_SUSPENDED_YIELD_FROM_LOCKED = 3, + FRAME_EXECUTING = 4, + FRAME_CLEARED = 5 } PyFrameState; -#define FRAME_STATE_SUSPENDED(S) ((S) == FRAME_SUSPENDED || (S) == FRAME_SUSPENDED_YIELD_FROM) -#define FRAME_STATE_FINISHED(S) ((S) >= FRAME_COMPLETED) - +#define FRAME_STATE_SUSPENDED(S) ((S) >= FRAME_SUSPENDED && (S) <= FRAME_SUSPENDED_YIELD_FROM_LOCKED) +#define FRAME_STATE_FINISHED(S) ((S) == FRAME_CLEARED) #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index e89f4b5c8a4ec10..2184f40956d4e24 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -27,10 +27,10 @@ _PyFunction_IsVersionValid(uint32_t version) return version >= FUNC_VERSION_FIRST_VALID; } -extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +// Exported for external JIT support +PyAPI_FUNC(uint32_t) _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); void _PyFunction_ClearCodeByVersion(uint32_t version); -PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_code); extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); @@ -47,6 +47,17 @@ static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) { #define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func)) +/* Get the callable wrapped by a classmethod. + Returns a borrowed reference. + The caller must ensure 'cm' is a classmethod object. */ +extern PyObject *_PyClassMethod_GetFunc(PyObject *cm); + +/* Get the callable wrapped by a staticmethod. + Returns a borrowed reference. + The caller must ensure 'sm' is a staticmethod object. */ +extern PyObject *_PyStaticMethod_GetFunc(PyObject *sm); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index fd284d0e4ecc2f5..84cbb56a9192156 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -207,10 +190,6 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { extern void _Py_ScheduleGC(PyThreadState *tstate); -#ifndef Py_GIL_DISABLED -extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); -#endif - /* Tell the GC to track this object. * @@ -220,7 +199,7 @@ extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -245,18 +224,13 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - PyGC_Head *generation0 = &gcstate->young.head; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ gcstate->visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; - gcstate->young.count++; /* number of tracked GC objects */ gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold) { - _Py_TriggerGC(gcstate); - } #endif } @@ -292,9 +266,6 @@ static inline void _PyObject_GC_UNTRACK( gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - if (gcstate->young.count > 0) { - gcstate->young.count--; - } gcstate->heap_size--; #endif } @@ -364,8 +335,9 @@ extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; // GC visit callback for tracked interpreter frames -extern int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); -extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// GH-150766: exported for greenlet +PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); #ifdef Py_GIL_DISABLED extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index b08c8c52221f4b2..c86ae242feac1ed 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,7 +22,7 @@ PyGenObject *_PyGen_GetGeneratorFromFrame(_PyInterpreterFrame *frame) } PyAPI_FUNC(PyObject *)_PyGen_yf(PyGenObject *); -extern void _PyGen_Finalize(PyObject *self); +extern int _PyGen_ClearFrame(PyGenObject *self); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); @@ -33,10 +33,15 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); + extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; +PyAPI_FUNC(PySendResult) _PyAsyncGenASend_Send(PyObject *iter, PyObject *arg, PyObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 92ded14891a1013..7d952a1e52561a6 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyObject_Dump(obj); } } #endif @@ -1326,18 +1326,32 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(AGEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CORO_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emax)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Emin)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(FINISHED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(False)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CLOSED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_CREATED)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_RUNNING)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(GEN_SUSPENDED)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(JSONDecodeError)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(PENDING)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(Py_Repr)); @@ -1432,6 +1446,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__iter__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__itruediv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__ixor__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_import__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__lazy_modules__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__le__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__len__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__length_hint__)); @@ -1560,13 +1576,16 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(add_done_callback)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(adobe)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_child)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(after_in_parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_interpreters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arg)); @@ -1578,6 +1597,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(asend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(athrow)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); @@ -1607,6 +1627,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_stack)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cache_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); @@ -1617,6 +1638,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -1650,9 +1672,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(co_varnames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(col_offset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(collector)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compression)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(config)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); @@ -1715,7 +1739,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_tb)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_type)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_val)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exc_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(excepthook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception)); @@ -1750,6 +1776,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flags)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(flush)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fold)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(foldspaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(follow_symlinks)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(format_spec)); @@ -1763,10 +1790,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_event_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(getattr)); @@ -1789,6 +1816,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignorechars)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(imag)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(implieslink)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(importlib)); @@ -1870,6 +1898,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); @@ -1906,6 +1935,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); @@ -1932,6 +1962,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcodes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(open)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opener)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(operation)); @@ -1945,7 +1976,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outpath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pad)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(padded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pair)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parameter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password)); @@ -1957,6 +1991,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(person)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pi_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pidfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pointer_bits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); @@ -1968,6 +2003,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress_routine)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(proto)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(protocol)); @@ -1988,6 +2024,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readline)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(readonly)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(real)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(recursive)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); @@ -1996,6 +2033,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repeat)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); @@ -2007,6 +2045,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); @@ -2046,9 +2085,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(spam)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(src_dir_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stack_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start_time_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(statement)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stats)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(status)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stderr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); @@ -2066,8 +2108,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(symmetric_difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tabsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tag)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(targetfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_frame)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_lasti)); @@ -2086,6 +2130,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(times)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timespec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timestamp_us)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timetuple)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(timeunit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(top)); @@ -2109,6 +2154,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(utcoffset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); @@ -2124,6 +2170,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(which)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(who)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(withdata)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wrapcol)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cd21b0847b7cdde..8a8bbc3b6d05bf7 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,21 +46,35 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") } literals; struct { + STRUCT_FOR_ID(AGEN_CLOSED) + STRUCT_FOR_ID(AGEN_CREATED) + STRUCT_FOR_ID(AGEN_RUNNING) + STRUCT_FOR_ID(AGEN_SUSPENDED) STRUCT_FOR_ID(CANCELLED) + STRUCT_FOR_ID(CORO_CLOSED) + STRUCT_FOR_ID(CORO_CREATED) + STRUCT_FOR_ID(CORO_RUNNING) + STRUCT_FOR_ID(CORO_SUSPENDED) STRUCT_FOR_ID(Emax) STRUCT_FOR_ID(Emin) STRUCT_FOR_ID(FINISHED) STRUCT_FOR_ID(False) + STRUCT_FOR_ID(GEN_CLOSED) + STRUCT_FOR_ID(GEN_CREATED) + STRUCT_FOR_ID(GEN_RUNNING) + STRUCT_FOR_ID(GEN_SUSPENDED) STRUCT_FOR_ID(JSONDecodeError) STRUCT_FOR_ID(PENDING) STRUCT_FOR_ID(Py_Repr) @@ -155,6 +169,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(__iter__) STRUCT_FOR_ID(__itruediv__) STRUCT_FOR_ID(__ixor__) + STRUCT_FOR_ID(__lazy_import__) + STRUCT_FOR_ID(__lazy_modules__) STRUCT_FOR_ID(__le__) STRUCT_FOR_ID(__len__) STRUCT_FOR_ID(__length_hint__) @@ -283,13 +299,16 @@ struct _Py_global_strings { STRUCT_FOR_ID(aclose) STRUCT_FOR_ID(add) STRUCT_FOR_ID(add_done_callback) + STRUCT_FOR_ID(adobe) STRUCT_FOR_ID(after_in_child) STRUCT_FOR_ID(after_in_parent) STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_interpreters) STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) + STRUCT_FOR_ID(alphabet) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) STRUCT_FOR_ID(arg) @@ -301,6 +320,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(asend) STRUCT_FOR_ID(ast) STRUCT_FOR_ID(athrow) + STRUCT_FOR_ID(attr) STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(autocommit) STRUCT_FOR_ID(backtick) @@ -330,6 +350,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_parameter_type) STRUCT_FOR_ID(c_return) STRUCT_FOR_ID(c_stack) + STRUCT_FOR_ID(cache_frames) STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) @@ -340,6 +361,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) + STRUCT_FOR_ID(canonical) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) @@ -373,9 +395,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(co_varnames) STRUCT_FOR_ID(code) STRUCT_FOR_ID(col_offset) + STRUCT_FOR_ID(collector) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) STRUCT_FOR_ID(compile_mode) + STRUCT_FOR_ID(compression) STRUCT_FOR_ID(config) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) @@ -438,7 +462,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) STRUCT_FOR_ID(exc) + STRUCT_FOR_ID(exc_tb) STRUCT_FOR_ID(exc_type) + STRUCT_FOR_ID(exc_val) STRUCT_FOR_ID(exc_value) STRUCT_FOR_ID(excepthook) STRUCT_FOR_ID(exception) @@ -473,6 +499,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(flags) STRUCT_FOR_ID(flush) STRUCT_FOR_ID(fold) + STRUCT_FOR_ID(foldspaces) STRUCT_FOR_ID(follow_symlinks) STRUCT_FOR_ID(format) STRUCT_FOR_ID(format_spec) @@ -486,10 +513,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) - STRUCT_FOR_ID(get_event_loop) STRUCT_FOR_ID(get_loop) STRUCT_FOR_ID(get_source) STRUCT_FOR_ID(getattr) @@ -512,6 +539,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) + STRUCT_FOR_ID(ignorechars) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(implieslink) STRUCT_FOR_ID(importlib) @@ -593,6 +621,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) + STRUCT_FOR_ID(max_threads) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) @@ -629,6 +658,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) @@ -655,6 +685,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) + STRUCT_FOR_ID(opcodes) STRUCT_FOR_ID(open) STRUCT_FOR_ID(opener) STRUCT_FOR_ID(operation) @@ -668,7 +699,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(outpath) STRUCT_FOR_ID(overlapped) STRUCT_FOR_ID(owner) + STRUCT_FOR_ID(pad) + STRUCT_FOR_ID(padded) STRUCT_FOR_ID(pages) + STRUCT_FOR_ID(pair) STRUCT_FOR_ID(parameter) STRUCT_FOR_ID(parent) STRUCT_FOR_ID(password) @@ -680,6 +714,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(person) STRUCT_FOR_ID(pi_factory) STRUCT_FOR_ID(pid) + STRUCT_FOR_ID(pidfd) STRUCT_FOR_ID(pointer_bits) STRUCT_FOR_ID(policy) STRUCT_FOR_ID(pos) @@ -691,6 +726,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) + STRUCT_FOR_ID(progress_callback) STRUCT_FOR_ID(progress_routine) STRUCT_FOR_ID(proto) STRUCT_FOR_ID(protocol) @@ -711,6 +747,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(readline) STRUCT_FOR_ID(readonly) STRUCT_FOR_ID(real) + STRUCT_FOR_ID(recursive) STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) @@ -719,6 +756,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(repeat) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(repr) STRUCT_FOR_ID(reqrefs) STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) @@ -730,6 +768,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reversed) STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) + STRUCT_FOR_ID(sample_interval_us) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) @@ -769,9 +808,12 @@ struct _Py_global_strings { STRUCT_FOR_ID(spam) STRUCT_FOR_ID(src) STRUCT_FOR_ID(src_dir_fd) + STRUCT_FOR_ID(stack_frames) STRUCT_FOR_ID(stacklevel) STRUCT_FOR_ID(start) + STRUCT_FOR_ID(start_time_us) STRUCT_FOR_ID(statement) + STRUCT_FOR_ID(stats) STRUCT_FOR_ID(status) STRUCT_FOR_ID(stderr) STRUCT_FOR_ID(stdin) @@ -789,8 +831,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(symmetric_difference_update) STRUCT_FOR_ID(tabsize) STRUCT_FOR_ID(tag) + STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) + STRUCT_FOR_ID(targetfd) STRUCT_FOR_ID(task) STRUCT_FOR_ID(tb_frame) STRUCT_FOR_ID(tb_lasti) @@ -809,6 +853,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(times) STRUCT_FOR_ID(timespec) STRUCT_FOR_ID(timestamp) + STRUCT_FOR_ID(timestamp_us) STRUCT_FOR_ID(timetuple) STRUCT_FOR_ID(timeunit) STRUCT_FOR_ID(top) @@ -832,6 +877,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) + STRUCT_FOR_ID(utcoffset) STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) @@ -847,6 +893,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(which) STRUCT_FOR_ID(who) STRUCT_FOR_ID(withdata) + STRUCT_FOR_ID(wrapcol) STRUCT_FOR_ID(writable) STRUCT_FOR_ID(write) STRUCT_FOR_ID(write_through) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index d64a18bb09e08f8..a1078828afa572e 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -32,6 +32,20 @@ extern int _PyImport_FixupBuiltin( PyObject *modules ); +extern PyObject * _PyImport_ResolveName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +extern PyObject * _PyImport_GetAbsName( + PyThreadState *tstate, PyObject *name, PyObject *globals, int level); +// Symbol is exported for the JIT on Windows builds. +PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( + PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name); +extern PyObject * _PyImport_LazyImportModuleLevelObject( + PyThreadState *tstate, PyObject *name, PyObject *builtins, + PyObject *globals, PyObject *locals, PyObject *fromlist, int level); + + #ifdef HAVE_DLOPEN # include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW @@ -69,11 +83,19 @@ extern void _PyImport_ClearModules(PyInterpreterState *interp); extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp); +extern PyObject * _PyImport_InitLazyModules( + PyInterpreterState *interp); +extern void _PyImport_ClearLazyModules(PyInterpreterState *interp); + extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp); extern int _PyImport_IsDefaultImportFunc( PyInterpreterState *interp, PyObject *func); +extern int _PyImport_IsDefaultLazyImportFunc( + PyInterpreterState *interp, + PyObject *func); + extern PyObject * _PyImport_GetImportlibLoader( PyInterpreterState *interp, const char *loader_name); @@ -128,11 +150,18 @@ PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); // state of the module argument: // - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, // call _PyEval_DisableGIL(). -// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already -// enabled permanently, issue a warning referencing the module's name. +// - Otherwise, call _PyImport_EnableGILAndWarn // // This function may raise an exception. extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), call _PyEval_EnableGILPermanent(). +// If the GIL was not already enabled permanently, issue a warning referencing +// the module's name. +// Leave a message in verbose mode. +// +// This function may raise an exception. +extern int _PyImport_EnableGILAndWarn(PyThreadState *, PyObject *module_name); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index f60c5510d200752..9ed87a544234c59 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -124,7 +124,7 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); -typedef PyModuleDef_Slot *(*PyModExportFunction)(void); +typedef PySlot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 368dafb90635d7b..183b2d45c5ede1c 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,10 +153,8 @@ typedef enum { } _PyConfigInitEnum; typedef enum { - /* For now, this means the GIL is enabled. - - gh-116329: This will eventually change to "the GIL is disabled but can - be re-enabled by loading an incompatible extension module." */ + /* In free threaded builds, this means that the GIL is disabled at startup, + but may be enabled by loading an incompatible extension module. */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index b5c927735374beb..6257b771fd9539c 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -73,6 +73,7 @@ int _PyInstructionSequence_SetAnnotationsCode(_PyInstructionSequence *seq, _PyInstructionSequence *annotations); int _PyInstructionSequence_AddNested(_PyInstructionSequence *seq, _PyInstructionSequence *nested); void PyInstructionSequence_Fini(_PyInstructionSequence *seq); +_PyInstruction _PyInstructionSequence_GetInstruction(_PyInstructionSequence *seq, int pos); extern PyTypeObject _PyInstructionSequence_Type; #define _PyInstructionSequence_Check(v) Py_IS_TYPE((v), &_PyInstructionSequence_Type) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index ebc5622912f0cbb..56b55e93a014cb5 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -64,25 +64,21 @@ PyAPI_FUNC(void) _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index); - -extern PyObject _PyInstrumentation_MISSING; -extern PyObject _PyInstrumentation_DISABLE; +PyAPI_DATA(PyObject) _PyInstrumentation_MISSING; +PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -122,6 +118,17 @@ typedef struct _PyCoMonitoringData { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); + +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9e4504479cd9f04..d3efac906aeb933 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,6 +14,7 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState +#include "pycore_uop.h" // _PyBloomFilter #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -68,7 +69,7 @@ struct code_arena_st; struct trampoline_api_st { void* (*init_state)(void); void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; Py_ssize_t code_padding; @@ -87,7 +88,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; - _PyFrameEvalFunction prev_eval_frame; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif @@ -174,56 +177,83 @@ struct gc_generation { generations */ }; -struct gc_collection_stats { - /* number of collected objects */ - Py_ssize_t collected; - /* total number of uncollectable objects (put into gc.garbage) */ - Py_ssize_t uncollectable; -}; - /* Running stats per generation */ struct gc_generation_stats { + PyTime_t ts_start; + PyTime_t ts_stop; /* total number of collections */ Py_ssize_t collections; /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; + // Total number of objects considered for collection and traversed: + Py_ssize_t candidates; + // Total duration of the collection in seconds: + double duration; + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 +#ifdef Py_GIL_DISABLED +#define GC_YOUNG_STATS_SIZE 1 +#define GC_OLD_STATS_SIZE 1 +#else +#define GC_YOUNG_STATS_SIZE 11 +#define GC_OLD_STATS_SIZE 3 +#endif +struct gc_young_stats_buffer { + struct gc_generation_stats items[GC_YOUNG_STATS_SIZE]; + int8_t index; +}; + +struct gc_old_stats_buffer { + struct gc_generation_stats items[GC_OLD_STATS_SIZE]; + int8_t index; }; /* If we change this, we need to change the default value in the signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 +struct gc_stats { + struct gc_young_stats_buffer young; + struct gc_old_stats_buffer old[2]; +}; + struct _gc_runtime_state { /* Is automatic collection enabled? */ int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; +#else struct gc_generation young; struct gc_generation old[2]; +#endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; - struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + struct gc_stats *generation_stats; +#ifdef Py_GIL_DISABLED + /* Protects generation_stats between gc_get_stats_impl() (reader) and + gc_collect_main() (writer); both resolve the stats slot (buffer index) + under this lock so they agree on which slot to touch (gh-151646). */ + PyMutex stats_mutex; +#endif /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; -#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -236,21 +266,31 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; - - /* Memory usage of the process (RSS + swap) after last GC. */ - Py_ssize_t last_mem; - - /* This accumulates the new object count whenever collection is deferred - due to the RSS increase condition not being meet. Reset on collection. */ - Py_ssize_t deferred_count; - - /* Mutex held for gc_should_collect_mem_usage(). */ - PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, \ + .heap_size = 0, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -274,8 +314,6 @@ struct _import_runtime_state { Modules are added there and looked up in _imp.find_extension(). */ struct _Py_hashtable_t *hashtable; } extensions; - /* Package context -- the full module name for package imports */ - const char * pkgcontext; }; struct _import_state { @@ -311,6 +349,22 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + PyObject *lazy_import_func; + int lazy_imports_mode; + PyObject *lazy_imports_filter; + PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. + PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; +#ifdef Py_GIL_DISABLED + PyMutex lazy_mutex; +#endif /* The global import lock. */ _PyRecursiveMutex lock; /* diagnostic info in PyImport_ImportModuleLevelObject() */ @@ -385,6 +439,28 @@ typedef struct _rare_events { uint8_t func_modification; } _rare_events; +// Optimization configuration for the interpreter. +// This groups all thresholds and optimization flags for both JIT and interpreter. +typedef struct _PyOptimizationConfig { + // Interpreter optimization thresholds + uint16_t jump_backward_initial_value; + uint16_t jump_backward_initial_backoff; + + uint16_t resume_initial_value; + uint16_t resume_initial_backoff; + + // JIT optimization thresholds + uint16_t side_exit_initial_value; + uint16_t side_exit_initial_backoff; + + // Trace fitness thresholds + uint16_t fitness_initial; + + // Optimization flags + bool specialization_enabled; + bool uops_optimize_enabled; +} _PyOptimizationConfig; + struct Bigint { struct Bigint *next; @@ -458,8 +534,13 @@ struct _py_func_state { /****** type state *********/ /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 + all the static builtin types will fit (for all builds). + If you add a new static type to the standard library, you may have to + update one of these numbers. + */ +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 120 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ + (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) @@ -755,6 +836,8 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +#define _PyInterpreterGuard_GUARDS_NOT_ALLOWED UINTPTR_MAX + /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -858,6 +941,7 @@ struct _is { PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; + int eval_frame_allow_specialization; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers @@ -929,14 +1013,19 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; - struct _PyExecutorObject *executor_list_head; + + // Optimization configuration (thresholds and flags for JIT and interpreter) + _PyOptimizationConfig opt_config; + _PyBloomFilter *executor_blooms; // Contiguous bloom filter array + struct _PyExecutorObject **executor_ptrs; // Corresponding executor pointer array + size_t executor_count; // Number of valid executors + size_t executor_capacity; // Array capacity struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; struct _PyExecutorObject *cold_dynamic_executor; - int executor_deletion_list_remaining_capacity; size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; @@ -975,6 +1064,11 @@ struct _is { #endif #endif + // The number of remaining finalization guards. + // If this is _PyInterpreterGuard_GUARDS_NOT_ALLOWED, then finalization + // guards can no longer be created. + uintptr_t finalization_guards; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 8949d6cc2fc4bb3..9809cd292995f0b 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -27,7 +27,7 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { // Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or // freed. Used by dump_frame() in Python/traceback.c. The function uses // heuristics to detect freed memory, it's not 100% reliable. -static inline PyCodeObject* +static inline PyCodeObject* _Py_NO_SANITIZE_THREAD _PyFrame_SafeGetCode(_PyInterpreterFrame *f) { // globals and builtins may be NULL on a legit frame, but it's unlikely. @@ -70,7 +70,7 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) // Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the // frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The // function uses heuristics to detect freed memory, it's not 100% reliable. -static inline int +static inline int _Py_NO_SANITIZE_THREAD _PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) { // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() @@ -102,10 +102,10 @@ static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } -static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f) { +static inline _PyStackRef _PyFrame_StackPeek(_PyInterpreterFrame *f, int depth) { assert(f->stackpointer > _PyFrame_Stackbase(f)); - assert(!PyStackRef_IsNull(f->stackpointer[-1])); - return f->stackpointer[-1]; + assert(!PyStackRef_IsNull(f->stackpointer[-depth])); + return f->stackpointer[-depth]; } static inline _PyStackRef _PyFrame_StackPop(_PyInterpreterFrame *f) { @@ -149,6 +149,12 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * int stacktop = (int)(src->stackpointer - src->localsplus); assert(stacktop >= 0); dest->stackpointer = dest->localsplus + stacktop; + // visited is GC bookkeeping for the current stack walk, not frame state. + dest->visited = 0; +#ifdef Py_DEBUG + dest->stackpointer_valid = src->stackpointer_valid; + dest->lltrace = src->lltrace; +#endif for (int i = 0; i < stacktop; i++) { dest->localsplus[i] = PyStackRef_MakeHeapSafe(src->localsplus[i]); } @@ -202,6 +208,7 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif @@ -225,21 +232,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - assert(frame->stackpointer != NULL); - _PyStackRef *sp = frame->stackpointer; -#ifndef NDEBUG - frame->stackpointer = NULL; -#endif - return sp; + return frame->stackpointer; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - assert(frame->stackpointer == NULL); frame->stackpointer = stack_pointer; } +static inline void +_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif + frame->stackpointer_valid = 1; +#endif +} + +static inline void +_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 1); +#endif + frame->stackpointer_valid = 0; +#endif +} + +static inline void +_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif +#endif +} + + /* Determine whether a frame is incomplete. * A frame is incomplete if it is part way through * creating cell objects or a generator or coroutine. @@ -277,9 +314,26 @@ _PyThreadState_GetFrame(PyThreadState *tstate) return _PyFrame_GetFirstComplete(tstate->current_frame); } +// Update last_profiled_frame for remote profiler frame caching. +// Only update if we're removing the exact frame that was last profiled. +// This avoids corrupting the cache when transient frames (called and returned +// between profiler samples) update last_profiled_frame to addresses the +// profiler never saw. +// The sequence distinguishes this anchor from a later frame that reuses the +// same _PyInterpreterFrame address. +#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) \ + do { \ + PyThreadState *tstate_ = (tstate); \ + _PyInterpreterFrame *frame_ = (frame); \ + if (tstate_->last_profiled_frame == frame_) { \ + tstate_->last_profiled_frame = (previous); \ + tstate_->last_profiled_frame_seq++; \ + } \ + } while (0) + /* For use by _PyFrame_GetFrameObject Do not call directly. */ -PyFrameObject * +PyAPI_FUNC(PyFrameObject *) _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame); /* Gets the PyFrameObject for this frame, lazily @@ -297,7 +351,8 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } -void +// Exported for external JIT support +PyAPI_FUNC(void) _PyFrame_ClearLocals(_PyInterpreterFrame *frame); /* Clears all references in the frame. @@ -308,8 +363,10 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame); * in the frame. * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. + * + * Exported for external JIT support */ -void + PyAPI_FUNC(void) _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int @@ -333,7 +390,8 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) size < tstate->datastack_limit - tstate->datastack_top; } -extern _PyInterpreterFrame * +// Exported for external JIT support +PyAPI_FUNC(_PyInterpreterFrame *) _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -383,6 +441,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif frame->return_offset = 0; @@ -395,6 +454,10 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous); +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194863f..4d267e35504b94d 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { @@ -45,7 +44,8 @@ struct _PyInterpreterFrame { char owner; #ifdef Py_DEBUG uint8_t visited:1; - uint8_t lltrace:7; + uint8_t stackpointer_valid:1; + uint8_t lltrace:6; #else uint8_t visited; #endif diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 39c2a30f6e979de..59a7b16073f886c 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,8 +18,9 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 +#define INTRINSIC_BUILD_FROZENSET 12 -#define MAX_INTRINSIC_1 11 +#define MAX_INTRINSIC_1 12 /* Binary Functions: */ diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607ba4bc..9ce6ec97b6090e4 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,12 +13,26 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT -typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef _Py_CODEUNIT *(*jit_func)( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, + _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 +); + +_Py_CODEUNIT *_PyJIT_Entry( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); +PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); +PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno); +PyAPI_FUNC(int) _Py_jit_assertion_failure(int line); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_publish.h b/Include/internal/pycore_jit_publish.h new file mode 100644 index 000000000000000..7ee77f5c996fafe --- /dev/null +++ b/Include/internal/pycore_jit_publish.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_JIT_PUBLISH_H +#define Py_INTERNAL_JIT_PUBLISH_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef struct _PyJitCodeRegistration _PyJitCodeRegistration; + +#ifdef _Py_JIT + +/* Publish JIT code to optional tooling backends. + * + * The return value is a backend-specific deregistration handle, not a + * success/failure indicator. NULL means there is nothing to unregister later: + * perf does not need a handle, and GDB/GNU backtrace registration failures + * are intentionally non-fatal because tooling support must not make JIT + * compilation fail. + */ +_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration); + +#endif // _Py_JIT + +#endif // Py_INTERNAL_JIT_PUBLISH_H diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h new file mode 100644 index 000000000000000..7099b88812ce7be --- /dev/null +++ b/Include/internal/pycore_jit_unwind.h @@ -0,0 +1,81 @@ +#ifndef Py_INTERNAL_JIT_UNWIND_H +#define Py_INTERNAL_JIT_UNWIND_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include + +#if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) +# define PY_HAVE_JIT_GDB_UNWIND +# if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) +# define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND +# endif +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) \ + || defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +extern PyMutex _Py_jit_debug_mutex; +#endif + +/* DWARF exception-handling pointer encodings shared by JIT unwind users. */ +enum { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + DWRF_EH_PE_indirect = 0x80 +}; + +/* Return the size of the generated .eh_frame data for the given encoding. */ +size_t _PyJitUnwind_EhFrameSize(int absolute_addr); + +/* + * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error. + * absolute_addr selects the FDE address encoding: + * - 0: PC-relative offsets (perf jitdump synthesized DSO). + * - nonzero: absolute addresses (GDB JIT in-memory ELF). + */ +size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr); + +void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJitUnwind_GdbUnregisterCode(void *handle); + +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, + size_t code_size); + +void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle); +#endif + +#endif // JIT unwind support + +#endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h new file mode 100644 index 000000000000000..b81e4211b08ff39 --- /dev/null +++ b/Include/internal/pycore_lazyimportobject.h @@ -0,0 +1,35 @@ +// Lazy object interface. + +#ifndef Py_INTERNAL_LAZYIMPORTOBJECT_H +#define Py_INTERNAL_LAZYIMPORTOBJECT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) PyLazyImport_Type; +#define PyLazyImport_CheckExact(op) Py_IS_TYPE((op), &PyLazyImport_Type) + +typedef struct { + PyObject_HEAD + PyObject *lz_builtins; + PyObject *lz_from; + PyObject *lz_attr; + // Frame information for the original import location. + PyCodeObject *lz_code; // Code object where the lazy import was created. + int lz_instr_offset; // Instruction offset where the lazy import was created. +} PyLazyImportObject; + + +PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +PyAPI_FUNC(PyObject *) _PyLazyImport_New( + struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_LAZYIMPORTOBJECT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index ffbcebdb7dfb50d..df0d00f752573b5 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -14,15 +14,17 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*); +PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyList_Concat(PyObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. -extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); +PyAPI_FUNC(PyObject *) _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #ifdef Py_GIL_DISABLED // Returns -1 in case of races with other threads. -extern int _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); +PyAPI_FUNC(int) _PyList_GetItemRefNoLock(PyListObject *, Py_ssize_t, _PyStackRef *); #endif #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index c4e007e744ce0f7..e31d8b4e5c68c92 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -70,6 +70,9 @@ PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) // error messages) otherwise returns 0. extern int _PyMutex_TryUnlock(PyMutex *m); +// Yield the processor to other threads (e.g., sched_yield). +extern void _Py_yield(void); + // PyEvent is a one-time event notification typedef struct { diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index d545ba0c3abb522..fb5622c99f7a135 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,7 +64,8 @@ PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -#define _PY_IS_SMALL_INT(val) ((val) >= 0 && (val) < 256 && (val) < _PY_NSMALLPOSINTS) +#define _PY_IS_SMALL_INT(val) \ + (-_PY_NSMALLNEGINTS <= (val) && (val) < _PY_NSMALLPOSINTS) // Return a reference to the immortal zero singleton. // The function cannot return NULL. @@ -231,6 +232,21 @@ _PyLong_IsPositive(const PyLongObject *op) return (op->long_value.lv_tag & SIGN_MASK) == 0; } +/* Return true if the argument is a small int */ +static inline bool +_PyLong_IsSmallInt(const PyLongObject *op) +{ + assert(PyLong_Check(op)); + bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + if (is_small_int) { + assert(PyLong_CheckExact(op)); + assert(_Py_IsImmortal(op)); + assert((_PyLong_IsCompact(op) + && _PY_IS_SMALL_INT(_PyLong_CompactValue(op)))); + } + return is_small_int; +} + static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { @@ -270,6 +286,14 @@ _PyLong_SameSign(const PyLongObject *a, const PyLongObject *b) return (a->long_value.lv_tag & SIGN_MASK) == (b->long_value.lv_tag & SIGN_MASK); } +/* Initialize the tag of a freshly-allocated int. */ +static inline void +_PyLong_InitTag(PyLongObject *op) +{ + assert(PyLong_Check(op)); + op->long_value.lv_tag = SIGN_ZERO; /* non-immortal zero */ +} + #define TAG_FROM_SIGN_AND_SIZE(sign, size) \ ((uintptr_t)(1 - (sign)) | ((uintptr_t)(size) << NON_SIZE_BITS)) @@ -279,6 +303,7 @@ _PyLong_SetSignAndDigitCount(PyLongObject *op, int sign, Py_ssize_t size) assert(size >= 0); assert(-1 <= sign && sign <= 1); assert(sign != 0 || size == 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = TAG_FROM_SIGN_AND_SIZE(sign, size); } @@ -286,13 +311,16 @@ static inline void _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) { assert(size >= 0); + assert(!_PyLong_IsSmallInt(op)); op->long_value.lv_tag = (((size_t)size) << NON_SIZE_BITS) | (op->long_value.lv_tag & SIGN_MASK); } #define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void -_PyLong_FlipSign(PyLongObject *op) { +_PyLong_FlipSign(PyLongObject *op) +{ + assert(!_PyLong_IsSmallInt(op)); unsigned int flipped_sign = 2 - (op->long_value.lv_tag & SIGN_MASK); op->long_value.lv_tag &= NON_SIZE_MASK; op->long_value.lv_tag |= flipped_sign; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c6955160..53ed570f99f118a 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,9 +286,25 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) + Python 3.15a4 3657 (Add BINARY_OP_SUBSCR_USTR_INT) + Python 3.15a4 3658 (Optimize bytecode for list/set called on genexp) + Python 3.15a4 3659 (Add CALL_FUNCTION_EX specialization) + Python 3.15a4 3660 (Change generator preamble code) + Python 3.15a4 3661 (Lazy imports IMPORT_NAME opcode changes) + Python 3.15a8 3662 (Add counter to RESUME) + Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) + Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) + Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) + Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations) + Python 3.16a0 3700 (Initial version) + Python 3.16a0 3701 (Add CONSTANT_EMPTY_TUPLE to LOAD_COMMON_CONSTANT) + Python 3.16a1 3702 (Replace DELETE_NAME with PUSH_NULL; STORE_NAME) + Python 3.16a1 3703 (Replace DELETE_GLOBAL with PUSH_NULL; STORE_GLOBAL) + Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) - Python 3.16 will start with 3700 + Python 3.17 will start with 3750 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -299,7 +315,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3704 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h new file mode 100644 index 000000000000000..c117cbd16283da9 --- /dev/null +++ b/Include/internal/pycore_mmap.h @@ -0,0 +1,47 @@ +#ifndef Py_INTERNAL_MMAP_H +#define Py_INTERNAL_MMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pystate.h" + +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +# include +# include +#endif + +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +static inline int +_PyAnnotateMemoryMap(void *addr, size_t size, const char *name) +{ +#ifndef Py_DEBUG + if (!_Py_GetConfig()->dev_mode) { + return 0; + } +#endif + // The name length cannot exceed 80 (including the '\0'). + assert(strlen(name) < 80); + int res = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name); + if (res < 0) { + return -1; + } + return 0; +} +#else +static inline int +_PyAnnotateMemoryMap(void *Py_UNUSED(addr), size_t Py_UNUSED(size), const char *Py_UNUSED(name)) +{ + return 0; +} +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MMAP_H diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 6eef6eaa5df8444..5bcfd17cec46271 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -19,6 +19,8 @@ extern int _PyModule_IsPossiblyShadowing(PyObject *); extern int _PyModule_IsExtension(PyObject *obj); +extern int _PyModule_InitModuleDictWatcher(PyInterpreterState *interp); + typedef int (*_Py_modexecfunc)(PyObject *); typedef struct { @@ -30,7 +32,7 @@ typedef struct { PyObject *md_name; bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif Py_ssize_t md_state_size; traverseproc md_state_traverse; @@ -51,11 +53,13 @@ static inline PyModuleDef *_PyModule_GetDefOrNull(PyObject *arg) { return NULL; } +// Get md_token. Used in _DuringGC functions; must have no side effects. static inline PyModuleDef *_PyModule_GetToken(PyObject *arg) { PyModuleObject *mod = _PyModule_CAST(arg); return (PyModuleDef *)mod->md_token; } +// Get md_state. Used in _DuringGC functions; must have no side effects. static inline void* _PyModule_GetState(PyObject* mod) { return _PyModule_CAST(mod)->md_state; } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 980d6d7764bd2cd..98880cd83780dca 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -144,7 +144,7 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; } # if SIZEOF_VOID_P > 4 - op->ob_refcnt = (PY_UINT32_T)new_refcnt; + op->ob_refcnt = (uint32_t)new_refcnt; # else op->ob_refcnt = new_refcnt; # endif @@ -252,25 +252,6 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) } } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - if (_Py_IsImmortal(op)) { - _Py_DECREF_IMMORTAL_STAT_INC(); - return; - } - _Py_DECREF_STAT_INC(); -#ifdef Py_REF_DEBUG - _Py_DEC_REFTOTAL(PyInterpreterState_Get()); -#endif - op->ob_refcnt--; -#ifdef Py_DEBUG - if (op->ob_refcnt <= 0) { - _Py_FatalRefcountError("Expected a positive remaining refcount"); - } -#endif -} - #else // TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build static inline void @@ -279,24 +260,12 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) Py_DECREF(op); } -static inline void -_Py_DECREF_NO_DEALLOC(PyObject *op) -{ - Py_DECREF(op); -} - static inline int _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) { return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; } -static inline int -_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) -{ - return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; -} - // Merge the local and shared reference count fields and add `extra` to the // refcount when merging. Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); @@ -496,6 +465,9 @@ static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruc #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct) #endif +#else // Py_GIL_DISABLED +# define Py_DECREF_MORTAL(op) Py_DECREF(op) +# define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF(op) #endif /* Inline functions trading binary compatibility for speed: @@ -858,13 +830,17 @@ _PyObject_IS_GC(PyObject *obj) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyObject_Hash() -static inline Py_hash_t -_PyObject_HashFast(PyObject *op) +// Fast inlined version of PyObject_Hash(). Dictionaries are very +// likely to include string keys (class and instance attributes, +// json, ...) so we include a fast path for strings. +// This function should not be used in a collection if str is not +// very likely, since it is slower than PyObject_Hash() on types +// other than str. See gh-137759. +static inline Py_ALWAYS_INLINE Py_hash_t +_PyObject_HashDictKey(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } @@ -902,14 +878,16 @@ PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); +// Exported for external JIT support +PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); -extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, unsigned int *); // Internal API to look for a name through the MRO. @@ -918,9 +896,13 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); -PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, PyObject *name, _PyStackRef *method); +// Like PyObject_GetAttr but returns a _PyStackRef. For types, this can +// return a deferred reference to reduce reference count contention. +PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyObject *obj, PyObject *name); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -929,7 +911,9 @@ PyAPI_FUNC(int) _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, // deferred reference counting. // // Returns 1 if the value was cached or 0 otherwise. -extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, +// +// Exported for external JIT support +PyAPI_FUNC(int) _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, unsigned int tp_version); @@ -1046,6 +1030,8 @@ static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) } #endif } +#else +# define _Py_INCREF_MORTAL(op) Py_INCREF(op) #endif /* Utility for the tp_traverse slot of mutable heap types that have no other diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index a7ba8f340737aab..f6b4bd90a234554 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -14,6 +14,12 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ #undef uint #define uint pymem_uint +/* NOTE: the following overviews were in the initial checkin, in 1998. In + * 2026, they're still helpful, but some details have changed. For example, + * we now use 32 size classes 16 bytes apart, and an arena is generally at + * least 1MB. Use sys._debugmallocstats() to see exact current details for + * the specific version of CPython used. + */ /* An object allocator for Python. @@ -208,7 +214,11 @@ typedef unsigned int pymem_uint; /* assuming >= 16 bits */ * mappings to reduce heap fragmentation. */ #ifdef USE_LARGE_ARENAS -#define ARENA_BITS 20 /* 1 MiB */ +# ifdef PYMALLOC_USE_HUGEPAGES +# define ARENA_BITS 21 /* 2 MiB */ +# else +# define ARENA_BITS 20 /* 1 MiB */ +# endif #else #define ARENA_BITS 18 /* 256 KiB */ #endif @@ -469,7 +479,7 @@ nfp free pools in usable_arenas. */ /* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the * `arenas` vector. */ #define INITIAL_ARENA_OBJECTS 16 @@ -512,7 +522,11 @@ struct _obmalloc_mgmt { memory address bit allocation for keys - 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size: + ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or + 21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled. All bit widths + below are derived from ARENA_BITS automatically. + + 64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default): 15 -> MAP_TOP_BITS 15 -> MAP_MID_BITS 14 -> MAP_BOT_BITS @@ -520,6 +534,14 @@ struct _obmalloc_mgmt { ---- 64 + 64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages): + 15 -> MAP_TOP_BITS + 15 -> MAP_MID_BITS + 13 -> MAP_BOT_BITS + 21 -> ideal aligned arena + ---- + 64 + 64-bit pointers, IGNORE_BITS=16, and 2^20 arena size: 16 -> IGNORE_BITS 10 -> MAP_TOP_BITS @@ -675,7 +697,11 @@ struct _obmalloc_state { /* Allocate memory directly from the O/S virtual memory system, - * where supported. Otherwise fallback on malloc */ + * where supported. Otherwise fallback on malloc. + * + * Large-page and huge-page backends may round the mapped size up + * internally, so pass the original requested size back to + * _PyObject_VirtualFree(). */ void *_PyObject_VirtualAlloc(size_t size); void _PyObject_VirtualFree(void *, size_t size); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 548627dc7982ec9..31d2e31573d8c53 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -66,6 +66,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case BINARY_OP_SUBSCR_TUPLE_INT: return 2; + case BINARY_OP_SUBSCR_USTR_INT: + return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; case BINARY_OP_SUBTRACT_INT: @@ -106,6 +108,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BUILTIN_O: return 2 + oparg; + case CALL_EX_NON_PY_GENERAL: + return 4; + case CALL_EX_PY: + return 4; case CALL_FUNCTION_EX: return 4; case CALL_INTRINSIC_1: @@ -151,7 +157,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 3; + return 4; case COMPARE_OP: return 2; case COMPARE_OP_FLOAT: @@ -172,16 +178,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1 + (oparg-1); case COPY_FREE_VARS: return 0; - case DELETE_ATTR: - return 1; case DELETE_DEREF: return 0; case DELETE_FAST: return 0; - case DELETE_GLOBAL: - return 0; - case DELETE_NAME: - return 0; case DELETE_SUBSCR: return 2; case DICT_MERGE: @@ -193,7 +193,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case END_FOR: return 1; case END_SEND: - return 2; + return 3; case ENTER_EXECUTOR: return 0; case EXIT_INIT_CHECK: @@ -214,6 +214,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case FOR_ITER_TUPLE: return 2; + case FOR_ITER_VIRTUAL: + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -222,9 +224,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case GET_ITER: return 1; - case GET_LEN: + case GET_ITER_SELF: + return 1; + case GET_ITER_VIRTUAL: return 1; - case GET_YIELD_FROM_ITER: + case GET_LEN: return 1; case IMPORT_FROM: return 1; @@ -241,7 +245,7 @@ int _PyOpcode_num_popped(int opcode, int oparg) { case INSTRUMENTED_END_FOR: return 3; case INSTRUMENTED_END_SEND: - return 2; + return 3; case INSTRUMENTED_FOR_ITER: return 2; case INSTRUMENTED_INSTRUCTION: @@ -420,14 +424,20 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 0; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: - return 2; + return 3; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -488,6 +498,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -532,7 +544,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case BINARY_OP_EXTEND: return 1; case BINARY_OP_INPLACE_ADD_UNICODE: - return 0; + return 1; case BINARY_OP_MULTIPLY_FLOAT: return 1; case BINARY_OP_MULTIPLY_INT: @@ -549,6 +561,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case BINARY_OP_SUBSCR_TUPLE_INT: return 1; + case BINARY_OP_SUBSCR_USTR_INT: + return 1; case BINARY_OP_SUBTRACT_FLOAT: return 1; case BINARY_OP_SUBTRACT_INT: @@ -589,6 +603,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_BUILTIN_O: return 1; + case CALL_EX_NON_PY_GENERAL: + return 1; + case CALL_EX_PY: + return 0; case CALL_FUNCTION_EX: return 1; case CALL_INTRINSIC_1: @@ -608,7 +626,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CALL_LEN: return 1; case CALL_LIST_APPEND: - return 0; + return 1; case CALL_METHOD_DESCRIPTOR_FAST: return 1; case CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: @@ -634,7 +652,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case CHECK_EXC_MATCH: return 2; case CLEANUP_THROW: - return 2; + return 3; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -655,16 +673,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2 + (oparg-1); case COPY_FREE_VARS: return 0; - case DELETE_ATTR: - return 0; case DELETE_DEREF: return 0; case DELETE_FAST: return 0; - case DELETE_GLOBAL: - return 0; - case DELETE_NAME: - return 0; case DELETE_SUBSCR: return 0; case DICT_MERGE: @@ -697,6 +709,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 3; case FOR_ITER_TUPLE: return 3; + case FOR_ITER_VIRTUAL: + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -705,10 +719,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case GET_ITER: return 2; + case GET_ITER_SELF: + return 2; + case GET_ITER_VIRTUAL: + return 2; case GET_LEN: return 2; - case GET_YIELD_FROM_ITER: - return 1; case IMPORT_FROM: return 2; case IMPORT_NAME: @@ -788,7 +804,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1; + return 0; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: @@ -903,14 +919,20 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case RESUME_CHECK: return 0; + case RESUME_CHECK_JIT: + return 0; case RETURN_GENERATOR: return 1; case RETURN_VALUE: return 1; case SEND: - return 2; + return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: - return 1; + return 2; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -971,6 +993,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1028,11 +1052,13 @@ enum InstructionFormat { #define HAS_ESCAPES_FLAG (512) #define HAS_EXIT_FLAG (1024) #define HAS_PURE_FLAG (2048) -#define HAS_ERROR_NO_POP_FLAG (4096) -#define HAS_NO_SAVE_IP_FLAG (8192) -#define HAS_PERIODIC_FLAG (16384) -#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) -#define HAS_NEEDS_GUARD_IP_FLAG (65536) +#define HAS_SYNC_SP_FLAG (4096) +#define HAS_ERROR_NO_POP_FLAG (8192) +#define HAS_NO_SAVE_IP_FLAG (16384) +#define HAS_PERIODIC_FLAG (32768) +#define HAS_UNPREDICTABLE_JUMP_FLAG (65536) +#define HAS_NEEDS_GUARD_IP_FLAG (131072) +#define HAS_RECORDS_VALUE_FLAG (262144) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1045,11 +1071,13 @@ enum InstructionFormat { #define OPCODE_HAS_ESCAPES(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ESCAPES_FLAG)) #define OPCODE_HAS_EXIT(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_EXIT_FLAG)) #define OPCODE_HAS_PURE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PURE_FLAG)) +#define OPCODE_HAS_SYNC_SP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_SYNC_SP_FLAG)) #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) #define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) #define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) +#define OPCODE_HAS_RECORDS_VALUE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_RECORDS_VALUE_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1069,24 +1097,25 @@ struct opcode_metadata { uint32_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; +PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { - [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1098,34 +1127,36 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EG_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CHECK_EXC_MATCH] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CLEANUP_THROW] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1133,20 +1164,17 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, [COMPARE_OP_STR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_DICT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CONTAINS_OP_SET] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, @@ -1155,22 +1183,24 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_SELF] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_VIRTUAL] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1178,7 +1208,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_NOT_TAKEN] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_POP_ITER] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, @@ -1186,9 +1216,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1197,21 +1227,21 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [JUMP_BACKWARD_NO_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, @@ -1230,14 +1260,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_MAPPING] = { true, INSTR_FMT_IX, 0 }, [MATCH_SEQUENCE] = { true, INSTR_FMT_IX, 0 }, @@ -1255,20 +1285,23 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, - [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, [STORE_FAST_LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG }, @@ -1277,26 +1310,27 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, - [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, + [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1311,7 +1345,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 10 +#define MAX_UOP_PER_EXPANSION 11 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1321,22 +1355,23 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BINARY_OP] = { .nuops = 1, .uops = { { _BINARY_OP, OPARG_SIMPLE, 4 } } }, - [BINARY_OP_ADD_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_EXTEND] = { .nuops = 2, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 } } }, + [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_ADD_UNICODE] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_EXTEND] = { .nuops = 4, .uops = { { _GUARD_BINARY_OP_EXTEND, 4, 1 }, { _BINARY_OP_EXTEND, 4, 1 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_MULTIPLY_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 3, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_MULTIPLY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 5, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 5, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 5, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_COMPACT_ASCII, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, @@ -1346,88 +1381,90 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, OPARG_SIMPLE, 0 } } }, [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, - [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 4, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 10, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BOUND_METHOD_GENERAL] = { .nuops = 7, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 2, .uops = { { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 } } }, - [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, + [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, - [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 9, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_PY_GENERAL] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_STR_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TYPE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, OPARG_SIMPLE, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, - [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, - [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_STR] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP] = { .nuops = 3, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [CONTAINS_OP_DICT] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [CONTAINS_OP_SET] = { .nuops = 4, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, - [DELETE_ATTR] = { .nuops = 1, .uops = { { _DELETE_ATTR, OPARG_SIMPLE, 0 } } }, [DELETE_DEREF] = { .nuops = 1, .uops = { { _DELETE_DEREF, OPARG_SIMPLE, 0 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { _DELETE_FAST, OPARG_SIMPLE, 0 } } }, - [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, OPARG_SIMPLE, 0 } } }, - [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, OPARG_SIMPLE, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [DICT_MERGE] = { .nuops = 1, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 } } }, - [DICT_UPDATE] = { .nuops = 1, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 } } }, + [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [END_FOR] = { .nuops = 1, .uops = { { _END_FOR, OPARG_SIMPLE, 0 } } }, [END_SEND] = { .nuops = 1, .uops = { { _END_SEND, OPARG_SIMPLE, 0 } } }, [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, OPARG_SIMPLE, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, - [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, - [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [FOR_ITER] = { .nuops = 2, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _FOR_ITER, OPARG_REPLACED, 0 } } }, + [FOR_ITER_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_TOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER] = { .nuops = 2, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER_SELF] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITERATOR, OPARG_SIMPLE, 1 }, { _PUSH_NULL, OPARG_SIMPLE, 1 } } }, + [GET_ITER_VIRTUAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _PUSH_TAGGED_ZERO, OPARG_SIMPLE, 1 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, OPARG_SIMPLE, 0 } } }, - [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [IS_OP] = { .nuops = 3, .uops = { { _IS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, - [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, + [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 4, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _GUARD_TYPE_VERSION, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_MODULE] = { .nuops = 3, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, [LOAD_CONST] = { .nuops = 1, .uops = { { _LOAD_CONST, OPARG_SIMPLE, 0 } } }, @@ -1445,13 +1482,13 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, - [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, + [LOAD_SUPER_ATTR_METHOD] = { .nuops = 3, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _GUARD_LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 }, { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 2, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, OPARG_SIMPLE, 0 } } }, - [MATCH_CLASS] = { .nuops = 1, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 } } }, + [MATCH_CLASS] = { .nuops = 4, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, OPARG_SIMPLE, 0 } } }, [MATCH_MAPPING] = { .nuops = 1, .uops = { { _MATCH_MAPPING, OPARG_SIMPLE, 0 } } }, [MATCH_SEQUENCE] = { .nuops = 1, .uops = { { _MATCH_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1466,38 +1503,40 @@ _PyOpcode_macro_expansion[256] = { [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, - [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 0 } } }, + [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, - [RETURN_VALUE] = { .nuops = 1, .uops = { { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, - [SEND_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, + [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, + [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, - [SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 } } }, + [SET_UPDATE] = { .nuops = 2, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, OPARG_SIMPLE, 3 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION_AND_LOCK, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, - [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [STORE_ATTR_WITH_HINT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _LOCK_OBJECT, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION_LOCKED, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_SLOT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, OPARG_SIMPLE, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, OPARG_SIMPLE, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _STORE_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST] = { .nuops = 2, .uops = { { _SWAP_FAST, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 3, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 4, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _SWAP_FAST, OPARG_BOTTOM, 0 }, { _POP_TOP, OPARG_BOTTOM, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, OPARG_SIMPLE, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 2, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 } } }, - [STORE_SUBSCR_LIST_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 4, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_STORE_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_LIST] = { .nuops = 2, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _TO_BOOL_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_LIST] = { .nuops = 3, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_STR] = { .nuops = 2, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 } } }, - [UNARY_INVERT] = { .nuops = 1, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 } } }, - [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 } } }, + [TO_BOOL_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 3 } } }, + [UNARY_INVERT] = { .nuops = 2, .uops = { { _UNARY_INVERT, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [UNARY_NEGATIVE] = { .nuops = 2, .uops = { { _UNARY_NEGATIVE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { _UNARY_NOT, OPARG_SIMPLE, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { _UNPACK_EX, OPARG_SIMPLE, 0 } } }, [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE, OPARG_SIMPLE, 0 } } }, @@ -1505,11 +1544,11 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TUPLE, OPARG_SIMPLE, 1 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 2, .uops = { { _GUARD_TOS_TUPLE, OPARG_SIMPLE, 0 }, { _UNPACK_SEQUENCE_TWO_TUPLE, OPARG_SIMPLE, 1 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, OPARG_SIMPLE, 0 } } }, - [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, + [YIELD_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _YIELD_VALUE, OPARG_SIMPLE, 0 } } }, }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[267]; +PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", @@ -1527,6 +1566,7 @@ const char *_PyOpcode_OpName[267] = { [BINARY_OP_SUBSCR_LIST_SLICE] = "BINARY_OP_SUBSCR_LIST_SLICE", [BINARY_OP_SUBSCR_STR_INT] = "BINARY_OP_SUBSCR_STR_INT", [BINARY_OP_SUBSCR_TUPLE_INT] = "BINARY_OP_SUBSCR_TUPLE_INT", + [BINARY_OP_SUBSCR_USTR_INT] = "BINARY_OP_SUBSCR_USTR_INT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SLICE] = "BINARY_SLICE", @@ -1547,6 +1587,8 @@ const char *_PyOpcode_OpName[267] = { [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_BUILTIN_O] = "CALL_BUILTIN_O", + [CALL_EX_NON_PY_GENERAL] = "CALL_EX_NON_PY_GENERAL", + [CALL_EX_PY] = "CALL_EX_PY", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", @@ -1580,11 +1622,8 @@ const char *_PyOpcode_OpName[267] = { [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", [COPY_FREE_VARS] = "COPY_FREE_VARS", - [DELETE_ATTR] = "DELETE_ATTR", [DELETE_DEREF] = "DELETE_DEREF", [DELETE_FAST] = "DELETE_FAST", - [DELETE_GLOBAL] = "DELETE_GLOBAL", - [DELETE_NAME] = "DELETE_NAME", [DELETE_SUBSCR] = "DELETE_SUBSCR", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", @@ -1601,12 +1640,14 @@ const char *_PyOpcode_OpName[267] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [FOR_ITER_VIRTUAL] = "FOR_ITER_VIRTUAL", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", [GET_ITER] = "GET_ITER", + [GET_ITER_SELF] = "GET_ITER_SELF", + [GET_ITER_VIRTUAL] = "GET_ITER_VIRTUAL", [GET_LEN] = "GET_LEN", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", @@ -1704,10 +1745,13 @@ const char *_PyOpcode_OpName[267] = { [RESERVED] = "RESERVED", [RESUME] = "RESUME", [RESUME_CHECK] = "RESUME_CHECK", + [RESUME_CHECK_JIT] = "RESUME_CHECK_JIT", [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_ASYNC_GEN] = "SEND_ASYNC_GEN", [SEND_GEN] = "SEND_GEN", + [SEND_VIRTUAL] = "SEND_VIRTUAL", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_FINALLY] = "SETUP_FINALLY", @@ -1738,6 +1782,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1751,12 +1796,14 @@ const char *_PyOpcode_OpName[267] = { }; #endif -extern const uint8_t _PyOpcode_Caches[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [RESUME] = 1, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, + [FOR_ITER] = 1, [UNPACK_SEQUENCE] = 1, [STORE_ATTR] = 4, [LOAD_GLOBAL] = 4, @@ -1769,16 +1816,21 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, - [FOR_ITER] = 1, + [GET_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, + [CALL_FUNCTION_EX] = 1, [BINARY_OP] = 5, }; #endif -extern const uint8_t _PyOpcode_Deopt[256]; +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [117] = 117, + [118] = 118, + [119] = 119, + [120] = 120, [121] = 121, [122] = 122, [123] = 123, @@ -1786,15 +1838,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [210] = 210, - [211] = 211, - [212] = 212, - [213] = 213, - [214] = 214, - [215] = 215, - [216] = 216, - [217] = 217, - [218] = 218, [219] = 219, [220] = 220, [221] = 221, @@ -1809,7 +1852,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1824,6 +1866,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_SUBSCR_LIST_SLICE] = BINARY_OP, [BINARY_OP_SUBSCR_STR_INT] = BINARY_OP, [BINARY_OP_SUBSCR_TUPLE_INT] = BINARY_OP, + [BINARY_OP_SUBSCR_USTR_INT] = BINARY_OP, [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, [BINARY_OP_SUBTRACT_INT] = BINARY_OP, [BINARY_SLICE] = BINARY_SLICE, @@ -1844,6 +1887,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_BUILTIN_O] = CALL, + [CALL_EX_NON_PY_GENERAL] = CALL_FUNCTION_EX, + [CALL_EX_PY] = CALL_FUNCTION_EX, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, @@ -1877,11 +1922,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [CONVERT_VALUE] = CONVERT_VALUE, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, - [DELETE_ATTR] = DELETE_ATTR, [DELETE_DEREF] = DELETE_DEREF, [DELETE_FAST] = DELETE_FAST, - [DELETE_GLOBAL] = DELETE_GLOBAL, - [DELETE_NAME] = DELETE_NAME, [DELETE_SUBSCR] = DELETE_SUBSCR, [DICT_MERGE] = DICT_MERGE, [DICT_UPDATE] = DICT_UPDATE, @@ -1898,12 +1940,14 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [FOR_ITER_TUPLE] = FOR_ITER, + [FOR_ITER_VIRTUAL] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, [GET_ITER] = GET_ITER, + [GET_ITER_SELF] = GET_ITER, + [GET_ITER_VIRTUAL] = GET_ITER, [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, @@ -1995,10 +2039,13 @@ const uint8_t _PyOpcode_Deopt[256] = { [RESERVED] = RESERVED, [RESUME] = RESUME, [RESUME_CHECK] = RESUME, + [RESUME_CHECK_JIT] = RESUME, [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, + [SEND_ASYNC_GEN] = SEND, [SEND_GEN] = SEND, + [SEND_VIRTUAL] = SEND, [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, [SET_ADD] = SET_ADD, [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, @@ -2025,6 +2072,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2040,6 +2088,10 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 117: \ + case 118: \ + case 119: \ + case 120: \ case 121: \ case 122: \ case 123: \ @@ -2047,15 +2099,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ case 219: \ case 220: \ case 221: \ @@ -2070,7 +2113,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 79a1a242556a52b..7067b48ec22cb3c 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -73,16 +73,30 @@ extern "C" { #define CONSTANT_BUILTIN_TUPLE 2 #define CONSTANT_BUILTIN_ALL 3 #define CONSTANT_BUILTIN_ANY 4 -#define NUM_COMMON_CONSTANTS 5 +#define CONSTANT_BUILTIN_LIST 5 +#define CONSTANT_BUILTIN_SET 6 +#define CONSTANT_NONE 7 +#define CONSTANT_EMPTY_STR 8 +#define CONSTANT_TRUE 9 +#define CONSTANT_FALSE 10 +#define CONSTANT_MINUS_ONE 11 +#define CONSTANT_BUILTIN_FROZENSET 12 +#define CONSTANT_EMPTY_TUPLE 13 +#define NUM_COMMON_CONSTANTS 14 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 #define RESUME_AFTER_YIELD 1 #define RESUME_AFTER_YIELD_FROM 2 #define RESUME_AFTER_AWAIT 3 +#define RESUME_AT_GEN_EXPR_START 4 -#define RESUME_OPARG_LOCATION_MASK 0x3 -#define RESUME_OPARG_DEPTH1_MASK 0x4 +#define RESUME_OPARG_LOCATION_MASK 0x7 +#define RESUME_OPARG_DEPTH1_MASK 0x8 + +#define GET_ITER_YIELD_FROM 1 +#define GET_ITER_YIELD_FROM_NO_CHECK 2 +#define GET_ITER_YIELD_FROM_CORO_CHECK 3 #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 653285a2c6b79b0..3d60638649dcb5a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,34 +9,181 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_jit_publish.h" #include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef +#include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define OPTIMIZER_EFFECTIVENESS 2 +#define MAX_TARGET_LENGTH (FITNESS_INITIAL / OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + + +typedef struct _PyJitUopBuffer { + _PyUOpInstruction *start; + _PyUOpInstruction *next; + _PyUOpInstruction *end; +} _PyJitUopBuffer; + +typedef struct _JitOptRefBuffer { + JitOptRef *used; + JitOptRef *end; +} _JitOptRefBuffer; + +typedef struct _JitOptContext { + char done; + char out_of_space; + bool contradiction; + // Has the builtins dict been watched? + bool builtins_watched; + // The current "executing" frame. + _Py_UOpsAbstractFrame *frame; + _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; + int curr_frame_depth; + + // Arena for the symbolic types. + ty_arena t_arena; + + /* To do -- We could make this more space efficient + * by using a single array and growing the stack and + * locals toward each other. */ + _JitOptRefBuffer locals; + _JitOptRefBuffer stack; + JitOptRef locals_array[ABSTRACT_INTERP_LOCALS_SIZE]; + JitOptRef stack_array[ABSTRACT_INTERP_STACK_SIZE]; + _PyJitUopBuffer out_buffer; + _PyBloomFilter *dependencies; +} JitOptContext; + + +static inline void +uop_buffer_init(_PyJitUopBuffer *trace, _PyUOpInstruction *start, uint32_t size) +{ + trace->next = trace->start = start; + trace->end = start + size; +} + +static inline _PyUOpInstruction * +uop_buffer_last(_PyJitUopBuffer *trace) +{ + assert(trace->next > trace->start); + return trace->next-1; +} + +static inline int +uop_buffer_length(_PyJitUopBuffer *trace) +{ + return (int)(trace->next - trace->start); +} + +static inline int +uop_buffer_remaining_space(_PyJitUopBuffer *trace) +{ + return (int)(trace->end - trace->next); +} + +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + struct _PyExecutorObject *executor; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +#define MAX_RECORDED_VALUES 3 +typedef struct _PyJitTracerPreviousState { + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + PyObject *recorded_values[MAX_RECORDED_VALUES]; // Strong, may be NULL + int recorded_count; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerTranslatorState { + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) +} _PyJitTracerTranslatorState; + +typedef struct _PyJitTracerState { + bool is_tracing; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; + _PyJitTracerTranslatorState translator_state; + JitOptContext opt_context; + _PyJitUopBuffer code_buffer; + _PyJitUopBuffer out_buffer; + _PyUOpInstruction uop_array[2 * UOP_MAX_TRACE_LENGTH]; +} _PyJitTracerState; typedef struct _PyExecutorLinkListNode { struct _PyExecutorObject *next; struct _PyExecutorObject *previous; } _PyExecutorLinkListNode; - typedef struct { uint8_t opcode; uint8_t oparg; - uint8_t valid:1; - uint8_t linked:1; - uint8_t chain_depth:6; // Must be big enough for MAX_CHAIN_DEPTH - 1. - bool warm; - int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). - _PyBloomFilter bloom; - _PyExecutorLinkListNode links; + uint8_t valid; + uint8_t chain_depth; // Must be big enough for MAX_CHAIN_DEPTH - 1. + bool cold; + uint8_t pending_deletion; + int32_t index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). + int32_t bloom_array_idx; // Index in interp->executor_blooms/executor_ptrs. + _PyExecutorLinkListNode links; // Used by deletion list. PyCodeObject *code; // Weak (NULL if no corresponding ENTER_EXECUTOR). } _PyVMData; typedef struct _PyExitData { uint32_t target; - uint16_t index:14; + uint16_t index:12; + uint16_t stack_cache:2; uint16_t is_dynamic:1; uint16_t is_control_flow:1; _Py_BackoffCounter temperature; @@ -51,23 +198,96 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + _PyJitCodeRegistration *jit_registration; _PyExitData exits[1]; } _PyExecutorObject; -/* If pending deletion list gets large enough, then scan, - * and free any executors that aren't executing - * i.e. any that aren't a thread's current_executor. */ -#define EXECUTOR_DELETE_LIST_MAX 100 - // Export for '_opcode' shared extension (JIT compiler). PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); -void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); +int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); -void _Py_BloomFilter_Init(_PyBloomFilter *); -void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); +/* We use a bloomfilter with k = 6, m = 256 + * The choice of k and the following constants + * could do with a more rigorous analysis, + * but here is a simple analysis: + * + * We want to keep the false positive rate low. + * For n = 5 (a trace depends on 5 objects), + * we expect 30 bits set, giving a false positive + * rate of (30/256)**6 == 2.5e-6 which is plenty + * good enough. + * + * However with n = 10 we expect 60 bits set (worst case), + * giving a false positive of (60/256)**6 == 0.0001 + * + * We choose k = 6, rather than a higher number as + * it means the false positive rate grows slower for high n. + * + * n = 5, k = 6 => fp = 2.6e-6 + * n = 5, k = 8 => fp = 3.5e-7 + * n = 10, k = 6 => fp = 1.6e-4 + * n = 10, k = 8 => fp = 0.9e-4 + * n = 15, k = 6 => fp = 0.18% + * n = 15, k = 8 => fp = 0.23% + * n = 20, k = 6 => fp = 1.1% + * n = 20, k = 8 => fp = 2.3% + * + * The above analysis assumes perfect hash functions, + * but those don't exist, so the real false positive + * rates may be worse. + */ + +#define _Py_BLOOM_FILTER_K 6 +#define _Py_BLOOM_FILTER_SEED 20221211 + +static inline uint64_t +address_to_hash(void *ptr) { + assert(ptr != NULL); + uint64_t uhash = _Py_BLOOM_FILTER_SEED; + uintptr_t addr = (uintptr_t)ptr; + for (int i = 0; i < SIZEOF_VOID_P; i++) { + uhash ^= addr & 255; + uhash *= (uint64_t)PyHASH_MULTIPLIER; + addr >>= 8; + } + return uhash; +} + +static inline void +_Py_BloomFilter_Init(_PyBloomFilter *bloom) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + bloom->bits[i] = 0; + } +} + +static inline void +_Py_BloomFilter_Add(_PyBloomFilter *bloom, void *ptr) +{ + uint64_t hash = address_to_hash(ptr); + assert(_Py_BLOOM_FILTER_K <= 8); + for (int i = 0; i < _Py_BLOOM_FILTER_K; i++) { + uint8_t bits = hash & 255; + bloom->bits[bits >> _Py_BLOOM_FILTER_WORD_SHIFT] |= + (_Py_bloom_filter_word_t)1 << (bits & (_Py_BLOOM_FILTER_BITS_PER_WORD - 1)); + hash >>= 8; + } +} + +static inline bool +bloom_filter_may_contain(const _PyBloomFilter *bloom, const _PyBloomFilter *hashes) +{ + for (int i = 0; i < _Py_BLOOM_FILTER_WORDS; i++) { + if ((bloom->bits[i] & hashes->bits[i]) != hashes->bits[i]) { + return false; + } + } + return true; +} + #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 @@ -79,7 +299,6 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #else # define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) # define _Py_Executors_InvalidateAll(A, B) ((void)0) -# define _Py_Executors_InvalidateCold(A) ((void)0) #endif @@ -89,11 +308,11 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define JIT_CLEANUP_THRESHOLD 1000 int _Py_uop_analyze_and_optimize( - PyFunctionObject *func, - _PyUOpInstruction *trace, int trace_len, int curr_stackentries, - _PyBloomFilter *dependencies); + _PyThreadStateImpl *tstate, + _PyUOpInstruction *input, int trace_len, int curr_stackentries, + _PyUOpInstruction *output, _PyBloomFilter *dependencies); -extern PyTypeObject _PyUOpExecutor_Type; +PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type; #define UOP_FORMAT_TARGET 0 @@ -117,90 +336,16 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) return inst->error_target; } -// Holds locals, stack, locals, stack ... co_consts (in that order) -#define MAX_ABSTRACT_INTERP_SIZE 4096 - -#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) - -// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (16) - -// The maximum number of side exits that we can take before requiring forward -// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this -// is the "maximum amount of polymorphism" that an isolated trace tree can -// handle before rejoining the rest of the program. -#define MAX_CHAIN_DEPTH 4 - -/* Symbols */ -/* See explanation in optimizer_symbols.c */ - - -typedef enum _JitSymType { - JIT_SYM_UNKNOWN_TAG = 1, - JIT_SYM_NULL_TAG = 2, - JIT_SYM_NON_NULL_TAG = 3, - JIT_SYM_BOTTOM_TAG = 4, - JIT_SYM_TYPE_VERSION_TAG = 5, - JIT_SYM_KNOWN_CLASS_TAG = 6, - JIT_SYM_KNOWN_VALUE_TAG = 7, - JIT_SYM_TUPLE_TAG = 8, - JIT_SYM_TRUTHINESS_TAG = 9, - JIT_SYM_COMPACT_INT = 10, -} JitSymType; - -typedef struct _jit_opt_known_class { - uint8_t tag; - uint32_t version; - PyTypeObject *type; -} JitOptKnownClass; - -typedef struct _jit_opt_known_version { - uint8_t tag; - uint32_t version; -} JitOptKnownVersion; - -typedef struct _jit_opt_known_value { - uint8_t tag; - PyObject *value; -} JitOptKnownValue; - -#define MAX_SYMBOLIC_TUPLE_SIZE 7 - -typedef struct _jit_opt_tuple { - uint8_t tag; - uint8_t length; - uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; -} JitOptTuple; - -typedef struct { - uint8_t tag; - bool invert; - uint16_t value; -} JitOptTruthiness; - -typedef struct { - uint8_t tag; -} JitOptCompactInt; - -typedef union _jit_opt_symbol { - uint8_t tag; - JitOptKnownClass cls; - JitOptKnownValue value; - JitOptKnownVersion version; - JitOptTuple tuple; - JitOptTruthiness truthiness; - JitOptCompactInt compact; -} JitOptSymbol; - - -// This mimics the _PyStackRef API -typedef union { - uintptr_t bits; -} JitOptRef; #define REF_IS_BORROWED 1 +#define REF_IS_UNIQUE 2 +#define REF_IS_INVALID 3 +#define REF_TAG_BITS 3 -#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_IS_BORROWED))) +#define REF_GET_TAG(x) ((uintptr_t)(x) & (REF_TAG_BITS)) +#define REF_CLEAR_TAG(x) ((uintptr_t)(x) & (~REF_TAG_BITS)) + +#define JIT_BITS_TO_PTR_MASKED(REF) ((JitOptSymbol *)(((REF).bits) & (~REF_TAG_BITS))) static inline JitOptSymbol * PyJitRef_Unwrap(JitOptRef ref) @@ -217,16 +362,58 @@ PyJitRef_Wrap(JitOptSymbol *sym) return (JitOptRef){.bits=(uintptr_t)sym}; } +static inline JitOptRef +PyJitRef_WrapInvalid(void *ptr) +{ + return (JitOptRef){.bits = REF_CLEAR_TAG((uintptr_t)ptr) | REF_IS_INVALID}; +} + +static inline bool +PyJitRef_IsInvalid(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_INVALID; +} + +static inline JitOptRef +PyJitRef_MakeUnique(JitOptRef ref) +{ + return (JitOptRef){ REF_CLEAR_TAG(ref.bits) | REF_IS_UNIQUE }; +} + +static inline bool +PyJitRef_IsUnique(JitOptRef ref) +{ + return REF_GET_TAG(ref.bits) == REF_IS_UNIQUE; +} + +static inline JitOptRef +PyJitRef_StripBorrowInfo(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + return ref; + } + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) }; +} + static inline JitOptRef PyJitRef_StripReferenceInfo(JitOptRef ref) { return PyJitRef_Wrap(PyJitRef_Unwrap(ref)); } +static inline JitOptRef +PyJitRef_RemoveUnique(JitOptRef ref) +{ + if (PyJitRef_IsUnique(ref)) { + ref = PyJitRef_StripReferenceInfo(ref); + } + return ref; +} + static inline JitOptRef PyJitRef_Borrow(JitOptRef ref) { - return (JitOptRef){ .bits = ref.bits | REF_IS_BORROWED }; + return (JitOptRef){ .bits = REF_CLEAR_TAG(ref.bits) | REF_IS_BORROWED }; } static const JitOptRef PyJitRef_NULL = {.bits = REF_IS_BORROWED}; @@ -240,51 +427,9 @@ PyJitRef_IsNull(JitOptRef ref) static inline int PyJitRef_IsBorrowed(JitOptRef ref) { - return (ref.bits & REF_IS_BORROWED) == REF_IS_BORROWED; + return REF_GET_TAG(ref.bits) == REF_IS_BORROWED; } -struct _Py_UOpsAbstractFrame { - bool globals_watched; - // The version number of the globals dicts, once checked. 0 if unchecked. - uint32_t globals_checked_version; - // Max stacklen - int stack_len; - int locals_len; - PyFunctionObject *func; - PyCodeObject *code; - - JitOptRef *stack_pointer; - JitOptRef *stack; - JitOptRef *locals; -}; - -typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; - -typedef struct ty_arena { - int ty_curr_number; - int ty_max_number; - JitOptSymbol arena[TY_ARENA_SIZE]; -} ty_arena; - -typedef struct _JitOptContext { - char done; - char out_of_space; - bool contradiction; - // Has the builtins dict been watched? - bool builtins_watched; - // The current "executing" frame. - _Py_UOpsAbstractFrame *frame; - _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH]; - int curr_frame_depth; - - // Arena for the symbolic types. - ty_arena t_arena; - - JitOptRef *n_consumed; - JitOptRef *limit; - JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE]; -} JitOptContext; - extern bool _Py_uop_sym_is_null(JitOptRef sym); extern bool _Py_uop_sym_is_not_null(JitOptRef sym); extern bool _Py_uop_sym_is_const(JitOptContext *ctx, JitOptRef sym); @@ -297,11 +442,13 @@ extern JitOptRef _Py_uop_sym_new_type( extern JitOptRef _Py_uop_sym_new_const(JitOptContext *ctx, PyObject *const_val); extern JitOptRef _Py_uop_sym_new_const_steal(JitOptContext *ctx, PyObject *const_val); bool _Py_uop_sym_is_safe_const(JitOptContext *ctx, JitOptRef sym); +bool _Py_uop_sym_is_not_container(JitOptRef sym); _PyStackRef _Py_uop_sym_get_const_as_stackref(JitOptContext *ctx, JitOptRef sym); extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); extern bool _Py_uop_sym_has_type(JitOptRef sym); extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern unsigned int _Py_uop_sym_get_type_version(JitOptRef sym); extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); @@ -317,17 +464,32 @@ extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value, extern bool _Py_uop_sym_is_compact_int(JitOptRef sym); extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx); extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx, JitOptRef sym); - -extern void _Py_uop_abstractcontext_init(JitOptContext *ctx); +extern JitOptRef _Py_uop_sym_new_predicate(JitOptContext *ctx, JitOptRef lhs_ref, JitOptRef rhs_ref, JitOptPredicateKind kind); +extern void _Py_uop_sym_apply_predicate_narrowing(JitOptContext *ctx, JitOptRef sym, bool branch_is_true); +extern void _Py_uop_sym_set_recorded_value(JitOptContext *ctx, JitOptRef sym, PyObject *value); +extern void _Py_uop_sym_set_recorded_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *type); +extern void _Py_uop_sym_set_recorded_gen_func(JitOptContext *ctx, JitOptRef ref, PyFunctionObject *value); +extern PyCodeObject *_Py_uop_sym_get_probable_func_code(JitOptRef sym); +extern PyObject *_Py_uop_sym_get_probable_value(JitOptRef sym); +extern PyTypeObject *_Py_uop_sym_get_probable_type(JitOptRef sym); +extern JitOptRef *_Py_uop_sym_set_stack_depth(JitOptContext *ctx, int stack_depth, JitOptRef *current_sp); + +extern void _Py_uop_abstractcontext_init(JitOptContext *ctx, _PyBloomFilter *dependencies); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( JitOptContext *ctx, PyCodeObject *co, - int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); + +extern _Py_UOpsAbstractFrame *_Py_uop_frame_new_from_symbol( + JitOptContext *ctx, + JitOptRef callable, + JitOptRef *args, + int arg_len); + +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); @@ -344,35 +506,51 @@ extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); -static inline int is_terminator(const _PyUOpInstruction *uop) -{ - int opcode = uop->opcode; - return ( - opcode == _EXIT_TRACE || - opcode == _DEOPT || - opcode == _JUMP_TO_TOP || - opcode == _DYNAMIC_EXIT - ); -} - extern void _PyExecutor_Free(_PyExecutorObject *self); PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); #ifdef _Py_TIER2 -extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); +PyAPI_FUNC(void) _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); +PyAPI_FUNC(int) _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); -int +PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, - _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, - int oparg); + _Py_CODEUNIT *close_loop_instr, _PyStackRef *stack_pointer, int chain_depth, _PyExitData *exit, + int oparg, _PyExecutorObject *current_executor); + +PyAPI_FUNC(void) _PyJit_FinalizeTracing(PyThreadState *tstate, int err); +PyAPI_FUNC(bool) _PyJit_EnterExecutorShouldStopTracing(int og_opcode); -void _PyJit_FinalizeTracing(PyThreadState *tstate); +void _PyPrintExecutor(_PyExecutorObject *executor, const _PyUOpInstruction *marker); +void _PyJit_TracerFree(_PyThreadStateImpl *_tstate); -void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); +#ifdef _Py_TIER2 +typedef void (*_Py_RecordFuncPtr)(_PyInterpreterFrame *frame, _PyStackRef *stackpointer, int oparg, PyObject **recorded_value); +PyAPI_DATA(const _Py_RecordFuncPtr) _PyOpcode_RecordFunctions[]; + +typedef struct { + uint8_t count; + uint8_t indices[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordEntry; + +typedef struct { + uint8_t count; + uint8_t transform_mask; + uint8_t slots[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordSlotMap; + +PyAPI_DATA(const _PyOpcodeRecordEntry) _PyOpcode_RecordEntries[256]; +PyAPI_DATA(const _PyOpcodeRecordSlotMap) _PyOpcode_RecordSlotMaps[256]; + +/* Convert a family-recorded value to the form a recorder uop expects. + * If no transform is needed, return the input value unchanged. + * Takes ownership of `value` and returns a new strong reference or NULL. + */ +PyAPI_FUNC(PyObject *) _PyOpcode_RecordTransformValue(int uop, PyObject *value); +#endif #ifdef __cplusplus } diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h new file mode 100644 index 000000000000000..a722652cc8163aa --- /dev/null +++ b/Include/internal/pycore_optimizer_types.h @@ -0,0 +1,169 @@ +#ifndef Py_INTERNAL_OPTIMIZER_TYPES_H +#define Py_INTERNAL_OPTIMIZER_TYPES_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include "pycore_uop.h" // UOP_MAX_TRACE_LENGTH + +#define ABSTRACT_INTERP_STACK_SIZE 256 +#define ABSTRACT_INTERP_LOCALS_SIZE 512 + + +#define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) + +// Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) +#define MAX_ABSTRACT_FRAME_DEPTH (16) + +// The maximum number of side exits that we can take before requiring forward +// progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this +// is the "maximum amount of polymorphism" that an isolated trace tree can +// handle before rejoining the rest of the program. +#define MAX_CHAIN_DEPTH 4 + +/* Symbols */ +/* See explanation in optimizer_symbols.c */ + + +typedef enum _JitSymType { + JIT_SYM_UNKNOWN_TAG = 1, + JIT_SYM_NULL_TAG = 2, + JIT_SYM_NON_NULL_TAG = 3, + JIT_SYM_BOTTOM_TAG = 4, + JIT_SYM_TYPE_VERSION_TAG = 5, + JIT_SYM_KNOWN_CLASS_TAG = 7, + JIT_SYM_KNOWN_VALUE_TAG = 8, + JIT_SYM_TUPLE_TAG = 9, + JIT_SYM_TRUTHINESS_TAG = 10, + JIT_SYM_COMPACT_INT = 11, + JIT_SYM_PREDICATE_TAG = 12, + JIT_SYM_RECORDED_VALUE_TAG = 13, + JIT_SYM_RECORDED_TYPE_TAG = 14, + JIT_SYM_RECORDED_GEN_FUNC_TAG = 15, +} JitSymType; + +typedef struct _jit_opt_known_class { + uint8_t tag; + uint32_t version; + PyTypeObject *type; +} JitOptKnownClass; + +typedef struct _jit_opt_known_version { + uint8_t tag; + uint32_t version; +} JitOptKnownVersion; + +typedef struct _jit_opt_known_func_version { + uint8_t tag; + uint32_t func_version; +} JitOptKnownFuncVersion; + +typedef struct _jit_opt_known_value { + uint8_t tag; + PyObject *value; +} JitOptKnownValue; + +#define MAX_SYMBOLIC_TUPLE_SIZE 7 + +typedef struct _jit_opt_tuple { + uint8_t tag; + uint8_t length; + uint16_t items[MAX_SYMBOLIC_TUPLE_SIZE]; +} JitOptTuple; + +typedef struct { + uint8_t tag; + bool invert; + uint16_t value; +} JitOptTruthiness; + +typedef enum { + JIT_PRED_IS, + JIT_PRED_IS_NOT, + JIT_PRED_EQ, + JIT_PRED_NE, +} JitOptPredicateKind; + +typedef struct { + uint8_t tag; + uint8_t kind; + uint16_t lhs; + uint16_t rhs; +} JitOptPredicate; + +typedef struct _jit_opt_recorded_value { + uint8_t tag; + bool known_type; + PyObject *value; +} JitOptRecordedValue; + +typedef struct _jit_opt_recorded_type { + uint8_t tag; + PyTypeObject *type; +} JitOptRecordedType; + +/* Represents a generator, but we record the + * function as the generator is emphemeral */ +typedef struct _jit_opt_recorded_gen_func { + uint8_t tag; + PyFunctionObject *func; +} JitOptRecordedGenFunc; + +typedef struct { + uint8_t tag; +} JitOptCompactInt; + +typedef union _jit_opt_symbol { + uint8_t tag; + JitOptKnownClass cls; + JitOptKnownValue value; + JitOptKnownVersion version; + JitOptKnownFuncVersion func_version; + JitOptTuple tuple; + JitOptTruthiness truthiness; + JitOptCompactInt compact; + JitOptPredicate predicate; + JitOptRecordedValue recorded_value; + JitOptRecordedType recorded_type; + JitOptRecordedGenFunc recorded_gen_func; +} JitOptSymbol; + +// This mimics the _PyStackRef API +typedef union { + uintptr_t bits; +} JitOptRef; + +typedef struct _Py_UOpsAbstractFrame { + bool globals_watched; + // The version number of the globals dicts, once checked. 0 if unchecked. + uint32_t globals_checked_version; + // Max stacklen + int stack_len; + int locals_len; + bool caller; // We have made a call from this frame during the trace + bool is_c_recursion_checked; + JitOptRef callable; + PyFunctionObject *func; + PyCodeObject *code; + + JitOptRef *stack_pointer; + JitOptRef *stack; + JitOptRef *locals; +} _Py_UOpsAbstractFrame; + +typedef struct ty_arena { + int ty_curr_number; + int ty_max_number; + JitOptSymbol arena[TY_ARENA_SIZE]; +} ty_arena; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_TYPES_H */ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2c46f59ab7da9fc..b89d02035db71f9 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -14,10 +14,8 @@ extern "C" { #include "pycore_pyarena.h" // PyArena _Py_DECLARE_STR(empty, "") -#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED) #define _parser_runtime_state_INIT \ { \ - .mutex = {0}, \ .dummy_name = { \ .kind = Name_kind, \ .v.Name.id = &_Py_STR(empty), \ @@ -28,20 +26,6 @@ _Py_DECLARE_STR(empty, "") .end_col_offset = 0, \ }, \ } -#else -#define _parser_runtime_state_INIT \ - { \ - .dummy_name = { \ - .kind = Name_kind, \ - .v.Name.id = &_Py_STR(empty), \ - .v.Name.ctx = Load, \ - .lineno = 1, \ - .col_offset = 0, \ - .end_lineno = 1, \ - .end_col_offset = 0, \ - }, \ - } -#endif extern struct _mod* _PyParser_ASTFromString( const char *str, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index c31c33657002ecc..d8ec306a0dae3fc 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -31,6 +31,8 @@ extern "C" { _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) \ + _Py_atomic_load_ptr_consume(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ @@ -39,12 +41,18 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) \ + _Py_atomic_load_int8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -53,14 +61,26 @@ extern "C" { _Py_atomic_store_ptr_release(&value, new_value) #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \ _Py_atomic_store_uintptr_release(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) \ + _Py_atomic_store_int8_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) \ + _Py_atomic_store_int8_release(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_AND_UINT64(value, new_value) \ + (void)_Py_atomic_and_uint64(&value, new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) \ + (void)_Py_atomic_or_uint64(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -77,10 +97,16 @@ extern "C" { _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_LOAD_INT(value) \ + _Py_atomic_load_int(&value) +#define FT_ATOMIC_STORE_INT(value, new_value) \ + _Py_atomic_store_int(&value, new_value) #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT(value) \ + _Py_atomic_load_uint(&value) #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ @@ -112,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -121,21 +148,31 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT8_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_AND_UINT64(value, new_value) (void)(value &= new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) (void)(value |= new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -144,8 +181,11 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT(value) value +#define FT_ATOMIC_STORE_INT(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT(value) value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value @@ -162,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f80808fcc8c4d7d..c1f9d71e40077c1 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -11,8 +11,8 @@ extern "C" { /* Error handling definitions */ -extern _PyErr_StackItem* _PyErr_GetTopmostException(PyThreadState *tstate); -extern PyObject* _PyErr_GetHandledException(PyThreadState *); +PyAPI_FUNC(_PyErr_StackItem*) _PyErr_GetTopmostException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetHandledException(PyThreadState *); extern void _PyErr_SetHandledException(PyThreadState *, PyObject *); extern void _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, PyObject **); @@ -29,7 +29,8 @@ PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause( ... ); -extern int _PyException_AddNote( +// Export for 'pyexpat' shared extension. +PyAPI_FUNC(int) _PyException_AddNote( PyObject *exc, PyObject *note); @@ -108,7 +109,7 @@ extern void _PyErr_Restore( PyObject *value, PyObject *traceback); -extern void _PyErr_SetObject( +PyAPI_FUNC(void) _PyErr_SetObject( PyThreadState *tstate, PyObject *type, PyObject *value); @@ -122,9 +123,11 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset, - PyObject *module); +// Export for test_peg_generator +PyAPI_FUNC(int) _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, + int lineno, int col_offset, + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); @@ -169,7 +172,8 @@ extern PyObject* _PyErr_FormatFromCauseTstate( const char *format, ...); -extern PyObject* _PyExc_CreateExceptionGroup( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyExc_CreateExceptionGroup( const char *msg, PyObject *excs); @@ -180,7 +184,8 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); -extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 84cb72fa6fd1b26..3056dc44cc0f1b1 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -27,14 +27,14 @@ _Py_HashPointerRaw(const void *ptr) * pppppppp ssssssss ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt + * eeeeeeee eeeeeeee eeeeeeee pyexpat XML hash salt * * memory layout on 32 bit systems * cccccccc cccccccc cccccccc uc * ppppssss ........ ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt + * eeeeeeee eeeeeeee eeee.... pyexpat XML hash salt * * (*) The siphash member may not be available on 32 bit platforms without * an unsigned int64 data type. @@ -58,7 +58,9 @@ typedef union { Py_hash_t suffix; } djbx33a; struct { - unsigned char padding[16]; + /* 16 bytes for XML_SetHashSalt16Bytes */ + uint8_t hashsalt16[16]; + /* 4/8 bytes for legacy XML_SetHashSalt */ Py_hash_t hashsalt; } expat; } _Py_HashSecret_t; diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8faf7a4d403f846..12e1e78526db35a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -18,9 +18,6 @@ extern int _Py_SetFileSystemEncoding( const char *errors); extern void _Py_ClearFileSystemEncoding(void); extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate); -#ifdef MS_WINDOWS -extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void); -#endif extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index 4fcac3aab8bf51c..532c5ceafb56395 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -146,17 +146,17 @@ extern void _Py_set_387controlword(unsigned short); unsigned int old_fpcr, new_fpcr #define _Py_SET_53BIT_PRECISION_START \ do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + __asm__ ("fmove.l %%fpcr,%0" : "=dm" (old_fpcr)); \ /* Set double precision / round to nearest. */ \ new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (new_fpcr)); \ } \ } while (0) #define _Py_SET_53BIT_PRECISION_END \ do { \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (old_fpcr)); \ } \ } while (0) #endif @@ -182,8 +182,7 @@ extern void _Py_set_387controlword(unsigned short); // (extended precision), and we don't know how to change // the rounding precision. #if !defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) && \ - !defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) + !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) # define _PY_SHORT_FLOAT_REPR 0 #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index ed943b510567a4a..05484e847f1195a 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -70,6 +70,27 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) #endif } +// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a +// pointer. +static inline int _PyMem_IsULongFreed(unsigned long value) +{ +#if SIZEOF_LONG == 8 + return (value == 0 + || value == (unsigned long)0xCDCDCDCDCDCDCDCD + || value == (unsigned long)0xDDDDDDDDDDDDDDDD + || value == (unsigned long)0xFDFDFDFDFDFDFDFD + || value == (unsigned long)0xFFFFFFFFFFFFFFFF); +#elif SIZEOF_LONG == 4 + return (value == 0 + || value == (unsigned long)0xCDCDCDCD + || value == (unsigned long)0xDDDDDDDD + || value == (unsigned long)0xFDFDFDFD + || value == (unsigned long)0xFFFFFFFF); +#else +# error "unknown long size" +#endif +} + extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index c593edc86d9952a..2a0e0817dcc7f86 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -30,6 +30,12 @@ extern void* _PyMem_MiCalloc(void *, size_t, size_t); extern void _PyMem_MiFree(void *, void *); extern void* _PyMem_MiRealloc(void *, void *, size_t); # define PYMEM_ALLOC {NULL, _PyMem_MiMalloc, _PyMem_MiCalloc, _PyMem_MiRealloc, _PyMem_MiFree} +extern void* _PyMem_MiRawMalloc(void *, size_t); +extern void* _PyMem_MiRawCalloc(void *, size_t, size_t); +extern void _PyMem_MiRawFree(void *, void *); +extern void* _PyMem_MiRawRealloc(void *, void *, size_t); +# undef PYRAW_ALLOC +# define PYRAW_ALLOC {NULL, _PyMem_MiRawMalloc, _PyMem_MiRawCalloc, _PyMem_MiRawRealloc, _PyMem_MiRawFree} #elif defined(WITH_PYMALLOC) extern void* _PyObject_Malloc(void *, size_t); extern void* _PyObject_Calloc(void *, size_t, size_t); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index cab458f84028e23..ca6819d2cd44730 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -266,6 +266,8 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ PyMutex_Unlock(&(runtime)->interpreters.mutex) +#define ASSERT_HEAD_IS_LOCKED(runtime) \ + assert(PyMutex_IsLocked(&(runtime)->interpreters.mutex)) #define _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) \ for (PyThreadState *t = interp->threads.head; t; t = t->next) @@ -314,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { -#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__) - return (uintptr_t)__builtin_frame_address(0); -#elif defined(_MSC_VER) - return (uintptr_t)_AddressOfReturnAddress(); + uintptr_t result; +#if defined(_M_ARM64) + result = __getReg(31); +#elif defined(_M_X64) || defined(_M_IX86) + result = (uintptr_t)_AddressOfReturnAddress(); +#elif defined(__aarch64__) + __asm__ ("mov %0, sp" : "=r" (result)); +#elif defined(__x86_64__) + __asm__("{movq %%rsp, %0" : "=r" (result)); #else char here; - /* Avoid compiler warning about returning stack address */ - return return_pointer_as_int(&here); + result = (uintptr_t)&here; #endif + return result; } static inline intptr_t @@ -331,9 +338,27 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } +/* PEP 788 structures. */ + +struct PyInterpreterGuard { + PyInterpreterState *interp; +}; + +struct PyInterpreterView { + int64_t id; +}; + +// Exports for '_testinternalcapi' shared extension +PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 04a557e1204064e..66dd7cd843b04fc 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -44,9 +44,10 @@ extern PyObject * _Py_CompileStringObjectWithModule( * no two calls to check recursion depth are more than this far * apart. In practice, that means it must be larger than the C * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define _PyOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) +#if (defined(Py_DEBUG) \ + || defined(_Py_ADDRESS_SANITIZER) \ + || defined(_Py_THREAD_SANITIZER)) \ + || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 1f9b3fcf777493f..eeca6fc472be37b 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -83,8 +83,9 @@ struct _qsbr_shared { // Minimum observed read sequence of all QSBR thread states uint64_t rd_seq; - // Array of QSBR thread states. + // Array of QSBR thread states (aligned to 64 bytes). struct _qsbr_pad *array; + void *array_raw; // raw allocation pointer (for free) Py_ssize_t size; // Freelist of unused _qsbr_thread_states (protected by mutex) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7fc7f343fe600f8..fcd2ae9b1d1f1ae 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -56,6 +56,29 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } +// Atomic so a thread that reads initialized=1 observes all writes +// from the initialization sequence (gh-146302). + +static inline int +_PyRuntimeState_GetCoreInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->core_initialized); +} + +static inline void +_PyRuntimeState_SetCoreInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->core_initialized, initialized); +} + +static inline int +_PyRuntimeState_GetInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->initialized); +} + +static inline void +_PyRuntimeState_SetInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->initialized, initialized); +} + #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b182f7825a2326f..6c48ac0dccfcb16 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -13,7 +13,7 @@ extern "C" { #include "pycore_debug_offsets.h" // _Py_DebugOffsets_INIT() #include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT -#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_floatobject.h" // _py_float_format_* #include "pycore_function.h" #include "pycore_hamt.h" // _PyHamt_BitmapNode_Type #include "pycore_import.h" // IMPORTS_INIT @@ -84,10 +84,6 @@ extern PyTypeObject _PyExc_MemoryError; .stoptheworld = { \ .is_global = 1, \ }, \ - .float_state = { \ - .float_format = _py_float_format_unknown, \ - .double_format = _py_float_format_unknown, \ - }, \ .types = { \ .next_version_tag = _Py_TYPE_VERSION_NEXT, \ }, \ @@ -134,13 +130,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ @@ -233,4 +223,4 @@ extern PyTypeObject _PyExc_MemoryError; #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ \ No newline at end of file +#endif /* !Py_INTERNAL_RUNTIME_INIT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 50d82d0a3650375..366d2d300fb4780 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1321,21 +1321,35 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ } #define _Py_str_identifiers_INIT { \ + INIT_ID(AGEN_CLOSED), \ + INIT_ID(AGEN_CREATED), \ + INIT_ID(AGEN_RUNNING), \ + INIT_ID(AGEN_SUSPENDED), \ INIT_ID(CANCELLED), \ + INIT_ID(CORO_CLOSED), \ + INIT_ID(CORO_CREATED), \ + INIT_ID(CORO_RUNNING), \ + INIT_ID(CORO_SUSPENDED), \ INIT_ID(Emax), \ INIT_ID(Emin), \ INIT_ID(FINISHED), \ INIT_ID(False), \ + INIT_ID(GEN_CLOSED), \ + INIT_ID(GEN_CREATED), \ + INIT_ID(GEN_RUNNING), \ + INIT_ID(GEN_SUSPENDED), \ INIT_ID(JSONDecodeError), \ INIT_ID(PENDING), \ INIT_ID(Py_Repr), \ @@ -1430,6 +1444,8 @@ extern "C" { INIT_ID(__iter__), \ INIT_ID(__itruediv__), \ INIT_ID(__ixor__), \ + INIT_ID(__lazy_import__), \ + INIT_ID(__lazy_modules__), \ INIT_ID(__le__), \ INIT_ID(__len__), \ INIT_ID(__length_hint__), \ @@ -1558,13 +1574,16 @@ extern "C" { INIT_ID(aclose), \ INIT_ID(add), \ INIT_ID(add_done_callback), \ + INIT_ID(adobe), \ INIT_ID(after_in_child), \ INIT_ID(after_in_parent), \ INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_interpreters), \ INIT_ID(all_threads), \ INIT_ID(allow_code), \ + INIT_ID(alphabet), \ INIT_ID(any), \ INIT_ID(append), \ INIT_ID(arg), \ @@ -1576,6 +1595,7 @@ extern "C" { INIT_ID(asend), \ INIT_ID(ast), \ INIT_ID(athrow), \ + INIT_ID(attr), \ INIT_ID(attribute), \ INIT_ID(autocommit), \ INIT_ID(backtick), \ @@ -1605,6 +1625,7 @@ extern "C" { INIT_ID(c_parameter_type), \ INIT_ID(c_return), \ INIT_ID(c_stack), \ + INIT_ID(cache_frames), \ INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ @@ -1615,6 +1636,7 @@ extern "C" { INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ + INIT_ID(canonical), \ INIT_ID(capath), \ INIT_ID(capitals), \ INIT_ID(category), \ @@ -1648,9 +1670,11 @@ extern "C" { INIT_ID(co_varnames), \ INIT_ID(code), \ INIT_ID(col_offset), \ + INIT_ID(collector), \ INIT_ID(command), \ INIT_ID(comment_factory), \ INIT_ID(compile_mode), \ + INIT_ID(compression), \ INIT_ID(config), \ INIT_ID(consts), \ INIT_ID(context), \ @@ -1713,7 +1737,9 @@ extern "C" { INIT_ID(event), \ INIT_ID(eventmask), \ INIT_ID(exc), \ + INIT_ID(exc_tb), \ INIT_ID(exc_type), \ + INIT_ID(exc_val), \ INIT_ID(exc_value), \ INIT_ID(excepthook), \ INIT_ID(exception), \ @@ -1748,6 +1774,7 @@ extern "C" { INIT_ID(flags), \ INIT_ID(flush), \ INIT_ID(fold), \ + INIT_ID(foldspaces), \ INIT_ID(follow_symlinks), \ INIT_ID(format), \ INIT_ID(format_spec), \ @@ -1761,10 +1788,10 @@ extern "C" { INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ - INIT_ID(get_event_loop), \ INIT_ID(get_loop), \ INIT_ID(get_source), \ INIT_ID(getattr), \ @@ -1787,6 +1814,7 @@ extern "C" { INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ + INIT_ID(ignorechars), \ INIT_ID(imag), \ INIT_ID(implieslink), \ INIT_ID(importlib), \ @@ -1868,6 +1896,7 @@ extern "C" { INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ + INIT_ID(max_threads), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ @@ -1904,6 +1933,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ @@ -1930,6 +1960,7 @@ extern "C" { INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ + INIT_ID(opcodes), \ INIT_ID(open), \ INIT_ID(opener), \ INIT_ID(operation), \ @@ -1943,7 +1974,10 @@ extern "C" { INIT_ID(outpath), \ INIT_ID(overlapped), \ INIT_ID(owner), \ + INIT_ID(pad), \ + INIT_ID(padded), \ INIT_ID(pages), \ + INIT_ID(pair), \ INIT_ID(parameter), \ INIT_ID(parent), \ INIT_ID(password), \ @@ -1955,6 +1989,7 @@ extern "C" { INIT_ID(person), \ INIT_ID(pi_factory), \ INIT_ID(pid), \ + INIT_ID(pidfd), \ INIT_ID(pointer_bits), \ INIT_ID(policy), \ INIT_ID(pos), \ @@ -1966,6 +2001,7 @@ extern "C" { INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ + INIT_ID(progress_callback), \ INIT_ID(progress_routine), \ INIT_ID(proto), \ INIT_ID(protocol), \ @@ -1986,6 +2022,7 @@ extern "C" { INIT_ID(readline), \ INIT_ID(readonly), \ INIT_ID(real), \ + INIT_ID(recursive), \ INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ @@ -1994,6 +2031,7 @@ extern "C" { INIT_ID(repeat), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(repr), \ INIT_ID(reqrefs), \ INIT_ID(require_ready), \ INIT_ID(reserved), \ @@ -2005,6 +2043,7 @@ extern "C" { INIT_ID(reversed), \ INIT_ID(rounding), \ INIT_ID(salt), \ + INIT_ID(sample_interval_us), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ @@ -2044,9 +2083,12 @@ extern "C" { INIT_ID(spam), \ INIT_ID(src), \ INIT_ID(src_dir_fd), \ + INIT_ID(stack_frames), \ INIT_ID(stacklevel), \ INIT_ID(start), \ + INIT_ID(start_time_us), \ INIT_ID(statement), \ + INIT_ID(stats), \ INIT_ID(status), \ INIT_ID(stderr), \ INIT_ID(stdin), \ @@ -2064,8 +2106,10 @@ extern "C" { INIT_ID(symmetric_difference_update), \ INIT_ID(tabsize), \ INIT_ID(tag), \ + INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ + INIT_ID(targetfd), \ INIT_ID(task), \ INIT_ID(tb_frame), \ INIT_ID(tb_lasti), \ @@ -2084,6 +2128,7 @@ extern "C" { INIT_ID(times), \ INIT_ID(timespec), \ INIT_ID(timestamp), \ + INIT_ID(timestamp_us), \ INIT_ID(timetuple), \ INIT_ID(timeunit), \ INIT_ID(top), \ @@ -2107,6 +2152,7 @@ extern "C" { INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ + INIT_ID(utcoffset), \ INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ @@ -2122,6 +2168,7 @@ extern "C" { INIT_ID(which), \ INIT_ID(who), \ INIT_ID(withdata), \ + INIT_ID(wrapcol), \ INIT_ID(writable), \ INIT_ID(write), \ INIT_ID(write_through), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 995f49e78dcda34..145e66de9984ca7 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -31,20 +31,10 @@ struct _pymem_allocators { debug_alloc_api_t obj; } debug; int is_debug_enabled; + int use_hugepages; PyObjectArenaAllocator obj_arena; }; -enum _py_float_format_type { - _py_float_format_unknown, - _py_float_format_ieee_big_endian, - _py_float_format_ieee_little_endian, -}; - -struct _Py_float_runtime_state { - enum _py_float_format_type float_format; - enum _py_float_format_type double_format; -}; - struct pyhash_runtime_state { struct { #ifndef MS_WINDOWS @@ -77,9 +67,7 @@ struct _fileutils_state { struct _parser_runtime_state { #ifdef Py_DEBUG long memo_statistics[_PYPEGEN_NSTATISTICS]; -#ifdef Py_GIL_DISABLED PyMutex mutex; -#endif #else int _not_used; #endif @@ -170,10 +158,18 @@ struct pyruntimestate { /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */ int preinitialized; - /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */ + /* Is Python core initialized? Set to 1 by _Py_InitializeCore(). + + Use _PyRuntimeState_GetCoreInitialized() and + _PyRuntimeState_SetCoreInitialized() to access it, + don't access it directly. */ int core_initialized; - /* Is Python fully initialized? Set to 1 by Py_Initialize() */ + /* Is Python fully initialized? Set to 1 by Py_Initialize(). + + Use _PyRuntimeState_GetInitialized() and + _PyRuntimeState_SetInitialized() to access it, + don't access it directly. */ int initialized; /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize() @@ -271,11 +267,20 @@ struct pyruntimestate { } audit_hooks; struct _py_object_runtime_state object_state; - struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _types_runtime_state types; struct _Py_time_runtime_state time; +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + // Used in "Python/emscripten_trampoline.c" to choose between wasm-gc + // trampoline and JavaScript trampoline. + PyObject* (*emscripten_trampoline)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); +#endif + /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; struct _Py_static_objects static_objects; diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 24d0135ed1aeca8..92d1a15177f79eb 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -35,6 +35,9 @@ extern void _PySet_ClearInternal(PySetObject *so); PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key); +PyObject * +_PySet_Freeze(PyObject *set); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index ba8b1f1cb27dee3..b6c821764886c36 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -12,7 +12,7 @@ extern "C" { /* runtime lifecycle */ PyAPI_FUNC(PyObject *) -_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); +_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop, PyObject *step); #ifdef __cplusplus } diff --git a/Include/internal/pycore_slots.h b/Include/internal/pycore_slots.h new file mode 100644 index 000000000000000..960a632aea17eea --- /dev/null +++ b/Include/internal/pycore_slots.h @@ -0,0 +1,138 @@ +#ifndef _Py_PYCORE_SLOTS_H +#define _Py_PYCORE_SLOTS_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +/* Slot data type */ +typedef enum _PySlot_DTYPE { + _PySlot_DTYPE_VOID, + _PySlot_DTYPE_FUNC, + _PySlot_DTYPE_PTR, + _PySlot_DTYPE_SIZE, + _PySlot_DTYPE_INT64, + _PySlot_DTYPE_UINT64, +}_PySlot_DTYPE; + +/* Slot kind, used to identify: + * - the thing the slot initializes (type/module/special) + * - the struct type (PySlot/PyType_Slot/PyModuleDef_Slot) + */ +typedef enum _PySlot_KIND { + _PySlot_KIND_TYPE, + _PySlot_KIND_MOD, + _PySlot_KIND_COMPAT, + _PySlot_KIND_SLOT, +} _PySlot_KIND; + +typedef enum _PySlot_PROBLEM_HANDLING { + _PySlot_PROBLEM_ALLOW, + _PySlot_PROBLEM_DEPRECATED, + _PySlot_PROBLEM_REJECT, +} _PySlot_PROBLEM_HANDLING; + +PyAPI_DATA(const char *const) _PySlot_names[]; + +#define _PySlot_MAX_NESTING 5 + +/* State for one nesting level of a slots iterator */ +typedef struct _PySlotIterator_state { + union { + // tagged by slot_struct_kind: + const PySlot *slot; // with _PySlot_KIND_SLOT + const PyType_Slot *tp_slot; // with _PySlot_KIND_TYPE + const PyModuleDef_Slot *mod_slot; // with _PySlot_KIND_MOD + const void *any_slot; + }; + _PySlot_KIND slot_struct_kind; +} _PySlotIterator_state; + +#define _PySlot_SEEN_ENTRY_BITS (8 * sizeof(unsigned int)) + +/* State for a slots iterator */ +typedef struct { + _PySlotIterator_state *state; + _PySlotIterator_state states[_PySlot_MAX_NESTING]; + unsigned int seen[_Py_slot_COUNT / _PySlot_SEEN_ENTRY_BITS + 1]; + _PySlot_KIND kind; + uint8_t recursion_level; + bool is_at_end :1; + bool is_first_run :1; + + // Name of the object (type/module) being defined, NULL if unknown. + // Must be set by the callers as soon as it's known. + const char *name; + + /* Output information: */ + + // The slot. Always a copy; may be modified by caller of the iterator. + PySlot current; + +} _PySlotIterator; + +/* Initialize an iterator using a PySlot array */ +PyAPI_FUNC(void) +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind); + +/* Initialize an iterator using a legacy slot array */ +PyAPI_FUNC(void) +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind); + +/* Reset a *successfully exhausted* iterator to the beginning. + * The *slots* must be the same as for the previous + * `_PySlotIterator_InitWithKind` call. + * (Unlike creating a new iterator, we can skip some validation after Rewind.) + */ +PyAPI_FUNC(void) _PySlotIterator_Rewind(_PySlotIterator *it, const void *slots); + +/* Iteration function. + * + * Return false at the end (when successfully exhausted). + * Otherwise (even on error), fill output information in `it` and return true. + * + * On error, set an exception and set `it->current.sl_id` to `Py_slot_invalid`. + */ +PyAPI_FUNC(bool) _PySlotIterator_Next(_PySlotIterator *it); + +/* Return 1 if given slot was "seen" by an earlier _PySlotIterator_Next call. + * (This state is not reset by rewinding.) + */ +PyAPI_FUNC(bool) _PySlotIterator_SawSlot(_PySlotIterator *, int); + +static inline const char * +_PySlot_GetName(uint16_t id) +{ + if (id >= _Py_slot_COUNT) { + return ""; + } + if (id == Py_slot_invalid) { + return "Py_slot_invalid"; + } + return _PySlot_names[id]; +} + +static inline void +_PySlot_err_bad_slot(char *kind, uint16_t id) +{ + if (id < _Py_slot_COUNT) { + PyErr_Format(PyExc_SystemError, "invalid %s slot %d (%s)", + kind, (unsigned int)id, _PySlot_names[id]); + } + else if (id == Py_slot_invalid) { + PyErr_Format(PyExc_SystemError, "invalid slot (Py_slot_invalid, %u)", + (unsigned int)id); + } + else { + PyErr_Format(PyExc_SystemError, "unknown %s slot ID %u", + kind, (unsigned int)id); + } +} + +#include "internal/pycore_slots_generated.h" + +#endif // _Py_PYCORE_SLOTS_H diff --git a/Include/internal/pycore_slots_generated.h b/Include/internal/pycore_slots_generated.h new file mode 100644 index 000000000000000..73a77070038cef4 --- /dev/null +++ b/Include/internal/pycore_slots_generated.h @@ -0,0 +1,958 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H +#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H + +static inline uint16_t +_PySlot_resolve_type_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_bf_getbuffer; + case 2: + return Py_bf_releasebuffer; + case 3: + return Py_mp_ass_subscript; + case 4: + return Py_mp_length; + case Py_slot_end: + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_doc: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_members: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + case Py_slot_subslots: + case Py_tp_slots: + case Py_tp_name: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_tp_metaclass: + case Py_tp_module: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline uint16_t +_PySlot_resolve_mod_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_mod_create; + case 2: + return Py_mod_exec; + case 3: + return Py_mod_multiple_interpreters; + case 4: + return Py_mod_gil; + case Py_slot_end: + case Py_mod_create: + case Py_mod_exec: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_mod_slots: + case Py_mod_name: + case Py_mod_doc: + case Py_mod_state_size: + case Py_mod_methods: + case Py_mod_state_traverse: + case Py_mod_state_clear: + case Py_mod_state_free: + case Py_mod_abi: + case Py_mod_token: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline void* +_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_subscript; + case Py_nb_absolute: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_absolute; + case Py_nb_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_add; + case Py_nb_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_and; + case Py_nb_bool: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_bool; + case Py_nb_divmod: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_divmod; + case Py_nb_float: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_float; + case Py_nb_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_floor_divide; + case Py_nb_index: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_index; + case Py_nb_inplace_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_add; + case Py_nb_inplace_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_and; + case Py_nb_inplace_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_floor_divide; + case Py_nb_inplace_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_lshift; + case Py_nb_inplace_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_multiply; + case Py_nb_inplace_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_or; + case Py_nb_inplace_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_power; + case Py_nb_inplace_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_remainder; + case Py_nb_inplace_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_rshift; + case Py_nb_inplace_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_subtract; + case Py_nb_inplace_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_true_divide; + case Py_nb_inplace_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_xor; + case Py_nb_int: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_int; + case Py_nb_invert: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_invert; + case Py_nb_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_lshift; + case Py_nb_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_multiply; + case Py_nb_negative: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_negative; + case Py_nb_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_or; + case Py_nb_positive: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_positive; + case Py_nb_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_power; + case Py_nb_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_remainder; + case Py_nb_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_rshift; + case Py_nb_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_subtract; + case Py_nb_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_true_divide; + case Py_nb_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_xor; + case Py_sq_ass_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_ass_item; + case Py_sq_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_concat; + case Py_sq_contains: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_contains; + case Py_sq_inplace_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_concat; + case Py_sq_inplace_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_repeat; + case Py_sq_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_item; + case Py_sq_length: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_length; + case Py_sq_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_repeat; + case Py_tp_alloc: + return (void*)tp->tp_alloc; + case Py_tp_base: + return (void*)tp->tp_base; + case Py_tp_bases: + return (void*)tp->tp_bases; + case Py_tp_call: + return (void*)tp->tp_call; + case Py_tp_clear: + return (void*)tp->tp_clear; + case Py_tp_dealloc: + return (void*)tp->tp_dealloc; + case Py_tp_del: + return (void*)tp->tp_del; + case Py_tp_descr_get: + return (void*)tp->tp_descr_get; + case Py_tp_descr_set: + return (void*)tp->tp_descr_set; + case Py_tp_doc: + return (void*)tp->tp_doc; + case Py_tp_getattr: + return (void*)tp->tp_getattr; + case Py_tp_getattro: + return (void*)tp->tp_getattro; + case Py_tp_hash: + return (void*)tp->tp_hash; + case Py_tp_init: + return (void*)tp->tp_init; + case Py_tp_is_gc: + return (void*)tp->tp_is_gc; + case Py_tp_iter: + return (void*)tp->tp_iter; + case Py_tp_iternext: + return (void*)tp->tp_iternext; + case Py_tp_methods: + return (void*)tp->tp_methods; + case Py_tp_new: + return (void*)tp->tp_new; + case Py_tp_repr: + return (void*)tp->tp_repr; + case Py_tp_richcompare: + return (void*)tp->tp_richcompare; + case Py_tp_setattr: + return (void*)tp->tp_setattr; + case Py_tp_setattro: + return (void*)tp->tp_setattro; + case Py_tp_str: + return (void*)tp->tp_str; + case Py_tp_traverse: + return (void*)tp->tp_traverse; + case Py_tp_members: + return (void*)tp->tp_members; + case Py_tp_getset: + return (void*)tp->tp_getset; + case Py_tp_free: + return (void*)tp->tp_free; + case Py_nb_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_matrix_multiply; + case Py_nb_inplace_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_matrix_multiply; + case Py_am_await: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_await; + case Py_am_aiter: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_aiter; + case Py_am_anext: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_anext; + case Py_tp_finalize: + return (void*)tp->tp_finalize; + case Py_am_send: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_send; + case Py_tp_vectorcall: + return (void*)tp->tp_vectorcall; + case Py_tp_token: + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) return NULL; + return (void*)((PyHeapTypeObject*)tp)->ht_token; + case Py_bf_getbuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_getbuffer; + case Py_bf_releasebuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_releasebuffer; + case Py_mp_ass_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_ass_subscript; + case Py_mp_length: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_length; + } + _PySlot_err_bad_slot("PyType_GetSlot", slot_id); + return NULL; +} + +static inline void +_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht, PySlot slot) +{ + switch (slot.sl_id) { + case Py_mp_subscript: + ht->as_mapping.mp_subscript = (binaryfunc)slot.sl_func; + break; + case Py_nb_absolute: + ht->as_number.nb_absolute = (unaryfunc)slot.sl_func; + break; + case Py_nb_add: + ht->as_number.nb_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_and: + ht->as_number.nb_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_bool: + ht->as_number.nb_bool = (inquiry)slot.sl_func; + break; + case Py_nb_divmod: + ht->as_number.nb_divmod = (binaryfunc)slot.sl_func; + break; + case Py_nb_float: + ht->as_number.nb_float = (unaryfunc)slot.sl_func; + break; + case Py_nb_floor_divide: + ht->as_number.nb_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_index: + ht->as_number.nb_index = (unaryfunc)slot.sl_func; + break; + case Py_nb_inplace_add: + ht->as_number.nb_inplace_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_and: + ht->as_number.nb_inplace_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_floor_divide: + ht->as_number.nb_inplace_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_lshift: + ht->as_number.nb_inplace_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_multiply: + ht->as_number.nb_inplace_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_or: + ht->as_number.nb_inplace_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_power: + ht->as_number.nb_inplace_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_inplace_remainder: + ht->as_number.nb_inplace_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_rshift: + ht->as_number.nb_inplace_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_subtract: + ht->as_number.nb_inplace_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_true_divide: + ht->as_number.nb_inplace_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_xor: + ht->as_number.nb_inplace_xor = (binaryfunc)slot.sl_func; + break; + case Py_nb_int: + ht->as_number.nb_int = (unaryfunc)slot.sl_func; + break; + case Py_nb_invert: + ht->as_number.nb_invert = (unaryfunc)slot.sl_func; + break; + case Py_nb_lshift: + ht->as_number.nb_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_multiply: + ht->as_number.nb_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_negative: + ht->as_number.nb_negative = (unaryfunc)slot.sl_func; + break; + case Py_nb_or: + ht->as_number.nb_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_positive: + ht->as_number.nb_positive = (unaryfunc)slot.sl_func; + break; + case Py_nb_power: + ht->as_number.nb_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_remainder: + ht->as_number.nb_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_rshift: + ht->as_number.nb_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_subtract: + ht->as_number.nb_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_true_divide: + ht->as_number.nb_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_xor: + ht->as_number.nb_xor = (binaryfunc)slot.sl_func; + break; + case Py_sq_ass_item: + ht->as_sequence.sq_ass_item = (ssizeobjargproc)slot.sl_func; + break; + case Py_sq_concat: + ht->as_sequence.sq_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_contains: + ht->as_sequence.sq_contains = (objobjproc)slot.sl_func; + break; + case Py_sq_inplace_concat: + ht->as_sequence.sq_inplace_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_inplace_repeat: + ht->as_sequence.sq_inplace_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_item: + ht->as_sequence.sq_item = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_length: + ht->as_sequence.sq_length = (lenfunc)slot.sl_func; + break; + case Py_sq_repeat: + ht->as_sequence.sq_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_tp_alloc: + ht->ht_type.tp_alloc = (allocfunc)slot.sl_func; + break; + case Py_tp_base: + ht->ht_type.tp_base = slot.sl_ptr; + break; + case Py_tp_bases: + ht->ht_type.tp_bases = slot.sl_ptr; + break; + case Py_tp_call: + ht->ht_type.tp_call = (ternaryfunc)slot.sl_func; + break; + case Py_tp_clear: + ht->ht_type.tp_clear = (inquiry)slot.sl_func; + break; + case Py_tp_dealloc: + ht->ht_type.tp_dealloc = (destructor)slot.sl_func; + break; + case Py_tp_del: + ht->ht_type.tp_del = (destructor)slot.sl_func; + break; + case Py_tp_descr_get: + ht->ht_type.tp_descr_get = (descrgetfunc)slot.sl_func; + break; + case Py_tp_descr_set: + ht->ht_type.tp_descr_set = (descrsetfunc)slot.sl_func; + break; + case Py_tp_doc: + ht->ht_type.tp_doc = slot.sl_ptr; + break; + case Py_tp_getattr: + ht->ht_type.tp_getattr = (getattrfunc)slot.sl_func; + break; + case Py_tp_getattro: + ht->ht_type.tp_getattro = (getattrofunc)slot.sl_func; + break; + case Py_tp_hash: + ht->ht_type.tp_hash = (hashfunc)slot.sl_func; + break; + case Py_tp_init: + ht->ht_type.tp_init = (initproc)slot.sl_func; + break; + case Py_tp_is_gc: + ht->ht_type.tp_is_gc = (inquiry)slot.sl_func; + break; + case Py_tp_iter: + ht->ht_type.tp_iter = (getiterfunc)slot.sl_func; + break; + case Py_tp_iternext: + ht->ht_type.tp_iternext = (iternextfunc)slot.sl_func; + break; + case Py_tp_methods: + ht->ht_type.tp_methods = slot.sl_ptr; + break; + case Py_tp_new: + ht->ht_type.tp_new = (newfunc)slot.sl_func; + break; + case Py_tp_repr: + ht->ht_type.tp_repr = (reprfunc)slot.sl_func; + break; + case Py_tp_richcompare: + ht->ht_type.tp_richcompare = (richcmpfunc)slot.sl_func; + break; + case Py_tp_setattr: + ht->ht_type.tp_setattr = (setattrfunc)slot.sl_func; + break; + case Py_tp_setattro: + ht->ht_type.tp_setattro = (setattrofunc)slot.sl_func; + break; + case Py_tp_str: + ht->ht_type.tp_str = (reprfunc)slot.sl_func; + break; + case Py_tp_traverse: + ht->ht_type.tp_traverse = (traverseproc)slot.sl_func; + break; + case Py_tp_members: + ht->ht_type.tp_members = slot.sl_ptr; + break; + case Py_tp_getset: + ht->ht_type.tp_getset = slot.sl_ptr; + break; + case Py_tp_free: + ht->ht_type.tp_free = (freefunc)slot.sl_func; + break; + case Py_nb_matrix_multiply: + ht->as_number.nb_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_matrix_multiply: + ht->as_number.nb_inplace_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_am_await: + ht->as_async.am_await = (unaryfunc)slot.sl_func; + break; + case Py_am_aiter: + ht->as_async.am_aiter = (unaryfunc)slot.sl_func; + break; + case Py_am_anext: + ht->as_async.am_anext = (unaryfunc)slot.sl_func; + break; + case Py_tp_finalize: + ht->ht_type.tp_finalize = (destructor)slot.sl_func; + break; + case Py_am_send: + ht->as_async.am_send = (sendfunc)slot.sl_func; + break; + case Py_tp_vectorcall: + ht->ht_type.tp_vectorcall = (vectorcallfunc)slot.sl_func; + break; + case Py_bf_getbuffer: + ht->as_buffer.bf_getbuffer = (getbufferproc)slot.sl_func; + break; + case Py_bf_releasebuffer: + ht->as_buffer.bf_releasebuffer = (releasebufferproc)slot.sl_func; + break; + case Py_mp_ass_subscript: + ht->as_mapping.mp_ass_subscript = (objobjargproc)slot.sl_func; + break; + case Py_mp_length: + ht->as_mapping.mp_length = (lenfunc)slot.sl_func; + break; + } +} + +static inline _PySlot_DTYPE +_PySlot_get_dtype(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: return _PySlot_DTYPE_VOID; + case Py_mp_subscript: return _PySlot_DTYPE_FUNC; + case Py_nb_absolute: return _PySlot_DTYPE_FUNC; + case Py_nb_add: return _PySlot_DTYPE_FUNC; + case Py_nb_and: return _PySlot_DTYPE_FUNC; + case Py_nb_bool: return _PySlot_DTYPE_FUNC; + case Py_nb_divmod: return _PySlot_DTYPE_FUNC; + case Py_nb_float: return _PySlot_DTYPE_FUNC; + case Py_nb_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_index: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_add: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_and: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_or: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_power: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_xor: return _PySlot_DTYPE_FUNC; + case Py_nb_int: return _PySlot_DTYPE_FUNC; + case Py_nb_invert: return _PySlot_DTYPE_FUNC; + case Py_nb_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_negative: return _PySlot_DTYPE_FUNC; + case Py_nb_or: return _PySlot_DTYPE_FUNC; + case Py_nb_positive: return _PySlot_DTYPE_FUNC; + case Py_nb_power: return _PySlot_DTYPE_FUNC; + case Py_nb_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_xor: return _PySlot_DTYPE_FUNC; + case Py_sq_ass_item: return _PySlot_DTYPE_FUNC; + case Py_sq_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_contains: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_repeat: return _PySlot_DTYPE_FUNC; + case Py_sq_item: return _PySlot_DTYPE_FUNC; + case Py_sq_length: return _PySlot_DTYPE_FUNC; + case Py_sq_repeat: return _PySlot_DTYPE_FUNC; + case Py_tp_alloc: return _PySlot_DTYPE_FUNC; + case Py_tp_base: return _PySlot_DTYPE_PTR; + case Py_tp_bases: return _PySlot_DTYPE_PTR; + case Py_tp_call: return _PySlot_DTYPE_FUNC; + case Py_tp_clear: return _PySlot_DTYPE_FUNC; + case Py_tp_dealloc: return _PySlot_DTYPE_FUNC; + case Py_tp_del: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_get: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_set: return _PySlot_DTYPE_FUNC; + case Py_tp_doc: return _PySlot_DTYPE_PTR; + case Py_tp_getattr: return _PySlot_DTYPE_FUNC; + case Py_tp_getattro: return _PySlot_DTYPE_FUNC; + case Py_tp_hash: return _PySlot_DTYPE_FUNC; + case Py_tp_init: return _PySlot_DTYPE_FUNC; + case Py_tp_is_gc: return _PySlot_DTYPE_FUNC; + case Py_tp_iter: return _PySlot_DTYPE_FUNC; + case Py_tp_iternext: return _PySlot_DTYPE_FUNC; + case Py_tp_methods: return _PySlot_DTYPE_PTR; + case Py_tp_new: return _PySlot_DTYPE_FUNC; + case Py_tp_repr: return _PySlot_DTYPE_FUNC; + case Py_tp_richcompare: return _PySlot_DTYPE_FUNC; + case Py_tp_setattr: return _PySlot_DTYPE_FUNC; + case Py_tp_setattro: return _PySlot_DTYPE_FUNC; + case Py_tp_str: return _PySlot_DTYPE_FUNC; + case Py_tp_traverse: return _PySlot_DTYPE_FUNC; + case Py_tp_members: return _PySlot_DTYPE_PTR; + case Py_tp_getset: return _PySlot_DTYPE_PTR; + case Py_tp_free: return _PySlot_DTYPE_FUNC; + case Py_nb_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_am_await: return _PySlot_DTYPE_FUNC; + case Py_am_aiter: return _PySlot_DTYPE_FUNC; + case Py_am_anext: return _PySlot_DTYPE_FUNC; + case Py_tp_finalize: return _PySlot_DTYPE_FUNC; + case Py_am_send: return _PySlot_DTYPE_FUNC; + case Py_tp_vectorcall: return _PySlot_DTYPE_FUNC; + case Py_tp_token: return _PySlot_DTYPE_PTR; + case Py_mod_create: return _PySlot_DTYPE_FUNC; + case Py_mod_exec: return _PySlot_DTYPE_FUNC; + case Py_mod_multiple_interpreters: return _PySlot_DTYPE_UINT64; + case Py_mod_gil: return _PySlot_DTYPE_UINT64; + case Py_bf_getbuffer: return _PySlot_DTYPE_FUNC; + case Py_bf_releasebuffer: return _PySlot_DTYPE_FUNC; + case Py_mp_ass_subscript: return _PySlot_DTYPE_FUNC; + case Py_mp_length: return _PySlot_DTYPE_FUNC; + case Py_slot_subslots: return _PySlot_DTYPE_PTR; + case Py_tp_slots: return _PySlot_DTYPE_PTR; + case Py_mod_slots: return _PySlot_DTYPE_PTR; + case Py_tp_name: return _PySlot_DTYPE_PTR; + case Py_tp_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_extra_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_itemsize: return _PySlot_DTYPE_SIZE; + case Py_tp_flags: return _PySlot_DTYPE_UINT64; + case Py_mod_name: return _PySlot_DTYPE_PTR; + case Py_mod_doc: return _PySlot_DTYPE_PTR; + case Py_mod_state_size: return _PySlot_DTYPE_SIZE; + case Py_mod_methods: return _PySlot_DTYPE_PTR; + case Py_mod_state_traverse: return _PySlot_DTYPE_FUNC; + case Py_mod_state_clear: return _PySlot_DTYPE_FUNC; + case Py_mod_state_free: return _PySlot_DTYPE_FUNC; + case Py_tp_metaclass: return _PySlot_DTYPE_PTR; + case Py_tp_module: return _PySlot_DTYPE_PTR; + case Py_mod_abi: return _PySlot_DTYPE_PTR; + case Py_mod_token: return _PySlot_DTYPE_PTR; + default: return _PySlot_DTYPE_VOID; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_duplicate_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + case Py_mod_exec: + case Py_mod_abi: + return _PySlot_PROBLEM_ALLOW; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_null_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: + case Py_tp_doc: + case Py_tp_token: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_tp_slots: + case Py_mod_slots: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_mod_state_size: + return _PySlot_PROBLEM_ALLOW; + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_mod_create: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline bool +_PySlot_get_must_be_static(uint16_t slot_id) +{ + switch (slot_id) { + case Py_tp_methods: return true; + case Py_tp_members: return true; + case Py_tp_getset: return true; + case Py_mod_methods: return true; + } + return false; +} + +#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index e59611c07fa7935..9495ccc8ac38896 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -71,11 +71,15 @@ static const _PyStackRef PyStackRef_NULL = { .index = 0 }; static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; #define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) -#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) -#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) #define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) +#define PyStackRef_ZERO_BITS PyStackRef_NULL + static inline _PyStackRef PyStackRef_Wrap(void *ptr) { @@ -259,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + PyObject *obj = _Py_stackref_get_object(ref); + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); +} +#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__) + static inline void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) { @@ -369,6 +385,10 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); static const _PyStackRef PyStackRef_ERROR = { .bits = Py_TAG_INVALID }; +/* For use in the JIT to clear an unused value. + * PyStackRef_ZERO_BITS has no meaning and should not be used other than by the JIT. */ +static const _PyStackRef PyStackRef_ZERO_BITS = { .bits = 0 }; + /* Wrap a pointer in a stack ref. * The resulting stack reference is not safe and should only be used * in the interpreter to pass values from one uop to another. @@ -446,191 +466,6 @@ PyStackRef_IncrementTaggedIntNoOverflow(_PyStackRef ref) return (_PyStackRef){ .bits = ref.bits + (1 << Py_TAGGED_SHIFT) }; } -#define PyStackRef_IsDeferredOrTaggedInt(ref) (((ref).bits & Py_TAG_REFCNT) != 0) - -#ifdef Py_GIL_DISABLED - -#define Py_TAG_DEFERRED Py_TAG_REFCNT - -#define Py_TAG_PTR ((uintptr_t)0) - - -static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; - -#define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) -#define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) -#define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) - -// Checks that mask out the deferred bit in the free threading build. -#define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None) -#define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) -#define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) - -#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) - -static inline PyObject * -PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) -{ - assert(!PyStackRef_IsTaggedInt(stackref)); - PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); - return cleared; -} - -#define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) - -static inline PyObject * -PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) -{ - assert(!PyStackRef_IsDeferred(stackref)); - return (PyObject *)stackref.bits; -} - -static inline PyObject * -PyStackRef_AsPyObjectSteal(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferred(stackref)) { - return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); - } - return PyStackRef_AsPyObjectBorrow(stackref); -} - -static inline _PyStackRef -_PyStackRef_FromPyObjectSteal(PyObject *obj) -{ - assert(obj != NULL); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} -# define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) - -static inline bool -PyStackRef_IsHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsDeferred(stackref)) { - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); - } - return true; -} - -static inline _PyStackRef -PyStackRef_MakeHeapSafe(_PyStackRef stackref) -{ - if (PyStackRef_IsHeapSafe(stackref)) { - return stackref; - } - PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectStealMortal(PyObject *obj) -{ - assert(obj != NULL); - assert(!_Py_IsImmortal(obj)); - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - return (_PyStackRef){ .bits = (uintptr_t)obj }; -} - -static inline _PyStackRef -PyStackRef_FromPyObjectNew(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - if (_PyObject_HasDeferredRefcount(obj)) { - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; - } - else { - return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; - } -} -#define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) - -static inline _PyStackRef -PyStackRef_FromPyObjectBorrow(PyObject *obj) -{ - // Make sure we don't take an already tagged value. - assert(((uintptr_t)obj & Py_TAG_BITS) == 0); - assert(obj != NULL); - return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; -} -#define PyStackRef_FromPyObjectBorrow(obj) PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj)) - -#define PyStackRef_CLOSE(REF) \ - do { \ - _PyStackRef _close_tmp = (REF); \ - assert(!PyStackRef_IsNull(_close_tmp)); \ - if (!PyStackRef_IsDeferredOrTaggedInt(_close_tmp)) { \ - Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ - } \ - } while (0) - -static inline void -PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) -{ - (void)destruct; - PyStackRef_CLOSE(ref); -} - -static inline int -PyStackRef_RefcountOnObject(_PyStackRef ref) -{ - return (ref.bits & Py_TAG_REFCNT) == 0; -} - -static inline _PyStackRef -PyStackRef_DUP(_PyStackRef stackref) -{ - assert(!PyStackRef_IsNull(stackref)); - if (PyStackRef_IsDeferredOrTaggedInt(stackref)) { - return stackref; - } - Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); - return stackref; -} - -static inline _PyStackRef -PyStackRef_Borrow(_PyStackRef stackref) -{ - return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED }; -} - -// Convert a possibly deferred reference to a strong reference. -static inline _PyStackRef -PyStackRef_AsStrongReference(_PyStackRef stackref) -{ - return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); -} - -#define PyStackRef_XCLOSE(stackref) \ - do { \ - _PyStackRef _tmp = (stackref); \ - if (!PyStackRef_IsNull(_tmp)) { \ - PyStackRef_CLOSE(_tmp); \ - } \ - } while (0); - -#define PyStackRef_CLEAR(op) \ - do { \ - _PyStackRef *_tmp_op_ptr = &(op); \ - _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ - if (!PyStackRef_IsNull(_tmp_old_op)) { \ - *_tmp_op_ptr = PyStackRef_NULL; \ - PyStackRef_CLOSE(_tmp_old_op); \ - } \ - } while (0) - -#define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew - -#else // Py_GIL_DISABLED - -// With GIL - /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ @@ -649,13 +484,24 @@ static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT }) #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT }) +#ifdef Py_GIL_DISABLED +// Checks that mask out the deferred bit in the free threading build. +#define PyStackRef_IsNone(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_NoneStruct) +#define PyStackRef_IsTrue(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_TrueStruct) +#define PyStackRef_IsFalse(REF) (((REF).bits & ~Py_TAG_REFCNT) == (uintptr_t)&_Py_FalseStruct) +#else #define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT)) #define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT)) +#endif -#ifdef Py_DEBUG +#define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) -static inline void PyStackRef_CheckValid(_PyStackRef ref) { +#if defined(Py_DEBUG) && !defined(Py_GIL_DISABLED) + +static inline void +PyStackRef_CheckValid(_PyStackRef ref) +{ assert(ref.bits != 0); int tag = ref.bits & Py_TAG_BITS; PyObject *obj = BITS_TO_PTR_MASKED(ref); @@ -706,6 +552,8 @@ PyStackRef_Borrow(_PyStackRef ref) static inline PyObject * PyStackRef_AsPyObjectSteal(_PyStackRef ref) { + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); if (PyStackRef_RefcountOnObject(ref)) { return BITS_TO_PTR(ref); } @@ -718,14 +566,18 @@ static inline _PyStackRef PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); -#if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +#ifdef Py_GIL_DISABLED + return (_PyStackRef){ .bits = (uintptr_t)obj }; #else +# if SIZEOF_VOID_P > 4 + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; +# else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; -#endif +# endif _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); PyStackRef_CheckValid(ref); return ref; +#endif } static inline _PyStackRef @@ -733,7 +585,7 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) { assert(obj != NULL); assert(!_Py_IsImmortal(obj)); - _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) }); + _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; } @@ -742,9 +594,15 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { assert(obj != NULL); +#ifdef Py_GIL_DISABLED + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; + } +#else if (_Py_IsImmortal(obj)) { - return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; } +#endif _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); @@ -767,6 +625,7 @@ _PyStackRef_FromPyObjectNewMortal(PyObject *obj) static inline _PyStackRef PyStackRef_FromPyObjectBorrow(PyObject *obj) { + assert(obj != NULL); return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } @@ -786,10 +645,27 @@ PyStackRef_DUP(_PyStackRef ref) } #endif +static inline _PyStackRef +PyStackRef_DupImmortal(_PyStackRef ref) +{ + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref))); + return ref; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#ifdef Py_GIL_DISABLED + if ((ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT) { + return true; + } + PyObject *obj = BITS_TO_PTR_MASKED(ref); + return obj == NULL || _PyObject_HasDeferredRefcount(obj); +#else + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); +#endif } static inline _PyStackRef @@ -805,6 +681,13 @@ PyStackRef_MakeHeapSafe(_PyStackRef ref) return ref; } +// Convert a possibly deferred reference to a strong reference. +static inline _PyStackRef +PyStackRef_AsStrongReference(_PyStackRef stackref) +{ + return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); +} + #ifdef _WIN32 #define PyStackRef_CLOSE(REF) \ do { \ @@ -822,12 +705,6 @@ PyStackRef_CLOSE(_PyStackRef ref) } #endif -static inline bool -PyStackRef_IsNullOrInt(_PyStackRef ref) -{ - return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); -} - static inline void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) { @@ -860,8 +737,6 @@ PyStackRef_XCLOSE(_PyStackRef ref) } while (0) -#endif // Py_GIL_DISABLED - // Note: this is a macro because MSVC (Windows) has trouble inlining it. #define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) @@ -918,6 +793,13 @@ _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) ref->ref = PyStackRef_NULL; } +static inline void +_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) +{ + _PyThreadState_PushCStackRef(tstate, ref); + ref->ref = PyStackRef_FromPyObjectNew(obj); +} + static inline void _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { @@ -929,13 +811,24 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) PyStackRef_XCLOSE(ref->ref); } +static inline _PyStackRef +_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + assert(tstate_impl->c_stack_refs == ref); + tstate_impl->c_stack_refs = ref->next; +#endif + return ref->ref; +} + #ifdef Py_GIL_DISABLED static inline int _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) { if (_PyObject_HasDeferredRefcount(op)) { - *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED }; + *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_REFCNT }; return 1; } if (_Py_TryIncrefCompare(src, op)) { @@ -975,6 +868,18 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) } \ } while (0) +static inline void +_PyStackRef_CloseStack(_PyStackRef *arguments, int total_args) +{ + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index 225f423912f2c27..656acae960ac0d2 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -10,28 +10,24 @@ extern "C" { // Returns a str() containing the hex representation of argbuf. // Export for '_hashlib' shared extension. -PyAPI_FUNC(PyObject*) _Py_strhex(const - char* argbuf, - const Py_ssize_t arglen); +PyAPI_FUNC(PyObject *) _Py_strhex(const char *argbuf, Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -extern PyObject* _Py_strhex_bytes( - const char* argbuf, - const Py_ssize_t arglen); +extern PyObject *_Py_strhex_bytes(const char *argbuf, Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -extern PyObject* _Py_strhex_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +extern PyObject *_Py_strhex_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); // Export for 'binascii' shared extension -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( - const char* argbuf, - const Py_ssize_t arglen, - PyObject* sep, - const int bytes_per_group); +PyAPI_FUNC(PyObject *) _Py_strhex_bytes_with_sep( + const char *argbuf, + Py_ssize_t arglen, + PyObject *sep, + Py_ssize_t bytes_per_group); #ifdef __cplusplus } diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 9dbfa913219afa1..c650a94a1eab2e1 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -90,6 +90,7 @@ typedef struct _symtable_entry { PyObject *ste_id; /* int: key in ste_table->st_blocks */ PyObject *ste_symbols; /* dict: variable names to flags */ PyObject *ste_name; /* string: name of current block */ + PyObject *ste_function_name; /* string or NULL: for annotation blocks: name of the corresponding functions */ PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ @@ -126,6 +127,7 @@ typedef struct _symtable_entry { unsigned ste_method : 1; /* true if block is a function block defined in class scope */ unsigned ste_has_conditional_annotations : 1; /* true if block has conditionally executed annotations */ unsigned ste_in_conditional_block : 1; /* set while we are inside a conditionally executed block */ + unsigned ste_in_try_block : 1; /* set while we are inside a try/except block */ unsigned ste_in_unevaluated_annotation : 1; /* set while we are processing an annotation that will not be evaluated */ int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ @@ -151,7 +153,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject extern void _PySymtable_Free(struct symtable *); extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name); -extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); + +// Export for '_pickle' shared extension +PyAPI_FUNC(PyObject *) +_Py_Mangle(PyObject *, PyObject *); +PyAPI_FUNC(int) +_Py_IsPrivateName(PyObject *); /* Flags for def-use information */ diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 8357cce9d899fbb..e016afaa5c56879 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,55 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. @@ -85,7 +36,8 @@ extern void _Py_DumpHexadecimal( uintptr_t value, Py_ssize_t width); -extern PyObject* _PyTraceBack_FromFrame( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 572e80258763197..9974ea3c4143fa6 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -21,7 +21,10 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the TABLES_LOCK(). */ + Variable protected by the TABLES_LOCK() and stored atomically. + Atomic store is used so that it can read without locking for the + general case of checking if tracemalloc is enabled. + */ int tracing; /* limit of the number of frames in a traceback, 1 by default. @@ -30,8 +33,8 @@ struct _PyTraceMalloc_Config { }; -/* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. */ +/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce + the memory footprint on 64-bit architectures: 12 bytes instead of 16. */ #if defined(_MSC_VER) #pragma pack(push, 4) #endif @@ -46,18 +49,22 @@ tracemalloc_frame { PyObject *filename; unsigned int lineno; }; -#ifdef _MSC_VER -#pragma pack(pop) -#endif -struct tracemalloc_traceback { +struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +tracemalloc_traceback { Py_uhash_t hash; /* Number of frames stored */ uint16_t nframe; /* Total number of frames the traceback had */ uint16_t total_nframe; - struct tracemalloc_frame frames[1]; + struct tracemalloc_frame frames[]; }; +#ifdef _MSC_VER +#pragma pack(pop) +#endif struct _tracemalloc_runtime_state { @@ -95,7 +102,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ _Py_hashtable_t *domains; - struct tracemalloc_traceback empty_traceback; + struct tracemalloc_traceback *empty_traceback; Py_tss_t reentrant_key; }; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 50048801b2e4eec..eb2b0c84acdc7c8 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr #include "pycore_uop.h" // struct _PyUOpInstruction @@ -22,37 +23,6 @@ struct _gc_thread_state { }; #endif -#if _Py_TIER2 -typedef struct _PyJitTracerInitialState { - int stack_depth; - int chain_depth; - struct _PyExitData *exit; - PyCodeObject *code; // Strong - PyFunctionObject *func; // Strong - _Py_CODEUNIT *start_instr; - _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *jump_backward_instr; -} _PyJitTracerInitialState; - -typedef struct _PyJitTracerPreviousState { - bool dependencies_still_valid; - bool instr_is_super; - int code_max_size; - int code_curr_size; - int instr_oparg; - int instr_stacklevel; - _Py_CODEUNIT *instr; - PyCodeObject *instr_code; // Strong - struct _PyInterpreterFrame *instr_frame; - _PyBloomFilter dependencies; -} _PyJitTracerPreviousState; - -typedef struct _PyJitTracerState { - _PyUOpInstruction *code_buffer; - _PyJitTracerInitialState initial_state; - _PyJitTracerPreviousState prev_state; -} _PyJitTracerState; -#endif // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields @@ -61,6 +31,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; + // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. Py_ssize_t refcount; @@ -77,6 +51,13 @@ typedef struct _PyThreadStateImpl { PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference + // Distinguishes between yield and return from PyEval_EvalFrame(). + // See gen_send_ex2() in Objects/genobject.c + enum { + GENERATOR_RETURN = 0, + GENERATOR_YIELD = 1, + } generator_return_kind; + /* Head of circular linked-list of all tasks which are instances of `asyncio.Task` or subclasses of it used in `asyncio.all_tasks`. */ @@ -119,7 +100,13 @@ typedef struct _PyThreadStateImpl { Py_ssize_t reftotal; // this thread's total refcount operations #endif #if _Py_TIER2 - _PyJitTracerState jit_tracer_state; + struct _PyJitTracerState *jit_tracer_state; +#endif + +#ifdef Py_GIL_DISABLED + // gh-144438: Add padding to ensure that the fields above don't share a + // cache line with other allocations. + char __padding[64]; #endif } _PyThreadStateImpl; diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 46db02593ad106c..e294b16b3df60f6 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -21,10 +21,18 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); /* other API */ +PyAPI_FUNC(void) _PyStolenTuple_Free(PyObject *self); + #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); +PyAPI_FUNC(PyObject *) _PyTuple_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Repeat(PyObject *self, Py_ssize_t n); + +PyAPI_FUNC(PyObject *) _PyTuple_FromPair(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_FromPairSteal(PyObject *, PyObject *); typedef struct { PyObject_HEAD diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 3661f171e2b013a..c8b888f1c395483 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_interp_structs.h" // managed_static_type_state #include "pycore_moduleobject.h" // PyModuleObject +#include "pycore_structs.h" // _PyStackRef /* state */ @@ -25,6 +26,7 @@ extern "C" { #define _Py_TYPE_VERSION_BYTEARRAY 9 #define _Py_TYPE_VERSION_BYTES 10 #define _Py_TYPE_VERSION_COMPLEX 11 +#define _Py_TYPE_VERSION_FROZENDICT 12 #define _Py_TYPE_VERSION_NEXT 16 @@ -58,7 +60,8 @@ extern void _PyStaticType_FiniBuiltin( extern void _PyStaticType_ClearWeakRefs( PyInterpreterState *interp, PyTypeObject *type); -extern managed_static_type_state * _PyStaticType_GetState( +// Exported for external JIT support +PyAPI_FUNC(managed_static_type_state *) _PyStaticType_GetState( PyInterpreterState *interp, PyTypeObject *type); @@ -94,7 +97,6 @@ PyAPI_FUNC(PyObject *) _PyType_LookupSubclasses(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_InitSubclasses(PyTypeObject *); extern PyObject * _PyType_GetBases(PyTypeObject *type); -extern PyObject * _PyType_GetMRO(PyTypeObject *type); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); extern int _PyType_HasSubclasses(PyTypeObject *); @@ -112,17 +114,25 @@ _PyType_IsReady(PyTypeObject *type) extern PyObject* _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); extern PyObject* _Py_type_getattro(PyObject *type, PyObject *name); +extern _PyStackRef _Py_type_getattro_stackref(PyTypeObject *type, PyObject *name, + int *suppress_missing_attribute); extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int op); extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +extern PyObject *_PySuper_LookupDescr(PyTypeObject *su_type, + PyTypeObject *su_obj_type, + PyObject *name); + extern PyObject* _PyType_GetFullyQualifiedName(PyTypeObject *type, char sep); // Perform the following operation, in a thread-safe way when required by the @@ -138,6 +148,9 @@ extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); +// Raise PyType_Modified with the type lock already held. +extern void _PyType_Modified_Unlocked(PyTypeObject *type); + PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); PyTypeObject *_PyType_LookupByVersion(unsigned int version); @@ -149,12 +162,18 @@ typedef int (*_py_validate_type)(PyTypeObject *); // It will verify the ``ty`` through user-defined validation function ``validate``, // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. -extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); -extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Exported for external JIT support +int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); // Precalculates count of non-unique slots and fills wrapperbase.name_count. extern int _PyType_InitSlotDefs(PyInterpreterState *interp); +// Like PyType_GetBaseByToken, but does not modify refcounts. +// Cannot fail; arguments must be valid. +PyAPI_FUNC(int) +_PyType_GetBaseByToken_Borrow(PyTypeObject *type, void *token, PyTypeObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index e7ca65a56b6ec3b..75d5068f815b918 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -32,6 +32,8 @@ extern PyObject* _PyUnicode_ResizeCompact( PyObject *unicode, Py_ssize_t length); extern PyObject* _PyUnicode_GetEmpty(void); +PyAPI_FUNC(PyObject*) _PyUnicode_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyUnicode_Repeat(PyObject *str, Py_ssize_t len); /* Generic helper macro to convert characters of different types. @@ -307,14 +309,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. @@ -333,7 +327,8 @@ extern PyObject* _PyUnicode_XStrip( /* Dedent a string. - Behaviour is expected to be an exact match of `textwrap.dedent`. + Intended to dedent Python source. Unlike `textwrap.dedent`, this + only supports spaces and tabs and doesn't normalize empty lines. Return a new reference on success, NULL with exception set on error. */ extern PyObject* _PyUnicode_Dedent(PyObject *unicode); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b4d920154b6e836..00d6297432b5fca 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -12,10 +12,42 @@ extern "C" { static inline void _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { PyObject *string; + string = &_Py_ID(AGEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(AGEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(CANCELLED); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(CORO_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(Emax); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -32,6 +64,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CLOSED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_CREATED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_RUNNING); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(GEN_SUSPENDED); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(JSONDecodeError); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -408,6 +456,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_import__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(__lazy_modules__); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(__le__); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -920,6 +976,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(adobe); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(after_in_child); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -940,6 +1000,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_interpreters); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(all_threads); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -948,6 +1012,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(alphabet); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(any); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -992,6 +1060,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(attr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(attribute); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1108,6 +1180,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cache_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_datetime_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1148,6 +1224,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(canonical); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1280,6 +1360,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(collector); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(command); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1292,6 +1376,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(compression); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(config); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1540,10 +1628,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_tb); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_type); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(exc_val); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(exc_value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1680,6 +1776,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(foldspaces); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(follow_symlinks); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1732,19 +1832,19 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(generation); + string = &_Py_ID(gc); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(get); + string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(get_debug); + string = &_Py_ID(get); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(get_event_loop); + string = &_Py_ID(get_debug); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); @@ -1836,6 +1936,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(ignorechars); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(imag); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2160,6 +2264,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(max_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2304,6 +2412,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2408,6 +2520,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(opcodes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(open); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2460,10 +2576,22 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pad); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(padded); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pages); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pair); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(parameter); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2508,6 +2636,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pidfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pointer_bits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2552,6 +2684,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(progress_callback); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(progress_routine); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2632,6 +2768,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(recursive); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reducer_override); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2664,6 +2804,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reqrefs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2708,6 +2852,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sample_interval_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2864,6 +3012,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stack_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(stacklevel); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2872,10 +3024,18 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(start_time_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(statement); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stats); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2944,6 +3104,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(take_bytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(target); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2952,6 +3116,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(targetfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3024,6 +3192,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(timestamp_us); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(timetuple); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3116,6 +3288,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(utcoffset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3176,6 +3352,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(wrapcol); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(writable); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3236,6 +3416,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3260,6 +3444,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4e1b15af42caa3d..9ff87faec40d0b4 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -31,20 +31,35 @@ typedef struct _PyUOpInstruction{ uint64_t operand0; // A cache entry uint64_t operand1; #ifdef Py_STATS + int32_t fitness; uint64_t execution_count; #endif } _PyUOpInstruction; -// This is the length of the trace we translate initially. -#define UOP_MAX_TRACE_LENGTH 3000 -#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +// Fitness is the target length of the trace we translate initially. The uop +// buffer has a small amount of extra space for entry/loop-closing overhead. +#define FITNESS_INITIAL 2500 + +#define UOP_TRACE_BUFFER_OVERHEAD 10 +#define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD) /* Bloom filter with m = 256 * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 +#ifdef HAVE_GCC_UINT128_T +#define _Py_BLOOM_FILTER_WORDS 2 +typedef __uint128_t _Py_bloom_filter_word_t; +#else +#define _Py_BLOOM_FILTER_WORDS 4 +typedef uint64_t _Py_bloom_filter_word_t; +#endif + +#define _Py_BLOOM_FILTER_BITS_PER_WORD \ + ((int)(sizeof(_Py_bloom_filter_word_t) * 8)) +#define _Py_BLOOM_FILTER_WORD_SHIFT \ + ((sizeof(_Py_bloom_filter_word_t) == 16) ? 7 : 6) typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; + _Py_bloom_filter_word_t bits[_Py_BLOOM_FILTER_WORDS]; } _PyBloomFilter; #ifdef __cplusplus diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7a33a5b84fd21ac..c30813f6d800f34 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -11,27 +11,42 @@ extern "C" { #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _BINARY_OP 302 -#define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS 304 -#define _BINARY_OP_ADD_INT 305 -#define _BINARY_OP_ADD_UNICODE 306 -#define _BINARY_OP_EXTEND 307 -#define _BINARY_OP_INPLACE_ADD_UNICODE 308 -#define _BINARY_OP_MULTIPLY_FLOAT 309 -#define _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS 310 -#define _BINARY_OP_MULTIPLY_INT 311 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 312 -#define _BINARY_OP_SUBSCR_DICT 313 -#define _BINARY_OP_SUBSCR_INIT_CALL 314 -#define _BINARY_OP_SUBSCR_LIST_INT 315 -#define _BINARY_OP_SUBSCR_LIST_SLICE 316 -#define _BINARY_OP_SUBSCR_STR_INT 317 -#define _BINARY_OP_SUBSCR_TUPLE_INT 318 -#define _BINARY_OP_SUBTRACT_FLOAT 319 -#define _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS 320 -#define _BINARY_OP_SUBTRACT_INT 321 -#define _BINARY_SLICE 322 +#define _ALLOCATE_OBJECT 302 +#define _BINARY_OP 303 +#define _BINARY_OP_ADD_FLOAT 304 +#define _BINARY_OP_ADD_FLOAT_INPLACE 305 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT 306 +#define _BINARY_OP_ADD_INT 307 +#define _BINARY_OP_ADD_INT_INPLACE 308 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT 309 +#define _BINARY_OP_ADD_UNICODE 310 +#define _BINARY_OP_EXTEND 311 +#define _BINARY_OP_INPLACE_ADD_UNICODE 312 +#define _BINARY_OP_MULTIPLY_FLOAT 313 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE 314 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT 315 +#define _BINARY_OP_MULTIPLY_INT 316 +#define _BINARY_OP_MULTIPLY_INT_INPLACE 317 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT 318 +#define _BINARY_OP_SUBSCR_CHECK_FUNC 319 +#define _BINARY_OP_SUBSCR_DICT 320 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH 321 +#define _BINARY_OP_SUBSCR_INIT_CALL 322 +#define _BINARY_OP_SUBSCR_LIST_INT 323 +#define _BINARY_OP_SUBSCR_LIST_SLICE 324 +#define _BINARY_OP_SUBSCR_STR_INT 325 +#define _BINARY_OP_SUBSCR_TUPLE_INT 326 +#define _BINARY_OP_SUBSCR_USTR_INT 327 +#define _BINARY_OP_SUBTRACT_FLOAT 328 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE 329 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT 330 +#define _BINARY_OP_SUBTRACT_INT 331 +#define _BINARY_OP_SUBTRACT_INT_INPLACE 332 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT 333 +#define _BINARY_OP_TRUEDIV_FLOAT 334 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE 335 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT 336 +#define _BINARY_SLICE 337 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -40,148 +55,194 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 323 -#define _CALL_BUILTIN_FAST 324 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 325 -#define _CALL_BUILTIN_O 326 -#define _CALL_INTRINSIC_1 CALL_INTRINSIC_1 -#define _CALL_INTRINSIC_2 CALL_INTRINSIC_2 -#define _CALL_ISINSTANCE 327 -#define _CALL_KW_NON_PY 328 -#define _CALL_LEN 329 -#define _CALL_LIST_APPEND 330 -#define _CALL_METHOD_DESCRIPTOR_FAST 331 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 332 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 333 -#define _CALL_METHOD_DESCRIPTOR_O 334 -#define _CALL_NON_PY_GENERAL 335 -#define _CALL_STR_1 336 -#define _CALL_TUPLE_1 337 -#define _CALL_TYPE_1 338 -#define _CHECK_AND_ALLOCATE_OBJECT 339 -#define _CHECK_ATTR_CLASS 340 -#define _CHECK_ATTR_METHOD_LAZY_DICT 341 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 342 +#define _CALL_BUILTIN_CLASS 338 +#define _CALL_BUILTIN_FAST 339 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 340 +#define _CALL_BUILTIN_O 341 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL 342 +#define _CALL_INTRINSIC_1 343 +#define _CALL_INTRINSIC_2 344 +#define _CALL_ISINSTANCE 345 +#define _CALL_KW_NON_PY 346 +#define _CALL_LEN 347 +#define _CALL_LIST_APPEND 348 +#define _CALL_METHOD_DESCRIPTOR_FAST 349 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE 350 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 351 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE 352 +#define _CALL_METHOD_DESCRIPTOR_NOARGS 353 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE 354 +#define _CALL_METHOD_DESCRIPTOR_O 355 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE 356 +#define _CALL_NON_PY_GENERAL 357 +#define _CALL_STR_1 358 +#define _CALL_TUPLE_1 359 +#define _CALL_TYPE_1 360 +#define _CHECK_ATTR_CLASS 361 +#define _CHECK_ATTR_METHOD_LAZY_DICT 362 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 363 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION_EXACT_ARGS 343 -#define _CHECK_FUNCTION_VERSION 344 -#define _CHECK_FUNCTION_VERSION_INLINE 345 -#define _CHECK_FUNCTION_VERSION_KW 346 -#define _CHECK_IS_NOT_PY_CALLABLE 347 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 348 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 349 -#define _CHECK_METHOD_VERSION 350 -#define _CHECK_METHOD_VERSION_KW 351 -#define _CHECK_PEP_523 352 -#define _CHECK_PERIODIC 353 -#define _CHECK_PERIODIC_AT_END 354 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 355 -#define _CHECK_RECURSION_REMAINING 356 -#define _CHECK_STACK_SPACE 357 -#define _CHECK_STACK_SPACE_OPERAND 358 -#define _CHECK_VALIDITY 359 -#define _COLD_DYNAMIC_EXIT 360 -#define _COLD_EXIT 361 -#define _COMPARE_OP 362 -#define _COMPARE_OP_FLOAT 363 -#define _COMPARE_OP_INT 364 -#define _COMPARE_OP_STR 365 -#define _CONTAINS_OP 366 -#define _CONTAINS_OP_DICT 367 -#define _CONTAINS_OP_SET 368 +#define _CHECK_FUNCTION_EXACT_ARGS 364 +#define _CHECK_FUNCTION_VERSION 365 +#define _CHECK_FUNCTION_VERSION_INLINE 366 +#define _CHECK_FUNCTION_VERSION_KW 367 +#define _CHECK_IS_NOT_PY_CALLABLE 368 +#define _CHECK_IS_NOT_PY_CALLABLE_EX 369 +#define _CHECK_IS_NOT_PY_CALLABLE_KW 370 +#define _CHECK_IS_PY_CALLABLE_EX 371 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 372 +#define _CHECK_METHOD_VERSION 373 +#define _CHECK_METHOD_VERSION_KW 374 +#define _CHECK_OBJECT 375 +#define _CHECK_PEP_523 376 +#define _CHECK_PERIODIC 377 +#define _CHECK_PERIODIC_AT_END 378 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 379 +#define _CHECK_RECURSION_LIMIT 380 +#define _CHECK_RECURSION_REMAINING 381 +#define _CHECK_STACK_SPACE 382 +#define _CHECK_STACK_SPACE_OPERAND 383 +#define _CHECK_VALIDITY 384 +#define _COLD_DYNAMIC_EXIT 385 +#define _COLD_EXIT 386 +#define _COMPARE_OP 387 +#define _COMPARE_OP_FLOAT 388 +#define _COMPARE_OP_INT 389 +#define _COMPARE_OP_STR 390 +#define _CONTAINS_OP 391 +#define _CONTAINS_OP_DICT 392 +#define _CONTAINS_OP_SET 393 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 369 -#define _COPY_1 370 -#define _COPY_2 371 -#define _COPY_3 372 +#define _COPY 394 +#define _COPY_1 395 +#define _COPY_2 396 +#define _COPY_3 397 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 373 -#define _DELETE_ATTR DELETE_ATTR +#define _CREATE_INIT_FRAME 398 #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 374 -#define _DICT_MERGE DICT_MERGE -#define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 375 -#define _DO_CALL_FUNCTION_EX 376 -#define _DO_CALL_KW 377 -#define _DYNAMIC_EXIT 378 +#define _DEOPT 399 +#define _DICT_MERGE 400 +#define _DICT_UPDATE 401 +#define _DO_CALL 402 +#define _DO_CALL_FUNCTION_EX 403 +#define _DO_CALL_KW 404 +#define _DYNAMIC_EXIT 405 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 379 +#define _ERROR_POP_N 406 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 380 -#define _EXPAND_METHOD_KW 381 -#define _FATAL_ERROR 382 +#define _EXPAND_METHOD 407 +#define _EXPAND_METHOD_KW 408 +#define _FATAL_ERROR 409 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 383 -#define _FOR_ITER_GEN_FRAME 384 -#define _FOR_ITER_TIER_TWO 385 +#define _FOR_ITER 410 +#define _FOR_ITER_GEN_FRAME 411 +#define _FOR_ITER_TIER_TWO 412 +#define _FOR_ITER_VIRTUAL 413 +#define _FOR_ITER_VIRTUAL_TIER_TWO 414 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _GET_ITER GET_ITER +#define _GET_ITER 415 +#define _GET_ITER_TRAD 416 #define _GET_LEN GET_LEN -#define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 386 -#define _GUARD_CALLABLE_ISINSTANCE 387 -#define _GUARD_CALLABLE_LEN 388 -#define _GUARD_CALLABLE_LIST_APPEND 389 -#define _GUARD_CALLABLE_STR_1 390 -#define _GUARD_CALLABLE_TUPLE_1 391 -#define _GUARD_CALLABLE_TYPE_1 392 -#define _GUARD_DORV_NO_DICT 393 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 -#define _GUARD_GLOBALS_VERSION 395 -#define _GUARD_IP_RETURN_GENERATOR 396 -#define _GUARD_IP_RETURN_VALUE 397 -#define _GUARD_IP_YIELD_VALUE 398 -#define _GUARD_IP__PUSH_FRAME 399 -#define _GUARD_IS_FALSE_POP 400 -#define _GUARD_IS_NONE_POP 401 -#define _GUARD_IS_NOT_NONE_POP 402 -#define _GUARD_IS_TRUE_POP 403 -#define _GUARD_KEYS_VERSION 404 -#define _GUARD_NOS_DICT 405 -#define _GUARD_NOS_FLOAT 406 -#define _GUARD_NOS_INT 407 -#define _GUARD_NOS_LIST 408 -#define _GUARD_NOS_NOT_NULL 409 -#define _GUARD_NOS_NULL 410 -#define _GUARD_NOS_OVERFLOWED 411 -#define _GUARD_NOS_TUPLE 412 -#define _GUARD_NOS_UNICODE 413 -#define _GUARD_NOT_EXHAUSTED_LIST 414 -#define _GUARD_NOT_EXHAUSTED_RANGE 415 -#define _GUARD_NOT_EXHAUSTED_TUPLE 416 -#define _GUARD_THIRD_NULL 417 -#define _GUARD_TOS_ANY_SET 418 -#define _GUARD_TOS_DICT 419 -#define _GUARD_TOS_FLOAT 420 -#define _GUARD_TOS_INT 421 -#define _GUARD_TOS_LIST 422 -#define _GUARD_TOS_OVERFLOWED 423 -#define _GUARD_TOS_SLICE 424 -#define _GUARD_TOS_TUPLE 425 -#define _GUARD_TOS_UNICODE 426 -#define _GUARD_TYPE_VERSION 427 -#define _GUARD_TYPE_VERSION_AND_LOCK 428 -#define _HANDLE_PENDING_AND_DEOPT 429 +#define _GUARD_3OS_ASYNC_GEN_ASEND 417 +#define _GUARD_BINARY_OP_EXTEND 418 +#define _GUARD_BINARY_OP_EXTEND_LHS 419 +#define _GUARD_BINARY_OP_EXTEND_RHS 420 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 421 +#define _GUARD_BIT_IS_SET_POP 422 +#define _GUARD_BIT_IS_SET_POP_4 423 +#define _GUARD_BIT_IS_SET_POP_5 424 +#define _GUARD_BIT_IS_SET_POP_6 425 +#define _GUARD_BIT_IS_SET_POP_7 426 +#define _GUARD_BIT_IS_UNSET_POP 427 +#define _GUARD_BIT_IS_UNSET_POP_4 428 +#define _GUARD_BIT_IS_UNSET_POP_5 429 +#define _GUARD_BIT_IS_UNSET_POP_6 430 +#define _GUARD_BIT_IS_UNSET_POP_7 431 +#define _GUARD_CALLABLE_BUILTIN_CLASS 432 +#define _GUARD_CALLABLE_BUILTIN_FAST 433 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 434 +#define _GUARD_CALLABLE_BUILTIN_O 435 +#define _GUARD_CALLABLE_ISINSTANCE 436 +#define _GUARD_CALLABLE_LEN 437 +#define _GUARD_CALLABLE_LIST_APPEND 438 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 439 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 440 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 441 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 442 +#define _GUARD_CALLABLE_STR_1 443 +#define _GUARD_CALLABLE_TUPLE_1 444 +#define _GUARD_CALLABLE_TYPE_1 445 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR 446 +#define _GUARD_CODE_VERSION_RETURN_VALUE 447 +#define _GUARD_CODE_VERSION_YIELD_VALUE 448 +#define _GUARD_CODE_VERSION__PUSH_FRAME 449 +#define _GUARD_DORV_NO_DICT 450 +#define _GUARD_GLOBALS_VERSION 451 +#define _GUARD_IP_RETURN_GENERATOR 452 +#define _GUARD_IP_RETURN_VALUE 453 +#define _GUARD_IP_YIELD_VALUE 454 +#define _GUARD_IP__PUSH_FRAME 455 +#define _GUARD_IS_FALSE_POP 456 +#define _GUARD_IS_NONE_POP 457 +#define _GUARD_IS_NOT_NONE_POP 458 +#define _GUARD_IS_TRUE_POP 459 +#define _GUARD_ITERATOR 460 +#define _GUARD_ITER_VIRTUAL 461 +#define _GUARD_LOAD_SUPER_ATTR_METHOD 462 +#define _GUARD_NOS_COMPACT_ASCII 463 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT 464 +#define _GUARD_NOS_DICT_SUBSCRIPT 465 +#define _GUARD_NOS_FLOAT 466 +#define _GUARD_NOS_INT 467 +#define _GUARD_NOS_ITER_VIRTUAL 468 +#define _GUARD_NOS_LIST 469 +#define _GUARD_NOS_NOT_NULL 470 +#define _GUARD_NOS_NULL 471 +#define _GUARD_NOS_OVERFLOWED 472 +#define _GUARD_NOS_TUPLE 473 +#define _GUARD_NOS_TYPE_VERSION 474 +#define _GUARD_NOS_UNICODE 475 +#define _GUARD_NOT_EXHAUSTED_LIST 476 +#define _GUARD_NOT_EXHAUSTED_RANGE 477 +#define _GUARD_NOT_EXHAUSTED_TUPLE 478 +#define _GUARD_THIRD_NULL 479 +#define _GUARD_TOS_ANY_DICT 480 +#define _GUARD_TOS_ANY_SET 481 +#define _GUARD_TOS_DICT 482 +#define _GUARD_TOS_FLOAT 483 +#define _GUARD_TOS_FROZENDICT 484 +#define _GUARD_TOS_FROZENSET 485 +#define _GUARD_TOS_INT 486 +#define _GUARD_TOS_IS_NONE 487 +#define _GUARD_TOS_LIST 488 +#define _GUARD_TOS_NOT_NULL 489 +#define _GUARD_TOS_OVERFLOWED 490 +#define _GUARD_TOS_SET 491 +#define _GUARD_TOS_SLICE 492 +#define _GUARD_TOS_TUPLE 493 +#define _GUARD_TOS_UNICODE 494 +#define _GUARD_TYPE 495 +#define _GUARD_TYPE_ITER 496 +#define _GUARD_TYPE_VERSION 497 +#define _GUARD_TYPE_VERSION_LOCKED 498 +#define _HANDLE_PENDING_AND_DEOPT 499 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 -#define _INIT_CALL_PY_EXACT_ARGS 431 -#define _INIT_CALL_PY_EXACT_ARGS_0 432 -#define _INIT_CALL_PY_EXACT_ARGS_1 433 -#define _INIT_CALL_PY_EXACT_ARGS_2 434 -#define _INIT_CALL_PY_EXACT_ARGS_3 435 -#define _INIT_CALL_PY_EXACT_ARGS_4 436 -#define _INSERT_NULL 437 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 500 +#define _INIT_CALL_PY_EXACT_ARGS 501 +#define _INIT_CALL_PY_EXACT_ARGS_0 502 +#define _INIT_CALL_PY_EXACT_ARGS_1 503 +#define _INIT_CALL_PY_EXACT_ARGS_2 504 +#define _INIT_CALL_PY_EXACT_ARGS_3 505 +#define _INIT_CALL_PY_EXACT_ARGS_4 506 +#define _INSERT_NULL 507 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -191,178 +252,1183 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 438 -#define _IS_OP IS_OP -#define _ITER_CHECK_LIST 439 -#define _ITER_CHECK_RANGE 440 -#define _ITER_CHECK_TUPLE 441 -#define _ITER_JUMP_LIST 442 -#define _ITER_JUMP_RANGE 443 -#define _ITER_JUMP_TUPLE 444 -#define _ITER_NEXT_LIST 445 -#define _ITER_NEXT_LIST_TIER_TWO 446 -#define _ITER_NEXT_RANGE 447 -#define _ITER_NEXT_TUPLE 448 +#define _IS_NONE 508 +#define _IS_OP 509 +#define _ITER_CHECK_LIST 510 +#define _ITER_CHECK_RANGE 511 +#define _ITER_CHECK_TUPLE 512 +#define _ITER_JUMP_LIST 513 +#define _ITER_JUMP_RANGE 514 +#define _ITER_JUMP_TUPLE 515 +#define _ITER_NEXT_INLINE 516 +#define _ITER_NEXT_LIST 517 +#define _ITER_NEXT_LIST_TIER_TWO 518 +#define _ITER_NEXT_RANGE 519 +#define _ITER_NEXT_TUPLE 520 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 449 +#define _JUMP_TO_TOP 521 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 450 -#define _LOAD_ATTR_CLASS 451 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 452 -#define _LOAD_ATTR_METHOD_LAZY_DICT 453 -#define _LOAD_ATTR_METHOD_NO_DICT 454 -#define _LOAD_ATTR_METHOD_WITH_VALUES 455 -#define _LOAD_ATTR_MODULE 456 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 -#define _LOAD_ATTR_PROPERTY_FRAME 459 -#define _LOAD_ATTR_SLOT 460 -#define _LOAD_ATTR_WITH_HINT 461 +#define _LIST_EXTEND 522 +#define _LOAD_ATTR 523 +#define _LOAD_ATTR_CLASS 524 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME 525 +#define _LOAD_ATTR_INSTANCE_VALUE 526 +#define _LOAD_ATTR_METHOD_LAZY_DICT 527 +#define _LOAD_ATTR_METHOD_NO_DICT 528 +#define _LOAD_ATTR_METHOD_WITH_VALUES 529 +#define _LOAD_ATTR_MODULE 530 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 531 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 532 +#define _LOAD_ATTR_PROPERTY_FRAME 533 +#define _LOAD_ATTR_SLOT 534 +#define _LOAD_ATTR_WITH_HINT 535 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 462 +#define _LOAD_BYTECODE 536 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 463 -#define _LOAD_CONST_INLINE_BORROW 464 -#define _LOAD_CONST_UNDER_INLINE 465 -#define _LOAD_CONST_UNDER_INLINE_BORROW 466 +#define _LOAD_CONST_INLINE 537 +#define _LOAD_CONST_INLINE_BORROW 538 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 467 -#define _LOAD_FAST_0 468 -#define _LOAD_FAST_1 469 -#define _LOAD_FAST_2 470 -#define _LOAD_FAST_3 471 -#define _LOAD_FAST_4 472 -#define _LOAD_FAST_5 473 -#define _LOAD_FAST_6 474 -#define _LOAD_FAST_7 475 +#define _LOAD_FAST 539 +#define _LOAD_FAST_0 540 +#define _LOAD_FAST_1 541 +#define _LOAD_FAST_2 542 +#define _LOAD_FAST_3 543 +#define _LOAD_FAST_4 544 +#define _LOAD_FAST_5 545 +#define _LOAD_FAST_6 546 +#define _LOAD_FAST_7 547 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 476 -#define _LOAD_FAST_BORROW_0 477 -#define _LOAD_FAST_BORROW_1 478 -#define _LOAD_FAST_BORROW_2 479 -#define _LOAD_FAST_BORROW_3 480 -#define _LOAD_FAST_BORROW_4 481 -#define _LOAD_FAST_BORROW_5 482 -#define _LOAD_FAST_BORROW_6 483 -#define _LOAD_FAST_BORROW_7 484 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW +#define _LOAD_FAST_BORROW 548 +#define _LOAD_FAST_BORROW_0 549 +#define _LOAD_FAST_BORROW_1 550 +#define _LOAD_FAST_BORROW_2 551 +#define _LOAD_FAST_BORROW_3 552 +#define _LOAD_FAST_BORROW_4 553 +#define _LOAD_FAST_BORROW_5 554 +#define _LOAD_FAST_BORROW_6 555 +#define _LOAD_FAST_BORROW_7 556 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK -#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 485 -#define _LOAD_GLOBAL_BUILTINS 486 -#define _LOAD_GLOBAL_MODULE 487 +#define _LOAD_GLOBAL 557 +#define _LOAD_GLOBAL_BUILTINS 558 +#define _LOAD_GLOBAL_MODULE 559 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 488 -#define _LOAD_SMALL_INT_0 489 -#define _LOAD_SMALL_INT_1 490 -#define _LOAD_SMALL_INT_2 491 -#define _LOAD_SMALL_INT_3 492 -#define _LOAD_SPECIAL 493 +#define _LOAD_SMALL_INT 560 +#define _LOAD_SMALL_INT_0 561 +#define _LOAD_SMALL_INT_1 562 +#define _LOAD_SMALL_INT_2 563 +#define _LOAD_SMALL_INT_3 564 +#define _LOAD_SPECIAL 565 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 494 +#define _LOAD_SUPER_ATTR_METHOD 566 +#define _LOCK_OBJECT 567 +#define _MAKE_CALLARGS_A_TUPLE 568 #define _MAKE_CELL MAKE_CELL -#define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 495 +#define _MAKE_FUNCTION 569 +#define _MAKE_HEAP_SAFE 570 +#define _MAKE_WARM 571 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS MATCH_CLASS +#define _MATCH_CLASS 572 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 496 -#define _MAYBE_EXPAND_METHOD_KW 497 -#define _MONITOR_CALL 498 -#define _MONITOR_CALL_KW 499 -#define _MONITOR_JUMP_BACKWARD 500 -#define _MONITOR_RESUME 501 +#define _MAYBE_EXPAND_METHOD 573 +#define _MAYBE_EXPAND_METHOD_KW 574 +#define _MONITOR_CALL 575 +#define _MONITOR_CALL_KW 576 +#define _MONITOR_JUMP_BACKWARD 577 +#define _MONITOR_RESUME 578 #define _NOP NOP -#define _POP_CALL 502 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 -#define _POP_CALL_ONE 504 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 -#define _POP_CALL_TWO 506 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 508 -#define _POP_JUMP_IF_TRUE 509 +#define _POP_JUMP_IF_FALSE 579 +#define _POP_JUMP_IF_TRUE 580 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 510 -#define _POP_TOP_INT 511 -#define _POP_TOP_LOAD_CONST_INLINE 512 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 -#define _POP_TOP_NOP 514 -#define _POP_TOP_UNICODE 515 -#define _POP_TWO 516 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 +#define _POP_TOP_FLOAT 581 +#define _POP_TOP_INT 582 +#define _POP_TOP_NOP 583 +#define _POP_TOP_OPARG 584 +#define _POP_TOP_UNICODE 585 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 518 +#define _PUSH_FRAME 586 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 519 -#define _PY_FRAME_GENERAL 520 -#define _PY_FRAME_KW 521 -#define _QUICKEN_RESUME 522 -#define _REPLACE_WITH_TRUE 523 -#define _RESUME_CHECK RESUME_CHECK +#define _PUSH_NULL_CONDITIONAL 587 +#define _PUSH_TAGGED_ZERO 588 +#define _PY_FRAME_EX 589 +#define _PY_FRAME_GENERAL 590 +#define _PY_FRAME_KW 591 +#define _RECORD_3OS_GEN_FUNC 592 +#define _RECORD_4OS 593 +#define _RECORD_BOUND_METHOD 594 +#define _RECORD_CALLABLE 595 +#define _RECORD_CALLABLE_KW 596 +#define _RECORD_CODE 597 +#define _RECORD_NOS 598 +#define _RECORD_NOS_GEN_FUNC 599 +#define _RECORD_NOS_TYPE 600 +#define _RECORD_TOS 601 +#define _RECORD_TOS_TYPE 602 +#define _REPLACE_WITH_TRUE 603 +#define _RESUME_CHECK 604 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 524 -#define _SEND 525 -#define _SEND_GEN_FRAME 526 +#define _RETURN_VALUE 605 +#define _RROT_3 606 +#define _SAVE_RETURN_OFFSET 607 +#define _SEND_ASYNC_GEN 608 +#define _SEND_ASYNC_GEN_TIER_TWO 609 +#define _SEND_GEN_FRAME 610 +#define _SEND_VIRTUAL 611 +#define _SEND_VIRTUAL_TIER_TWO 612 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 527 -#define _STORE_ATTR 528 -#define _STORE_ATTR_INSTANCE_VALUE 529 -#define _STORE_ATTR_SLOT 530 -#define _STORE_ATTR_WITH_HINT 531 +#define _SET_UPDATE 613 +#define _SPILL_OR_RELOAD 614 +#define _START_EXECUTOR 615 +#define _STORE_ATTR 616 +#define _STORE_ATTR_INSTANCE_VALUE 617 +#define _STORE_ATTR_SLOT 618 +#define _STORE_ATTR_WITH_HINT 619 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 532 -#define _STORE_FAST_0 533 -#define _STORE_FAST_1 534 -#define _STORE_FAST_2 535 -#define _STORE_FAST_3 536 -#define _STORE_FAST_4 537 -#define _STORE_FAST_5 538 -#define _STORE_FAST_6 539 -#define _STORE_FAST_7 540 -#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST -#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 541 -#define _STORE_SUBSCR 542 -#define _STORE_SUBSCR_DICT 543 -#define _STORE_SUBSCR_LIST_INT 544 -#define _SWAP 545 -#define _SWAP_2 546 -#define _SWAP_3 547 -#define _TIER2_RESUME_CHECK 548 -#define _TO_BOOL 549 +#define _STORE_SLICE 620 +#define _STORE_SUBSCR 621 +#define _STORE_SUBSCR_DICT 622 +#define _STORE_SUBSCR_DICT_KNOWN_HASH 623 +#define _STORE_SUBSCR_LIST_INT 624 +#define _SWAP 625 +#define _SWAP_2 626 +#define _SWAP_3 627 +#define _SWAP_FAST 628 +#define _SWAP_FAST_0 629 +#define _SWAP_FAST_1 630 +#define _SWAP_FAST_2 631 +#define _SWAP_FAST_3 632 +#define _SWAP_FAST_4 633 +#define _SWAP_FAST_5 634 +#define _SWAP_FAST_6 635 +#define _SWAP_FAST_7 636 +#define _TIER2_RESUME_CHECK 637 +#define _TO_BOOL 638 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 550 +#define _TO_BOOL_INT 639 +#define _TO_BOOL_LIST 640 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 551 -#define _UNARY_INVERT UNARY_INVERT -#define _UNARY_NEGATIVE UNARY_NEGATIVE +#define _TO_BOOL_STR 641 +#define _TRACE_RECORD TRACE_RECORD +#define _UNARY_INVERT 642 +#define _UNARY_NEGATIVE 643 +#define _UNARY_NEGATIVE_FLOAT_INPLACE 644 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 552 -#define _UNPACK_SEQUENCE_LIST 553 -#define _UNPACK_SEQUENCE_TUPLE 554 -#define _UNPACK_SEQUENCE_TWO_TUPLE 555 +#define _UNPACK_SEQUENCE 645 +#define _UNPACK_SEQUENCE_LIST 646 +#define _UNPACK_SEQUENCE_TUPLE 647 +#define _UNPACK_SEQUENCE_TWO_TUPLE 648 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 649 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 650 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 651 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 555 +#define _YIELD_VALUE 652 +#define MAX_UOP_ID 652 +#define _ALLOCATE_OBJECT_r00 653 +#define _BINARY_OP_r23 654 +#define _BINARY_OP_ADD_FLOAT_r03 655 +#define _BINARY_OP_ADD_FLOAT_r13 656 +#define _BINARY_OP_ADD_FLOAT_r23 657 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 658 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 659 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 660 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 661 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 662 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 663 +#define _BINARY_OP_ADD_INT_r03 664 +#define _BINARY_OP_ADD_INT_r13 665 +#define _BINARY_OP_ADD_INT_r23 666 +#define _BINARY_OP_ADD_INT_INPLACE_r03 667 +#define _BINARY_OP_ADD_INT_INPLACE_r13 668 +#define _BINARY_OP_ADD_INT_INPLACE_r23 669 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 670 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 671 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 672 +#define _BINARY_OP_ADD_UNICODE_r03 673 +#define _BINARY_OP_ADD_UNICODE_r13 674 +#define _BINARY_OP_ADD_UNICODE_r23 675 +#define _BINARY_OP_EXTEND_r23 676 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 677 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 678 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 679 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 680 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 681 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 682 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 683 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 684 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 685 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 686 +#define _BINARY_OP_MULTIPLY_INT_r03 687 +#define _BINARY_OP_MULTIPLY_INT_r13 688 +#define _BINARY_OP_MULTIPLY_INT_r23 689 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 690 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 691 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 692 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 693 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 694 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 695 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 696 +#define _BINARY_OP_SUBSCR_DICT_r23 697 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 698 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 699 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 700 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 701 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 702 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 703 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 704 +#define _BINARY_OP_SUBSCR_STR_INT_r23 705 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 706 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 707 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 708 +#define _BINARY_OP_SUBSCR_USTR_INT_r23 709 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 710 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 711 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 712 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 713 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 714 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 715 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 716 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 717 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 718 +#define _BINARY_OP_SUBTRACT_INT_r03 719 +#define _BINARY_OP_SUBTRACT_INT_r13 720 +#define _BINARY_OP_SUBTRACT_INT_r23 721 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 722 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 723 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 724 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 725 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 726 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 727 +#define _BINARY_OP_TRUEDIV_FLOAT_r23 728 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 729 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 730 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 731 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 732 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 733 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 734 +#define _BINARY_SLICE_r31 735 +#define _BUILD_INTERPOLATION_r01 736 +#define _BUILD_LIST_r01 737 +#define _BUILD_MAP_r01 738 +#define _BUILD_SET_r01 739 +#define _BUILD_SLICE_r01 740 +#define _BUILD_STRING_r01 741 +#define _BUILD_TEMPLATE_r21 742 +#define _BUILD_TUPLE_r01 743 +#define _CALL_BUILTIN_CLASS_r00 744 +#define _CALL_BUILTIN_FAST_r00 745 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 746 +#define _CALL_BUILTIN_O_r03 747 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 748 +#define _CALL_INTRINSIC_1_r12 749 +#define _CALL_INTRINSIC_2_r23 750 +#define _CALL_ISINSTANCE_r31 751 +#define _CALL_KW_NON_PY_r11 752 +#define _CALL_LEN_r33 753 +#define _CALL_LIST_APPEND_r03 754 +#define _CALL_LIST_APPEND_r13 755 +#define _CALL_LIST_APPEND_r23 756 +#define _CALL_LIST_APPEND_r33 757 +#define _CALL_METHOD_DESCRIPTOR_FAST_r00 758 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 759 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 760 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 761 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 762 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 763 +#define _CALL_METHOD_DESCRIPTOR_O_r03 764 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 765 +#define _CALL_NON_PY_GENERAL_r01 766 +#define _CALL_STR_1_r32 767 +#define _CALL_TUPLE_1_r32 768 +#define _CALL_TYPE_1_r02 769 +#define _CALL_TYPE_1_r12 770 +#define _CALL_TYPE_1_r22 771 +#define _CALL_TYPE_1_r32 772 +#define _CHECK_ATTR_CLASS_r01 773 +#define _CHECK_ATTR_CLASS_r11 774 +#define _CHECK_ATTR_CLASS_r22 775 +#define _CHECK_ATTR_CLASS_r33 776 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 777 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 778 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 779 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 780 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 781 +#define _CHECK_EG_MATCH_r22 782 +#define _CHECK_EXC_MATCH_r22 783 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 784 +#define _CHECK_FUNCTION_VERSION_r00 785 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 786 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 787 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 788 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 789 +#define _CHECK_FUNCTION_VERSION_KW_r11 790 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 791 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 792 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 793 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 794 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 795 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 796 +#define _CHECK_IS_PY_CALLABLE_EX_r03 797 +#define _CHECK_IS_PY_CALLABLE_EX_r13 798 +#define _CHECK_IS_PY_CALLABLE_EX_r23 799 +#define _CHECK_IS_PY_CALLABLE_EX_r33 800 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 801 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 802 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 803 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 804 +#define _CHECK_METHOD_VERSION_r00 805 +#define _CHECK_METHOD_VERSION_KW_r11 806 +#define _CHECK_OBJECT_r00 807 +#define _CHECK_PEP_523_r00 808 +#define _CHECK_PEP_523_r11 809 +#define _CHECK_PEP_523_r22 810 +#define _CHECK_PEP_523_r33 811 +#define _CHECK_PERIODIC_r00 812 +#define _CHECK_PERIODIC_AT_END_r00 813 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 814 +#define _CHECK_RECURSION_LIMIT_r00 815 +#define _CHECK_RECURSION_LIMIT_r11 816 +#define _CHECK_RECURSION_LIMIT_r22 817 +#define _CHECK_RECURSION_LIMIT_r33 818 +#define _CHECK_RECURSION_REMAINING_r00 819 +#define _CHECK_RECURSION_REMAINING_r11 820 +#define _CHECK_RECURSION_REMAINING_r22 821 +#define _CHECK_RECURSION_REMAINING_r33 822 +#define _CHECK_STACK_SPACE_r00 823 +#define _CHECK_STACK_SPACE_OPERAND_r00 824 +#define _CHECK_STACK_SPACE_OPERAND_r11 825 +#define _CHECK_STACK_SPACE_OPERAND_r22 826 +#define _CHECK_STACK_SPACE_OPERAND_r33 827 +#define _CHECK_VALIDITY_r00 828 +#define _CHECK_VALIDITY_r11 829 +#define _CHECK_VALIDITY_r22 830 +#define _CHECK_VALIDITY_r33 831 +#define _COLD_DYNAMIC_EXIT_r00 832 +#define _COLD_EXIT_r00 833 +#define _COMPARE_OP_r21 834 +#define _COMPARE_OP_FLOAT_r03 835 +#define _COMPARE_OP_FLOAT_r13 836 +#define _COMPARE_OP_FLOAT_r23 837 +#define _COMPARE_OP_INT_r23 838 +#define _COMPARE_OP_STR_r23 839 +#define _CONTAINS_OP_r23 840 +#define _CONTAINS_OP_DICT_r23 841 +#define _CONTAINS_OP_SET_r23 842 +#define _CONVERT_VALUE_r11 843 +#define _COPY_r01 844 +#define _COPY_1_r02 845 +#define _COPY_1_r12 846 +#define _COPY_1_r23 847 +#define _COPY_2_r03 848 +#define _COPY_2_r13 849 +#define _COPY_2_r23 850 +#define _COPY_3_r03 851 +#define _COPY_3_r13 852 +#define _COPY_3_r23 853 +#define _COPY_3_r33 854 +#define _COPY_FREE_VARS_r00 855 +#define _COPY_FREE_VARS_r11 856 +#define _COPY_FREE_VARS_r22 857 +#define _COPY_FREE_VARS_r33 858 +#define _CREATE_INIT_FRAME_r01 859 +#define _DELETE_DEREF_r00 860 +#define _DELETE_FAST_r00 861 +#define _DELETE_SUBSCR_r20 862 +#define _DEOPT_r00 863 +#define _DEOPT_r10 864 +#define _DEOPT_r20 865 +#define _DEOPT_r30 866 +#define _DICT_MERGE_r11 867 +#define _DICT_UPDATE_r11 868 +#define _DO_CALL_r01 869 +#define _DO_CALL_FUNCTION_EX_r31 870 +#define _DO_CALL_KW_r11 871 +#define _DYNAMIC_EXIT_r00 872 +#define _DYNAMIC_EXIT_r10 873 +#define _DYNAMIC_EXIT_r20 874 +#define _DYNAMIC_EXIT_r30 875 +#define _END_FOR_r10 876 +#define _END_SEND_r31 877 +#define _ERROR_POP_N_r00 878 +#define _EXIT_INIT_CHECK_r10 879 +#define _EXIT_TRACE_r00 880 +#define _EXIT_TRACE_r10 881 +#define _EXIT_TRACE_r20 882 +#define _EXIT_TRACE_r30 883 +#define _EXPAND_METHOD_r00 884 +#define _EXPAND_METHOD_KW_r11 885 +#define _FATAL_ERROR_r00 886 +#define _FATAL_ERROR_r11 887 +#define _FATAL_ERROR_r22 888 +#define _FATAL_ERROR_r33 889 +#define _FORMAT_SIMPLE_r11 890 +#define _FORMAT_WITH_SPEC_r21 891 +#define _FOR_ITER_r23 892 +#define _FOR_ITER_GEN_FRAME_r03 893 +#define _FOR_ITER_GEN_FRAME_r13 894 +#define _FOR_ITER_GEN_FRAME_r23 895 +#define _FOR_ITER_TIER_TWO_r23 896 +#define _FOR_ITER_VIRTUAL_r23 897 +#define _FOR_ITER_VIRTUAL_TIER_TWO_r23 898 +#define _GET_AITER_r11 899 +#define _GET_ANEXT_r12 900 +#define _GET_AWAITABLE_r11 901 +#define _GET_ITER_r12 902 +#define _GET_ITER_TRAD_r12 903 +#define _GET_LEN_r12 904 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r03 905 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r13 906 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r23 907 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r33 908 +#define _GUARD_BINARY_OP_EXTEND_r22 909 +#define _GUARD_BINARY_OP_EXTEND_LHS_r02 910 +#define _GUARD_BINARY_OP_EXTEND_LHS_r12 911 +#define _GUARD_BINARY_OP_EXTEND_LHS_r22 912 +#define _GUARD_BINARY_OP_EXTEND_LHS_r33 913 +#define _GUARD_BINARY_OP_EXTEND_RHS_r02 914 +#define _GUARD_BINARY_OP_EXTEND_RHS_r12 915 +#define _GUARD_BINARY_OP_EXTEND_RHS_r22 916 +#define _GUARD_BINARY_OP_EXTEND_RHS_r33 917 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 918 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 919 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 920 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 921 +#define _GUARD_BIT_IS_SET_POP_r00 922 +#define _GUARD_BIT_IS_SET_POP_r10 923 +#define _GUARD_BIT_IS_SET_POP_r21 924 +#define _GUARD_BIT_IS_SET_POP_r32 925 +#define _GUARD_BIT_IS_SET_POP_4_r00 926 +#define _GUARD_BIT_IS_SET_POP_4_r10 927 +#define _GUARD_BIT_IS_SET_POP_4_r21 928 +#define _GUARD_BIT_IS_SET_POP_4_r32 929 +#define _GUARD_BIT_IS_SET_POP_5_r00 930 +#define _GUARD_BIT_IS_SET_POP_5_r10 931 +#define _GUARD_BIT_IS_SET_POP_5_r21 932 +#define _GUARD_BIT_IS_SET_POP_5_r32 933 +#define _GUARD_BIT_IS_SET_POP_6_r00 934 +#define _GUARD_BIT_IS_SET_POP_6_r10 935 +#define _GUARD_BIT_IS_SET_POP_6_r21 936 +#define _GUARD_BIT_IS_SET_POP_6_r32 937 +#define _GUARD_BIT_IS_SET_POP_7_r00 938 +#define _GUARD_BIT_IS_SET_POP_7_r10 939 +#define _GUARD_BIT_IS_SET_POP_7_r21 940 +#define _GUARD_BIT_IS_SET_POP_7_r32 941 +#define _GUARD_BIT_IS_UNSET_POP_r00 942 +#define _GUARD_BIT_IS_UNSET_POP_r10 943 +#define _GUARD_BIT_IS_UNSET_POP_r21 944 +#define _GUARD_BIT_IS_UNSET_POP_r32 945 +#define _GUARD_BIT_IS_UNSET_POP_4_r00 946 +#define _GUARD_BIT_IS_UNSET_POP_4_r10 947 +#define _GUARD_BIT_IS_UNSET_POP_4_r21 948 +#define _GUARD_BIT_IS_UNSET_POP_4_r32 949 +#define _GUARD_BIT_IS_UNSET_POP_5_r00 950 +#define _GUARD_BIT_IS_UNSET_POP_5_r10 951 +#define _GUARD_BIT_IS_UNSET_POP_5_r21 952 +#define _GUARD_BIT_IS_UNSET_POP_5_r32 953 +#define _GUARD_BIT_IS_UNSET_POP_6_r00 954 +#define _GUARD_BIT_IS_UNSET_POP_6_r10 955 +#define _GUARD_BIT_IS_UNSET_POP_6_r21 956 +#define _GUARD_BIT_IS_UNSET_POP_6_r32 957 +#define _GUARD_BIT_IS_UNSET_POP_7_r00 958 +#define _GUARD_BIT_IS_UNSET_POP_7_r10 959 +#define _GUARD_BIT_IS_UNSET_POP_7_r21 960 +#define _GUARD_BIT_IS_UNSET_POP_7_r32 961 +#define _GUARD_CALLABLE_BUILTIN_CLASS_r00 962 +#define _GUARD_CALLABLE_BUILTIN_FAST_r00 963 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 964 +#define _GUARD_CALLABLE_BUILTIN_O_r00 965 +#define _GUARD_CALLABLE_ISINSTANCE_r03 966 +#define _GUARD_CALLABLE_ISINSTANCE_r13 967 +#define _GUARD_CALLABLE_ISINSTANCE_r23 968 +#define _GUARD_CALLABLE_ISINSTANCE_r33 969 +#define _GUARD_CALLABLE_LEN_r03 970 +#define _GUARD_CALLABLE_LEN_r13 971 +#define _GUARD_CALLABLE_LEN_r23 972 +#define _GUARD_CALLABLE_LEN_r33 973 +#define _GUARD_CALLABLE_LIST_APPEND_r03 974 +#define _GUARD_CALLABLE_LIST_APPEND_r13 975 +#define _GUARD_CALLABLE_LIST_APPEND_r23 976 +#define _GUARD_CALLABLE_LIST_APPEND_r33 977 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 978 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 979 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 980 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 981 +#define _GUARD_CALLABLE_STR_1_r03 982 +#define _GUARD_CALLABLE_STR_1_r13 983 +#define _GUARD_CALLABLE_STR_1_r23 984 +#define _GUARD_CALLABLE_STR_1_r33 985 +#define _GUARD_CALLABLE_TUPLE_1_r03 986 +#define _GUARD_CALLABLE_TUPLE_1_r13 987 +#define _GUARD_CALLABLE_TUPLE_1_r23 988 +#define _GUARD_CALLABLE_TUPLE_1_r33 989 +#define _GUARD_CALLABLE_TYPE_1_r03 990 +#define _GUARD_CALLABLE_TYPE_1_r13 991 +#define _GUARD_CALLABLE_TYPE_1_r23 992 +#define _GUARD_CALLABLE_TYPE_1_r33 993 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 994 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 995 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 996 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 997 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 998 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 999 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 1000 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 1001 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 1002 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 1003 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 1004 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 1005 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 1006 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 1007 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 1008 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 1009 +#define _GUARD_DORV_NO_DICT_r01 1010 +#define _GUARD_DORV_NO_DICT_r11 1011 +#define _GUARD_DORV_NO_DICT_r22 1012 +#define _GUARD_DORV_NO_DICT_r33 1013 +#define _GUARD_GLOBALS_VERSION_r00 1014 +#define _GUARD_GLOBALS_VERSION_r11 1015 +#define _GUARD_GLOBALS_VERSION_r22 1016 +#define _GUARD_GLOBALS_VERSION_r33 1017 +#define _GUARD_IP_RETURN_GENERATOR_r00 1018 +#define _GUARD_IP_RETURN_GENERATOR_r11 1019 +#define _GUARD_IP_RETURN_GENERATOR_r22 1020 +#define _GUARD_IP_RETURN_GENERATOR_r33 1021 +#define _GUARD_IP_RETURN_VALUE_r00 1022 +#define _GUARD_IP_RETURN_VALUE_r11 1023 +#define _GUARD_IP_RETURN_VALUE_r22 1024 +#define _GUARD_IP_RETURN_VALUE_r33 1025 +#define _GUARD_IP_YIELD_VALUE_r00 1026 +#define _GUARD_IP_YIELD_VALUE_r11 1027 +#define _GUARD_IP_YIELD_VALUE_r22 1028 +#define _GUARD_IP_YIELD_VALUE_r33 1029 +#define _GUARD_IP__PUSH_FRAME_r00 1030 +#define _GUARD_IP__PUSH_FRAME_r11 1031 +#define _GUARD_IP__PUSH_FRAME_r22 1032 +#define _GUARD_IP__PUSH_FRAME_r33 1033 +#define _GUARD_IS_FALSE_POP_r00 1034 +#define _GUARD_IS_FALSE_POP_r10 1035 +#define _GUARD_IS_FALSE_POP_r21 1036 +#define _GUARD_IS_FALSE_POP_r32 1037 +#define _GUARD_IS_NONE_POP_r00 1038 +#define _GUARD_IS_NONE_POP_r10 1039 +#define _GUARD_IS_NONE_POP_r21 1040 +#define _GUARD_IS_NONE_POP_r32 1041 +#define _GUARD_IS_NOT_NONE_POP_r10 1042 +#define _GUARD_IS_TRUE_POP_r00 1043 +#define _GUARD_IS_TRUE_POP_r10 1044 +#define _GUARD_IS_TRUE_POP_r21 1045 +#define _GUARD_IS_TRUE_POP_r32 1046 +#define _GUARD_ITERATOR_r01 1047 +#define _GUARD_ITERATOR_r11 1048 +#define _GUARD_ITERATOR_r22 1049 +#define _GUARD_ITERATOR_r33 1050 +#define _GUARD_ITER_VIRTUAL_r01 1051 +#define _GUARD_ITER_VIRTUAL_r11 1052 +#define _GUARD_ITER_VIRTUAL_r22 1053 +#define _GUARD_ITER_VIRTUAL_r33 1054 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1055 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1056 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1057 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1058 +#define _GUARD_NOS_COMPACT_ASCII_r02 1059 +#define _GUARD_NOS_COMPACT_ASCII_r12 1060 +#define _GUARD_NOS_COMPACT_ASCII_r22 1061 +#define _GUARD_NOS_COMPACT_ASCII_r33 1062 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 1063 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 1064 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 1065 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 1066 +#define _GUARD_NOS_DICT_SUBSCRIPT_r02 1067 +#define _GUARD_NOS_DICT_SUBSCRIPT_r12 1068 +#define _GUARD_NOS_DICT_SUBSCRIPT_r22 1069 +#define _GUARD_NOS_DICT_SUBSCRIPT_r33 1070 +#define _GUARD_NOS_FLOAT_r02 1071 +#define _GUARD_NOS_FLOAT_r12 1072 +#define _GUARD_NOS_FLOAT_r22 1073 +#define _GUARD_NOS_FLOAT_r33 1074 +#define _GUARD_NOS_INT_r02 1075 +#define _GUARD_NOS_INT_r12 1076 +#define _GUARD_NOS_INT_r22 1077 +#define _GUARD_NOS_INT_r33 1078 +#define _GUARD_NOS_ITER_VIRTUAL_r02 1079 +#define _GUARD_NOS_ITER_VIRTUAL_r12 1080 +#define _GUARD_NOS_ITER_VIRTUAL_r22 1081 +#define _GUARD_NOS_ITER_VIRTUAL_r33 1082 +#define _GUARD_NOS_LIST_r02 1083 +#define _GUARD_NOS_LIST_r12 1084 +#define _GUARD_NOS_LIST_r22 1085 +#define _GUARD_NOS_LIST_r33 1086 +#define _GUARD_NOS_NOT_NULL_r02 1087 +#define _GUARD_NOS_NOT_NULL_r12 1088 +#define _GUARD_NOS_NOT_NULL_r22 1089 +#define _GUARD_NOS_NOT_NULL_r33 1090 +#define _GUARD_NOS_NULL_r02 1091 +#define _GUARD_NOS_NULL_r12 1092 +#define _GUARD_NOS_NULL_r22 1093 +#define _GUARD_NOS_NULL_r33 1094 +#define _GUARD_NOS_OVERFLOWED_r02 1095 +#define _GUARD_NOS_OVERFLOWED_r12 1096 +#define _GUARD_NOS_OVERFLOWED_r22 1097 +#define _GUARD_NOS_OVERFLOWED_r33 1098 +#define _GUARD_NOS_TUPLE_r02 1099 +#define _GUARD_NOS_TUPLE_r12 1100 +#define _GUARD_NOS_TUPLE_r22 1101 +#define _GUARD_NOS_TUPLE_r33 1102 +#define _GUARD_NOS_TYPE_VERSION_r02 1103 +#define _GUARD_NOS_TYPE_VERSION_r12 1104 +#define _GUARD_NOS_TYPE_VERSION_r22 1105 +#define _GUARD_NOS_TYPE_VERSION_r33 1106 +#define _GUARD_NOS_UNICODE_r02 1107 +#define _GUARD_NOS_UNICODE_r12 1108 +#define _GUARD_NOS_UNICODE_r22 1109 +#define _GUARD_NOS_UNICODE_r33 1110 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1111 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1112 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1113 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1114 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1115 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1116 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1117 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1118 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1119 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1120 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1121 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1122 +#define _GUARD_THIRD_NULL_r03 1123 +#define _GUARD_THIRD_NULL_r13 1124 +#define _GUARD_THIRD_NULL_r23 1125 +#define _GUARD_THIRD_NULL_r33 1126 +#define _GUARD_TOS_ANY_DICT_r01 1127 +#define _GUARD_TOS_ANY_DICT_r11 1128 +#define _GUARD_TOS_ANY_DICT_r22 1129 +#define _GUARD_TOS_ANY_DICT_r33 1130 +#define _GUARD_TOS_ANY_SET_r01 1131 +#define _GUARD_TOS_ANY_SET_r11 1132 +#define _GUARD_TOS_ANY_SET_r22 1133 +#define _GUARD_TOS_ANY_SET_r33 1134 +#define _GUARD_TOS_DICT_r01 1135 +#define _GUARD_TOS_DICT_r11 1136 +#define _GUARD_TOS_DICT_r22 1137 +#define _GUARD_TOS_DICT_r33 1138 +#define _GUARD_TOS_FLOAT_r01 1139 +#define _GUARD_TOS_FLOAT_r11 1140 +#define _GUARD_TOS_FLOAT_r22 1141 +#define _GUARD_TOS_FLOAT_r33 1142 +#define _GUARD_TOS_FROZENDICT_r01 1143 +#define _GUARD_TOS_FROZENDICT_r11 1144 +#define _GUARD_TOS_FROZENDICT_r22 1145 +#define _GUARD_TOS_FROZENDICT_r33 1146 +#define _GUARD_TOS_FROZENSET_r01 1147 +#define _GUARD_TOS_FROZENSET_r11 1148 +#define _GUARD_TOS_FROZENSET_r22 1149 +#define _GUARD_TOS_FROZENSET_r33 1150 +#define _GUARD_TOS_INT_r01 1151 +#define _GUARD_TOS_INT_r11 1152 +#define _GUARD_TOS_INT_r22 1153 +#define _GUARD_TOS_INT_r33 1154 +#define _GUARD_TOS_IS_NONE_r01 1155 +#define _GUARD_TOS_IS_NONE_r11 1156 +#define _GUARD_TOS_IS_NONE_r22 1157 +#define _GUARD_TOS_IS_NONE_r33 1158 +#define _GUARD_TOS_LIST_r01 1159 +#define _GUARD_TOS_LIST_r11 1160 +#define _GUARD_TOS_LIST_r22 1161 +#define _GUARD_TOS_LIST_r33 1162 +#define _GUARD_TOS_NOT_NULL_r01 1163 +#define _GUARD_TOS_NOT_NULL_r11 1164 +#define _GUARD_TOS_NOT_NULL_r22 1165 +#define _GUARD_TOS_NOT_NULL_r33 1166 +#define _GUARD_TOS_OVERFLOWED_r01 1167 +#define _GUARD_TOS_OVERFLOWED_r11 1168 +#define _GUARD_TOS_OVERFLOWED_r22 1169 +#define _GUARD_TOS_OVERFLOWED_r33 1170 +#define _GUARD_TOS_SET_r01 1171 +#define _GUARD_TOS_SET_r11 1172 +#define _GUARD_TOS_SET_r22 1173 +#define _GUARD_TOS_SET_r33 1174 +#define _GUARD_TOS_SLICE_r01 1175 +#define _GUARD_TOS_SLICE_r11 1176 +#define _GUARD_TOS_SLICE_r22 1177 +#define _GUARD_TOS_SLICE_r33 1178 +#define _GUARD_TOS_TUPLE_r01 1179 +#define _GUARD_TOS_TUPLE_r11 1180 +#define _GUARD_TOS_TUPLE_r22 1181 +#define _GUARD_TOS_TUPLE_r33 1182 +#define _GUARD_TOS_UNICODE_r01 1183 +#define _GUARD_TOS_UNICODE_r11 1184 +#define _GUARD_TOS_UNICODE_r22 1185 +#define _GUARD_TOS_UNICODE_r33 1186 +#define _GUARD_TYPE_r01 1187 +#define _GUARD_TYPE_r11 1188 +#define _GUARD_TYPE_r22 1189 +#define _GUARD_TYPE_r33 1190 +#define _GUARD_TYPE_ITER_r02 1191 +#define _GUARD_TYPE_ITER_r12 1192 +#define _GUARD_TYPE_ITER_r22 1193 +#define _GUARD_TYPE_ITER_r33 1194 +#define _GUARD_TYPE_VERSION_r01 1195 +#define _GUARD_TYPE_VERSION_r11 1196 +#define _GUARD_TYPE_VERSION_r22 1197 +#define _GUARD_TYPE_VERSION_r33 1198 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1199 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1200 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1201 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1202 +#define _HANDLE_PENDING_AND_DEOPT_r00 1203 +#define _HANDLE_PENDING_AND_DEOPT_r10 1204 +#define _HANDLE_PENDING_AND_DEOPT_r20 1205 +#define _HANDLE_PENDING_AND_DEOPT_r30 1206 +#define _IMPORT_FROM_r12 1207 +#define _IMPORT_NAME_r21 1208 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1209 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1210 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1211 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1212 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1213 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1214 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1215 +#define _INSERT_NULL_r10 1216 +#define _INSTRUMENTED_FOR_ITER_r23 1217 +#define _INSTRUMENTED_INSTRUCTION_r00 1218 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1219 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1220 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1221 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1222 +#define _INSTRUMENTED_LINE_r00 1223 +#define _INSTRUMENTED_NOT_TAKEN_r00 1224 +#define _INSTRUMENTED_NOT_TAKEN_r11 1225 +#define _INSTRUMENTED_NOT_TAKEN_r22 1226 +#define _INSTRUMENTED_NOT_TAKEN_r33 1227 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1228 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1229 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1230 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1231 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1232 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1233 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1234 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1235 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1236 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1237 +#define _IS_NONE_r11 1238 +#define _IS_OP_r03 1239 +#define _IS_OP_r13 1240 +#define _IS_OP_r23 1241 +#define _ITER_CHECK_LIST_r02 1242 +#define _ITER_CHECK_LIST_r12 1243 +#define _ITER_CHECK_LIST_r22 1244 +#define _ITER_CHECK_LIST_r33 1245 +#define _ITER_CHECK_RANGE_r02 1246 +#define _ITER_CHECK_RANGE_r12 1247 +#define _ITER_CHECK_RANGE_r22 1248 +#define _ITER_CHECK_RANGE_r33 1249 +#define _ITER_CHECK_TUPLE_r02 1250 +#define _ITER_CHECK_TUPLE_r12 1251 +#define _ITER_CHECK_TUPLE_r22 1252 +#define _ITER_CHECK_TUPLE_r33 1253 +#define _ITER_JUMP_LIST_r02 1254 +#define _ITER_JUMP_LIST_r12 1255 +#define _ITER_JUMP_LIST_r22 1256 +#define _ITER_JUMP_LIST_r33 1257 +#define _ITER_JUMP_RANGE_r02 1258 +#define _ITER_JUMP_RANGE_r12 1259 +#define _ITER_JUMP_RANGE_r22 1260 +#define _ITER_JUMP_RANGE_r33 1261 +#define _ITER_JUMP_TUPLE_r02 1262 +#define _ITER_JUMP_TUPLE_r12 1263 +#define _ITER_JUMP_TUPLE_r22 1264 +#define _ITER_JUMP_TUPLE_r33 1265 +#define _ITER_NEXT_INLINE_r23 1266 +#define _ITER_NEXT_LIST_r23 1267 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1268 +#define _ITER_NEXT_RANGE_r03 1269 +#define _ITER_NEXT_RANGE_r13 1270 +#define _ITER_NEXT_RANGE_r23 1271 +#define _ITER_NEXT_TUPLE_r03 1272 +#define _ITER_NEXT_TUPLE_r13 1273 +#define _ITER_NEXT_TUPLE_r23 1274 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1275 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1276 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1277 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1278 +#define _JUMP_TO_TOP_r00 1279 +#define _LIST_APPEND_r10 1280 +#define _LIST_EXTEND_r11 1281 +#define _LOAD_ATTR_r10 1282 +#define _LOAD_ATTR_CLASS_r11 1283 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1284 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1285 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1286 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1287 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1288 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1289 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1290 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1291 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1292 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1293 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1294 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1295 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1296 +#define _LOAD_ATTR_MODULE_r12 1297 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1298 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1299 +#define _LOAD_ATTR_PROPERTY_FRAME_r01 1300 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1301 +#define _LOAD_ATTR_PROPERTY_FRAME_r22 1302 +#define _LOAD_ATTR_PROPERTY_FRAME_r33 1303 +#define _LOAD_ATTR_SLOT_r02 1304 +#define _LOAD_ATTR_SLOT_r12 1305 +#define _LOAD_ATTR_SLOT_r23 1306 +#define _LOAD_ATTR_WITH_HINT_r12 1307 +#define _LOAD_BUILD_CLASS_r01 1308 +#define _LOAD_BYTECODE_r00 1309 +#define _LOAD_COMMON_CONSTANT_r01 1310 +#define _LOAD_COMMON_CONSTANT_r12 1311 +#define _LOAD_COMMON_CONSTANT_r23 1312 +#define _LOAD_CONST_r01 1313 +#define _LOAD_CONST_r12 1314 +#define _LOAD_CONST_r23 1315 +#define _LOAD_CONST_INLINE_r01 1316 +#define _LOAD_CONST_INLINE_r12 1317 +#define _LOAD_CONST_INLINE_r23 1318 +#define _LOAD_CONST_INLINE_BORROW_r01 1319 +#define _LOAD_CONST_INLINE_BORROW_r12 1320 +#define _LOAD_CONST_INLINE_BORROW_r23 1321 +#define _LOAD_DEREF_r01 1322 +#define _LOAD_FAST_r01 1323 +#define _LOAD_FAST_r12 1324 +#define _LOAD_FAST_r23 1325 +#define _LOAD_FAST_0_r01 1326 +#define _LOAD_FAST_0_r12 1327 +#define _LOAD_FAST_0_r23 1328 +#define _LOAD_FAST_1_r01 1329 +#define _LOAD_FAST_1_r12 1330 +#define _LOAD_FAST_1_r23 1331 +#define _LOAD_FAST_2_r01 1332 +#define _LOAD_FAST_2_r12 1333 +#define _LOAD_FAST_2_r23 1334 +#define _LOAD_FAST_3_r01 1335 +#define _LOAD_FAST_3_r12 1336 +#define _LOAD_FAST_3_r23 1337 +#define _LOAD_FAST_4_r01 1338 +#define _LOAD_FAST_4_r12 1339 +#define _LOAD_FAST_4_r23 1340 +#define _LOAD_FAST_5_r01 1341 +#define _LOAD_FAST_5_r12 1342 +#define _LOAD_FAST_5_r23 1343 +#define _LOAD_FAST_6_r01 1344 +#define _LOAD_FAST_6_r12 1345 +#define _LOAD_FAST_6_r23 1346 +#define _LOAD_FAST_7_r01 1347 +#define _LOAD_FAST_7_r12 1348 +#define _LOAD_FAST_7_r23 1349 +#define _LOAD_FAST_AND_CLEAR_r01 1350 +#define _LOAD_FAST_AND_CLEAR_r12 1351 +#define _LOAD_FAST_AND_CLEAR_r23 1352 +#define _LOAD_FAST_BORROW_r01 1353 +#define _LOAD_FAST_BORROW_r12 1354 +#define _LOAD_FAST_BORROW_r23 1355 +#define _LOAD_FAST_BORROW_0_r01 1356 +#define _LOAD_FAST_BORROW_0_r12 1357 +#define _LOAD_FAST_BORROW_0_r23 1358 +#define _LOAD_FAST_BORROW_1_r01 1359 +#define _LOAD_FAST_BORROW_1_r12 1360 +#define _LOAD_FAST_BORROW_1_r23 1361 +#define _LOAD_FAST_BORROW_2_r01 1362 +#define _LOAD_FAST_BORROW_2_r12 1363 +#define _LOAD_FAST_BORROW_2_r23 1364 +#define _LOAD_FAST_BORROW_3_r01 1365 +#define _LOAD_FAST_BORROW_3_r12 1366 +#define _LOAD_FAST_BORROW_3_r23 1367 +#define _LOAD_FAST_BORROW_4_r01 1368 +#define _LOAD_FAST_BORROW_4_r12 1369 +#define _LOAD_FAST_BORROW_4_r23 1370 +#define _LOAD_FAST_BORROW_5_r01 1371 +#define _LOAD_FAST_BORROW_5_r12 1372 +#define _LOAD_FAST_BORROW_5_r23 1373 +#define _LOAD_FAST_BORROW_6_r01 1374 +#define _LOAD_FAST_BORROW_6_r12 1375 +#define _LOAD_FAST_BORROW_6_r23 1376 +#define _LOAD_FAST_BORROW_7_r01 1377 +#define _LOAD_FAST_BORROW_7_r12 1378 +#define _LOAD_FAST_BORROW_7_r23 1379 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1380 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1381 +#define _LOAD_FAST_CHECK_r01 1382 +#define _LOAD_FAST_CHECK_r12 1383 +#define _LOAD_FAST_CHECK_r23 1384 +#define _LOAD_FAST_LOAD_FAST_r02 1385 +#define _LOAD_FAST_LOAD_FAST_r13 1386 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1387 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1388 +#define _LOAD_GLOBAL_r00 1389 +#define _LOAD_GLOBAL_BUILTINS_r01 1390 +#define _LOAD_GLOBAL_MODULE_r01 1391 +#define _LOAD_LOCALS_r01 1392 +#define _LOAD_LOCALS_r12 1393 +#define _LOAD_LOCALS_r23 1394 +#define _LOAD_NAME_r01 1395 +#define _LOAD_SMALL_INT_r01 1396 +#define _LOAD_SMALL_INT_r12 1397 +#define _LOAD_SMALL_INT_r23 1398 +#define _LOAD_SMALL_INT_0_r01 1399 +#define _LOAD_SMALL_INT_0_r12 1400 +#define _LOAD_SMALL_INT_0_r23 1401 +#define _LOAD_SMALL_INT_1_r01 1402 +#define _LOAD_SMALL_INT_1_r12 1403 +#define _LOAD_SMALL_INT_1_r23 1404 +#define _LOAD_SMALL_INT_2_r01 1405 +#define _LOAD_SMALL_INT_2_r12 1406 +#define _LOAD_SMALL_INT_2_r23 1407 +#define _LOAD_SMALL_INT_3_r01 1408 +#define _LOAD_SMALL_INT_3_r12 1409 +#define _LOAD_SMALL_INT_3_r23 1410 +#define _LOAD_SPECIAL_r00 1411 +#define _LOAD_SUPER_ATTR_ATTR_r31 1412 +#define _LOAD_SUPER_ATTR_METHOD_r32 1413 +#define _LOCK_OBJECT_r01 1414 +#define _LOCK_OBJECT_r11 1415 +#define _LOCK_OBJECT_r22 1416 +#define _LOCK_OBJECT_r33 1417 +#define _MAKE_CALLARGS_A_TUPLE_r33 1418 +#define _MAKE_CELL_r00 1419 +#define _MAKE_FUNCTION_r12 1420 +#define _MAKE_HEAP_SAFE_r01 1421 +#define _MAKE_HEAP_SAFE_r11 1422 +#define _MAKE_HEAP_SAFE_r22 1423 +#define _MAKE_HEAP_SAFE_r33 1424 +#define _MAKE_WARM_r00 1425 +#define _MAKE_WARM_r11 1426 +#define _MAKE_WARM_r22 1427 +#define _MAKE_WARM_r33 1428 +#define _MAP_ADD_r20 1429 +#define _MATCH_CLASS_r33 1430 +#define _MATCH_KEYS_r23 1431 +#define _MATCH_MAPPING_r02 1432 +#define _MATCH_MAPPING_r12 1433 +#define _MATCH_MAPPING_r23 1434 +#define _MATCH_SEQUENCE_r02 1435 +#define _MATCH_SEQUENCE_r12 1436 +#define _MATCH_SEQUENCE_r23 1437 +#define _MAYBE_EXPAND_METHOD_r00 1438 +#define _MAYBE_EXPAND_METHOD_KW_r11 1439 +#define _MONITOR_CALL_r00 1440 +#define _MONITOR_CALL_KW_r11 1441 +#define _MONITOR_JUMP_BACKWARD_r00 1442 +#define _MONITOR_JUMP_BACKWARD_r11 1443 +#define _MONITOR_JUMP_BACKWARD_r22 1444 +#define _MONITOR_JUMP_BACKWARD_r33 1445 +#define _MONITOR_RESUME_r00 1446 +#define _NOP_r00 1447 +#define _NOP_r11 1448 +#define _NOP_r22 1449 +#define _NOP_r33 1450 +#define _POP_EXCEPT_r10 1451 +#define _POP_ITER_r20 1452 +#define _POP_JUMP_IF_FALSE_r00 1453 +#define _POP_JUMP_IF_FALSE_r10 1454 +#define _POP_JUMP_IF_FALSE_r21 1455 +#define _POP_JUMP_IF_FALSE_r32 1456 +#define _POP_JUMP_IF_TRUE_r00 1457 +#define _POP_JUMP_IF_TRUE_r10 1458 +#define _POP_JUMP_IF_TRUE_r21 1459 +#define _POP_JUMP_IF_TRUE_r32 1460 +#define _POP_TOP_r10 1461 +#define _POP_TOP_FLOAT_r00 1462 +#define _POP_TOP_FLOAT_r10 1463 +#define _POP_TOP_FLOAT_r21 1464 +#define _POP_TOP_FLOAT_r32 1465 +#define _POP_TOP_INT_r00 1466 +#define _POP_TOP_INT_r10 1467 +#define _POP_TOP_INT_r21 1468 +#define _POP_TOP_INT_r32 1469 +#define _POP_TOP_NOP_r00 1470 +#define _POP_TOP_NOP_r10 1471 +#define _POP_TOP_NOP_r21 1472 +#define _POP_TOP_NOP_r32 1473 +#define _POP_TOP_OPARG_r00 1474 +#define _POP_TOP_UNICODE_r00 1475 +#define _POP_TOP_UNICODE_r10 1476 +#define _POP_TOP_UNICODE_r21 1477 +#define _POP_TOP_UNICODE_r32 1478 +#define _PUSH_EXC_INFO_r02 1479 +#define _PUSH_EXC_INFO_r12 1480 +#define _PUSH_EXC_INFO_r23 1481 +#define _PUSH_FRAME_r10 1482 +#define _PUSH_NULL_r01 1483 +#define _PUSH_NULL_r12 1484 +#define _PUSH_NULL_r23 1485 +#define _PUSH_NULL_CONDITIONAL_r00 1486 +#define _PUSH_TAGGED_ZERO_r01 1487 +#define _PUSH_TAGGED_ZERO_r12 1488 +#define _PUSH_TAGGED_ZERO_r23 1489 +#define _PY_FRAME_EX_r31 1490 +#define _PY_FRAME_GENERAL_r01 1491 +#define _PY_FRAME_KW_r11 1492 +#define _REPLACE_WITH_TRUE_r02 1493 +#define _REPLACE_WITH_TRUE_r12 1494 +#define _REPLACE_WITH_TRUE_r23 1495 +#define _RESUME_CHECK_r00 1496 +#define _RESUME_CHECK_r11 1497 +#define _RESUME_CHECK_r22 1498 +#define _RESUME_CHECK_r33 1499 +#define _RETURN_GENERATOR_r01 1500 +#define _RETURN_VALUE_r11 1501 +#define _RROT_3_r03 1502 +#define _RROT_3_r13 1503 +#define _RROT_3_r23 1504 +#define _RROT_3_r33 1505 +#define _SAVE_RETURN_OFFSET_r00 1506 +#define _SAVE_RETURN_OFFSET_r11 1507 +#define _SAVE_RETURN_OFFSET_r22 1508 +#define _SAVE_RETURN_OFFSET_r33 1509 +#define _SEND_ASYNC_GEN_r33 1510 +#define _SEND_ASYNC_GEN_TIER_TWO_r33 1511 +#define _SEND_GEN_FRAME_r33 1512 +#define _SEND_VIRTUAL_r33 1513 +#define _SEND_VIRTUAL_TIER_TWO_r03 1514 +#define _SEND_VIRTUAL_TIER_TWO_r13 1515 +#define _SEND_VIRTUAL_TIER_TWO_r23 1516 +#define _SEND_VIRTUAL_TIER_TWO_r33 1517 +#define _SETUP_ANNOTATIONS_r00 1518 +#define _SET_ADD_r10 1519 +#define _SET_FUNCTION_ATTRIBUTE_r01 1520 +#define _SET_FUNCTION_ATTRIBUTE_r11 1521 +#define _SET_FUNCTION_ATTRIBUTE_r21 1522 +#define _SET_FUNCTION_ATTRIBUTE_r32 1523 +#define _SET_IP_r00 1524 +#define _SET_IP_r11 1525 +#define _SET_IP_r22 1526 +#define _SET_IP_r33 1527 +#define _SET_UPDATE_r11 1528 +#define _SPILL_OR_RELOAD_r01 1529 +#define _SPILL_OR_RELOAD_r02 1530 +#define _SPILL_OR_RELOAD_r03 1531 +#define _SPILL_OR_RELOAD_r10 1532 +#define _SPILL_OR_RELOAD_r12 1533 +#define _SPILL_OR_RELOAD_r13 1534 +#define _SPILL_OR_RELOAD_r20 1535 +#define _SPILL_OR_RELOAD_r21 1536 +#define _SPILL_OR_RELOAD_r23 1537 +#define _SPILL_OR_RELOAD_r30 1538 +#define _SPILL_OR_RELOAD_r31 1539 +#define _SPILL_OR_RELOAD_r32 1540 +#define _START_EXECUTOR_r00 1541 +#define _STORE_ATTR_r20 1542 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1543 +#define _STORE_ATTR_SLOT_r21 1544 +#define _STORE_ATTR_WITH_HINT_r21 1545 +#define _STORE_DEREF_r10 1546 +#define _STORE_FAST_LOAD_FAST_r11 1547 +#define _STORE_FAST_STORE_FAST_r20 1548 +#define _STORE_GLOBAL_r10 1549 +#define _STORE_NAME_r10 1550 +#define _STORE_SLICE_r30 1551 +#define _STORE_SUBSCR_r30 1552 +#define _STORE_SUBSCR_DICT_r31 1553 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1554 +#define _STORE_SUBSCR_LIST_INT_r32 1555 +#define _SWAP_r11 1556 +#define _SWAP_2_r02 1557 +#define _SWAP_2_r12 1558 +#define _SWAP_2_r22 1559 +#define _SWAP_2_r33 1560 +#define _SWAP_3_r03 1561 +#define _SWAP_3_r13 1562 +#define _SWAP_3_r23 1563 +#define _SWAP_3_r33 1564 +#define _SWAP_FAST_r01 1565 +#define _SWAP_FAST_r11 1566 +#define _SWAP_FAST_r22 1567 +#define _SWAP_FAST_r33 1568 +#define _SWAP_FAST_0_r01 1569 +#define _SWAP_FAST_0_r11 1570 +#define _SWAP_FAST_0_r22 1571 +#define _SWAP_FAST_0_r33 1572 +#define _SWAP_FAST_1_r01 1573 +#define _SWAP_FAST_1_r11 1574 +#define _SWAP_FAST_1_r22 1575 +#define _SWAP_FAST_1_r33 1576 +#define _SWAP_FAST_2_r01 1577 +#define _SWAP_FAST_2_r11 1578 +#define _SWAP_FAST_2_r22 1579 +#define _SWAP_FAST_2_r33 1580 +#define _SWAP_FAST_3_r01 1581 +#define _SWAP_FAST_3_r11 1582 +#define _SWAP_FAST_3_r22 1583 +#define _SWAP_FAST_3_r33 1584 +#define _SWAP_FAST_4_r01 1585 +#define _SWAP_FAST_4_r11 1586 +#define _SWAP_FAST_4_r22 1587 +#define _SWAP_FAST_4_r33 1588 +#define _SWAP_FAST_5_r01 1589 +#define _SWAP_FAST_5_r11 1590 +#define _SWAP_FAST_5_r22 1591 +#define _SWAP_FAST_5_r33 1592 +#define _SWAP_FAST_6_r01 1593 +#define _SWAP_FAST_6_r11 1594 +#define _SWAP_FAST_6_r22 1595 +#define _SWAP_FAST_6_r33 1596 +#define _SWAP_FAST_7_r01 1597 +#define _SWAP_FAST_7_r11 1598 +#define _SWAP_FAST_7_r22 1599 +#define _SWAP_FAST_7_r33 1600 +#define _TIER2_RESUME_CHECK_r00 1601 +#define _TIER2_RESUME_CHECK_r11 1602 +#define _TIER2_RESUME_CHECK_r22 1603 +#define _TIER2_RESUME_CHECK_r33 1604 +#define _TO_BOOL_r11 1605 +#define _TO_BOOL_BOOL_r01 1606 +#define _TO_BOOL_BOOL_r11 1607 +#define _TO_BOOL_BOOL_r22 1608 +#define _TO_BOOL_BOOL_r33 1609 +#define _TO_BOOL_INT_r02 1610 +#define _TO_BOOL_INT_r12 1611 +#define _TO_BOOL_INT_r23 1612 +#define _TO_BOOL_LIST_r02 1613 +#define _TO_BOOL_LIST_r12 1614 +#define _TO_BOOL_LIST_r23 1615 +#define _TO_BOOL_NONE_r01 1616 +#define _TO_BOOL_NONE_r11 1617 +#define _TO_BOOL_NONE_r22 1618 +#define _TO_BOOL_NONE_r33 1619 +#define _TO_BOOL_STR_r02 1620 +#define _TO_BOOL_STR_r12 1621 +#define _TO_BOOL_STR_r23 1622 +#define _TRACE_RECORD_r00 1623 +#define _UNARY_INVERT_r12 1624 +#define _UNARY_NEGATIVE_r12 1625 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1626 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1627 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1628 +#define _UNARY_NOT_r01 1629 +#define _UNARY_NOT_r11 1630 +#define _UNARY_NOT_r22 1631 +#define _UNARY_NOT_r33 1632 +#define _UNPACK_EX_r10 1633 +#define _UNPACK_SEQUENCE_r10 1634 +#define _UNPACK_SEQUENCE_LIST_r10 1635 +#define _UNPACK_SEQUENCE_TUPLE_r10 1636 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1637 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1638 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1639 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1640 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1641 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1642 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1643 +#define _WITH_EXCEPT_START_r33 1644 +#define _YIELD_VALUE_r11 1645 +#define MAX_UOP_REGS_ID 1645 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index d5a3c362d875e65..990706c0b329223 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,13 +11,28 @@ extern "C" { #include #include "pycore_uop_ids.h" +#define MAX_CACHED_REGISTER 3 extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; -extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; +extern const char * const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); +typedef struct _pyuop_tos_cache_entry { + /* input depth is implicit in position */ + int8_t output; + int8_t exit; + int16_t opcode; +} _PyUopTOSentry; +typedef struct _PyUopCachingInfo { + uint8_t best[MAX_CACHED_REGISTER + 1]; + _PyUopTOSentry entries[MAX_CACHED_REGISTER + 1]; +} _PyUopCachingInfo; +extern const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1]; +extern const uint16_t _PyUop_SpillsAndReloads[4][4]; +extern const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1]; + #ifdef NEED_OPCODE_METADATA const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, @@ -44,50 +59,48 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_FAST_BORROW_7] = HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG, [_LOAD_FAST_AND_CLEAR] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_LOAD_CONST] = HAS_ARG_FLAG | HAS_CONST_FLAG, [_LOAD_SMALL_INT_0] = 0, [_LOAD_SMALL_INT_1] = 0, [_LOAD_SMALL_INT_2] = 0, [_LOAD_SMALL_INT_3] = 0, [_LOAD_SMALL_INT] = HAS_ARG_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_LOAD_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_SWAP_FAST_0] = HAS_LOCAL_FLAG, + [_SWAP_FAST_1] = HAS_LOCAL_FLAG, + [_SWAP_FAST_2] = HAS_LOCAL_FLAG, + [_SWAP_FAST_3] = HAS_LOCAL_FLAG, + [_SWAP_FAST_4] = HAS_LOCAL_FLAG, + [_SWAP_FAST_5] = HAS_LOCAL_FLAG, + [_SWAP_FAST_6] = HAS_LOCAL_FLAG, + [_SWAP_FAST_7] = HAS_LOCAL_FLAG, + [_SWAP_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, [_POP_TOP_FLOAT] = 0, [_POP_TOP_UNICODE] = 0, - [_POP_TWO] = HAS_ESCAPES_FLAG, + [_POP_TOP_OPARG] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_POP_ITER] = HAS_ESCAPES_FLAG, [_END_SEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, - [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNARY_NOT] = HAS_PURE_FLAG, + [_UNARY_NEGATIVE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = 0, + [_UNARY_NOT] = 0, [_TO_BOOL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_TO_BOOL_BOOL] = HAS_EXIT_FLAG, - [_TO_BOOL_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_INT] = 0, [_GUARD_NOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_LIST] = HAS_EXIT_FLAG, [_GUARD_TOS_SLICE] = HAS_EXIT_FLAG, - [_TO_BOOL_LIST] = HAS_ESCAPES_FLAG, + [_TO_BOOL_LIST] = 0, [_TO_BOOL_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_COMPACT_ASCII] = HAS_EXIT_FLAG, [_GUARD_NOS_UNICODE] = HAS_EXIT_FLAG, [_GUARD_TOS_UNICODE] = HAS_EXIT_FLAG, - [_TO_BOOL_STR] = HAS_ESCAPES_FLAG, - [_REPLACE_WITH_TRUE] = HAS_ESCAPES_FLAG, - [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_TO_BOOL_STR] = 0, + [_REPLACE_WITH_TRUE] = 0, + [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, @@ -95,59 +108,86 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = HAS_EXIT_FLAG, [_GUARD_NOS_FLOAT] = HAS_EXIT_FLAG, [_GUARD_TOS_FLOAT] = HAS_EXIT_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG, + [_BINARY_OP_MULTIPLY_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_ADD_FLOAT_INPLACE] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = 0, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_TRUEDIV_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_EXTEND_LHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND_RHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_EXTEND] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_LIST_SLICE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_STR_INT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_USTR_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, + [_GUARD_NOS_DICT_SUBSCRIPT] = HAS_DEOPT_FLAG, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = HAS_DEOPT_FLAG, + [_GUARD_TOS_ANY_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_DEOPT_FLAG, + [_GUARD_TOS_FROZENDICT] = HAS_EXIT_FLAG, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_INIT_CALL] = 0, [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_HEAP_SAFE] = 0, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, + [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, + [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = 0, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = 0, + [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = HAS_ARG_FLAG, [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -167,53 +207,68 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BUILD_TEMPLATE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_TUPLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_BUILD_LIST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LIST_EXTEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_SET_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_MAP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SETUP_ANNOTATIONS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_DICT_UPDATE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_DICT_MERGE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAP_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_SUPER_ATTR_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_TYPE_VERSION] = HAS_EXIT_FLAG, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_SUPER_ATTR_METHOD] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, - [_GUARD_TYPE_VERSION_AND_LOCK] = HAS_EXIT_FLAG, - [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_DEOPT_FLAG, - [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_VERSION_LOCKED] = HAS_EXIT_FLAG, + [_GUARD_TYPE] = HAS_EXIT_FLAG, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_EXIT_FLAG, + [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, + [_LOAD_ATTR_MODULE] = HAS_EXIT_FLAG, + [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_SLOT] = HAS_EXIT_FLAG, [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, - [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, + [_LOCK_OBJECT] = HAS_DEOPT_FLAG, [_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, - [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, - [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_IS_OP] = HAS_ARG_FLAG, + [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TOS_ANY_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_SET] = HAS_EXIT_FLAG, + [_GUARD_TOS_FROZENSET] = HAS_EXIT_FLAG, + [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CONTAINS_OP_DICT] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EG_MATCH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GET_ITER] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ITERATOR] = HAS_EXIT_FLAG, + [_GUARD_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_PUSH_TAGGED_ZERO] = 0, + [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_ITER] = HAS_EXIT_FLAG, + [_ITER_NEXT_INLINE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_GUARD_TOS_NOT_NULL] = HAS_EXIT_FLAG, + [_FOR_ITER_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -223,21 +278,19 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ITER_CHECK_RANGE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_RANGE] = HAS_EXIT_FLAG, [_ITER_NEXT_RANGE] = HAS_ERROR_FLAG, - [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_FOR_ITER_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INSERT_NULL] = 0, [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_DEOPT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, + [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_MAYBE_EXPAND_METHOD] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_FUNCTION_VERSION_INLINE] = HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -256,44 +309,61 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, - [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, - [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, - [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, - [_GUARD_CALLABLE_TYPE_1] = HAS_DEOPT_FLAG, - [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_STR_1] = HAS_DEOPT_FLAG, - [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_TUPLE_1] = HAS_DEOPT_FLAG, - [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, + [_GUARD_THIRD_NULL] = HAS_EXIT_FLAG, + [_GUARD_CALLABLE_TYPE_1] = HAS_EXIT_FLAG, + [_CALL_TYPE_1] = HAS_ARG_FLAG, + [_GUARD_CALLABLE_STR_1] = HAS_EXIT_FLAG, + [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_TUPLE_1] = HAS_EXIT_FLAG, + [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LEN] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LEN] = HAS_EXIT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_ISINSTANCE] = HAS_DEOPT_FLAG, + [_GUARD_CALLABLE_ISINSTANCE] = HAS_EXIT_FLAG, [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_GUARD_CALLABLE_LIST_APPEND] = HAS_DEOPT_FLAG, - [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_LIST_APPEND] = HAS_EXIT_FLAG, + [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, - [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CHECK_METHOD_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_IS_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_PY_FRAME_EX] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = HAS_EXIT_FLAG, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -308,6 +378,16 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SWAP] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_GUARD_IS_TRUE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_SET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_4] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_5] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_6] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP_7] = HAS_EXIT_FLAG, + [_GUARD_BIT_IS_UNSET_POP] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_JUMP_TO_TOP] = 0, @@ -318,367 +398,5751 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_CALL] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_LOAD_CONST_UNDER_INLINE] = 0, - [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, + [_RROT_3] = HAS_PURE_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, - [_DEOPT] = 0, - [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, - [_ERROR_POP_N] = HAS_ARG_FLAG, + [_DEOPT] = HAS_SYNC_SP_FLAG, + [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, + [_SPILL_OR_RELOAD] = 0, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = 0, - [_COLD_DYNAMIC_EXIT] = 0, + [_COLD_EXIT] = HAS_SYNC_SP_FLAG, + [_COLD_DYNAMIC_EXIT] = HAS_SYNC_SP_FLAG, + [_GUARD_CODE_VERSION__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = HAS_EXIT_FLAG, [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, + [_RECORD_TOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_TOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_TYPE] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_3OS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_4OS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE_KW] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_BOUND_METHOD] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CODE] = HAS_RECORDS_VALUE_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST] = { 0, 8 }, [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, - [_STORE_FAST] = { 0, 8 }, + [_SWAP_FAST] = { 0, 8 }, [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, + [_GUARD_BIT_IS_SET_POP] = { 4, 8 }, + [_GUARD_BIT_IS_UNSET_POP] = { 4, 8 }, +}; + +const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { + [_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _NOP_r00 }, + { 1, 1, _NOP_r11 }, + { 2, 2, _NOP_r22 }, + { 3, 3, _NOP_r33 }, + }, + }, + [_CHECK_PERIODIC] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _RESUME_CHECK_r00 }, + { 1, 1, _RESUME_CHECK_r11 }, + { 2, 2, _RESUME_CHECK_r22 }, + { 3, 3, _RESUME_CHECK_r33 }, + }, + }, + [_LOAD_FAST_CHECK] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_CHECK_r01 }, + { 2, 1, _LOAD_FAST_CHECK_r12 }, + { 3, 2, _LOAD_FAST_CHECK_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_0_r01 }, + { 2, 1, _LOAD_FAST_0_r12 }, + { 3, 2, _LOAD_FAST_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_1_r01 }, + { 2, 1, _LOAD_FAST_1_r12 }, + { 3, 2, _LOAD_FAST_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_2_r01 }, + { 2, 1, _LOAD_FAST_2_r12 }, + { 3, 2, _LOAD_FAST_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_3_r01 }, + { 2, 1, _LOAD_FAST_3_r12 }, + { 3, 2, _LOAD_FAST_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_4_r01 }, + { 2, 1, _LOAD_FAST_4_r12 }, + { 3, 2, _LOAD_FAST_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_5_r01 }, + { 2, 1, _LOAD_FAST_5_r12 }, + { 3, 2, _LOAD_FAST_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_6_r01 }, + { 2, 1, _LOAD_FAST_6_r12 }, + { 3, 2, _LOAD_FAST_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_7_r01 }, + { 2, 1, _LOAD_FAST_7_r12 }, + { 3, 2, _LOAD_FAST_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_r01 }, + { 2, 1, _LOAD_FAST_r12 }, + { 3, 2, _LOAD_FAST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_0_r01 }, + { 2, 1, _LOAD_FAST_BORROW_0_r12 }, + { 3, 2, _LOAD_FAST_BORROW_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_1_r01 }, + { 2, 1, _LOAD_FAST_BORROW_1_r12 }, + { 3, 2, _LOAD_FAST_BORROW_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_2_r01 }, + { 2, 1, _LOAD_FAST_BORROW_2_r12 }, + { 3, 2, _LOAD_FAST_BORROW_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_3_r01 }, + { 2, 1, _LOAD_FAST_BORROW_3_r12 }, + { 3, 2, _LOAD_FAST_BORROW_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_4] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_4_r01 }, + { 2, 1, _LOAD_FAST_BORROW_4_r12 }, + { 3, 2, _LOAD_FAST_BORROW_4_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_5] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_5_r01 }, + { 2, 1, _LOAD_FAST_BORROW_5_r12 }, + { 3, 2, _LOAD_FAST_BORROW_5_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_6] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_6_r01 }, + { 2, 1, _LOAD_FAST_BORROW_6_r12 }, + { 3, 2, _LOAD_FAST_BORROW_6_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW_7] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_7_r01 }, + { 2, 1, _LOAD_FAST_BORROW_7_r12 }, + { 3, 2, _LOAD_FAST_BORROW_7_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_BORROW_r01 }, + { 2, 1, _LOAD_FAST_BORROW_r12 }, + { 3, 2, _LOAD_FAST_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FAST_AND_CLEAR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_FAST_AND_CLEAR_r01 }, + { 2, 1, _LOAD_FAST_AND_CLEAR_r12 }, + { 3, 2, _LOAD_FAST_AND_CLEAR_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_r01 }, + { 2, 1, _LOAD_CONST_r12 }, + { 3, 2, _LOAD_CONST_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_0] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_0_r01 }, + { 2, 1, _LOAD_SMALL_INT_0_r12 }, + { 3, 2, _LOAD_SMALL_INT_0_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_1_r01 }, + { 2, 1, _LOAD_SMALL_INT_1_r12 }, + { 3, 2, _LOAD_SMALL_INT_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_2_r01 }, + { 2, 1, _LOAD_SMALL_INT_2_r12 }, + { 3, 2, _LOAD_SMALL_INT_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT_3] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_3_r01 }, + { 2, 1, _LOAD_SMALL_INT_3_r12 }, + { 3, 2, _LOAD_SMALL_INT_3_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SMALL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_SMALL_INT_r01 }, + { 2, 1, _LOAD_SMALL_INT_r12 }, + { 3, 2, _LOAD_SMALL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_FAST_0] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_0_r01 }, + { 1, 1, _SWAP_FAST_0_r11 }, + { 2, 2, _SWAP_FAST_0_r22 }, + { 3, 3, _SWAP_FAST_0_r33 }, + }, + }, + [_SWAP_FAST_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_1_r01 }, + { 1, 1, _SWAP_FAST_1_r11 }, + { 2, 2, _SWAP_FAST_1_r22 }, + { 3, 3, _SWAP_FAST_1_r33 }, + }, + }, + [_SWAP_FAST_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_2_r01 }, + { 1, 1, _SWAP_FAST_2_r11 }, + { 2, 2, _SWAP_FAST_2_r22 }, + { 3, 3, _SWAP_FAST_2_r33 }, + }, + }, + [_SWAP_FAST_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_3_r01 }, + { 1, 1, _SWAP_FAST_3_r11 }, + { 2, 2, _SWAP_FAST_3_r22 }, + { 3, 3, _SWAP_FAST_3_r33 }, + }, + }, + [_SWAP_FAST_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_4_r01 }, + { 1, 1, _SWAP_FAST_4_r11 }, + { 2, 2, _SWAP_FAST_4_r22 }, + { 3, 3, _SWAP_FAST_4_r33 }, + }, + }, + [_SWAP_FAST_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_5_r01 }, + { 1, 1, _SWAP_FAST_5_r11 }, + { 2, 2, _SWAP_FAST_5_r22 }, + { 3, 3, _SWAP_FAST_5_r33 }, + }, + }, + [_SWAP_FAST_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_6_r01 }, + { 1, 1, _SWAP_FAST_6_r11 }, + { 2, 2, _SWAP_FAST_6_r22 }, + { 3, 3, _SWAP_FAST_6_r33 }, + }, + }, + [_SWAP_FAST_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_7_r01 }, + { 1, 1, _SWAP_FAST_7_r11 }, + { 2, 2, _SWAP_FAST_7_r22 }, + { 3, 3, _SWAP_FAST_7_r33 }, + }, + }, + [_SWAP_FAST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SWAP_FAST_r01 }, + { 1, 1, _SWAP_FAST_r11 }, + { 2, 2, _SWAP_FAST_r22 }, + { 3, 3, _SWAP_FAST_r33 }, + }, + }, + [_POP_TOP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_TOP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_TOP_NOP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_NOP_r00 }, + { 0, 1, _POP_TOP_NOP_r10 }, + { 1, 2, _POP_TOP_NOP_r21 }, + { 2, 3, _POP_TOP_NOP_r32 }, + }, + }, + [_POP_TOP_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_INT_r00 }, + { 0, 1, _POP_TOP_INT_r10 }, + { 1, 2, _POP_TOP_INT_r21 }, + { 2, 3, _POP_TOP_INT_r32 }, + }, + }, + [_POP_TOP_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_FLOAT_r00 }, + { 0, 1, _POP_TOP_FLOAT_r10 }, + { 1, 2, _POP_TOP_FLOAT_r21 }, + { 2, 3, _POP_TOP_FLOAT_r32 }, + }, + }, + [_POP_TOP_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _POP_TOP_UNICODE_r00 }, + { 0, 1, _POP_TOP_UNICODE_r10 }, + { 1, 2, _POP_TOP_UNICODE_r21 }, + { 2, 3, _POP_TOP_UNICODE_r32 }, + }, + }, + [_POP_TOP_OPARG] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _POP_TOP_OPARG_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_NULL_r01 }, + { 2, 1, _PUSH_NULL_r12 }, + { 3, 2, _PUSH_NULL_r23 }, + { -1, -1, -1 }, + }, + }, + [_END_FOR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _END_FOR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_ITER] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _POP_ITER_r20 }, + { -1, -1, -1 }, + }, + }, + [_END_SEND] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _END_SEND_r31 }, + }, + }, + [_UNARY_NEGATIVE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_NEGATIVE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NEGATIVE_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNARY_NEGATIVE_FLOAT_INPLACE_r02 }, + { 2, 1, _UNARY_NEGATIVE_FLOAT_INPLACE_r12 }, + { 3, 2, _UNARY_NEGATIVE_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_NOT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _UNARY_NOT_r01 }, + { 1, 1, _UNARY_NOT_r11 }, + { 2, 2, _UNARY_NOT_r22 }, + { 3, 3, _UNARY_NOT_r33 }, + }, + }, + [_TO_BOOL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _TO_BOOL_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_BOOL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_BOOL_r01 }, + { 1, 1, _TO_BOOL_BOOL_r11 }, + { 2, 2, _TO_BOOL_BOOL_r22 }, + { 3, 3, _TO_BOOL_BOOL_r33 }, + }, + }, + [_TO_BOOL_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_INT_r02 }, + { 2, 1, _TO_BOOL_INT_r12 }, + { 3, 2, _TO_BOOL_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_LIST_r02 }, + { 2, 1, _GUARD_NOS_LIST_r12 }, + { 2, 2, _GUARD_NOS_LIST_r22 }, + { 3, 3, _GUARD_NOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_LIST_r01 }, + { 1, 1, _GUARD_TOS_LIST_r11 }, + { 2, 2, _GUARD_TOS_LIST_r22 }, + { 3, 3, _GUARD_TOS_LIST_r33 }, + }, + }, + [_GUARD_TOS_SLICE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SLICE_r01 }, + { 1, 1, _GUARD_TOS_SLICE_r11 }, + { 2, 2, _GUARD_TOS_SLICE_r22 }, + { 3, 3, _GUARD_TOS_SLICE_r33 }, + }, + }, + [_TO_BOOL_LIST] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_LIST_r02 }, + { 2, 1, _TO_BOOL_LIST_r12 }, + { 3, 2, _TO_BOOL_LIST_r23 }, + { -1, -1, -1 }, + }, + }, + [_TO_BOOL_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _TO_BOOL_NONE_r01 }, + { 1, 1, _TO_BOOL_NONE_r11 }, + { 2, 2, _TO_BOOL_NONE_r22 }, + { 3, 3, _TO_BOOL_NONE_r33 }, + }, + }, + [_GUARD_NOS_COMPACT_ASCII] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_COMPACT_ASCII_r02 }, + { 2, 1, _GUARD_NOS_COMPACT_ASCII_r12 }, + { 2, 2, _GUARD_NOS_COMPACT_ASCII_r22 }, + { 3, 3, _GUARD_NOS_COMPACT_ASCII_r33 }, + }, + }, + [_GUARD_NOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_UNICODE_r02 }, + { 2, 1, _GUARD_NOS_UNICODE_r12 }, + { 2, 2, _GUARD_NOS_UNICODE_r22 }, + { 3, 3, _GUARD_NOS_UNICODE_r33 }, + }, + }, + [_GUARD_TOS_UNICODE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_UNICODE_r01 }, + { 1, 1, _GUARD_TOS_UNICODE_r11 }, + { 2, 2, _GUARD_TOS_UNICODE_r22 }, + { 3, 3, _GUARD_TOS_UNICODE_r33 }, + }, + }, + [_TO_BOOL_STR] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _TO_BOOL_STR_r02 }, + { 2, 1, _TO_BOOL_STR_r12 }, + { 3, 2, _TO_BOOL_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_REPLACE_WITH_TRUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _REPLACE_WITH_TRUE_r02 }, + { 2, 1, _REPLACE_WITH_TRUE_r12 }, + { 3, 2, _REPLACE_WITH_TRUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNARY_INVERT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNARY_INVERT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_INT_r02 }, + { 2, 1, _GUARD_NOS_INT_r12 }, + { 2, 2, _GUARD_NOS_INT_r22 }, + { 3, 3, _GUARD_NOS_INT_r33 }, + }, + }, + [_GUARD_TOS_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_INT_r01 }, + { 1, 1, _GUARD_TOS_INT_r11 }, + { 2, 2, _GUARD_TOS_INT_r22 }, + { 3, 3, _GUARD_TOS_INT_r33 }, + }, + }, + [_GUARD_NOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_OVERFLOWED_r02 }, + { 2, 1, _GUARD_NOS_OVERFLOWED_r12 }, + { 2, 2, _GUARD_NOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_NOS_OVERFLOWED_r33 }, + }, + }, + [_GUARD_TOS_OVERFLOWED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_OVERFLOWED_r01 }, + { 1, 1, _GUARD_TOS_OVERFLOWED_r11 }, + { 2, 2, _GUARD_TOS_OVERFLOWED_r22 }, + { 3, 3, _GUARD_TOS_OVERFLOWED_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_FLOAT_r02 }, + { 2, 1, _GUARD_NOS_FLOAT_r12 }, + { 2, 2, _GUARD_NOS_FLOAT_r22 }, + { 3, 3, _GUARD_NOS_FLOAT_r33 }, + }, + }, + [_GUARD_TOS_FLOAT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FLOAT_r01 }, + { 1, 1, _GUARD_TOS_FLOAT_r11 }, + { 2, 2, _GUARD_TOS_FLOAT_r22 }, + { 3, 3, _GUARD_TOS_FLOAT_r33 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_ADD_UNICODE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_ADD_UNICODE_r03 }, + { 3, 1, _BINARY_OP_ADD_UNICODE_r13 }, + { 3, 2, _BINARY_OP_ADD_UNICODE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_INPLACE_ADD_UNICODE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BINARY_OP_INPLACE_ADD_UNICODE_r21 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_LHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_LHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_LHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_LHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_LHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_RHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_RHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_RHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_RHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_RHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_r22 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_EXTEND] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_EXTEND_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _BINARY_SLICE_r31 }, + }, + }, + [_STORE_SLICE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SLICE_r30 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_LIST_SLICE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_LIST_SLICE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_STR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_STR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_USTR_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_USTR_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TUPLE_r02 }, + { 2, 1, _GUARD_NOS_TUPLE_r12 }, + { 2, 2, _GUARD_NOS_TUPLE_r22 }, + { 3, 3, _GUARD_NOS_TUPLE_r33 }, + }, + }, + [_GUARD_TOS_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_TUPLE_r01 }, + { 1, 1, _GUARD_TOS_TUPLE_r11 }, + { 2, 2, _GUARD_TOS_TUPLE_r22 }, + { 3, 3, _GUARD_TOS_TUPLE_r33 }, + }, + }, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 }, + { 2, 1, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 }, + { 2, 2, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 }, + { 3, 3, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_TUPLE_INT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_SUBSCR_TUPLE_INT_r03 }, + { 3, 1, _BINARY_OP_SUBSCR_TUPLE_INT_r13 }, + { 3, 2, _BINARY_OP_SUBSCR_TUPLE_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_DICT_SUBSCRIPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_DICT_SUBSCRIPT_r02 }, + { 2, 1, _GUARD_NOS_DICT_SUBSCRIPT_r12 }, + { 2, 2, _GUARD_NOS_DICT_SUBSCRIPT_r22 }, + { 3, 3, _GUARD_NOS_DICT_SUBSCRIPT_r33 }, + }, + }, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 }, + { 3, 1, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 }, + { 3, 2, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 }, + { 3, 3, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 }, + }, + }, + [_GUARD_TOS_ANY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_DICT_r01 }, + { 1, 1, _GUARD_TOS_ANY_DICT_r11 }, + { 2, 2, _GUARD_TOS_ANY_DICT_r22 }, + { 3, 3, _GUARD_TOS_ANY_DICT_r33 }, + }, + }, + [_GUARD_TOS_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_DICT_r01 }, + { 1, 1, _GUARD_TOS_DICT_r11 }, + { 2, 2, _GUARD_TOS_DICT_r22 }, + { 3, 3, _GUARD_TOS_DICT_r33 }, + }, + }, + [_GUARD_TOS_FROZENDICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENDICT_r01 }, + { 1, 1, _GUARD_TOS_FROZENDICT_r11 }, + { 2, 2, _GUARD_TOS_FROZENDICT_r22 }, + { 3, 3, _GUARD_TOS_FROZENDICT_r33 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_CHECK_FUNC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_SUBSCR_CHECK_FUNC_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_SUBSCR_INIT_CALL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _BINARY_OP_SUBSCR_INIT_CALL_r01 }, + { 1, 1, _BINARY_OP_SUBSCR_INIT_CALL_r11 }, + { 1, 2, _BINARY_OP_SUBSCR_INIT_CALL_r21 }, + { 1, 3, _BINARY_OP_SUBSCR_INIT_CALL_r31 }, + }, + }, + [_LIST_APPEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LIST_APPEND_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_ADD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _SET_ADD_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_SUBSCR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 3, _STORE_SUBSCR_r30 }, + }, + }, + [_STORE_SUBSCR_LIST_INT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _STORE_SUBSCR_LIST_INT_r32 }, + }, + }, + [_STORE_SUBSCR_DICT] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_r31 }, + }, + }, + [_STORE_SUBSCR_DICT_KNOWN_HASH] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _STORE_SUBSCR_DICT_KNOWN_HASH_r31 }, + }, + }, + [_DELETE_SUBSCR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _DELETE_SUBSCR_r20 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_1] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _CALL_INTRINSIC_1_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_INTRINSIC_2] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CALL_INTRINSIC_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_HEAP_SAFE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _MAKE_HEAP_SAFE_r01 }, + { 1, 1, _MAKE_HEAP_SAFE_r11 }, + { 2, 2, _MAKE_HEAP_SAFE_r22 }, + { 3, 3, _MAKE_HEAP_SAFE_r33 }, + }, + }, + [_RETURN_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _RETURN_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AITER_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_ANEXT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ANEXT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_AWAITABLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _GET_AWAITABLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SEND_GEN_FRAME] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_GEN_FRAME_r33 }, + }, + }, + [_GUARD_TOS_IS_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_IS_NONE_r01 }, + { 1, 1, _GUARD_TOS_IS_NONE_r11 }, + { 2, 2, _GUARD_TOS_IS_NONE_r22 }, + { 3, 3, _GUARD_TOS_IS_NONE_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, + { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_SEND_VIRTUAL_TIER_TWO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SEND_VIRTUAL_TIER_TWO_r03 }, + { 3, 1, _SEND_VIRTUAL_TIER_TWO_r13 }, + { 3, 2, _SEND_VIRTUAL_TIER_TWO_r23 }, + { 3, 3, _SEND_VIRTUAL_TIER_TWO_r33 }, + }, + }, + [_GUARD_3OS_ASYNC_GEN_ASEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_3OS_ASYNC_GEN_ASEND_r03 }, + { 3, 1, _GUARD_3OS_ASYNC_GEN_ASEND_r13 }, + { 3, 2, _GUARD_3OS_ASYNC_GEN_ASEND_r23 }, + { 3, 3, _GUARD_3OS_ASYNC_GEN_ASEND_r33 }, + }, + }, + [_SEND_ASYNC_GEN_TIER_TWO] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, + }, + }, + [_YIELD_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _YIELD_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_POP_EXCEPT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _POP_EXCEPT_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_COMMON_CONSTANT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_COMMON_CONSTANT_r01 }, + { 2, 1, _LOAD_COMMON_CONSTANT_r12 }, + { 3, 2, _LOAD_COMMON_CONSTANT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_BUILD_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_BUILD_CLASS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_NAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_NAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TWO_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _UNPACK_SEQUENCE_TWO_TUPLE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 }, + { 2, 1, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 }, + { 3, 2, _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = { + .best = { 0, 1, 1, 1 }, + .entries = { + { 3, 0, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 }, + { 3, 1, _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_SEQUENCE_LIST] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_SEQUENCE_LIST_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_UNPACK_EX] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _UNPACK_EX_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _STORE_ATTR_r20 }, + { -1, -1, -1 }, + }, + }, + [_STORE_GLOBAL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_GLOBAL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_LOCALS] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_LOCALS_r01 }, + { 2, 1, _LOAD_LOCALS_r12 }, + { 3, 2, _LOAD_LOCALS_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_NAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_NAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_GLOBAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_NULL_CONDITIONAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _PUSH_NULL_CONDITIONAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_GLOBALS_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_GLOBALS_VERSION_r00 }, + { 1, 1, _GUARD_GLOBALS_VERSION_r11 }, + { 2, 2, _GUARD_GLOBALS_VERSION_r22 }, + { 3, 3, _GUARD_GLOBALS_VERSION_r33 }, + }, + }, + [_LOAD_GLOBAL_MODULE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_MODULE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_GLOBAL_BUILTINS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_GLOBAL_BUILTINS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CELL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAKE_CELL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DELETE_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _DELETE_DEREF_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_FROM_DICT_OR_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_FROM_DICT_OR_DEREF_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_DEREF] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _LOAD_DEREF_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_STORE_DEREF] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _STORE_DEREF_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COPY_FREE_VARS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _COPY_FREE_VARS_r00 }, + { 1, 1, _COPY_FREE_VARS_r11 }, + { 2, 2, _COPY_FREE_VARS_r22 }, + { 3, 3, _COPY_FREE_VARS_r33 }, + }, + }, + [_BUILD_STRING] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_STRING_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_INTERPOLATION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_INTERPOLATION_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TEMPLATE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _BUILD_TEMPLATE_r21 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_TUPLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_TUPLE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_LIST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_LIST_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LIST_EXTEND] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LIST_EXTEND_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SET_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SET] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SET_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_MAP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_MAP_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SETUP_ANNOTATIONS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _SETUP_ANNOTATIONS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_UPDATE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_UPDATE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_DICT_MERGE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _DICT_MERGE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAP_ADD] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 0, 2, _MAP_ADD_r20 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SUPER_ATTR_ATTR] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _LOAD_SUPER_ATTR_ATTR_r31 }, + }, + }, + [_GUARD_NOS_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_TYPE_VERSION_r02 }, + { 2, 1, _GUARD_NOS_TYPE_VERSION_r12 }, + { 2, 2, _GUARD_NOS_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_NOS_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_LOAD_SUPER_ATTR_METHOD] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_LOAD_SUPER_ATTR_METHOD_r03 }, + { 3, 1, _GUARD_LOAD_SUPER_ATTR_METHOD_r13 }, + { 3, 2, _GUARD_LOAD_SUPER_ATTR_METHOD_r23 }, + { 3, 3, _GUARD_LOAD_SUPER_ATTR_METHOD_r33 }, + }, + }, + [_LOAD_SUPER_ATTR_METHOD] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _LOAD_SUPER_ATTR_METHOD_r32 }, + }, + }, + [_LOAD_ATTR] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _LOAD_ATTR_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_VERSION] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_r33 }, + }, + }, + [_GUARD_TYPE_VERSION_LOCKED] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_VERSION_LOCKED_r01 }, + { 1, 1, _GUARD_TYPE_VERSION_LOCKED_r11 }, + { 2, 2, _GUARD_TYPE_VERSION_LOCKED_r22 }, + { 3, 3, _GUARD_TYPE_VERSION_LOCKED_r33 }, + }, + }, + [_GUARD_TYPE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_r01 }, + { 1, 1, _GUARD_TYPE_r11 }, + { 2, 2, _GUARD_TYPE_r22 }, + { 3, 3, _GUARD_TYPE_r33 }, + }, + }, + [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 }, + { 1, 1, _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 }, + { 2, 2, _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 }, + { 3, 3, _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 }, + }, + }, + [_LOAD_ATTR_INSTANCE_VALUE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_INSTANCE_VALUE_r02 }, + { 2, 1, _LOAD_ATTR_INSTANCE_VALUE_r12 }, + { 3, 2, _LOAD_ATTR_INSTANCE_VALUE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_MODULE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_MODULE_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_WITH_HINT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _LOAD_ATTR_WITH_HINT_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_SLOT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_SLOT_r02 }, + { 2, 1, _LOAD_ATTR_SLOT_r12 }, + { 3, 2, _LOAD_ATTR_SLOT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_CLASS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_CLASS_r01 }, + { 1, 1, _CHECK_ATTR_CLASS_r11 }, + { 2, 2, _CHECK_ATTR_CLASS_r22 }, + { 3, 3, _CHECK_ATTR_CLASS_r33 }, + }, + }, + [_LOAD_ATTR_CLASS] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_CLASS_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_PROPERTY_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOAD_ATTR_PROPERTY_FRAME_r01 }, + { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { 2, 2, _LOAD_ATTR_PROPERTY_FRAME_r22 }, + { 3, 3, _LOAD_ATTR_PROPERTY_FRAME_r33 }, + }, + }, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_DORV_NO_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_DORV_NO_DICT_r01 }, + { 1, 1, _GUARD_DORV_NO_DICT_r11 }, + { 2, 2, _GUARD_DORV_NO_DICT_r22 }, + { 3, 3, _GUARD_DORV_NO_DICT_r33 }, + }, + }, + [_STORE_ATTR_INSTANCE_VALUE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_INSTANCE_VALUE_r21 }, + { -1, -1, -1 }, + }, + }, + [_LOCK_OBJECT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOCK_OBJECT_r01 }, + { 1, 1, _LOCK_OBJECT_r11 }, + { 2, 2, _LOCK_OBJECT_r22 }, + { 3, 3, _LOCK_OBJECT_r33 }, + }, + }, + [_STORE_ATTR_WITH_HINT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_WITH_HINT_r21 }, + { -1, -1, -1 }, + }, + }, + [_STORE_ATTR_SLOT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _STORE_ATTR_SLOT_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _COMPARE_OP_r21 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_FLOAT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COMPARE_OP_FLOAT_r03 }, + { 3, 1, _COMPARE_OP_FLOAT_r13 }, + { 3, 2, _COMPARE_OP_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_INT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_INT_r23 }, + { -1, -1, -1 }, + }, + }, + [_COMPARE_OP_STR] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _COMPARE_OP_STR_r23 }, + { -1, -1, -1 }, + }, + }, + [_IS_OP] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _IS_OP_r03 }, + { 3, 1, _IS_OP_r13 }, + { 3, 2, _IS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TOS_ANY_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_ANY_SET_r01 }, + { 1, 1, _GUARD_TOS_ANY_SET_r11 }, + { 2, 2, _GUARD_TOS_ANY_SET_r22 }, + { 3, 3, _GUARD_TOS_ANY_SET_r33 }, + }, + }, + [_GUARD_TOS_SET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_SET_r01 }, + { 1, 1, _GUARD_TOS_SET_r11 }, + { 2, 2, _GUARD_TOS_SET_r22 }, + { 3, 3, _GUARD_TOS_SET_r33 }, + }, + }, + [_GUARD_TOS_FROZENSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_FROZENSET_r01 }, + { 1, 1, _GUARD_TOS_FROZENSET_r11 }, + { 2, 2, _GUARD_TOS_FROZENSET_r22 }, + { 3, 3, _GUARD_TOS_FROZENSET_r33 }, + }, + }, + [_CONTAINS_OP_SET] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_SET_r23 }, + { -1, -1, -1 }, + }, + }, + [_CONTAINS_OP_DICT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _CONTAINS_OP_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EG_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EG_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_EXC_MATCH] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 2, _CHECK_EXC_MATCH_r22 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_NAME] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _IMPORT_NAME_r21 }, + { -1, -1, -1 }, + }, + }, + [_IMPORT_FROM] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _IMPORT_FROM_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_IS_NONE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _IS_NONE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GET_LEN] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_LEN_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_CLASS] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MATCH_CLASS_r33 }, + }, + }, + [_MATCH_MAPPING] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_MAPPING_r02 }, + { 2, 1, _MATCH_MAPPING_r12 }, + { 3, 2, _MATCH_MAPPING_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_SEQUENCE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _MATCH_SEQUENCE_r02 }, + { 2, 1, _MATCH_SEQUENCE_r12 }, + { 3, 2, _MATCH_SEQUENCE_r23 }, + { -1, -1, -1 }, + }, + }, + [_MATCH_KEYS] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _MATCH_KEYS_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_ITERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITERATOR_r01 }, + { 1, 1, _GUARD_ITERATOR_r11 }, + { 2, 2, _GUARD_ITERATOR_r22 }, + { 3, 3, _GUARD_ITERATOR_r33 }, + }, + }, + [_GUARD_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITER_VIRTUAL_r01 }, + { 1, 1, _GUARD_ITER_VIRTUAL_r11 }, + { 2, 2, _GUARD_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_ITER_VIRTUAL_r33 }, + }, + }, + [_PUSH_TAGGED_ZERO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_TAGGED_ZERO_r01 }, + { 2, 1, _PUSH_TAGGED_ZERO_r12 }, + { 3, 2, _PUSH_TAGGED_ZERO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER_TRAD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_TRAD_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_TYPE_ITER] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TYPE_ITER_r02 }, + { 2, 1, _GUARD_TYPE_ITER_r12 }, + { 2, 2, _GUARD_TYPE_ITER_r22 }, + { 3, 3, _GUARD_TYPE_ITER_r33 }, + }, + }, + [_ITER_NEXT_INLINE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_ITER_VIRTUAL_r02 }, + { 2, 1, _GUARD_NOS_ITER_VIRTUAL_r12 }, + { 2, 2, _GUARD_NOS_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_NOS_ITER_VIRTUAL_r33 }, + }, + }, + [_GUARD_TOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_NOT_NULL_r01 }, + { 1, 1, _GUARD_TOS_NOT_NULL_r11 }, + { 2, 2, _GUARD_TOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_TOS_NOT_NULL_r33 }, + }, + }, + [_FOR_ITER_VIRTUAL_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_VIRTUAL_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_LIST_r02 }, + { 2, 1, _ITER_CHECK_LIST_r12 }, + { 2, 2, _ITER_CHECK_LIST_r22 }, + { 3, 3, _ITER_CHECK_LIST_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_LIST] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_LIST_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_LIST_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_LIST_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_LIST_r33 }, + }, + }, + [_ITER_NEXT_LIST_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_LIST_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_TUPLE_r02 }, + { 2, 1, _ITER_CHECK_TUPLE_r12 }, + { 2, 2, _ITER_CHECK_TUPLE_r22 }, + { 3, 3, _ITER_CHECK_TUPLE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_TUPLE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_TUPLE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_TUPLE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_TUPLE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_TUPLE_r33 }, + }, + }, + [_ITER_NEXT_TUPLE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_TUPLE_r03 }, + { 3, 1, _ITER_NEXT_TUPLE_r13 }, + { 3, 2, _ITER_NEXT_TUPLE_r23 }, + { -1, -1, -1 }, + }, + }, + [_ITER_CHECK_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _ITER_CHECK_RANGE_r02 }, + { 2, 1, _ITER_CHECK_RANGE_r12 }, + { 2, 2, _ITER_CHECK_RANGE_r22 }, + { 3, 3, _ITER_CHECK_RANGE_r33 }, + }, + }, + [_GUARD_NOT_EXHAUSTED_RANGE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOT_EXHAUSTED_RANGE_r02 }, + { 2, 1, _GUARD_NOT_EXHAUSTED_RANGE_r12 }, + { 2, 2, _GUARD_NOT_EXHAUSTED_RANGE_r22 }, + { 3, 3, _GUARD_NOT_EXHAUSTED_RANGE_r33 }, + }, + }, + [_ITER_NEXT_RANGE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _ITER_NEXT_RANGE_r03 }, + { 3, 1, _ITER_NEXT_RANGE_r13 }, + { 3, 2, _ITER_NEXT_RANGE_r23 }, + { -1, -1, -1 }, + }, + }, + [_FOR_ITER_GEN_FRAME] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _FOR_ITER_GEN_FRAME_r03 }, + { 3, 1, _FOR_ITER_GEN_FRAME_r13 }, + { 3, 2, _FOR_ITER_GEN_FRAME_r23 }, + { -1, -1, -1 }, + }, + }, + [_INSERT_NULL] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _INSERT_NULL_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_SPECIAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _LOAD_SPECIAL_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_WITH_EXCEPT_START] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _WITH_EXCEPT_START_r33 }, + }, + }, + [_PUSH_EXC_INFO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _PUSH_EXC_INFO_r02 }, + { 2, 1, _PUSH_EXC_INFO_r12 }, + { 3, 2, _PUSH_EXC_INFO_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_METHOD_WITH_VALUES] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_WITH_VALUES_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_WITH_VALUES_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_WITH_VALUES_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_METHOD_NO_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_NO_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_NO_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_NO_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _CHECK_ATTR_METHOD_LAZY_DICT_r01 }, + { 1, 1, _CHECK_ATTR_METHOD_LAZY_DICT_r11 }, + { 2, 2, _CHECK_ATTR_METHOD_LAZY_DICT_r22 }, + { 3, 3, _CHECK_ATTR_METHOD_LAZY_DICT_r33 }, + }, + }, + [_LOAD_ATTR_METHOD_LAZY_DICT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _LOAD_ATTR_METHOD_LAZY_DICT_r02 }, + { 2, 1, _LOAD_ATTR_METHOD_LAZY_DICT_r12 }, + { 3, 2, _LOAD_ATTR_METHOD_LAZY_DICT_r23 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _MAYBE_EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _PY_FRAME_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_INLINE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_VERSION_INLINE_r00 }, + { 1, 1, _CHECK_FUNCTION_VERSION_INLINE_r11 }, + { 2, 2, _CHECK_FUNCTION_VERSION_INLINE_r22 }, + { 3, 3, _CHECK_FUNCTION_VERSION_INLINE_r33 }, + }, + }, + [_CHECK_METHOD_VERSION] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_METHOD_VERSION_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _EXPAND_METHOD_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_IS_NOT_PY_CALLABLE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_NON_PY_GENERAL] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _CALL_NON_PY_GENERAL_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_PEP_523] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_PEP_523_r00 }, + { 1, 1, _CHECK_PEP_523_r11 }, + { 2, 2, _CHECK_PEP_523_r22 }, + { 3, 3, _CHECK_PEP_523_r33 }, + }, + }, + [_CHECK_FUNCTION_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_FUNCTION_EXACT_ARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_STACK_SPACE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_REMAINING] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_REMAINING_r00 }, + { 1, 1, _CHECK_RECURSION_REMAINING_r11 }, + { 2, 2, _CHECK_RECURSION_REMAINING_r22 }, + { 3, 3, _CHECK_RECURSION_REMAINING_r33 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_0] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_0_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_1] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_1_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_2] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_2_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_3] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_3_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS_4] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_4_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_INIT_CALL_PY_EXACT_ARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _INIT_CALL_PY_EXACT_ARGS_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PUSH_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _PUSH_FRAME_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NULL_r02 }, + { 2, 1, _GUARD_NOS_NULL_r12 }, + { 2, 2, _GUARD_NOS_NULL_r22 }, + { 3, 3, _GUARD_NOS_NULL_r33 }, + }, + }, + [_GUARD_THIRD_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_THIRD_NULL_r03 }, + { 3, 1, _GUARD_THIRD_NULL_r13 }, + { 3, 2, _GUARD_THIRD_NULL_r23 }, + { 3, 3, _GUARD_THIRD_NULL_r33 }, + }, + }, + [_GUARD_CALLABLE_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TYPE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TYPE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TYPE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TYPE_1_r33 }, + }, + }, + [_CALL_TYPE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _CALL_TYPE_1_r02 }, + { 2, 1, _CALL_TYPE_1_r12 }, + { 2, 2, _CALL_TYPE_1_r22 }, + { 2, 3, _CALL_TYPE_1_r32 }, + }, + }, + [_GUARD_CALLABLE_STR_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_STR_1_r03 }, + { 3, 1, _GUARD_CALLABLE_STR_1_r13 }, + { 3, 2, _GUARD_CALLABLE_STR_1_r23 }, + { 3, 3, _GUARD_CALLABLE_STR_1_r33 }, + }, + }, + [_CALL_STR_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_STR_1_r32 }, + }, + }, + [_GUARD_CALLABLE_TUPLE_1] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_TUPLE_1_r03 }, + { 3, 1, _GUARD_CALLABLE_TUPLE_1_r13 }, + { 3, 2, _GUARD_CALLABLE_TUPLE_1_r23 }, + { 3, 3, _GUARD_CALLABLE_TUPLE_1_r33 }, + }, + }, + [_CALL_TUPLE_1] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 2, 3, _CALL_TUPLE_1_r32 }, + }, + }, + [_CHECK_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_ALLOCATE_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ALLOCATE_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CREATE_INIT_FRAME] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _CREATE_INIT_FRAME_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXIT_INIT_CHECK] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 1, _EXIT_INIT_CHECK_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_BUILTIN_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_LEN] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LEN_r03 }, + { 3, 1, _GUARD_CALLABLE_LEN_r13 }, + { 3, 2, _GUARD_CALLABLE_LEN_r23 }, + { 3, 3, _GUARD_CALLABLE_LEN_r33 }, + }, + }, + [_CALL_LEN] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _CALL_LEN_r33 }, + }, + }, + [_GUARD_CALLABLE_ISINSTANCE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_ISINSTANCE_r03 }, + { 3, 1, _GUARD_CALLABLE_ISINSTANCE_r13 }, + { 3, 2, _GUARD_CALLABLE_ISINSTANCE_r23 }, + { 3, 3, _GUARD_CALLABLE_ISINSTANCE_r33 }, + }, + }, + [_CALL_ISINSTANCE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_ISINSTANCE_r31 }, + }, + }, + [_GUARD_CALLABLE_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_CALLABLE_LIST_APPEND_r03 }, + { 3, 1, _GUARD_CALLABLE_LIST_APPEND_r13 }, + { 3, 2, _GUARD_CALLABLE_LIST_APPEND_r23 }, + { 3, 3, _GUARD_CALLABLE_LIST_APPEND_r33 }, + }, + }, + [_CALL_LIST_APPEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CALL_LIST_APPEND_r03 }, + { 3, 1, _CALL_LIST_APPEND_r13 }, + { 3, 2, _CALL_LIST_APPEND_r23 }, + { 3, 3, _CALL_LIST_APPEND_r33 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_RECURSION_LIMIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_RECURSION_LIMIT_r00 }, + { 1, 1, _CHECK_RECURSION_LIMIT_r11 }, + { 2, 2, _CHECK_RECURSION_LIMIT_r22 }, + { 3, 3, _CHECK_RECURSION_LIMIT_r33 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 3, 0, _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAYBE_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _MAYBE_EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_PY_FRAME_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_FUNCTION_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_FUNCTION_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_METHOD_VERSION_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_METHOD_VERSION_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_EXPAND_METHOD_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _EXPAND_METHOD_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_KW] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CHECK_IS_NOT_PY_CALLABLE_KW_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CALL_KW_NON_PY] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CALL_KW_NON_PY_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_CALLARGS_A_TUPLE] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _MAKE_CALLARGS_A_TUPLE_r33 }, + }, + }, + [_CHECK_IS_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_PY_CALLABLE_EX_r33 }, + }, + }, + [_PY_FRAME_EX] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 1, _PY_FRAME_EX_r31 }, + }, + }, + [_CHECK_IS_NOT_PY_CALLABLE_EX] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _CHECK_IS_NOT_PY_CALLABLE_EX_r03 }, + { 3, 1, _CHECK_IS_NOT_PY_CALLABLE_EX_r13 }, + { 3, 2, _CHECK_IS_NOT_PY_CALLABLE_EX_r23 }, + { 3, 3, _CHECK_IS_NOT_PY_CALLABLE_EX_r33 }, + }, + }, + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 3, _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 }, + }, + }, + [_MAKE_FUNCTION] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _MAKE_FUNCTION_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_FUNCTION_ATTRIBUTE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _SET_FUNCTION_ATTRIBUTE_r01 }, + { 1, 1, _SET_FUNCTION_ATTRIBUTE_r11 }, + { 1, 2, _SET_FUNCTION_ATTRIBUTE_r21 }, + { 2, 3, _SET_FUNCTION_ATTRIBUTE_r32 }, + }, + }, + [_RETURN_GENERATOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 1, _RETURN_GENERATOR_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BUILD_SLICE] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _BUILD_SLICE_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_CONVERT_VALUE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _CONVERT_VALUE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_SIMPLE] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _FORMAT_SIMPLE_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_FORMAT_WITH_SPEC] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 1, 2, _FORMAT_WITH_SPEC_r21 }, + { -1, -1, -1 }, + }, + }, + [_COPY_1] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 2, 0, _COPY_1_r02 }, + { 2, 1, _COPY_1_r12 }, + { 3, 2, _COPY_1_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_2] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _COPY_2_r03 }, + { 3, 1, _COPY_2_r13 }, + { 3, 2, _COPY_2_r23 }, + { -1, -1, -1 }, + }, + }, + [_COPY_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _COPY_3_r03 }, + { 3, 1, _COPY_3_r13 }, + { 3, 2, _COPY_3_r23 }, + { 3, 3, _COPY_3_r33 }, + }, + }, + [_COPY] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 1, 0, _COPY_r01 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_r23 }, + { -1, -1, -1 }, + }, + }, + [_SWAP_2] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _SWAP_2_r02 }, + { 2, 1, _SWAP_2_r12 }, + { 2, 2, _SWAP_2_r22 }, + { 3, 3, _SWAP_2_r33 }, + }, + }, + [_SWAP_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SWAP_3_r03 }, + { 3, 1, _SWAP_3_r13 }, + { 3, 2, _SWAP_3_r23 }, + { 3, 3, _SWAP_3_r33 }, + }, + }, + [_SWAP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 1, 1, _SWAP_r11 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_IS_TRUE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_TRUE_POP_r00 }, + { 0, 0, _GUARD_IS_TRUE_POP_r10 }, + { 1, 1, _GUARD_IS_TRUE_POP_r21 }, + { 2, 2, _GUARD_IS_TRUE_POP_r32 }, + }, + }, + [_GUARD_IS_FALSE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_FALSE_POP_r00 }, + { 0, 0, _GUARD_IS_FALSE_POP_r10 }, + { 1, 1, _GUARD_IS_FALSE_POP_r21 }, + { 2, 2, _GUARD_IS_FALSE_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_SET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_SET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_SET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_SET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_SET_POP_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_4] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_4_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_4_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_4_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_5] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_5_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_5_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_5_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_6] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_6_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_6_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_6_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP_7] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_7_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_7_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_7_r32 }, + }, + }, + [_GUARD_BIT_IS_UNSET_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r00 }, + { 0, 0, _GUARD_BIT_IS_UNSET_POP_r10 }, + { 1, 1, _GUARD_BIT_IS_UNSET_POP_r21 }, + { 2, 2, _GUARD_BIT_IS_UNSET_POP_r32 }, + }, + }, + [_GUARD_IS_NONE_POP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IS_NONE_POP_r00 }, + { 0, 0, _GUARD_IS_NONE_POP_r10 }, + { 1, 1, _GUARD_IS_NONE_POP_r21 }, + { 2, 2, _GUARD_IS_NONE_POP_r32 }, + }, + }, + [_GUARD_IS_NOT_NONE_POP] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 0, 0, _GUARD_IS_NOT_NONE_POP_r10 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_JUMP_TO_TOP] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _JUMP_TO_TOP_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_SET_IP] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SET_IP_r00 }, + { 1, 1, _SET_IP_r11 }, + { 2, 2, _SET_IP_r22 }, + { 3, 3, _SET_IP_r33 }, + }, + }, + [_CHECK_STACK_SPACE_OPERAND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_STACK_SPACE_OPERAND_r00 }, + { 1, 1, _CHECK_STACK_SPACE_OPERAND_r11 }, + { 2, 2, _CHECK_STACK_SPACE_OPERAND_r22 }, + { 3, 3, _CHECK_STACK_SPACE_OPERAND_r33 }, + }, + }, + [_SAVE_RETURN_OFFSET] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _SAVE_RETURN_OFFSET_r00 }, + { 1, 1, _SAVE_RETURN_OFFSET_r11 }, + { 2, 2, _SAVE_RETURN_OFFSET_r22 }, + { 3, 3, _SAVE_RETURN_OFFSET_r33 }, + }, + }, + [_EXIT_TRACE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _EXIT_TRACE_r00 }, + { 0, 0, _EXIT_TRACE_r10 }, + { 0, 0, _EXIT_TRACE_r20 }, + { 0, 0, _EXIT_TRACE_r30 }, + }, + }, + [_DYNAMIC_EXIT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DYNAMIC_EXIT_r00 }, + { 0, 0, _DYNAMIC_EXIT_r10 }, + { 0, 0, _DYNAMIC_EXIT_r20 }, + { 0, 0, _DYNAMIC_EXIT_r30 }, + }, + }, + [_CHECK_VALIDITY] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _CHECK_VALIDITY_r00 }, + { 1, 1, _CHECK_VALIDITY_r11 }, + { 2, 2, _CHECK_VALIDITY_r22 }, + { 3, 3, _CHECK_VALIDITY_r33 }, + }, + }, + [_LOAD_CONST_INLINE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_r01 }, + { 2, 1, _LOAD_CONST_INLINE_r12 }, + { 3, 2, _LOAD_CONST_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_LOAD_CONST_INLINE_BORROW] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, + { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, + { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + }, + }, + [_RROT_3] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _RROT_3_r03 }, + { 3, 1, _RROT_3_r13 }, + { 3, 2, _RROT_3_r23 }, + { 3, 3, _RROT_3_r33 }, + }, + }, + [_START_EXECUTOR] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _START_EXECUTOR_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_MAKE_WARM] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _MAKE_WARM_r00 }, + { 1, 1, _MAKE_WARM_r11 }, + { 2, 2, _MAKE_WARM_r22 }, + { 3, 3, _MAKE_WARM_r33 }, + }, + }, + [_FATAL_ERROR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _FATAL_ERROR_r00 }, + { 1, 1, _FATAL_ERROR_r11 }, + { 2, 2, _FATAL_ERROR_r22 }, + { 3, 3, _FATAL_ERROR_r33 }, + }, + }, + [_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _DEOPT_r00 }, + { 0, 0, _DEOPT_r10 }, + { 0, 0, _DEOPT_r20 }, + { 0, 0, _DEOPT_r30 }, + }, + }, + [_HANDLE_PENDING_AND_DEOPT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r00 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r10 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r20 }, + { 0, 0, _HANDLE_PENDING_AND_DEOPT_r30 }, + }, + }, + [_ERROR_POP_N] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _ERROR_POP_N_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_TIER2_RESUME_CHECK] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _TIER2_RESUME_CHECK_r00 }, + { 1, 1, _TIER2_RESUME_CHECK_r11 }, + { 2, 2, _TIER2_RESUME_CHECK_r22 }, + { 3, 3, _TIER2_RESUME_CHECK_r33 }, + }, + }, + [_COLD_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_COLD_DYNAMIC_EXIT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _COLD_DYNAMIC_EXIT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_CODE_VERSION__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_CODE_VERSION__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_CODE_VERSION__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_CODE_VERSION__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_CODE_VERSION_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 }, + }, + }, + [_GUARD_IP__PUSH_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP__PUSH_FRAME_r00 }, + { 1, 1, _GUARD_IP__PUSH_FRAME_r11 }, + { 2, 2, _GUARD_IP__PUSH_FRAME_r22 }, + { 3, 3, _GUARD_IP__PUSH_FRAME_r33 }, + }, + }, + [_GUARD_IP_YIELD_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_YIELD_VALUE_r00 }, + { 1, 1, _GUARD_IP_YIELD_VALUE_r11 }, + { 2, 2, _GUARD_IP_YIELD_VALUE_r22 }, + { 3, 3, _GUARD_IP_YIELD_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_VALUE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_VALUE_r00 }, + { 1, 1, _GUARD_IP_RETURN_VALUE_r11 }, + { 2, 2, _GUARD_IP_RETURN_VALUE_r22 }, + { 3, 3, _GUARD_IP_RETURN_VALUE_r33 }, + }, + }, + [_GUARD_IP_RETURN_GENERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_IP_RETURN_GENERATOR_r00 }, + { 1, 1, _GUARD_IP_RETURN_GENERATOR_r11 }, + { 2, 2, _GUARD_IP_RETURN_GENERATOR_r22 }, + { 3, 3, _GUARD_IP_RETURN_GENERATOR_r33 }, + }, + }, +}; + +const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { + [_NOP_r00] = _NOP, + [_NOP_r11] = _NOP, + [_NOP_r22] = _NOP, + [_NOP_r33] = _NOP, + [_CHECK_PERIODIC_r00] = _CHECK_PERIODIC, + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + [_RESUME_CHECK_r00] = _RESUME_CHECK, + [_RESUME_CHECK_r11] = _RESUME_CHECK, + [_RESUME_CHECK_r22] = _RESUME_CHECK, + [_RESUME_CHECK_r33] = _RESUME_CHECK, + [_LOAD_FAST_CHECK_r01] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r12] = _LOAD_FAST_CHECK, + [_LOAD_FAST_CHECK_r23] = _LOAD_FAST_CHECK, + [_LOAD_FAST_0_r01] = _LOAD_FAST_0, + [_LOAD_FAST_0_r12] = _LOAD_FAST_0, + [_LOAD_FAST_0_r23] = _LOAD_FAST_0, + [_LOAD_FAST_1_r01] = _LOAD_FAST_1, + [_LOAD_FAST_1_r12] = _LOAD_FAST_1, + [_LOAD_FAST_1_r23] = _LOAD_FAST_1, + [_LOAD_FAST_2_r01] = _LOAD_FAST_2, + [_LOAD_FAST_2_r12] = _LOAD_FAST_2, + [_LOAD_FAST_2_r23] = _LOAD_FAST_2, + [_LOAD_FAST_3_r01] = _LOAD_FAST_3, + [_LOAD_FAST_3_r12] = _LOAD_FAST_3, + [_LOAD_FAST_3_r23] = _LOAD_FAST_3, + [_LOAD_FAST_4_r01] = _LOAD_FAST_4, + [_LOAD_FAST_4_r12] = _LOAD_FAST_4, + [_LOAD_FAST_4_r23] = _LOAD_FAST_4, + [_LOAD_FAST_5_r01] = _LOAD_FAST_5, + [_LOAD_FAST_5_r12] = _LOAD_FAST_5, + [_LOAD_FAST_5_r23] = _LOAD_FAST_5, + [_LOAD_FAST_6_r01] = _LOAD_FAST_6, + [_LOAD_FAST_6_r12] = _LOAD_FAST_6, + [_LOAD_FAST_6_r23] = _LOAD_FAST_6, + [_LOAD_FAST_7_r01] = _LOAD_FAST_7, + [_LOAD_FAST_7_r12] = _LOAD_FAST_7, + [_LOAD_FAST_7_r23] = _LOAD_FAST_7, + [_LOAD_FAST_r01] = _LOAD_FAST, + [_LOAD_FAST_r12] = _LOAD_FAST, + [_LOAD_FAST_r23] = _LOAD_FAST, + [_LOAD_FAST_BORROW_0_r01] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r12] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_0_r23] = _LOAD_FAST_BORROW_0, + [_LOAD_FAST_BORROW_1_r01] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r12] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_1_r23] = _LOAD_FAST_BORROW_1, + [_LOAD_FAST_BORROW_2_r01] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r12] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_2_r23] = _LOAD_FAST_BORROW_2, + [_LOAD_FAST_BORROW_3_r01] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r12] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_3_r23] = _LOAD_FAST_BORROW_3, + [_LOAD_FAST_BORROW_4_r01] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r12] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_4_r23] = _LOAD_FAST_BORROW_4, + [_LOAD_FAST_BORROW_5_r01] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r12] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_5_r23] = _LOAD_FAST_BORROW_5, + [_LOAD_FAST_BORROW_6_r01] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r12] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_6_r23] = _LOAD_FAST_BORROW_6, + [_LOAD_FAST_BORROW_7_r01] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r12] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_7_r23] = _LOAD_FAST_BORROW_7, + [_LOAD_FAST_BORROW_r01] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r12] = _LOAD_FAST_BORROW, + [_LOAD_FAST_BORROW_r23] = _LOAD_FAST_BORROW, + [_LOAD_FAST_AND_CLEAR_r01] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r12] = _LOAD_FAST_AND_CLEAR, + [_LOAD_FAST_AND_CLEAR_r23] = _LOAD_FAST_AND_CLEAR, + [_LOAD_CONST_r01] = _LOAD_CONST, + [_LOAD_CONST_r12] = _LOAD_CONST, + [_LOAD_CONST_r23] = _LOAD_CONST, + [_LOAD_SMALL_INT_0_r01] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r12] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_0_r23] = _LOAD_SMALL_INT_0, + [_LOAD_SMALL_INT_1_r01] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r12] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_1_r23] = _LOAD_SMALL_INT_1, + [_LOAD_SMALL_INT_2_r01] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r12] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_2_r23] = _LOAD_SMALL_INT_2, + [_LOAD_SMALL_INT_3_r01] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r12] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_3_r23] = _LOAD_SMALL_INT_3, + [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, + [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, + [_SWAP_FAST_0_r01] = _SWAP_FAST_0, + [_SWAP_FAST_0_r11] = _SWAP_FAST_0, + [_SWAP_FAST_0_r22] = _SWAP_FAST_0, + [_SWAP_FAST_0_r33] = _SWAP_FAST_0, + [_SWAP_FAST_1_r01] = _SWAP_FAST_1, + [_SWAP_FAST_1_r11] = _SWAP_FAST_1, + [_SWAP_FAST_1_r22] = _SWAP_FAST_1, + [_SWAP_FAST_1_r33] = _SWAP_FAST_1, + [_SWAP_FAST_2_r01] = _SWAP_FAST_2, + [_SWAP_FAST_2_r11] = _SWAP_FAST_2, + [_SWAP_FAST_2_r22] = _SWAP_FAST_2, + [_SWAP_FAST_2_r33] = _SWAP_FAST_2, + [_SWAP_FAST_3_r01] = _SWAP_FAST_3, + [_SWAP_FAST_3_r11] = _SWAP_FAST_3, + [_SWAP_FAST_3_r22] = _SWAP_FAST_3, + [_SWAP_FAST_3_r33] = _SWAP_FAST_3, + [_SWAP_FAST_4_r01] = _SWAP_FAST_4, + [_SWAP_FAST_4_r11] = _SWAP_FAST_4, + [_SWAP_FAST_4_r22] = _SWAP_FAST_4, + [_SWAP_FAST_4_r33] = _SWAP_FAST_4, + [_SWAP_FAST_5_r01] = _SWAP_FAST_5, + [_SWAP_FAST_5_r11] = _SWAP_FAST_5, + [_SWAP_FAST_5_r22] = _SWAP_FAST_5, + [_SWAP_FAST_5_r33] = _SWAP_FAST_5, + [_SWAP_FAST_6_r01] = _SWAP_FAST_6, + [_SWAP_FAST_6_r11] = _SWAP_FAST_6, + [_SWAP_FAST_6_r22] = _SWAP_FAST_6, + [_SWAP_FAST_6_r33] = _SWAP_FAST_6, + [_SWAP_FAST_7_r01] = _SWAP_FAST_7, + [_SWAP_FAST_7_r11] = _SWAP_FAST_7, + [_SWAP_FAST_7_r22] = _SWAP_FAST_7, + [_SWAP_FAST_7_r33] = _SWAP_FAST_7, + [_SWAP_FAST_r01] = _SWAP_FAST, + [_SWAP_FAST_r11] = _SWAP_FAST, + [_SWAP_FAST_r22] = _SWAP_FAST, + [_SWAP_FAST_r33] = _SWAP_FAST, + [_POP_TOP_r10] = _POP_TOP, + [_POP_TOP_NOP_r00] = _POP_TOP_NOP, + [_POP_TOP_NOP_r10] = _POP_TOP_NOP, + [_POP_TOP_NOP_r21] = _POP_TOP_NOP, + [_POP_TOP_NOP_r32] = _POP_TOP_NOP, + [_POP_TOP_INT_r00] = _POP_TOP_INT, + [_POP_TOP_INT_r10] = _POP_TOP_INT, + [_POP_TOP_INT_r21] = _POP_TOP_INT, + [_POP_TOP_INT_r32] = _POP_TOP_INT, + [_POP_TOP_FLOAT_r00] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r10] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r21] = _POP_TOP_FLOAT, + [_POP_TOP_FLOAT_r32] = _POP_TOP_FLOAT, + [_POP_TOP_UNICODE_r00] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, + [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, + [_POP_TOP_OPARG_r00] = _POP_TOP_OPARG, + [_PUSH_NULL_r01] = _PUSH_NULL, + [_PUSH_NULL_r12] = _PUSH_NULL, + [_PUSH_NULL_r23] = _PUSH_NULL, + [_END_FOR_r10] = _END_FOR, + [_POP_ITER_r20] = _POP_ITER, + [_END_SEND_r31] = _END_SEND, + [_UNARY_NEGATIVE_r12] = _UNARY_NEGATIVE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = _UNARY_NEGATIVE_FLOAT_INPLACE, + [_UNARY_NOT_r01] = _UNARY_NOT, + [_UNARY_NOT_r11] = _UNARY_NOT, + [_UNARY_NOT_r22] = _UNARY_NOT, + [_UNARY_NOT_r33] = _UNARY_NOT, + [_TO_BOOL_r11] = _TO_BOOL, + [_TO_BOOL_BOOL_r01] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r11] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r22] = _TO_BOOL_BOOL, + [_TO_BOOL_BOOL_r33] = _TO_BOOL_BOOL, + [_TO_BOOL_INT_r02] = _TO_BOOL_INT, + [_TO_BOOL_INT_r12] = _TO_BOOL_INT, + [_TO_BOOL_INT_r23] = _TO_BOOL_INT, + [_GUARD_NOS_LIST_r02] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r12] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r22] = _GUARD_NOS_LIST, + [_GUARD_NOS_LIST_r33] = _GUARD_NOS_LIST, + [_GUARD_TOS_LIST_r01] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r11] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r22] = _GUARD_TOS_LIST, + [_GUARD_TOS_LIST_r33] = _GUARD_TOS_LIST, + [_GUARD_TOS_SLICE_r01] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r11] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r22] = _GUARD_TOS_SLICE, + [_GUARD_TOS_SLICE_r33] = _GUARD_TOS_SLICE, + [_TO_BOOL_LIST_r02] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r12] = _TO_BOOL_LIST, + [_TO_BOOL_LIST_r23] = _TO_BOOL_LIST, + [_TO_BOOL_NONE_r01] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r11] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r22] = _TO_BOOL_NONE, + [_TO_BOOL_NONE_r33] = _TO_BOOL_NONE, + [_GUARD_NOS_COMPACT_ASCII_r02] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r12] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r22] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_COMPACT_ASCII_r33] = _GUARD_NOS_COMPACT_ASCII, + [_GUARD_NOS_UNICODE_r02] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r12] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r22] = _GUARD_NOS_UNICODE, + [_GUARD_NOS_UNICODE_r33] = _GUARD_NOS_UNICODE, + [_GUARD_TOS_UNICODE_r01] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r11] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r22] = _GUARD_TOS_UNICODE, + [_GUARD_TOS_UNICODE_r33] = _GUARD_TOS_UNICODE, + [_TO_BOOL_STR_r02] = _TO_BOOL_STR, + [_TO_BOOL_STR_r12] = _TO_BOOL_STR, + [_TO_BOOL_STR_r23] = _TO_BOOL_STR, + [_REPLACE_WITH_TRUE_r02] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r12] = _REPLACE_WITH_TRUE, + [_REPLACE_WITH_TRUE_r23] = _REPLACE_WITH_TRUE, + [_UNARY_INVERT_r12] = _UNARY_INVERT, + [_GUARD_NOS_INT_r02] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r12] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r22] = _GUARD_NOS_INT, + [_GUARD_NOS_INT_r33] = _GUARD_NOS_INT, + [_GUARD_TOS_INT_r01] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, + [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, + [_GUARD_NOS_OVERFLOWED_r33] = _GUARD_NOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r01] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r11] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r22] = _GUARD_TOS_OVERFLOWED, + [_GUARD_TOS_OVERFLOWED_r33] = _GUARD_TOS_OVERFLOWED, + [_BINARY_OP_MULTIPLY_INT_r03] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r13] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_MULTIPLY_INT_r23] = _BINARY_OP_MULTIPLY_INT, + [_BINARY_OP_ADD_INT_r03] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r13] = _BINARY_OP_ADD_INT, + [_BINARY_OP_ADD_INT_r23] = _BINARY_OP_ADD_INT, + [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, + [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_r23] = _BINARY_OP_ADD_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + [_GUARD_NOS_FLOAT_r02] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r12] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r22] = _GUARD_NOS_FLOAT, + [_GUARD_NOS_FLOAT_r33] = _GUARD_NOS_FLOAT, + [_GUARD_TOS_FLOAT_r01] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r11] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r22] = _GUARD_TOS_FLOAT, + [_GUARD_TOS_FLOAT_r33] = _GUARD_TOS_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r03] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r13] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_MULTIPLY_FLOAT_r23] = _BINARY_OP_MULTIPLY_FLOAT, + [_BINARY_OP_ADD_FLOAT_r03] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r13] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_ADD_FLOAT_r23] = _BINARY_OP_ADD_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r03] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r13] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_SUBTRACT_FLOAT_r23] = _BINARY_OP_SUBTRACT_FLOAT, + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = _BINARY_OP_ADD_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_r23] = _BINARY_OP_TRUEDIV_FLOAT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_ADD_UNICODE_r03] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r13] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_ADD_UNICODE_r23] = _BINARY_OP_ADD_UNICODE, + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_r22] = _GUARD_BINARY_OP_EXTEND, + [_BINARY_OP_EXTEND_r23] = _BINARY_OP_EXTEND, + [_BINARY_SLICE_r31] = _BINARY_SLICE, + [_STORE_SLICE_r30] = _STORE_SLICE, + [_BINARY_OP_SUBSCR_LIST_INT_r23] = _BINARY_OP_SUBSCR_LIST_INT, + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = _BINARY_OP_SUBSCR_LIST_SLICE, + [_BINARY_OP_SUBSCR_STR_INT_r23] = _BINARY_OP_SUBSCR_STR_INT, + [_BINARY_OP_SUBSCR_USTR_INT_r23] = _BINARY_OP_SUBSCR_USTR_INT, + [_GUARD_NOS_TUPLE_r02] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r12] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r22] = _GUARD_NOS_TUPLE, + [_GUARD_NOS_TUPLE_r33] = _GUARD_NOS_TUPLE, + [_GUARD_TOS_TUPLE_r01] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, + [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_TOS_ANY_DICT_r01] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r11] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r22] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_ANY_DICT_r33] = _GUARD_TOS_ANY_DICT, + [_GUARD_TOS_DICT_r01] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r11] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r22] = _GUARD_TOS_DICT, + [_GUARD_TOS_DICT_r33] = _GUARD_TOS_DICT, + [_GUARD_TOS_FROZENDICT_r01] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r11] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r22] = _GUARD_TOS_FROZENDICT, + [_GUARD_TOS_FROZENDICT_r33] = _GUARD_TOS_FROZENDICT, + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, + [_BINARY_OP_SUBSCR_DICT_r23] = _BINARY_OP_SUBSCR_DICT, + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = _BINARY_OP_SUBSCR_CHECK_FUNC, + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = _BINARY_OP_SUBSCR_INIT_CALL, + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = _BINARY_OP_SUBSCR_INIT_CALL, + [_LIST_APPEND_r10] = _LIST_APPEND, + [_SET_ADD_r10] = _SET_ADD, + [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, + [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, + [_DELETE_SUBSCR_r20] = _DELETE_SUBSCR, + [_CALL_INTRINSIC_1_r12] = _CALL_INTRINSIC_1, + [_CALL_INTRINSIC_2_r23] = _CALL_INTRINSIC_2, + [_MAKE_HEAP_SAFE_r01] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r11] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r22] = _MAKE_HEAP_SAFE, + [_MAKE_HEAP_SAFE_r33] = _MAKE_HEAP_SAFE, + [_RETURN_VALUE_r11] = _RETURN_VALUE, + [_GET_AITER_r11] = _GET_AITER, + [_GET_ANEXT_r12] = _GET_ANEXT, + [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, + [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r22] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r33] = _GUARD_TOS_IS_NONE, + [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_SEND_VIRTUAL_TIER_TWO_r03] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r13] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r23] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r33] = _SEND_VIRTUAL_TIER_TWO, + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, + [_YIELD_VALUE_r11] = _YIELD_VALUE, + [_POP_EXCEPT_r10] = _POP_EXCEPT, + [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r12] = _LOAD_COMMON_CONSTANT, + [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, + [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, + [_STORE_NAME_r10] = _STORE_NAME, + [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + [_UNPACK_SEQUENCE_TUPLE_r10] = _UNPACK_SEQUENCE_TUPLE, + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = _UNPACK_SEQUENCE_UNIQUE_TUPLE, + [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, + [_UNPACK_EX_r10] = _UNPACK_EX, + [_STORE_ATTR_r20] = _STORE_ATTR, + [_STORE_GLOBAL_r10] = _STORE_GLOBAL, + [_LOAD_LOCALS_r01] = _LOAD_LOCALS, + [_LOAD_LOCALS_r12] = _LOAD_LOCALS, + [_LOAD_LOCALS_r23] = _LOAD_LOCALS, + [_LOAD_NAME_r01] = _LOAD_NAME, + [_LOAD_GLOBAL_r00] = _LOAD_GLOBAL, + [_PUSH_NULL_CONDITIONAL_r00] = _PUSH_NULL_CONDITIONAL, + [_GUARD_GLOBALS_VERSION_r00] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r11] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r22] = _GUARD_GLOBALS_VERSION, + [_GUARD_GLOBALS_VERSION_r33] = _GUARD_GLOBALS_VERSION, + [_LOAD_GLOBAL_MODULE_r01] = _LOAD_GLOBAL_MODULE, + [_LOAD_GLOBAL_BUILTINS_r01] = _LOAD_GLOBAL_BUILTINS, + [_DELETE_FAST_r00] = _DELETE_FAST, + [_MAKE_CELL_r00] = _MAKE_CELL, + [_DELETE_DEREF_r00] = _DELETE_DEREF, + [_LOAD_FROM_DICT_OR_DEREF_r11] = _LOAD_FROM_DICT_OR_DEREF, + [_LOAD_DEREF_r01] = _LOAD_DEREF, + [_STORE_DEREF_r10] = _STORE_DEREF, + [_COPY_FREE_VARS_r00] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r11] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r22] = _COPY_FREE_VARS, + [_COPY_FREE_VARS_r33] = _COPY_FREE_VARS, + [_BUILD_STRING_r01] = _BUILD_STRING, + [_BUILD_INTERPOLATION_r01] = _BUILD_INTERPOLATION, + [_BUILD_TEMPLATE_r21] = _BUILD_TEMPLATE, + [_BUILD_TUPLE_r01] = _BUILD_TUPLE, + [_BUILD_LIST_r01] = _BUILD_LIST, + [_LIST_EXTEND_r11] = _LIST_EXTEND, + [_SET_UPDATE_r11] = _SET_UPDATE, + [_BUILD_SET_r01] = _BUILD_SET, + [_BUILD_MAP_r01] = _BUILD_MAP, + [_SETUP_ANNOTATIONS_r00] = _SETUP_ANNOTATIONS, + [_DICT_UPDATE_r11] = _DICT_UPDATE, + [_DICT_MERGE_r11] = _DICT_MERGE, + [_MAP_ADD_r20] = _MAP_ADD, + [_LOAD_SUPER_ATTR_ATTR_r31] = _LOAD_SUPER_ATTR_ATTR, + [_GUARD_NOS_TYPE_VERSION_r02] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r12] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r22] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_NOS_TYPE_VERSION_r33] = _GUARD_NOS_TYPE_VERSION, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = _GUARD_LOAD_SUPER_ATTR_METHOD, + [_LOAD_SUPER_ATTR_METHOD_r32] = _LOAD_SUPER_ATTR_METHOD, + [_LOAD_ATTR_r10] = _LOAD_ATTR, + [_GUARD_TYPE_VERSION_r01] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r11] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r22] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_r33] = _GUARD_TYPE_VERSION, + [_GUARD_TYPE_VERSION_LOCKED_r01] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r11] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r22] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_VERSION_LOCKED_r33] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_r01] = _GUARD_TYPE, + [_GUARD_TYPE_r11] = _GUARD_TYPE, + [_GUARD_TYPE_r22] = _GUARD_TYPE, + [_GUARD_TYPE_r33] = _GUARD_TYPE, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = _CHECK_MANAGED_OBJECT_HAS_VALUES, + [_LOAD_ATTR_INSTANCE_VALUE_r02] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r12] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_INSTANCE_VALUE_r23] = _LOAD_ATTR_INSTANCE_VALUE, + [_LOAD_ATTR_MODULE_r12] = _LOAD_ATTR_MODULE, + [_LOAD_ATTR_WITH_HINT_r12] = _LOAD_ATTR_WITH_HINT, + [_LOAD_ATTR_SLOT_r02] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r12] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r23] = _LOAD_ATTR_SLOT, + [_CHECK_ATTR_CLASS_r01] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, + [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, + [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r01] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r22] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r33] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, + [_GUARD_DORV_NO_DICT_r01] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, + [_GUARD_DORV_NO_DICT_r33] = _GUARD_DORV_NO_DICT, + [_STORE_ATTR_INSTANCE_VALUE_r21] = _STORE_ATTR_INSTANCE_VALUE, + [_LOCK_OBJECT_r01] = _LOCK_OBJECT, + [_LOCK_OBJECT_r11] = _LOCK_OBJECT, + [_LOCK_OBJECT_r22] = _LOCK_OBJECT, + [_LOCK_OBJECT_r33] = _LOCK_OBJECT, + [_STORE_ATTR_WITH_HINT_r21] = _STORE_ATTR_WITH_HINT, + [_STORE_ATTR_SLOT_r21] = _STORE_ATTR_SLOT, + [_COMPARE_OP_r21] = _COMPARE_OP, + [_COMPARE_OP_FLOAT_r03] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r13] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r23] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_INT_r23] = _COMPARE_OP_INT, + [_COMPARE_OP_STR_r23] = _COMPARE_OP_STR, + [_IS_OP_r03] = _IS_OP, + [_IS_OP_r13] = _IS_OP, + [_IS_OP_r23] = _IS_OP, + [_CONTAINS_OP_r23] = _CONTAINS_OP, + [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r22] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_ANY_SET_r33] = _GUARD_TOS_ANY_SET, + [_GUARD_TOS_SET_r01] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r11] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r22] = _GUARD_TOS_SET, + [_GUARD_TOS_SET_r33] = _GUARD_TOS_SET, + [_GUARD_TOS_FROZENSET_r01] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r11] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r22] = _GUARD_TOS_FROZENSET, + [_GUARD_TOS_FROZENSET_r33] = _GUARD_TOS_FROZENSET, + [_CONTAINS_OP_SET_r23] = _CONTAINS_OP_SET, + [_CONTAINS_OP_DICT_r23] = _CONTAINS_OP_DICT, + [_CHECK_EG_MATCH_r22] = _CHECK_EG_MATCH, + [_CHECK_EXC_MATCH_r22] = _CHECK_EXC_MATCH, + [_IMPORT_NAME_r21] = _IMPORT_NAME, + [_IMPORT_FROM_r12] = _IMPORT_FROM, + [_IS_NONE_r11] = _IS_NONE, + [_GET_LEN_r12] = _GET_LEN, + [_MATCH_CLASS_r33] = _MATCH_CLASS, + [_MATCH_MAPPING_r02] = _MATCH_MAPPING, + [_MATCH_MAPPING_r12] = _MATCH_MAPPING, + [_MATCH_MAPPING_r23] = _MATCH_MAPPING, + [_MATCH_SEQUENCE_r02] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r12] = _MATCH_SEQUENCE, + [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, + [_MATCH_KEYS_r23] = _MATCH_KEYS, + [_GET_ITER_r12] = _GET_ITER, + [_GUARD_ITERATOR_r01] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r11] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r22] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r33] = _GUARD_ITERATOR, + [_GUARD_ITER_VIRTUAL_r01] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r11] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r22] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r33] = _GUARD_ITER_VIRTUAL, + [_PUSH_TAGGED_ZERO_r01] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r12] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, + [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, + [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_GUARD_TYPE_ITER_r02] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r12] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r22] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r33] = _GUARD_TYPE_ITER, + [_ITER_NEXT_INLINE_r23] = _ITER_NEXT_INLINE, + [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r22] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r33] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_TOS_NOT_NULL_r01] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r11] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r22] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r33] = _GUARD_TOS_NOT_NULL, + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = _FOR_ITER_VIRTUAL_TIER_TWO, + [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, + [_ITER_CHECK_LIST_r33] = _ITER_CHECK_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r02] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r12] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r22] = _GUARD_NOT_EXHAUSTED_LIST, + [_GUARD_NOT_EXHAUSTED_LIST_r33] = _GUARD_NOT_EXHAUSTED_LIST, + [_ITER_NEXT_LIST_TIER_TWO_r23] = _ITER_NEXT_LIST_TIER_TWO, + [_ITER_CHECK_TUPLE_r02] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r12] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r22] = _ITER_CHECK_TUPLE, + [_ITER_CHECK_TUPLE_r33] = _ITER_CHECK_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = _GUARD_NOT_EXHAUSTED_TUPLE, + [_ITER_NEXT_TUPLE_r03] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r13] = _ITER_NEXT_TUPLE, + [_ITER_NEXT_TUPLE_r23] = _ITER_NEXT_TUPLE, + [_ITER_CHECK_RANGE_r02] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r12] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r22] = _ITER_CHECK_RANGE, + [_ITER_CHECK_RANGE_r33] = _ITER_CHECK_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = _GUARD_NOT_EXHAUSTED_RANGE, + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = _GUARD_NOT_EXHAUSTED_RANGE, + [_ITER_NEXT_RANGE_r03] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r13] = _ITER_NEXT_RANGE, + [_ITER_NEXT_RANGE_r23] = _ITER_NEXT_RANGE, + [_FOR_ITER_GEN_FRAME_r03] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r13] = _FOR_ITER_GEN_FRAME, + [_FOR_ITER_GEN_FRAME_r23] = _FOR_ITER_GEN_FRAME, + [_INSERT_NULL_r10] = _INSERT_NULL, + [_LOAD_SPECIAL_r00] = _LOAD_SPECIAL, + [_WITH_EXCEPT_START_r33] = _WITH_EXCEPT_START, + [_PUSH_EXC_INFO_r02] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, + [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, + [_LOAD_ATTR_METHOD_NO_DICT_r02] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r12] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_METHOD_NO_DICT_r23] = _LOAD_ATTR_METHOD_NO_DICT, + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = _CHECK_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = _LOAD_ATTR_METHOD_LAZY_DICT, + [_MAYBE_EXPAND_METHOD_r00] = _MAYBE_EXPAND_METHOD, + [_PY_FRAME_GENERAL_r01] = _PY_FRAME_GENERAL, + [_CHECK_FUNCTION_VERSION_r00] = _CHECK_FUNCTION_VERSION, + [_CHECK_FUNCTION_VERSION_INLINE_r00] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r11] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r22] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_FUNCTION_VERSION_INLINE_r33] = _CHECK_FUNCTION_VERSION_INLINE, + [_CHECK_METHOD_VERSION_r00] = _CHECK_METHOD_VERSION, + [_EXPAND_METHOD_r00] = _EXPAND_METHOD, + [_CHECK_IS_NOT_PY_CALLABLE_r00] = _CHECK_IS_NOT_PY_CALLABLE, + [_CALL_NON_PY_GENERAL_r01] = _CALL_NON_PY_GENERAL, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + [_CHECK_PEP_523_r00] = _CHECK_PEP_523, + [_CHECK_PEP_523_r11] = _CHECK_PEP_523, + [_CHECK_PEP_523_r22] = _CHECK_PEP_523, + [_CHECK_PEP_523_r33] = _CHECK_PEP_523, + [_CHECK_FUNCTION_EXACT_ARGS_r00] = _CHECK_FUNCTION_EXACT_ARGS, + [_CHECK_STACK_SPACE_r00] = _CHECK_STACK_SPACE, + [_CHECK_RECURSION_REMAINING_r00] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r11] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r22] = _CHECK_RECURSION_REMAINING, + [_CHECK_RECURSION_REMAINING_r33] = _CHECK_RECURSION_REMAINING, + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = _INIT_CALL_PY_EXACT_ARGS_0, + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = _INIT_CALL_PY_EXACT_ARGS_1, + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = _INIT_CALL_PY_EXACT_ARGS_2, + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = _INIT_CALL_PY_EXACT_ARGS_3, + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = _INIT_CALL_PY_EXACT_ARGS_4, + [_INIT_CALL_PY_EXACT_ARGS_r01] = _INIT_CALL_PY_EXACT_ARGS, + [_PUSH_FRAME_r10] = _PUSH_FRAME, + [_GUARD_NOS_NULL_r02] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r12] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, + [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, + [_GUARD_THIRD_NULL_r03] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r13] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r23] = _GUARD_THIRD_NULL, + [_GUARD_THIRD_NULL_r33] = _GUARD_THIRD_NULL, + [_GUARD_CALLABLE_TYPE_1_r03] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r13] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r23] = _GUARD_CALLABLE_TYPE_1, + [_GUARD_CALLABLE_TYPE_1_r33] = _GUARD_CALLABLE_TYPE_1, + [_CALL_TYPE_1_r02] = _CALL_TYPE_1, + [_CALL_TYPE_1_r12] = _CALL_TYPE_1, + [_CALL_TYPE_1_r22] = _CALL_TYPE_1, + [_CALL_TYPE_1_r32] = _CALL_TYPE_1, + [_GUARD_CALLABLE_STR_1_r03] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r13] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r23] = _GUARD_CALLABLE_STR_1, + [_GUARD_CALLABLE_STR_1_r33] = _GUARD_CALLABLE_STR_1, + [_CALL_STR_1_r32] = _CALL_STR_1, + [_GUARD_CALLABLE_TUPLE_1_r03] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r13] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r23] = _GUARD_CALLABLE_TUPLE_1, + [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, + [_CALL_TUPLE_1_r32] = _CALL_TUPLE_1, + [_CHECK_OBJECT_r00] = _CHECK_OBJECT, + [_ALLOCATE_OBJECT_r00] = _ALLOCATE_OBJECT, + [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, + [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = _GUARD_CALLABLE_BUILTIN_CLASS, + [_CALL_BUILTIN_CLASS_r00] = _CALL_BUILTIN_CLASS, + [_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O, + [_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O, + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_r00] = _CALL_BUILTIN_FAST, + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_GUARD_CALLABLE_LEN_r03] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r23] = _GUARD_CALLABLE_LEN, + [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, + [_CALL_LEN_r33] = _CALL_LEN, + [_GUARD_CALLABLE_ISINSTANCE_r03] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r13] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r23] = _GUARD_CALLABLE_ISINSTANCE, + [_GUARD_CALLABLE_ISINSTANCE_r33] = _GUARD_CALLABLE_ISINSTANCE, + [_CALL_ISINSTANCE_r31] = _CALL_ISINSTANCE, + [_GUARD_CALLABLE_LIST_APPEND_r03] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r13] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r23] = _GUARD_CALLABLE_LIST_APPEND, + [_GUARD_CALLABLE_LIST_APPEND_r33] = _GUARD_CALLABLE_LIST_APPEND, + [_CALL_LIST_APPEND_r03] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r13] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r23] = _CALL_LIST_APPEND, + [_CALL_LIST_APPEND_r33] = _CALL_LIST_APPEND, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_O_r03] = _CALL_METHOD_DESCRIPTOR_O, + [_CHECK_RECURSION_LIMIT_r00] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r11] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r22] = _CHECK_RECURSION_LIMIT, + [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS, + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, + [_PY_FRAME_KW_r11] = _PY_FRAME_KW, + [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, + [_CHECK_METHOD_VERSION_KW_r11] = _CHECK_METHOD_VERSION_KW, + [_EXPAND_METHOD_KW_r11] = _EXPAND_METHOD_KW, + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = _CHECK_IS_NOT_PY_CALLABLE_KW, + [_CALL_KW_NON_PY_r11] = _CALL_KW_NON_PY, + [_MAKE_CALLARGS_A_TUPLE_r33] = _MAKE_CALLARGS_A_TUPLE, + [_CHECK_IS_PY_CALLABLE_EX_r03] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r13] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r23] = _CHECK_IS_PY_CALLABLE_EX, + [_CHECK_IS_PY_CALLABLE_EX_r33] = _CHECK_IS_PY_CALLABLE_EX, + [_PY_FRAME_EX_r31] = _PY_FRAME_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = _CHECK_IS_NOT_PY_CALLABLE_EX, + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = _CALL_FUNCTION_EX_NON_PY_GENERAL, + [_MAKE_FUNCTION_r12] = _MAKE_FUNCTION, + [_SET_FUNCTION_ATTRIBUTE_r01] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r11] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, + [_SET_FUNCTION_ATTRIBUTE_r32] = _SET_FUNCTION_ATTRIBUTE, + [_RETURN_GENERATOR_r01] = _RETURN_GENERATOR, + [_BUILD_SLICE_r01] = _BUILD_SLICE, + [_CONVERT_VALUE_r11] = _CONVERT_VALUE, + [_FORMAT_SIMPLE_r11] = _FORMAT_SIMPLE, + [_FORMAT_WITH_SPEC_r21] = _FORMAT_WITH_SPEC, + [_COPY_1_r02] = _COPY_1, + [_COPY_1_r12] = _COPY_1, + [_COPY_1_r23] = _COPY_1, + [_COPY_2_r03] = _COPY_2, + [_COPY_2_r13] = _COPY_2, + [_COPY_2_r23] = _COPY_2, + [_COPY_3_r03] = _COPY_3, + [_COPY_3_r13] = _COPY_3, + [_COPY_3_r23] = _COPY_3, + [_COPY_3_r33] = _COPY_3, + [_COPY_r01] = _COPY, + [_BINARY_OP_r23] = _BINARY_OP, + [_SWAP_2_r02] = _SWAP_2, + [_SWAP_2_r12] = _SWAP_2, + [_SWAP_2_r22] = _SWAP_2, + [_SWAP_2_r33] = _SWAP_2, + [_SWAP_3_r03] = _SWAP_3, + [_SWAP_3_r13] = _SWAP_3, + [_SWAP_3_r23] = _SWAP_3, + [_SWAP_3_r33] = _SWAP_3, + [_SWAP_r11] = _SWAP, + [_GUARD_IS_TRUE_POP_r00] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r10] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r21] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_TRUE_POP_r32] = _GUARD_IS_TRUE_POP, + [_GUARD_IS_FALSE_POP_r00] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r10] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r21] = _GUARD_IS_FALSE_POP, + [_GUARD_IS_FALSE_POP_r32] = _GUARD_IS_FALSE_POP, + [_GUARD_BIT_IS_SET_POP_4_r00] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r10] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r21] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_4_r32] = _GUARD_BIT_IS_SET_POP_4, + [_GUARD_BIT_IS_SET_POP_5_r00] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r10] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r21] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_5_r32] = _GUARD_BIT_IS_SET_POP_5, + [_GUARD_BIT_IS_SET_POP_6_r00] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r10] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r21] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_6_r32] = _GUARD_BIT_IS_SET_POP_6, + [_GUARD_BIT_IS_SET_POP_7_r00] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r10] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r21] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_7_r32] = _GUARD_BIT_IS_SET_POP_7, + [_GUARD_BIT_IS_SET_POP_r00] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r10] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r21] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_SET_POP_r32] = _GUARD_BIT_IS_SET_POP, + [_GUARD_BIT_IS_UNSET_POP_4_r00] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r10] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r21] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_4_r32] = _GUARD_BIT_IS_UNSET_POP_4, + [_GUARD_BIT_IS_UNSET_POP_5_r00] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r10] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r21] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_5_r32] = _GUARD_BIT_IS_UNSET_POP_5, + [_GUARD_BIT_IS_UNSET_POP_6_r00] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r10] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r21] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_6_r32] = _GUARD_BIT_IS_UNSET_POP_6, + [_GUARD_BIT_IS_UNSET_POP_7_r00] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r10] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r21] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_7_r32] = _GUARD_BIT_IS_UNSET_POP_7, + [_GUARD_BIT_IS_UNSET_POP_r00] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r10] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r21] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_BIT_IS_UNSET_POP_r32] = _GUARD_BIT_IS_UNSET_POP, + [_GUARD_IS_NONE_POP_r00] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r10] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r21] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NONE_POP_r32] = _GUARD_IS_NONE_POP, + [_GUARD_IS_NOT_NONE_POP_r10] = _GUARD_IS_NOT_NONE_POP, + [_JUMP_TO_TOP_r00] = _JUMP_TO_TOP, + [_SET_IP_r00] = _SET_IP, + [_SET_IP_r11] = _SET_IP, + [_SET_IP_r22] = _SET_IP, + [_SET_IP_r33] = _SET_IP, + [_CHECK_STACK_SPACE_OPERAND_r00] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r11] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r22] = _CHECK_STACK_SPACE_OPERAND, + [_CHECK_STACK_SPACE_OPERAND_r33] = _CHECK_STACK_SPACE_OPERAND, + [_SAVE_RETURN_OFFSET_r00] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r11] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r22] = _SAVE_RETURN_OFFSET, + [_SAVE_RETURN_OFFSET_r33] = _SAVE_RETURN_OFFSET, + [_EXIT_TRACE_r00] = _EXIT_TRACE, + [_EXIT_TRACE_r10] = _EXIT_TRACE, + [_EXIT_TRACE_r20] = _EXIT_TRACE, + [_EXIT_TRACE_r30] = _EXIT_TRACE, + [_DYNAMIC_EXIT_r00] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r10] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r20] = _DYNAMIC_EXIT, + [_DYNAMIC_EXIT_r30] = _DYNAMIC_EXIT, + [_CHECK_VALIDITY_r00] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r11] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r22] = _CHECK_VALIDITY, + [_CHECK_VALIDITY_r33] = _CHECK_VALIDITY, + [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, + [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, + [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, + [_RROT_3_r03] = _RROT_3, + [_RROT_3_r13] = _RROT_3, + [_RROT_3_r23] = _RROT_3, + [_RROT_3_r33] = _RROT_3, + [_START_EXECUTOR_r00] = _START_EXECUTOR, + [_MAKE_WARM_r00] = _MAKE_WARM, + [_MAKE_WARM_r11] = _MAKE_WARM, + [_MAKE_WARM_r22] = _MAKE_WARM, + [_MAKE_WARM_r33] = _MAKE_WARM, + [_FATAL_ERROR_r00] = _FATAL_ERROR, + [_FATAL_ERROR_r11] = _FATAL_ERROR, + [_FATAL_ERROR_r22] = _FATAL_ERROR, + [_FATAL_ERROR_r33] = _FATAL_ERROR, + [_DEOPT_r00] = _DEOPT, + [_DEOPT_r10] = _DEOPT, + [_DEOPT_r20] = _DEOPT, + [_DEOPT_r30] = _DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r00] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r10] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r20] = _HANDLE_PENDING_AND_DEOPT, + [_HANDLE_PENDING_AND_DEOPT_r30] = _HANDLE_PENDING_AND_DEOPT, + [_ERROR_POP_N_r00] = _ERROR_POP_N, + [_SPILL_OR_RELOAD_r01] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r02] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r03] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r10] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r12] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r13] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r20] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r21] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r23] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r30] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r31] = _SPILL_OR_RELOAD, + [_SPILL_OR_RELOAD_r32] = _SPILL_OR_RELOAD, + [_TIER2_RESUME_CHECK_r00] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r11] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r22] = _TIER2_RESUME_CHECK, + [_TIER2_RESUME_CHECK_r33] = _TIER2_RESUME_CHECK, + [_COLD_EXIT_r00] = _COLD_EXIT, + [_COLD_DYNAMIC_EXIT_r00] = _COLD_DYNAMIC_EXIT, + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = _GUARD_CODE_VERSION__PUSH_FRAME, + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = _GUARD_CODE_VERSION_YIELD_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = _GUARD_CODE_VERSION_RETURN_VALUE, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = _GUARD_CODE_VERSION_RETURN_GENERATOR, + [_GUARD_IP__PUSH_FRAME_r00] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r11] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r22] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP__PUSH_FRAME_r33] = _GUARD_IP__PUSH_FRAME, + [_GUARD_IP_YIELD_VALUE_r00] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r11] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r22] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_YIELD_VALUE_r33] = _GUARD_IP_YIELD_VALUE, + [_GUARD_IP_RETURN_VALUE_r00] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r11] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r22] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_VALUE_r33] = _GUARD_IP_RETURN_VALUE, + [_GUARD_IP_RETURN_GENERATOR_r00] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r11] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r22] = _GUARD_IP_RETURN_GENERATOR, + [_GUARD_IP_RETURN_GENERATOR_r33] = _GUARD_IP_RETURN_GENERATOR, +}; + +const uint16_t _PyUop_SpillsAndReloads[4][4] = { + [0][1] = _SPILL_OR_RELOAD_r01, + [0][2] = _SPILL_OR_RELOAD_r02, + [0][3] = _SPILL_OR_RELOAD_r03, + [1][0] = _SPILL_OR_RELOAD_r10, + [1][2] = _SPILL_OR_RELOAD_r12, + [1][3] = _SPILL_OR_RELOAD_r13, + [2][0] = _SPILL_OR_RELOAD_r20, + [2][1] = _SPILL_OR_RELOAD_r21, + [2][3] = _SPILL_OR_RELOAD_r23, + [3][0] = _SPILL_OR_RELOAD_r30, + [3][1] = _SPILL_OR_RELOAD_r31, + [3][2] = _SPILL_OR_RELOAD_r32, }; -const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { +const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { + [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", + [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", [_BINARY_OP] = "_BINARY_OP", + [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", - [_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_ADD_FLOAT_r03] = "_BINARY_OP_ADD_FLOAT_r03", + [_BINARY_OP_ADD_FLOAT_r13] = "_BINARY_OP_ADD_FLOAT_r13", + [_BINARY_OP_ADD_FLOAT_r23] = "_BINARY_OP_ADD_FLOAT_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE] = "_BINARY_OP_ADD_FLOAT_INPLACE", + [_BINARY_OP_ADD_FLOAT_INPLACE_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_r23", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", + [_BINARY_OP_ADD_INT_r03] = "_BINARY_OP_ADD_INT_r03", + [_BINARY_OP_ADD_INT_r13] = "_BINARY_OP_ADD_INT_r13", + [_BINARY_OP_ADD_INT_r23] = "_BINARY_OP_ADD_INT_r23", + [_BINARY_OP_ADD_INT_INPLACE] = "_BINARY_OP_ADD_INT_INPLACE", + [_BINARY_OP_ADD_INT_INPLACE_r03] = "_BINARY_OP_ADD_INT_INPLACE_r03", + [_BINARY_OP_ADD_INT_INPLACE_r13] = "_BINARY_OP_ADD_INT_INPLACE_r13", + [_BINARY_OP_ADD_INT_INPLACE_r23] = "_BINARY_OP_ADD_INT_INPLACE_r23", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_ADD_INT_INPLACE_RIGHT_r23", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_BINARY_OP_ADD_UNICODE_r03] = "_BINARY_OP_ADD_UNICODE_r03", + [_BINARY_OP_ADD_UNICODE_r13] = "_BINARY_OP_ADD_UNICODE_r13", + [_BINARY_OP_ADD_UNICODE_r23] = "_BINARY_OP_ADD_UNICODE_r23", [_BINARY_OP_EXTEND] = "_BINARY_OP_EXTEND", + [_BINARY_OP_EXTEND_r23] = "_BINARY_OP_EXTEND_r23", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = "_BINARY_OP_INPLACE_ADD_UNICODE_r21", [_BINARY_OP_MULTIPLY_FLOAT] = "_BINARY_OP_MULTIPLY_FLOAT", - [_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_MULTIPLY_FLOAT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", + [_BINARY_OP_MULTIPLY_INT_r03] = "_BINARY_OP_MULTIPLY_INT_r03", + [_BINARY_OP_MULTIPLY_INT_r13] = "_BINARY_OP_MULTIPLY_INT_r13", + [_BINARY_OP_MULTIPLY_INT_r23] = "_BINARY_OP_MULTIPLY_INT_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE] = "_BINARY_OP_MULTIPLY_INT_INPLACE", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_r23", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBSCR_CHECK_FUNC] = "_BINARY_OP_SUBSCR_CHECK_FUNC", + [_BINARY_OP_SUBSCR_CHECK_FUNC_r23] = "_BINARY_OP_SUBSCR_CHECK_FUNC_r23", [_BINARY_OP_SUBSCR_DICT] = "_BINARY_OP_SUBSCR_DICT", + [_BINARY_OP_SUBSCR_DICT_r23] = "_BINARY_OP_SUBSCR_DICT_r23", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH", + [_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23] = "_BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23", [_BINARY_OP_SUBSCR_INIT_CALL] = "_BINARY_OP_SUBSCR_INIT_CALL", + [_BINARY_OP_SUBSCR_INIT_CALL_r01] = "_BINARY_OP_SUBSCR_INIT_CALL_r01", + [_BINARY_OP_SUBSCR_INIT_CALL_r11] = "_BINARY_OP_SUBSCR_INIT_CALL_r11", + [_BINARY_OP_SUBSCR_INIT_CALL_r21] = "_BINARY_OP_SUBSCR_INIT_CALL_r21", + [_BINARY_OP_SUBSCR_INIT_CALL_r31] = "_BINARY_OP_SUBSCR_INIT_CALL_r31", [_BINARY_OP_SUBSCR_LIST_INT] = "_BINARY_OP_SUBSCR_LIST_INT", + [_BINARY_OP_SUBSCR_LIST_INT_r23] = "_BINARY_OP_SUBSCR_LIST_INT_r23", [_BINARY_OP_SUBSCR_LIST_SLICE] = "_BINARY_OP_SUBSCR_LIST_SLICE", + [_BINARY_OP_SUBSCR_LIST_SLICE_r23] = "_BINARY_OP_SUBSCR_LIST_SLICE_r23", [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", + [_BINARY_OP_SUBSCR_STR_INT_r23] = "_BINARY_OP_SUBSCR_STR_INT_r23", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = "_BINARY_OP_SUBSCR_TUPLE_INT_r03", + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = "_BINARY_OP_SUBSCR_TUPLE_INT_r13", + [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = "_BINARY_OP_SUBSCR_TUPLE_INT_r23", + [_BINARY_OP_SUBSCR_USTR_INT] = "_BINARY_OP_SUBSCR_USTR_INT", + [_BINARY_OP_SUBSCR_USTR_INT_r23] = "_BINARY_OP_SUBSCR_USTR_INT_r23", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", - [_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS] = "_BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS", + [_BINARY_OP_SUBTRACT_FLOAT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", + [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", + [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", + [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r23", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13", + [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23", + [_BINARY_OP_TRUEDIV_FLOAT] = "_BINARY_OP_TRUEDIV_FLOAT", + [_BINARY_OP_TRUEDIV_FLOAT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23", [_BINARY_SLICE] = "_BINARY_SLICE", + [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", + [_BUILD_INTERPOLATION_r01] = "_BUILD_INTERPOLATION_r01", [_BUILD_LIST] = "_BUILD_LIST", + [_BUILD_LIST_r01] = "_BUILD_LIST_r01", [_BUILD_MAP] = "_BUILD_MAP", + [_BUILD_MAP_r01] = "_BUILD_MAP_r01", [_BUILD_SET] = "_BUILD_SET", + [_BUILD_SET_r01] = "_BUILD_SET_r01", [_BUILD_SLICE] = "_BUILD_SLICE", + [_BUILD_SLICE_r01] = "_BUILD_SLICE_r01", [_BUILD_STRING] = "_BUILD_STRING", + [_BUILD_STRING_r01] = "_BUILD_STRING_r01", [_BUILD_TEMPLATE] = "_BUILD_TEMPLATE", + [_BUILD_TEMPLATE_r21] = "_BUILD_TEMPLATE_r21", [_BUILD_TUPLE] = "_BUILD_TUPLE", + [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", + [_CALL_BUILTIN_CLASS_r00] = "_CALL_BUILTIN_CLASS_r00", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", + [_CALL_BUILTIN_FAST_r00] = "_CALL_BUILTIN_FAST_r00", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", + [_CALL_BUILTIN_O_r03] = "_CALL_BUILTIN_O_r03", + [_CALL_FUNCTION_EX_NON_PY_GENERAL] = "_CALL_FUNCTION_EX_NON_PY_GENERAL", + [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = "_CALL_FUNCTION_EX_NON_PY_GENERAL_r31", [_CALL_INTRINSIC_1] = "_CALL_INTRINSIC_1", + [_CALL_INTRINSIC_1_r12] = "_CALL_INTRINSIC_1_r12", [_CALL_INTRINSIC_2] = "_CALL_INTRINSIC_2", + [_CALL_INTRINSIC_2_r23] = "_CALL_INTRINSIC_2_r23", [_CALL_ISINSTANCE] = "_CALL_ISINSTANCE", + [_CALL_ISINSTANCE_r31] = "_CALL_ISINSTANCE_r31", [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", + [_CALL_KW_NON_PY_r11] = "_CALL_KW_NON_PY_r11", [_CALL_LEN] = "_CALL_LEN", + [_CALL_LEN_r33] = "_CALL_LEN_r33", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", + [_CALL_LIST_APPEND_r03] = "_CALL_LIST_APPEND_r03", + [_CALL_LIST_APPEND_r13] = "_CALL_LIST_APPEND_r13", + [_CALL_LIST_APPEND_r23] = "_CALL_LIST_APPEND_r23", + [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", + [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r03", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", + [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_METHOD_DESCRIPTOR_O_r03] = "_CALL_METHOD_DESCRIPTOR_O_r03", + [_CALL_METHOD_DESCRIPTOR_O_INLINE] = "_CALL_METHOD_DESCRIPTOR_O_INLINE", + [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = "_CALL_METHOD_DESCRIPTOR_O_INLINE_r03", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", + [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", + [_CALL_STR_1_r32] = "_CALL_STR_1_r32", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", + [_CALL_TUPLE_1_r32] = "_CALL_TUPLE_1_r32", [_CALL_TYPE_1] = "_CALL_TYPE_1", - [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", + [_CALL_TYPE_1_r02] = "_CALL_TYPE_1_r02", + [_CALL_TYPE_1_r12] = "_CALL_TYPE_1_r12", + [_CALL_TYPE_1_r22] = "_CALL_TYPE_1_r22", + [_CALL_TYPE_1_r32] = "_CALL_TYPE_1_r32", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", + [_CHECK_ATTR_CLASS_r01] = "_CHECK_ATTR_CLASS_r01", + [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", + [_CHECK_ATTR_CLASS_r22] = "_CHECK_ATTR_CLASS_r22", + [_CHECK_ATTR_CLASS_r33] = "_CHECK_ATTR_CLASS_r33", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", + [_CHECK_ATTR_METHOD_LAZY_DICT_r01] = "_CHECK_ATTR_METHOD_LAZY_DICT_r01", + [_CHECK_ATTR_METHOD_LAZY_DICT_r11] = "_CHECK_ATTR_METHOD_LAZY_DICT_r11", + [_CHECK_ATTR_METHOD_LAZY_DICT_r22] = "_CHECK_ATTR_METHOD_LAZY_DICT_r22", + [_CHECK_ATTR_METHOD_LAZY_DICT_r33] = "_CHECK_ATTR_METHOD_LAZY_DICT_r33", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_CHECK_EG_MATCH] = "_CHECK_EG_MATCH", + [_CHECK_EG_MATCH_r22] = "_CHECK_EG_MATCH_r22", [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", + [_CHECK_EXC_MATCH_r22] = "_CHECK_EXC_MATCH_r22", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_EXACT_ARGS_r00] = "_CHECK_FUNCTION_EXACT_ARGS_r00", [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_FUNCTION_VERSION_r00] = "_CHECK_FUNCTION_VERSION_r00", [_CHECK_FUNCTION_VERSION_INLINE] = "_CHECK_FUNCTION_VERSION_INLINE", + [_CHECK_FUNCTION_VERSION_INLINE_r00] = "_CHECK_FUNCTION_VERSION_INLINE_r00", + [_CHECK_FUNCTION_VERSION_INLINE_r11] = "_CHECK_FUNCTION_VERSION_INLINE_r11", + [_CHECK_FUNCTION_VERSION_INLINE_r22] = "_CHECK_FUNCTION_VERSION_INLINE_r22", + [_CHECK_FUNCTION_VERSION_INLINE_r33] = "_CHECK_FUNCTION_VERSION_INLINE_r33", [_CHECK_FUNCTION_VERSION_KW] = "_CHECK_FUNCTION_VERSION_KW", + [_CHECK_FUNCTION_VERSION_KW_r11] = "_CHECK_FUNCTION_VERSION_KW_r11", [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", + [_CHECK_IS_NOT_PY_CALLABLE_r00] = "_CHECK_IS_NOT_PY_CALLABLE_r00", + [_CHECK_IS_NOT_PY_CALLABLE_EX] = "_CHECK_IS_NOT_PY_CALLABLE_EX", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r03] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r03", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r13] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r13", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r23", + [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = "_CHECK_IS_NOT_PY_CALLABLE_EX_r33", [_CHECK_IS_NOT_PY_CALLABLE_KW] = "_CHECK_IS_NOT_PY_CALLABLE_KW", + [_CHECK_IS_NOT_PY_CALLABLE_KW_r11] = "_CHECK_IS_NOT_PY_CALLABLE_KW_r11", + [_CHECK_IS_PY_CALLABLE_EX] = "_CHECK_IS_PY_CALLABLE_EX", + [_CHECK_IS_PY_CALLABLE_EX_r03] = "_CHECK_IS_PY_CALLABLE_EX_r03", + [_CHECK_IS_PY_CALLABLE_EX_r13] = "_CHECK_IS_PY_CALLABLE_EX_r13", + [_CHECK_IS_PY_CALLABLE_EX_r23] = "_CHECK_IS_PY_CALLABLE_EX_r23", + [_CHECK_IS_PY_CALLABLE_EX_r33] = "_CHECK_IS_PY_CALLABLE_EX_r33", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r01", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r11", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r22", + [_CHECK_MANAGED_OBJECT_HAS_VALUES_r33] = "_CHECK_MANAGED_OBJECT_HAS_VALUES_r33", [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", + [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", + [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", + [_CHECK_OBJECT] = "_CHECK_OBJECT", + [_CHECK_OBJECT_r00] = "_CHECK_OBJECT_r00", [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", + [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", + [_CHECK_PEP_523_r22] = "_CHECK_PEP_523_r22", + [_CHECK_PEP_523_r33] = "_CHECK_PEP_523_r33", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", + [_CHECK_PERIODIC_r00] = "_CHECK_PERIODIC_r00", [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM", + [_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00] = "_CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00", + [_CHECK_RECURSION_LIMIT] = "_CHECK_RECURSION_LIMIT", + [_CHECK_RECURSION_LIMIT_r00] = "_CHECK_RECURSION_LIMIT_r00", + [_CHECK_RECURSION_LIMIT_r11] = "_CHECK_RECURSION_LIMIT_r11", + [_CHECK_RECURSION_LIMIT_r22] = "_CHECK_RECURSION_LIMIT_r22", + [_CHECK_RECURSION_LIMIT_r33] = "_CHECK_RECURSION_LIMIT_r33", [_CHECK_RECURSION_REMAINING] = "_CHECK_RECURSION_REMAINING", + [_CHECK_RECURSION_REMAINING_r00] = "_CHECK_RECURSION_REMAINING_r00", + [_CHECK_RECURSION_REMAINING_r11] = "_CHECK_RECURSION_REMAINING_r11", + [_CHECK_RECURSION_REMAINING_r22] = "_CHECK_RECURSION_REMAINING_r22", + [_CHECK_RECURSION_REMAINING_r33] = "_CHECK_RECURSION_REMAINING_r33", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_CHECK_STACK_SPACE_r00] = "_CHECK_STACK_SPACE_r00", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", + [_CHECK_STACK_SPACE_OPERAND_r00] = "_CHECK_STACK_SPACE_OPERAND_r00", + [_CHECK_STACK_SPACE_OPERAND_r11] = "_CHECK_STACK_SPACE_OPERAND_r11", + [_CHECK_STACK_SPACE_OPERAND_r22] = "_CHECK_STACK_SPACE_OPERAND_r22", + [_CHECK_STACK_SPACE_OPERAND_r33] = "_CHECK_STACK_SPACE_OPERAND_r33", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_CHECK_VALIDITY_r00] = "_CHECK_VALIDITY_r00", + [_CHECK_VALIDITY_r11] = "_CHECK_VALIDITY_r11", + [_CHECK_VALIDITY_r22] = "_CHECK_VALIDITY_r22", + [_CHECK_VALIDITY_r33] = "_CHECK_VALIDITY_r33", [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", + [_COLD_DYNAMIC_EXIT_r00] = "_COLD_DYNAMIC_EXIT_r00", [_COLD_EXIT] = "_COLD_EXIT", + [_COLD_EXIT_r00] = "_COLD_EXIT_r00", [_COMPARE_OP] = "_COMPARE_OP", + [_COMPARE_OP_r21] = "_COMPARE_OP_r21", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", + [_COMPARE_OP_FLOAT_r03] = "_COMPARE_OP_FLOAT_r03", + [_COMPARE_OP_FLOAT_r13] = "_COMPARE_OP_FLOAT_r13", + [_COMPARE_OP_FLOAT_r23] = "_COMPARE_OP_FLOAT_r23", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", + [_COMPARE_OP_INT_r23] = "_COMPARE_OP_INT_r23", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", + [_COMPARE_OP_STR_r23] = "_COMPARE_OP_STR_r23", [_CONTAINS_OP] = "_CONTAINS_OP", + [_CONTAINS_OP_r23] = "_CONTAINS_OP_r23", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", + [_CONTAINS_OP_DICT_r23] = "_CONTAINS_OP_DICT_r23", [_CONTAINS_OP_SET] = "_CONTAINS_OP_SET", + [_CONTAINS_OP_SET_r23] = "_CONTAINS_OP_SET_r23", [_CONVERT_VALUE] = "_CONVERT_VALUE", + [_CONVERT_VALUE_r11] = "_CONVERT_VALUE_r11", [_COPY] = "_COPY", + [_COPY_r01] = "_COPY_r01", [_COPY_1] = "_COPY_1", + [_COPY_1_r02] = "_COPY_1_r02", + [_COPY_1_r12] = "_COPY_1_r12", + [_COPY_1_r23] = "_COPY_1_r23", [_COPY_2] = "_COPY_2", + [_COPY_2_r03] = "_COPY_2_r03", + [_COPY_2_r13] = "_COPY_2_r13", + [_COPY_2_r23] = "_COPY_2_r23", [_COPY_3] = "_COPY_3", + [_COPY_3_r03] = "_COPY_3_r03", + [_COPY_3_r13] = "_COPY_3_r13", + [_COPY_3_r23] = "_COPY_3_r23", + [_COPY_3_r33] = "_COPY_3_r33", [_COPY_FREE_VARS] = "_COPY_FREE_VARS", + [_COPY_FREE_VARS_r00] = "_COPY_FREE_VARS_r00", + [_COPY_FREE_VARS_r11] = "_COPY_FREE_VARS_r11", + [_COPY_FREE_VARS_r22] = "_COPY_FREE_VARS_r22", + [_COPY_FREE_VARS_r33] = "_COPY_FREE_VARS_r33", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", - [_DELETE_ATTR] = "_DELETE_ATTR", + [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", [_DELETE_DEREF] = "_DELETE_DEREF", + [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", - [_DELETE_GLOBAL] = "_DELETE_GLOBAL", - [_DELETE_NAME] = "_DELETE_NAME", + [_DELETE_FAST_r00] = "_DELETE_FAST_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", + [_DELETE_SUBSCR_r20] = "_DELETE_SUBSCR_r20", [_DEOPT] = "_DEOPT", + [_DEOPT_r00] = "_DEOPT_r00", + [_DEOPT_r10] = "_DEOPT_r10", + [_DEOPT_r20] = "_DEOPT_r20", + [_DEOPT_r30] = "_DEOPT_r30", [_DICT_MERGE] = "_DICT_MERGE", + [_DICT_MERGE_r11] = "_DICT_MERGE_r11", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DICT_UPDATE_r11] = "_DICT_UPDATE_r11", [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", + [_DYNAMIC_EXIT_r00] = "_DYNAMIC_EXIT_r00", + [_DYNAMIC_EXIT_r10] = "_DYNAMIC_EXIT_r10", + [_DYNAMIC_EXIT_r20] = "_DYNAMIC_EXIT_r20", + [_DYNAMIC_EXIT_r30] = "_DYNAMIC_EXIT_r30", [_END_FOR] = "_END_FOR", + [_END_FOR_r10] = "_END_FOR_r10", [_END_SEND] = "_END_SEND", + [_END_SEND_r31] = "_END_SEND_r31", [_ERROR_POP_N] = "_ERROR_POP_N", + [_ERROR_POP_N_r00] = "_ERROR_POP_N_r00", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", + [_EXIT_INIT_CHECK_r10] = "_EXIT_INIT_CHECK_r10", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXIT_TRACE_r00] = "_EXIT_TRACE_r00", + [_EXIT_TRACE_r10] = "_EXIT_TRACE_r10", + [_EXIT_TRACE_r20] = "_EXIT_TRACE_r20", + [_EXIT_TRACE_r30] = "_EXIT_TRACE_r30", [_EXPAND_METHOD] = "_EXPAND_METHOD", + [_EXPAND_METHOD_r00] = "_EXPAND_METHOD_r00", [_EXPAND_METHOD_KW] = "_EXPAND_METHOD_KW", + [_EXPAND_METHOD_KW_r11] = "_EXPAND_METHOD_KW_r11", [_FATAL_ERROR] = "_FATAL_ERROR", + [_FATAL_ERROR_r00] = "_FATAL_ERROR_r00", + [_FATAL_ERROR_r11] = "_FATAL_ERROR_r11", + [_FATAL_ERROR_r22] = "_FATAL_ERROR_r22", + [_FATAL_ERROR_r33] = "_FATAL_ERROR_r33", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", + [_FORMAT_SIMPLE_r11] = "_FORMAT_SIMPLE_r11", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", + [_FORMAT_WITH_SPEC_r21] = "_FORMAT_WITH_SPEC_r21", [_FOR_ITER_GEN_FRAME] = "_FOR_ITER_GEN_FRAME", + [_FOR_ITER_GEN_FRAME_r03] = "_FOR_ITER_GEN_FRAME_r03", + [_FOR_ITER_GEN_FRAME_r13] = "_FOR_ITER_GEN_FRAME_r13", + [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", + [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", + [_FOR_ITER_VIRTUAL_TIER_TWO] = "_FOR_ITER_VIRTUAL_TIER_TWO", + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = "_FOR_ITER_VIRTUAL_TIER_TWO_r23", [_GET_AITER] = "_GET_AITER", + [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", + [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", + [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", [_GET_ITER] = "_GET_ITER", + [_GET_ITER_r12] = "_GET_ITER_r12", + [_GET_ITER_TRAD] = "_GET_ITER_TRAD", + [_GET_ITER_TRAD_r12] = "_GET_ITER_TRAD_r12", [_GET_LEN] = "_GET_LEN", - [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GET_LEN_r12] = "_GET_LEN_r12", + [_GUARD_3OS_ASYNC_GEN_ASEND] = "_GUARD_3OS_ASYNC_GEN_ASEND", + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = "_GUARD_3OS_ASYNC_GEN_ASEND_r03", + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", + [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", + [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = "_GUARD_BINARY_OP_EXTEND_LHS_r02", + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = "_GUARD_BINARY_OP_EXTEND_LHS_r12", + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = "_GUARD_BINARY_OP_EXTEND_LHS_r22", + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = "_GUARD_BINARY_OP_EXTEND_LHS_r33", + [_GUARD_BINARY_OP_EXTEND_RHS] = "_GUARD_BINARY_OP_EXTEND_RHS", + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = "_GUARD_BINARY_OP_EXTEND_RHS_r02", + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = "_GUARD_BINARY_OP_EXTEND_RHS_r12", + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = "_GUARD_BINARY_OP_EXTEND_RHS_r22", + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = "_GUARD_BINARY_OP_EXTEND_RHS_r33", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33", + [_GUARD_BIT_IS_SET_POP] = "_GUARD_BIT_IS_SET_POP", + [_GUARD_BIT_IS_SET_POP_r00] = "_GUARD_BIT_IS_SET_POP_r00", + [_GUARD_BIT_IS_SET_POP_r10] = "_GUARD_BIT_IS_SET_POP_r10", + [_GUARD_BIT_IS_SET_POP_r21] = "_GUARD_BIT_IS_SET_POP_r21", + [_GUARD_BIT_IS_SET_POP_r32] = "_GUARD_BIT_IS_SET_POP_r32", + [_GUARD_BIT_IS_SET_POP_4] = "_GUARD_BIT_IS_SET_POP_4", + [_GUARD_BIT_IS_SET_POP_4_r00] = "_GUARD_BIT_IS_SET_POP_4_r00", + [_GUARD_BIT_IS_SET_POP_4_r10] = "_GUARD_BIT_IS_SET_POP_4_r10", + [_GUARD_BIT_IS_SET_POP_4_r21] = "_GUARD_BIT_IS_SET_POP_4_r21", + [_GUARD_BIT_IS_SET_POP_4_r32] = "_GUARD_BIT_IS_SET_POP_4_r32", + [_GUARD_BIT_IS_SET_POP_5] = "_GUARD_BIT_IS_SET_POP_5", + [_GUARD_BIT_IS_SET_POP_5_r00] = "_GUARD_BIT_IS_SET_POP_5_r00", + [_GUARD_BIT_IS_SET_POP_5_r10] = "_GUARD_BIT_IS_SET_POP_5_r10", + [_GUARD_BIT_IS_SET_POP_5_r21] = "_GUARD_BIT_IS_SET_POP_5_r21", + [_GUARD_BIT_IS_SET_POP_5_r32] = "_GUARD_BIT_IS_SET_POP_5_r32", + [_GUARD_BIT_IS_SET_POP_6] = "_GUARD_BIT_IS_SET_POP_6", + [_GUARD_BIT_IS_SET_POP_6_r00] = "_GUARD_BIT_IS_SET_POP_6_r00", + [_GUARD_BIT_IS_SET_POP_6_r10] = "_GUARD_BIT_IS_SET_POP_6_r10", + [_GUARD_BIT_IS_SET_POP_6_r21] = "_GUARD_BIT_IS_SET_POP_6_r21", + [_GUARD_BIT_IS_SET_POP_6_r32] = "_GUARD_BIT_IS_SET_POP_6_r32", + [_GUARD_BIT_IS_SET_POP_7] = "_GUARD_BIT_IS_SET_POP_7", + [_GUARD_BIT_IS_SET_POP_7_r00] = "_GUARD_BIT_IS_SET_POP_7_r00", + [_GUARD_BIT_IS_SET_POP_7_r10] = "_GUARD_BIT_IS_SET_POP_7_r10", + [_GUARD_BIT_IS_SET_POP_7_r21] = "_GUARD_BIT_IS_SET_POP_7_r21", + [_GUARD_BIT_IS_SET_POP_7_r32] = "_GUARD_BIT_IS_SET_POP_7_r32", + [_GUARD_BIT_IS_UNSET_POP] = "_GUARD_BIT_IS_UNSET_POP", + [_GUARD_BIT_IS_UNSET_POP_r00] = "_GUARD_BIT_IS_UNSET_POP_r00", + [_GUARD_BIT_IS_UNSET_POP_r10] = "_GUARD_BIT_IS_UNSET_POP_r10", + [_GUARD_BIT_IS_UNSET_POP_r21] = "_GUARD_BIT_IS_UNSET_POP_r21", + [_GUARD_BIT_IS_UNSET_POP_r32] = "_GUARD_BIT_IS_UNSET_POP_r32", + [_GUARD_BIT_IS_UNSET_POP_4] = "_GUARD_BIT_IS_UNSET_POP_4", + [_GUARD_BIT_IS_UNSET_POP_4_r00] = "_GUARD_BIT_IS_UNSET_POP_4_r00", + [_GUARD_BIT_IS_UNSET_POP_4_r10] = "_GUARD_BIT_IS_UNSET_POP_4_r10", + [_GUARD_BIT_IS_UNSET_POP_4_r21] = "_GUARD_BIT_IS_UNSET_POP_4_r21", + [_GUARD_BIT_IS_UNSET_POP_4_r32] = "_GUARD_BIT_IS_UNSET_POP_4_r32", + [_GUARD_BIT_IS_UNSET_POP_5] = "_GUARD_BIT_IS_UNSET_POP_5", + [_GUARD_BIT_IS_UNSET_POP_5_r00] = "_GUARD_BIT_IS_UNSET_POP_5_r00", + [_GUARD_BIT_IS_UNSET_POP_5_r10] = "_GUARD_BIT_IS_UNSET_POP_5_r10", + [_GUARD_BIT_IS_UNSET_POP_5_r21] = "_GUARD_BIT_IS_UNSET_POP_5_r21", + [_GUARD_BIT_IS_UNSET_POP_5_r32] = "_GUARD_BIT_IS_UNSET_POP_5_r32", + [_GUARD_BIT_IS_UNSET_POP_6] = "_GUARD_BIT_IS_UNSET_POP_6", + [_GUARD_BIT_IS_UNSET_POP_6_r00] = "_GUARD_BIT_IS_UNSET_POP_6_r00", + [_GUARD_BIT_IS_UNSET_POP_6_r10] = "_GUARD_BIT_IS_UNSET_POP_6_r10", + [_GUARD_BIT_IS_UNSET_POP_6_r21] = "_GUARD_BIT_IS_UNSET_POP_6_r21", + [_GUARD_BIT_IS_UNSET_POP_6_r32] = "_GUARD_BIT_IS_UNSET_POP_6_r32", + [_GUARD_BIT_IS_UNSET_POP_7] = "_GUARD_BIT_IS_UNSET_POP_7", + [_GUARD_BIT_IS_UNSET_POP_7_r00] = "_GUARD_BIT_IS_UNSET_POP_7_r00", + [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", + [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", + [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_CALLABLE_BUILTIN_CLASS] = "_GUARD_CALLABLE_BUILTIN_CLASS", + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = "_GUARD_CALLABLE_BUILTIN_CLASS_r00", + [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", + [_GUARD_CALLABLE_BUILTIN_FAST_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_r00", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_BUILTIN_O] = "_GUARD_CALLABLE_BUILTIN_O", + [_GUARD_CALLABLE_BUILTIN_O_r00] = "_GUARD_CALLABLE_BUILTIN_O_r00", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", + [_GUARD_CALLABLE_ISINSTANCE_r03] = "_GUARD_CALLABLE_ISINSTANCE_r03", + [_GUARD_CALLABLE_ISINSTANCE_r13] = "_GUARD_CALLABLE_ISINSTANCE_r13", + [_GUARD_CALLABLE_ISINSTANCE_r23] = "_GUARD_CALLABLE_ISINSTANCE_r23", + [_GUARD_CALLABLE_ISINSTANCE_r33] = "_GUARD_CALLABLE_ISINSTANCE_r33", [_GUARD_CALLABLE_LEN] = "_GUARD_CALLABLE_LEN", + [_GUARD_CALLABLE_LEN_r03] = "_GUARD_CALLABLE_LEN_r03", + [_GUARD_CALLABLE_LEN_r13] = "_GUARD_CALLABLE_LEN_r13", + [_GUARD_CALLABLE_LEN_r23] = "_GUARD_CALLABLE_LEN_r23", + [_GUARD_CALLABLE_LEN_r33] = "_GUARD_CALLABLE_LEN_r33", [_GUARD_CALLABLE_LIST_APPEND] = "_GUARD_CALLABLE_LIST_APPEND", + [_GUARD_CALLABLE_LIST_APPEND_r03] = "_GUARD_CALLABLE_LIST_APPEND_r03", + [_GUARD_CALLABLE_LIST_APPEND_r13] = "_GUARD_CALLABLE_LIST_APPEND_r13", + [_GUARD_CALLABLE_LIST_APPEND_r23] = "_GUARD_CALLABLE_LIST_APPEND_r23", + [_GUARD_CALLABLE_LIST_APPEND_r33] = "_GUARD_CALLABLE_LIST_APPEND_r33", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O", + [_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00] = "_GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00", [_GUARD_CALLABLE_STR_1] = "_GUARD_CALLABLE_STR_1", + [_GUARD_CALLABLE_STR_1_r03] = "_GUARD_CALLABLE_STR_1_r03", + [_GUARD_CALLABLE_STR_1_r13] = "_GUARD_CALLABLE_STR_1_r13", + [_GUARD_CALLABLE_STR_1_r23] = "_GUARD_CALLABLE_STR_1_r23", + [_GUARD_CALLABLE_STR_1_r33] = "_GUARD_CALLABLE_STR_1_r33", [_GUARD_CALLABLE_TUPLE_1] = "_GUARD_CALLABLE_TUPLE_1", + [_GUARD_CALLABLE_TUPLE_1_r03] = "_GUARD_CALLABLE_TUPLE_1_r03", + [_GUARD_CALLABLE_TUPLE_1_r13] = "_GUARD_CALLABLE_TUPLE_1_r13", + [_GUARD_CALLABLE_TUPLE_1_r23] = "_GUARD_CALLABLE_TUPLE_1_r23", + [_GUARD_CALLABLE_TUPLE_1_r33] = "_GUARD_CALLABLE_TUPLE_1_r33", [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", + [_GUARD_CALLABLE_TYPE_1_r03] = "_GUARD_CALLABLE_TYPE_1_r03", + [_GUARD_CALLABLE_TYPE_1_r13] = "_GUARD_CALLABLE_TYPE_1_r13", + [_GUARD_CALLABLE_TYPE_1_r23] = "_GUARD_CALLABLE_TYPE_1_r23", + [_GUARD_CALLABLE_TYPE_1_r33] = "_GUARD_CALLABLE_TYPE_1_r33", + [_GUARD_CODE_VERSION_RETURN_GENERATOR] = "_GUARD_CODE_VERSION_RETURN_GENERATOR", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r00] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r00", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r11] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r11", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r22] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r22", + [_GUARD_CODE_VERSION_RETURN_GENERATOR_r33] = "_GUARD_CODE_VERSION_RETURN_GENERATOR_r33", + [_GUARD_CODE_VERSION_RETURN_VALUE] = "_GUARD_CODE_VERSION_RETURN_VALUE", + [_GUARD_CODE_VERSION_RETURN_VALUE_r00] = "_GUARD_CODE_VERSION_RETURN_VALUE_r00", + [_GUARD_CODE_VERSION_RETURN_VALUE_r11] = "_GUARD_CODE_VERSION_RETURN_VALUE_r11", + [_GUARD_CODE_VERSION_RETURN_VALUE_r22] = "_GUARD_CODE_VERSION_RETURN_VALUE_r22", + [_GUARD_CODE_VERSION_RETURN_VALUE_r33] = "_GUARD_CODE_VERSION_RETURN_VALUE_r33", + [_GUARD_CODE_VERSION_YIELD_VALUE] = "_GUARD_CODE_VERSION_YIELD_VALUE", + [_GUARD_CODE_VERSION_YIELD_VALUE_r00] = "_GUARD_CODE_VERSION_YIELD_VALUE_r00", + [_GUARD_CODE_VERSION_YIELD_VALUE_r11] = "_GUARD_CODE_VERSION_YIELD_VALUE_r11", + [_GUARD_CODE_VERSION_YIELD_VALUE_r22] = "_GUARD_CODE_VERSION_YIELD_VALUE_r22", + [_GUARD_CODE_VERSION_YIELD_VALUE_r33] = "_GUARD_CODE_VERSION_YIELD_VALUE_r33", + [_GUARD_CODE_VERSION__PUSH_FRAME] = "_GUARD_CODE_VERSION__PUSH_FRAME", + [_GUARD_CODE_VERSION__PUSH_FRAME_r00] = "_GUARD_CODE_VERSION__PUSH_FRAME_r00", + [_GUARD_CODE_VERSION__PUSH_FRAME_r11] = "_GUARD_CODE_VERSION__PUSH_FRAME_r11", + [_GUARD_CODE_VERSION__PUSH_FRAME_r22] = "_GUARD_CODE_VERSION__PUSH_FRAME_r22", + [_GUARD_CODE_VERSION__PUSH_FRAME_r33] = "_GUARD_CODE_VERSION__PUSH_FRAME_r33", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_DORV_NO_DICT_r01] = "_GUARD_DORV_NO_DICT_r01", + [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", + [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", + [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", + [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", + [_GUARD_GLOBALS_VERSION_r22] = "_GUARD_GLOBALS_VERSION_r22", + [_GUARD_GLOBALS_VERSION_r33] = "_GUARD_GLOBALS_VERSION_r33", [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_GENERATOR_r00] = "_GUARD_IP_RETURN_GENERATOR_r00", + [_GUARD_IP_RETURN_GENERATOR_r11] = "_GUARD_IP_RETURN_GENERATOR_r11", + [_GUARD_IP_RETURN_GENERATOR_r22] = "_GUARD_IP_RETURN_GENERATOR_r22", + [_GUARD_IP_RETURN_GENERATOR_r33] = "_GUARD_IP_RETURN_GENERATOR_r33", [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_RETURN_VALUE_r00] = "_GUARD_IP_RETURN_VALUE_r00", + [_GUARD_IP_RETURN_VALUE_r11] = "_GUARD_IP_RETURN_VALUE_r11", + [_GUARD_IP_RETURN_VALUE_r22] = "_GUARD_IP_RETURN_VALUE_r22", + [_GUARD_IP_RETURN_VALUE_r33] = "_GUARD_IP_RETURN_VALUE_r33", [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP_YIELD_VALUE_r00] = "_GUARD_IP_YIELD_VALUE_r00", + [_GUARD_IP_YIELD_VALUE_r11] = "_GUARD_IP_YIELD_VALUE_r11", + [_GUARD_IP_YIELD_VALUE_r22] = "_GUARD_IP_YIELD_VALUE_r22", + [_GUARD_IP_YIELD_VALUE_r33] = "_GUARD_IP_YIELD_VALUE_r33", [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", + [_GUARD_IP__PUSH_FRAME_r00] = "_GUARD_IP__PUSH_FRAME_r00", + [_GUARD_IP__PUSH_FRAME_r11] = "_GUARD_IP__PUSH_FRAME_r11", + [_GUARD_IP__PUSH_FRAME_r22] = "_GUARD_IP__PUSH_FRAME_r22", + [_GUARD_IP__PUSH_FRAME_r33] = "_GUARD_IP__PUSH_FRAME_r33", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", + [_GUARD_IS_FALSE_POP_r00] = "_GUARD_IS_FALSE_POP_r00", + [_GUARD_IS_FALSE_POP_r10] = "_GUARD_IS_FALSE_POP_r10", + [_GUARD_IS_FALSE_POP_r21] = "_GUARD_IS_FALSE_POP_r21", + [_GUARD_IS_FALSE_POP_r32] = "_GUARD_IS_FALSE_POP_r32", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", + [_GUARD_IS_NONE_POP_r00] = "_GUARD_IS_NONE_POP_r00", + [_GUARD_IS_NONE_POP_r10] = "_GUARD_IS_NONE_POP_r10", + [_GUARD_IS_NONE_POP_r21] = "_GUARD_IS_NONE_POP_r21", + [_GUARD_IS_NONE_POP_r32] = "_GUARD_IS_NONE_POP_r32", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", + [_GUARD_IS_NOT_NONE_POP_r10] = "_GUARD_IS_NOT_NONE_POP_r10", [_GUARD_IS_TRUE_POP] = "_GUARD_IS_TRUE_POP", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", + [_GUARD_IS_TRUE_POP_r00] = "_GUARD_IS_TRUE_POP_r00", + [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", + [_GUARD_IS_TRUE_POP_r21] = "_GUARD_IS_TRUE_POP_r21", + [_GUARD_IS_TRUE_POP_r32] = "_GUARD_IS_TRUE_POP_r32", + [_GUARD_ITERATOR] = "_GUARD_ITERATOR", + [_GUARD_ITERATOR_r01] = "_GUARD_ITERATOR_r01", + [_GUARD_ITERATOR_r11] = "_GUARD_ITERATOR_r11", + [_GUARD_ITERATOR_r22] = "_GUARD_ITERATOR_r22", + [_GUARD_ITERATOR_r33] = "_GUARD_ITERATOR_r33", + [_GUARD_ITER_VIRTUAL] = "_GUARD_ITER_VIRTUAL", + [_GUARD_ITER_VIRTUAL_r01] = "_GUARD_ITER_VIRTUAL_r01", + [_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11", + [_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22", + [_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33", + [_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r23", + [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r33", + [_GUARD_NOS_COMPACT_ASCII] = "_GUARD_NOS_COMPACT_ASCII", + [_GUARD_NOS_COMPACT_ASCII_r02] = "_GUARD_NOS_COMPACT_ASCII_r02", + [_GUARD_NOS_COMPACT_ASCII_r12] = "_GUARD_NOS_COMPACT_ASCII_r12", + [_GUARD_NOS_COMPACT_ASCII_r22] = "_GUARD_NOS_COMPACT_ASCII_r22", + [_GUARD_NOS_COMPACT_ASCII_r33] = "_GUARD_NOS_COMPACT_ASCII_r33", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33", + [_GUARD_NOS_DICT_SUBSCRIPT] = "_GUARD_NOS_DICT_SUBSCRIPT", + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = "_GUARD_NOS_DICT_SUBSCRIPT_r02", + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = "_GUARD_NOS_DICT_SUBSCRIPT_r12", + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = "_GUARD_NOS_DICT_SUBSCRIPT_r22", + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_SUBSCRIPT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", + [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", + [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", + [_GUARD_NOS_FLOAT_r22] = "_GUARD_NOS_FLOAT_r22", + [_GUARD_NOS_FLOAT_r33] = "_GUARD_NOS_FLOAT_r33", [_GUARD_NOS_INT] = "_GUARD_NOS_INT", + [_GUARD_NOS_INT_r02] = "_GUARD_NOS_INT_r02", + [_GUARD_NOS_INT_r12] = "_GUARD_NOS_INT_r12", + [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", + [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", + [_GUARD_NOS_ITER_VIRTUAL] = "_GUARD_NOS_ITER_VIRTUAL", + [_GUARD_NOS_ITER_VIRTUAL_r02] = "_GUARD_NOS_ITER_VIRTUAL_r02", + [_GUARD_NOS_ITER_VIRTUAL_r12] = "_GUARD_NOS_ITER_VIRTUAL_r12", + [_GUARD_NOS_ITER_VIRTUAL_r22] = "_GUARD_NOS_ITER_VIRTUAL_r22", + [_GUARD_NOS_ITER_VIRTUAL_r33] = "_GUARD_NOS_ITER_VIRTUAL_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", + [_GUARD_NOS_LIST_r02] = "_GUARD_NOS_LIST_r02", + [_GUARD_NOS_LIST_r12] = "_GUARD_NOS_LIST_r12", + [_GUARD_NOS_LIST_r22] = "_GUARD_NOS_LIST_r22", + [_GUARD_NOS_LIST_r33] = "_GUARD_NOS_LIST_r33", [_GUARD_NOS_NOT_NULL] = "_GUARD_NOS_NOT_NULL", + [_GUARD_NOS_NOT_NULL_r02] = "_GUARD_NOS_NOT_NULL_r02", + [_GUARD_NOS_NOT_NULL_r12] = "_GUARD_NOS_NOT_NULL_r12", + [_GUARD_NOS_NOT_NULL_r22] = "_GUARD_NOS_NOT_NULL_r22", + [_GUARD_NOS_NOT_NULL_r33] = "_GUARD_NOS_NOT_NULL_r33", [_GUARD_NOS_NULL] = "_GUARD_NOS_NULL", + [_GUARD_NOS_NULL_r02] = "_GUARD_NOS_NULL_r02", + [_GUARD_NOS_NULL_r12] = "_GUARD_NOS_NULL_r12", + [_GUARD_NOS_NULL_r22] = "_GUARD_NOS_NULL_r22", + [_GUARD_NOS_NULL_r33] = "_GUARD_NOS_NULL_r33", [_GUARD_NOS_OVERFLOWED] = "_GUARD_NOS_OVERFLOWED", + [_GUARD_NOS_OVERFLOWED_r02] = "_GUARD_NOS_OVERFLOWED_r02", + [_GUARD_NOS_OVERFLOWED_r12] = "_GUARD_NOS_OVERFLOWED_r12", + [_GUARD_NOS_OVERFLOWED_r22] = "_GUARD_NOS_OVERFLOWED_r22", + [_GUARD_NOS_OVERFLOWED_r33] = "_GUARD_NOS_OVERFLOWED_r33", [_GUARD_NOS_TUPLE] = "_GUARD_NOS_TUPLE", + [_GUARD_NOS_TUPLE_r02] = "_GUARD_NOS_TUPLE_r02", + [_GUARD_NOS_TUPLE_r12] = "_GUARD_NOS_TUPLE_r12", + [_GUARD_NOS_TUPLE_r22] = "_GUARD_NOS_TUPLE_r22", + [_GUARD_NOS_TUPLE_r33] = "_GUARD_NOS_TUPLE_r33", + [_GUARD_NOS_TYPE_VERSION] = "_GUARD_NOS_TYPE_VERSION", + [_GUARD_NOS_TYPE_VERSION_r02] = "_GUARD_NOS_TYPE_VERSION_r02", + [_GUARD_NOS_TYPE_VERSION_r12] = "_GUARD_NOS_TYPE_VERSION_r12", + [_GUARD_NOS_TYPE_VERSION_r22] = "_GUARD_NOS_TYPE_VERSION_r22", + [_GUARD_NOS_TYPE_VERSION_r33] = "_GUARD_NOS_TYPE_VERSION_r33", [_GUARD_NOS_UNICODE] = "_GUARD_NOS_UNICODE", + [_GUARD_NOS_UNICODE_r02] = "_GUARD_NOS_UNICODE_r02", + [_GUARD_NOS_UNICODE_r12] = "_GUARD_NOS_UNICODE_r12", + [_GUARD_NOS_UNICODE_r22] = "_GUARD_NOS_UNICODE_r22", + [_GUARD_NOS_UNICODE_r33] = "_GUARD_NOS_UNICODE_r33", [_GUARD_NOT_EXHAUSTED_LIST] = "_GUARD_NOT_EXHAUSTED_LIST", + [_GUARD_NOT_EXHAUSTED_LIST_r02] = "_GUARD_NOT_EXHAUSTED_LIST_r02", + [_GUARD_NOT_EXHAUSTED_LIST_r12] = "_GUARD_NOT_EXHAUSTED_LIST_r12", + [_GUARD_NOT_EXHAUSTED_LIST_r22] = "_GUARD_NOT_EXHAUSTED_LIST_r22", + [_GUARD_NOT_EXHAUSTED_LIST_r33] = "_GUARD_NOT_EXHAUSTED_LIST_r33", [_GUARD_NOT_EXHAUSTED_RANGE] = "_GUARD_NOT_EXHAUSTED_RANGE", + [_GUARD_NOT_EXHAUSTED_RANGE_r02] = "_GUARD_NOT_EXHAUSTED_RANGE_r02", + [_GUARD_NOT_EXHAUSTED_RANGE_r12] = "_GUARD_NOT_EXHAUSTED_RANGE_r12", + [_GUARD_NOT_EXHAUSTED_RANGE_r22] = "_GUARD_NOT_EXHAUSTED_RANGE_r22", + [_GUARD_NOT_EXHAUSTED_RANGE_r33] = "_GUARD_NOT_EXHAUSTED_RANGE_r33", [_GUARD_NOT_EXHAUSTED_TUPLE] = "_GUARD_NOT_EXHAUSTED_TUPLE", + [_GUARD_NOT_EXHAUSTED_TUPLE_r02] = "_GUARD_NOT_EXHAUSTED_TUPLE_r02", + [_GUARD_NOT_EXHAUSTED_TUPLE_r12] = "_GUARD_NOT_EXHAUSTED_TUPLE_r12", + [_GUARD_NOT_EXHAUSTED_TUPLE_r22] = "_GUARD_NOT_EXHAUSTED_TUPLE_r22", + [_GUARD_NOT_EXHAUSTED_TUPLE_r33] = "_GUARD_NOT_EXHAUSTED_TUPLE_r33", [_GUARD_THIRD_NULL] = "_GUARD_THIRD_NULL", + [_GUARD_THIRD_NULL_r03] = "_GUARD_THIRD_NULL_r03", + [_GUARD_THIRD_NULL_r13] = "_GUARD_THIRD_NULL_r13", + [_GUARD_THIRD_NULL_r23] = "_GUARD_THIRD_NULL_r23", + [_GUARD_THIRD_NULL_r33] = "_GUARD_THIRD_NULL_r33", + [_GUARD_TOS_ANY_DICT] = "_GUARD_TOS_ANY_DICT", + [_GUARD_TOS_ANY_DICT_r01] = "_GUARD_TOS_ANY_DICT_r01", + [_GUARD_TOS_ANY_DICT_r11] = "_GUARD_TOS_ANY_DICT_r11", + [_GUARD_TOS_ANY_DICT_r22] = "_GUARD_TOS_ANY_DICT_r22", + [_GUARD_TOS_ANY_DICT_r33] = "_GUARD_TOS_ANY_DICT_r33", [_GUARD_TOS_ANY_SET] = "_GUARD_TOS_ANY_SET", + [_GUARD_TOS_ANY_SET_r01] = "_GUARD_TOS_ANY_SET_r01", + [_GUARD_TOS_ANY_SET_r11] = "_GUARD_TOS_ANY_SET_r11", + [_GUARD_TOS_ANY_SET_r22] = "_GUARD_TOS_ANY_SET_r22", + [_GUARD_TOS_ANY_SET_r33] = "_GUARD_TOS_ANY_SET_r33", [_GUARD_TOS_DICT] = "_GUARD_TOS_DICT", + [_GUARD_TOS_DICT_r01] = "_GUARD_TOS_DICT_r01", + [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", + [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", + [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", + [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", + [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", + [_GUARD_TOS_FLOAT_r22] = "_GUARD_TOS_FLOAT_r22", + [_GUARD_TOS_FLOAT_r33] = "_GUARD_TOS_FLOAT_r33", + [_GUARD_TOS_FROZENDICT] = "_GUARD_TOS_FROZENDICT", + [_GUARD_TOS_FROZENDICT_r01] = "_GUARD_TOS_FROZENDICT_r01", + [_GUARD_TOS_FROZENDICT_r11] = "_GUARD_TOS_FROZENDICT_r11", + [_GUARD_TOS_FROZENDICT_r22] = "_GUARD_TOS_FROZENDICT_r22", + [_GUARD_TOS_FROZENDICT_r33] = "_GUARD_TOS_FROZENDICT_r33", + [_GUARD_TOS_FROZENSET] = "_GUARD_TOS_FROZENSET", + [_GUARD_TOS_FROZENSET_r01] = "_GUARD_TOS_FROZENSET_r01", + [_GUARD_TOS_FROZENSET_r11] = "_GUARD_TOS_FROZENSET_r11", + [_GUARD_TOS_FROZENSET_r22] = "_GUARD_TOS_FROZENSET_r22", + [_GUARD_TOS_FROZENSET_r33] = "_GUARD_TOS_FROZENSET_r33", [_GUARD_TOS_INT] = "_GUARD_TOS_INT", + [_GUARD_TOS_INT_r01] = "_GUARD_TOS_INT_r01", + [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", + [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", + [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", + [_GUARD_TOS_IS_NONE] = "_GUARD_TOS_IS_NONE", + [_GUARD_TOS_IS_NONE_r01] = "_GUARD_TOS_IS_NONE_r01", + [_GUARD_TOS_IS_NONE_r11] = "_GUARD_TOS_IS_NONE_r11", + [_GUARD_TOS_IS_NONE_r22] = "_GUARD_TOS_IS_NONE_r22", + [_GUARD_TOS_IS_NONE_r33] = "_GUARD_TOS_IS_NONE_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", + [_GUARD_TOS_LIST_r01] = "_GUARD_TOS_LIST_r01", + [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", + [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", + [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", + [_GUARD_TOS_NOT_NULL] = "_GUARD_TOS_NOT_NULL", + [_GUARD_TOS_NOT_NULL_r01] = "_GUARD_TOS_NOT_NULL_r01", + [_GUARD_TOS_NOT_NULL_r11] = "_GUARD_TOS_NOT_NULL_r11", + [_GUARD_TOS_NOT_NULL_r22] = "_GUARD_TOS_NOT_NULL_r22", + [_GUARD_TOS_NOT_NULL_r33] = "_GUARD_TOS_NOT_NULL_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", + [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", + [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", + [_GUARD_TOS_OVERFLOWED_r22] = "_GUARD_TOS_OVERFLOWED_r22", + [_GUARD_TOS_OVERFLOWED_r33] = "_GUARD_TOS_OVERFLOWED_r33", + [_GUARD_TOS_SET] = "_GUARD_TOS_SET", + [_GUARD_TOS_SET_r01] = "_GUARD_TOS_SET_r01", + [_GUARD_TOS_SET_r11] = "_GUARD_TOS_SET_r11", + [_GUARD_TOS_SET_r22] = "_GUARD_TOS_SET_r22", + [_GUARD_TOS_SET_r33] = "_GUARD_TOS_SET_r33", [_GUARD_TOS_SLICE] = "_GUARD_TOS_SLICE", + [_GUARD_TOS_SLICE_r01] = "_GUARD_TOS_SLICE_r01", + [_GUARD_TOS_SLICE_r11] = "_GUARD_TOS_SLICE_r11", + [_GUARD_TOS_SLICE_r22] = "_GUARD_TOS_SLICE_r22", + [_GUARD_TOS_SLICE_r33] = "_GUARD_TOS_SLICE_r33", [_GUARD_TOS_TUPLE] = "_GUARD_TOS_TUPLE", + [_GUARD_TOS_TUPLE_r01] = "_GUARD_TOS_TUPLE_r01", + [_GUARD_TOS_TUPLE_r11] = "_GUARD_TOS_TUPLE_r11", + [_GUARD_TOS_TUPLE_r22] = "_GUARD_TOS_TUPLE_r22", + [_GUARD_TOS_TUPLE_r33] = "_GUARD_TOS_TUPLE_r33", [_GUARD_TOS_UNICODE] = "_GUARD_TOS_UNICODE", + [_GUARD_TOS_UNICODE_r01] = "_GUARD_TOS_UNICODE_r01", + [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", + [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", + [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", + [_GUARD_TYPE] = "_GUARD_TYPE", + [_GUARD_TYPE_r01] = "_GUARD_TYPE_r01", + [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", + [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", + [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", + [_GUARD_TYPE_ITER] = "_GUARD_TYPE_ITER", + [_GUARD_TYPE_ITER_r02] = "_GUARD_TYPE_ITER_r02", + [_GUARD_TYPE_ITER_r12] = "_GUARD_TYPE_ITER_r12", + [_GUARD_TYPE_ITER_r22] = "_GUARD_TYPE_ITER_r22", + [_GUARD_TYPE_ITER_r33] = "_GUARD_TYPE_ITER_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", - [_GUARD_TYPE_VERSION_AND_LOCK] = "_GUARD_TYPE_VERSION_AND_LOCK", + [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", + [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", + [_GUARD_TYPE_VERSION_r22] = "_GUARD_TYPE_VERSION_r22", + [_GUARD_TYPE_VERSION_r33] = "_GUARD_TYPE_VERSION_r33", + [_GUARD_TYPE_VERSION_LOCKED] = "_GUARD_TYPE_VERSION_LOCKED", + [_GUARD_TYPE_VERSION_LOCKED_r01] = "_GUARD_TYPE_VERSION_LOCKED_r01", + [_GUARD_TYPE_VERSION_LOCKED_r11] = "_GUARD_TYPE_VERSION_LOCKED_r11", + [_GUARD_TYPE_VERSION_LOCKED_r22] = "_GUARD_TYPE_VERSION_LOCKED_r22", + [_GUARD_TYPE_VERSION_LOCKED_r33] = "_GUARD_TYPE_VERSION_LOCKED_r33", [_HANDLE_PENDING_AND_DEOPT] = "_HANDLE_PENDING_AND_DEOPT", + [_HANDLE_PENDING_AND_DEOPT_r00] = "_HANDLE_PENDING_AND_DEOPT_r00", + [_HANDLE_PENDING_AND_DEOPT_r10] = "_HANDLE_PENDING_AND_DEOPT_r10", + [_HANDLE_PENDING_AND_DEOPT_r20] = "_HANDLE_PENDING_AND_DEOPT_r20", + [_HANDLE_PENDING_AND_DEOPT_r30] = "_HANDLE_PENDING_AND_DEOPT_r30", [_IMPORT_FROM] = "_IMPORT_FROM", + [_IMPORT_FROM_r12] = "_IMPORT_FROM_r12", [_IMPORT_NAME] = "_IMPORT_NAME", + [_IMPORT_NAME_r21] = "_IMPORT_NAME_r21", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", + [_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_INIT_CALL_PY_EXACT_ARGS_r01] = "_INIT_CALL_PY_EXACT_ARGS_r01", [_INIT_CALL_PY_EXACT_ARGS_0] = "_INIT_CALL_PY_EXACT_ARGS_0", + [_INIT_CALL_PY_EXACT_ARGS_0_r01] = "_INIT_CALL_PY_EXACT_ARGS_0_r01", [_INIT_CALL_PY_EXACT_ARGS_1] = "_INIT_CALL_PY_EXACT_ARGS_1", + [_INIT_CALL_PY_EXACT_ARGS_1_r01] = "_INIT_CALL_PY_EXACT_ARGS_1_r01", [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", + [_INIT_CALL_PY_EXACT_ARGS_2_r01] = "_INIT_CALL_PY_EXACT_ARGS_2_r01", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", + [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", + [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", [_INSERT_NULL] = "_INSERT_NULL", + [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", + [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", + [_IS_OP_r03] = "_IS_OP_r03", + [_IS_OP_r13] = "_IS_OP_r13", + [_IS_OP_r23] = "_IS_OP_r23", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", + [_ITER_CHECK_LIST_r02] = "_ITER_CHECK_LIST_r02", + [_ITER_CHECK_LIST_r12] = "_ITER_CHECK_LIST_r12", + [_ITER_CHECK_LIST_r22] = "_ITER_CHECK_LIST_r22", + [_ITER_CHECK_LIST_r33] = "_ITER_CHECK_LIST_r33", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", + [_ITER_CHECK_RANGE_r02] = "_ITER_CHECK_RANGE_r02", + [_ITER_CHECK_RANGE_r12] = "_ITER_CHECK_RANGE_r12", + [_ITER_CHECK_RANGE_r22] = "_ITER_CHECK_RANGE_r22", + [_ITER_CHECK_RANGE_r33] = "_ITER_CHECK_RANGE_r33", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_CHECK_TUPLE_r02] = "_ITER_CHECK_TUPLE_r02", + [_ITER_CHECK_TUPLE_r12] = "_ITER_CHECK_TUPLE_r12", + [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", + [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", + [_ITER_NEXT_INLINE] = "_ITER_NEXT_INLINE", + [_ITER_NEXT_INLINE_r23] = "_ITER_NEXT_INLINE_r23", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", + [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_ITER_NEXT_RANGE_r03] = "_ITER_NEXT_RANGE_r03", + [_ITER_NEXT_RANGE_r13] = "_ITER_NEXT_RANGE_r13", + [_ITER_NEXT_RANGE_r23] = "_ITER_NEXT_RANGE_r23", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_ITER_NEXT_TUPLE_r03] = "_ITER_NEXT_TUPLE_r03", + [_ITER_NEXT_TUPLE_r13] = "_ITER_NEXT_TUPLE_r13", + [_ITER_NEXT_TUPLE_r23] = "_ITER_NEXT_TUPLE_r23", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", + [_JUMP_TO_TOP_r00] = "_JUMP_TO_TOP_r00", [_LIST_APPEND] = "_LIST_APPEND", + [_LIST_APPEND_r10] = "_LIST_APPEND_r10", [_LIST_EXTEND] = "_LIST_EXTEND", + [_LIST_EXTEND_r11] = "_LIST_EXTEND_r11", [_LOAD_ATTR] = "_LOAD_ATTR", + [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", + [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", + [_LOAD_ATTR_INSTANCE_VALUE_r02] = "_LOAD_ATTR_INSTANCE_VALUE_r02", + [_LOAD_ATTR_INSTANCE_VALUE_r12] = "_LOAD_ATTR_INSTANCE_VALUE_r12", + [_LOAD_ATTR_INSTANCE_VALUE_r23] = "_LOAD_ATTR_INSTANCE_VALUE_r23", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_LOAD_ATTR_METHOD_LAZY_DICT_r02] = "_LOAD_ATTR_METHOD_LAZY_DICT_r02", + [_LOAD_ATTR_METHOD_LAZY_DICT_r12] = "_LOAD_ATTR_METHOD_LAZY_DICT_r12", + [_LOAD_ATTR_METHOD_LAZY_DICT_r23] = "_LOAD_ATTR_METHOD_LAZY_DICT_r23", [_LOAD_ATTR_METHOD_NO_DICT] = "_LOAD_ATTR_METHOD_NO_DICT", + [_LOAD_ATTR_METHOD_NO_DICT_r02] = "_LOAD_ATTR_METHOD_NO_DICT_r02", + [_LOAD_ATTR_METHOD_NO_DICT_r12] = "_LOAD_ATTR_METHOD_NO_DICT_r12", + [_LOAD_ATTR_METHOD_NO_DICT_r23] = "_LOAD_ATTR_METHOD_NO_DICT_r23", [_LOAD_ATTR_METHOD_WITH_VALUES] = "_LOAD_ATTR_METHOD_WITH_VALUES", + [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = "_LOAD_ATTR_METHOD_WITH_VALUES_r02", + [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = "_LOAD_ATTR_METHOD_WITH_VALUES_r12", + [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = "_LOAD_ATTR_METHOD_WITH_VALUES_r23", [_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE", + [_LOAD_ATTR_MODULE_r12] = "_LOAD_ATTR_MODULE_r12", [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r01] = "_LOAD_ATTR_PROPERTY_FRAME_r01", + [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", + [_LOAD_ATTR_PROPERTY_FRAME_r22] = "_LOAD_ATTR_PROPERTY_FRAME_r22", + [_LOAD_ATTR_PROPERTY_FRAME_r33] = "_LOAD_ATTR_PROPERTY_FRAME_r33", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", + [_LOAD_ATTR_SLOT_r02] = "_LOAD_ATTR_SLOT_r02", + [_LOAD_ATTR_SLOT_r12] = "_LOAD_ATTR_SLOT_r12", + [_LOAD_ATTR_SLOT_r23] = "_LOAD_ATTR_SLOT_r23", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", + [_LOAD_ATTR_WITH_HINT_r12] = "_LOAD_ATTR_WITH_HINT_r12", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", + [_LOAD_BUILD_CLASS_r01] = "_LOAD_BUILD_CLASS_r01", [_LOAD_COMMON_CONSTANT] = "_LOAD_COMMON_CONSTANT", + [_LOAD_COMMON_CONSTANT_r01] = "_LOAD_COMMON_CONSTANT_r01", + [_LOAD_COMMON_CONSTANT_r12] = "_LOAD_COMMON_CONSTANT_r12", + [_LOAD_COMMON_CONSTANT_r23] = "_LOAD_COMMON_CONSTANT_r23", [_LOAD_CONST] = "_LOAD_CONST", + [_LOAD_CONST_r01] = "_LOAD_CONST_r01", + [_LOAD_CONST_r12] = "_LOAD_CONST_r12", + [_LOAD_CONST_r23] = "_LOAD_CONST_r23", [_LOAD_CONST_INLINE] = "_LOAD_CONST_INLINE", + [_LOAD_CONST_INLINE_r01] = "_LOAD_CONST_INLINE_r01", + [_LOAD_CONST_INLINE_r12] = "_LOAD_CONST_INLINE_r12", + [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", - [_LOAD_CONST_UNDER_INLINE] = "_LOAD_CONST_UNDER_INLINE", - [_LOAD_CONST_UNDER_INLINE_BORROW] = "_LOAD_CONST_UNDER_INLINE_BORROW", + [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", + [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", + [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", + [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", + [_LOAD_FAST_r01] = "_LOAD_FAST_r01", + [_LOAD_FAST_r12] = "_LOAD_FAST_r12", + [_LOAD_FAST_r23] = "_LOAD_FAST_r23", [_LOAD_FAST_0] = "_LOAD_FAST_0", + [_LOAD_FAST_0_r01] = "_LOAD_FAST_0_r01", + [_LOAD_FAST_0_r12] = "_LOAD_FAST_0_r12", + [_LOAD_FAST_0_r23] = "_LOAD_FAST_0_r23", [_LOAD_FAST_1] = "_LOAD_FAST_1", + [_LOAD_FAST_1_r01] = "_LOAD_FAST_1_r01", + [_LOAD_FAST_1_r12] = "_LOAD_FAST_1_r12", + [_LOAD_FAST_1_r23] = "_LOAD_FAST_1_r23", [_LOAD_FAST_2] = "_LOAD_FAST_2", + [_LOAD_FAST_2_r01] = "_LOAD_FAST_2_r01", + [_LOAD_FAST_2_r12] = "_LOAD_FAST_2_r12", + [_LOAD_FAST_2_r23] = "_LOAD_FAST_2_r23", [_LOAD_FAST_3] = "_LOAD_FAST_3", + [_LOAD_FAST_3_r01] = "_LOAD_FAST_3_r01", + [_LOAD_FAST_3_r12] = "_LOAD_FAST_3_r12", + [_LOAD_FAST_3_r23] = "_LOAD_FAST_3_r23", [_LOAD_FAST_4] = "_LOAD_FAST_4", + [_LOAD_FAST_4_r01] = "_LOAD_FAST_4_r01", + [_LOAD_FAST_4_r12] = "_LOAD_FAST_4_r12", + [_LOAD_FAST_4_r23] = "_LOAD_FAST_4_r23", [_LOAD_FAST_5] = "_LOAD_FAST_5", + [_LOAD_FAST_5_r01] = "_LOAD_FAST_5_r01", + [_LOAD_FAST_5_r12] = "_LOAD_FAST_5_r12", + [_LOAD_FAST_5_r23] = "_LOAD_FAST_5_r23", [_LOAD_FAST_6] = "_LOAD_FAST_6", + [_LOAD_FAST_6_r01] = "_LOAD_FAST_6_r01", + [_LOAD_FAST_6_r12] = "_LOAD_FAST_6_r12", + [_LOAD_FAST_6_r23] = "_LOAD_FAST_6_r23", [_LOAD_FAST_7] = "_LOAD_FAST_7", + [_LOAD_FAST_7_r01] = "_LOAD_FAST_7_r01", + [_LOAD_FAST_7_r12] = "_LOAD_FAST_7_r12", + [_LOAD_FAST_7_r23] = "_LOAD_FAST_7_r23", [_LOAD_FAST_AND_CLEAR] = "_LOAD_FAST_AND_CLEAR", + [_LOAD_FAST_AND_CLEAR_r01] = "_LOAD_FAST_AND_CLEAR_r01", + [_LOAD_FAST_AND_CLEAR_r12] = "_LOAD_FAST_AND_CLEAR_r12", + [_LOAD_FAST_AND_CLEAR_r23] = "_LOAD_FAST_AND_CLEAR_r23", [_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_r01] = "_LOAD_FAST_BORROW_r01", + [_LOAD_FAST_BORROW_r12] = "_LOAD_FAST_BORROW_r12", + [_LOAD_FAST_BORROW_r23] = "_LOAD_FAST_BORROW_r23", [_LOAD_FAST_BORROW_0] = "_LOAD_FAST_BORROW_0", + [_LOAD_FAST_BORROW_0_r01] = "_LOAD_FAST_BORROW_0_r01", + [_LOAD_FAST_BORROW_0_r12] = "_LOAD_FAST_BORROW_0_r12", + [_LOAD_FAST_BORROW_0_r23] = "_LOAD_FAST_BORROW_0_r23", [_LOAD_FAST_BORROW_1] = "_LOAD_FAST_BORROW_1", + [_LOAD_FAST_BORROW_1_r01] = "_LOAD_FAST_BORROW_1_r01", + [_LOAD_FAST_BORROW_1_r12] = "_LOAD_FAST_BORROW_1_r12", + [_LOAD_FAST_BORROW_1_r23] = "_LOAD_FAST_BORROW_1_r23", [_LOAD_FAST_BORROW_2] = "_LOAD_FAST_BORROW_2", + [_LOAD_FAST_BORROW_2_r01] = "_LOAD_FAST_BORROW_2_r01", + [_LOAD_FAST_BORROW_2_r12] = "_LOAD_FAST_BORROW_2_r12", + [_LOAD_FAST_BORROW_2_r23] = "_LOAD_FAST_BORROW_2_r23", [_LOAD_FAST_BORROW_3] = "_LOAD_FAST_BORROW_3", + [_LOAD_FAST_BORROW_3_r01] = "_LOAD_FAST_BORROW_3_r01", + [_LOAD_FAST_BORROW_3_r12] = "_LOAD_FAST_BORROW_3_r12", + [_LOAD_FAST_BORROW_3_r23] = "_LOAD_FAST_BORROW_3_r23", [_LOAD_FAST_BORROW_4] = "_LOAD_FAST_BORROW_4", + [_LOAD_FAST_BORROW_4_r01] = "_LOAD_FAST_BORROW_4_r01", + [_LOAD_FAST_BORROW_4_r12] = "_LOAD_FAST_BORROW_4_r12", + [_LOAD_FAST_BORROW_4_r23] = "_LOAD_FAST_BORROW_4_r23", [_LOAD_FAST_BORROW_5] = "_LOAD_FAST_BORROW_5", + [_LOAD_FAST_BORROW_5_r01] = "_LOAD_FAST_BORROW_5_r01", + [_LOAD_FAST_BORROW_5_r12] = "_LOAD_FAST_BORROW_5_r12", + [_LOAD_FAST_BORROW_5_r23] = "_LOAD_FAST_BORROW_5_r23", [_LOAD_FAST_BORROW_6] = "_LOAD_FAST_BORROW_6", + [_LOAD_FAST_BORROW_6_r01] = "_LOAD_FAST_BORROW_6_r01", + [_LOAD_FAST_BORROW_6_r12] = "_LOAD_FAST_BORROW_6_r12", + [_LOAD_FAST_BORROW_6_r23] = "_LOAD_FAST_BORROW_6_r23", [_LOAD_FAST_BORROW_7] = "_LOAD_FAST_BORROW_7", - [_LOAD_FAST_BORROW_LOAD_FAST_BORROW] = "_LOAD_FAST_BORROW_LOAD_FAST_BORROW", + [_LOAD_FAST_BORROW_7_r01] = "_LOAD_FAST_BORROW_7_r01", + [_LOAD_FAST_BORROW_7_r12] = "_LOAD_FAST_BORROW_7_r12", + [_LOAD_FAST_BORROW_7_r23] = "_LOAD_FAST_BORROW_7_r23", [_LOAD_FAST_CHECK] = "_LOAD_FAST_CHECK", - [_LOAD_FAST_LOAD_FAST] = "_LOAD_FAST_LOAD_FAST", + [_LOAD_FAST_CHECK_r01] = "_LOAD_FAST_CHECK_r01", + [_LOAD_FAST_CHECK_r12] = "_LOAD_FAST_CHECK_r12", + [_LOAD_FAST_CHECK_r23] = "_LOAD_FAST_CHECK_r23", [_LOAD_FROM_DICT_OR_DEREF] = "_LOAD_FROM_DICT_OR_DEREF", + [_LOAD_FROM_DICT_OR_DEREF_r11] = "_LOAD_FROM_DICT_OR_DEREF_r11", [_LOAD_GLOBAL] = "_LOAD_GLOBAL", + [_LOAD_GLOBAL_r00] = "_LOAD_GLOBAL_r00", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_LOAD_GLOBAL_BUILTINS_r01] = "_LOAD_GLOBAL_BUILTINS_r01", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", + [_LOAD_GLOBAL_MODULE_r01] = "_LOAD_GLOBAL_MODULE_r01", [_LOAD_LOCALS] = "_LOAD_LOCALS", + [_LOAD_LOCALS_r01] = "_LOAD_LOCALS_r01", + [_LOAD_LOCALS_r12] = "_LOAD_LOCALS_r12", + [_LOAD_LOCALS_r23] = "_LOAD_LOCALS_r23", [_LOAD_NAME] = "_LOAD_NAME", + [_LOAD_NAME_r01] = "_LOAD_NAME_r01", [_LOAD_SMALL_INT] = "_LOAD_SMALL_INT", + [_LOAD_SMALL_INT_r01] = "_LOAD_SMALL_INT_r01", + [_LOAD_SMALL_INT_r12] = "_LOAD_SMALL_INT_r12", + [_LOAD_SMALL_INT_r23] = "_LOAD_SMALL_INT_r23", [_LOAD_SMALL_INT_0] = "_LOAD_SMALL_INT_0", + [_LOAD_SMALL_INT_0_r01] = "_LOAD_SMALL_INT_0_r01", + [_LOAD_SMALL_INT_0_r12] = "_LOAD_SMALL_INT_0_r12", + [_LOAD_SMALL_INT_0_r23] = "_LOAD_SMALL_INT_0_r23", [_LOAD_SMALL_INT_1] = "_LOAD_SMALL_INT_1", + [_LOAD_SMALL_INT_1_r01] = "_LOAD_SMALL_INT_1_r01", + [_LOAD_SMALL_INT_1_r12] = "_LOAD_SMALL_INT_1_r12", + [_LOAD_SMALL_INT_1_r23] = "_LOAD_SMALL_INT_1_r23", [_LOAD_SMALL_INT_2] = "_LOAD_SMALL_INT_2", + [_LOAD_SMALL_INT_2_r01] = "_LOAD_SMALL_INT_2_r01", + [_LOAD_SMALL_INT_2_r12] = "_LOAD_SMALL_INT_2_r12", + [_LOAD_SMALL_INT_2_r23] = "_LOAD_SMALL_INT_2_r23", [_LOAD_SMALL_INT_3] = "_LOAD_SMALL_INT_3", + [_LOAD_SMALL_INT_3_r01] = "_LOAD_SMALL_INT_3_r01", + [_LOAD_SMALL_INT_3_r12] = "_LOAD_SMALL_INT_3_r12", + [_LOAD_SMALL_INT_3_r23] = "_LOAD_SMALL_INT_3_r23", [_LOAD_SPECIAL] = "_LOAD_SPECIAL", + [_LOAD_SPECIAL_r00] = "_LOAD_SPECIAL_r00", [_LOAD_SUPER_ATTR_ATTR] = "_LOAD_SUPER_ATTR_ATTR", + [_LOAD_SUPER_ATTR_ATTR_r31] = "_LOAD_SUPER_ATTR_ATTR_r31", [_LOAD_SUPER_ATTR_METHOD] = "_LOAD_SUPER_ATTR_METHOD", + [_LOAD_SUPER_ATTR_METHOD_r32] = "_LOAD_SUPER_ATTR_METHOD_r32", + [_LOCK_OBJECT] = "_LOCK_OBJECT", + [_LOCK_OBJECT_r01] = "_LOCK_OBJECT_r01", + [_LOCK_OBJECT_r11] = "_LOCK_OBJECT_r11", + [_LOCK_OBJECT_r22] = "_LOCK_OBJECT_r22", + [_LOCK_OBJECT_r33] = "_LOCK_OBJECT_r33", [_MAKE_CALLARGS_A_TUPLE] = "_MAKE_CALLARGS_A_TUPLE", + [_MAKE_CALLARGS_A_TUPLE_r33] = "_MAKE_CALLARGS_A_TUPLE_r33", [_MAKE_CELL] = "_MAKE_CELL", + [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", + [_MAKE_FUNCTION_r12] = "_MAKE_FUNCTION_r12", + [_MAKE_HEAP_SAFE] = "_MAKE_HEAP_SAFE", + [_MAKE_HEAP_SAFE_r01] = "_MAKE_HEAP_SAFE_r01", + [_MAKE_HEAP_SAFE_r11] = "_MAKE_HEAP_SAFE_r11", + [_MAKE_HEAP_SAFE_r22] = "_MAKE_HEAP_SAFE_r22", + [_MAKE_HEAP_SAFE_r33] = "_MAKE_HEAP_SAFE_r33", [_MAKE_WARM] = "_MAKE_WARM", + [_MAKE_WARM_r00] = "_MAKE_WARM_r00", + [_MAKE_WARM_r11] = "_MAKE_WARM_r11", + [_MAKE_WARM_r22] = "_MAKE_WARM_r22", + [_MAKE_WARM_r33] = "_MAKE_WARM_r33", [_MAP_ADD] = "_MAP_ADD", + [_MAP_ADD_r20] = "_MAP_ADD_r20", [_MATCH_CLASS] = "_MATCH_CLASS", + [_MATCH_CLASS_r33] = "_MATCH_CLASS_r33", [_MATCH_KEYS] = "_MATCH_KEYS", + [_MATCH_KEYS_r23] = "_MATCH_KEYS_r23", [_MATCH_MAPPING] = "_MATCH_MAPPING", + [_MATCH_MAPPING_r02] = "_MATCH_MAPPING_r02", + [_MATCH_MAPPING_r12] = "_MATCH_MAPPING_r12", + [_MATCH_MAPPING_r23] = "_MATCH_MAPPING_r23", [_MATCH_SEQUENCE] = "_MATCH_SEQUENCE", + [_MATCH_SEQUENCE_r02] = "_MATCH_SEQUENCE_r02", + [_MATCH_SEQUENCE_r12] = "_MATCH_SEQUENCE_r12", + [_MATCH_SEQUENCE_r23] = "_MATCH_SEQUENCE_r23", [_MAYBE_EXPAND_METHOD] = "_MAYBE_EXPAND_METHOD", + [_MAYBE_EXPAND_METHOD_r00] = "_MAYBE_EXPAND_METHOD_r00", [_MAYBE_EXPAND_METHOD_KW] = "_MAYBE_EXPAND_METHOD_KW", + [_MAYBE_EXPAND_METHOD_KW_r11] = "_MAYBE_EXPAND_METHOD_KW_r11", [_NOP] = "_NOP", - [_POP_CALL] = "_POP_CALL", - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_ONE] = "_POP_CALL_ONE", - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_TWO] = "_POP_CALL_TWO", - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", + [_NOP_r00] = "_NOP_r00", + [_NOP_r11] = "_NOP_r11", + [_NOP_r22] = "_NOP_r22", + [_NOP_r33] = "_NOP_r33", [_POP_EXCEPT] = "_POP_EXCEPT", + [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", + [_POP_ITER_r20] = "_POP_ITER_r20", [_POP_TOP] = "_POP_TOP", + [_POP_TOP_r10] = "_POP_TOP_r10", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", + [_POP_TOP_FLOAT_r00] = "_POP_TOP_FLOAT_r00", + [_POP_TOP_FLOAT_r10] = "_POP_TOP_FLOAT_r10", + [_POP_TOP_FLOAT_r21] = "_POP_TOP_FLOAT_r21", + [_POP_TOP_FLOAT_r32] = "_POP_TOP_FLOAT_r32", [_POP_TOP_INT] = "_POP_TOP_INT", - [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_INT_r00] = "_POP_TOP_INT_r00", + [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", + [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", + [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", [_POP_TOP_NOP] = "_POP_TOP_NOP", + [_POP_TOP_NOP_r00] = "_POP_TOP_NOP_r00", + [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", + [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", + [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", + [_POP_TOP_OPARG] = "_POP_TOP_OPARG", + [_POP_TOP_OPARG_r00] = "_POP_TOP_OPARG_r00", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", - [_POP_TWO] = "_POP_TWO", - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", + [_POP_TOP_UNICODE_r00] = "_POP_TOP_UNICODE_r00", + [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", + [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", + [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", + [_PUSH_EXC_INFO_r02] = "_PUSH_EXC_INFO_r02", + [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", + [_PUSH_EXC_INFO_r23] = "_PUSH_EXC_INFO_r23", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_r10] = "_PUSH_FRAME_r10", [_PUSH_NULL] = "_PUSH_NULL", + [_PUSH_NULL_r01] = "_PUSH_NULL_r01", + [_PUSH_NULL_r12] = "_PUSH_NULL_r12", + [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", + [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", + [_PUSH_TAGGED_ZERO] = "_PUSH_TAGGED_ZERO", + [_PUSH_TAGGED_ZERO_r01] = "_PUSH_TAGGED_ZERO_r01", + [_PUSH_TAGGED_ZERO_r12] = "_PUSH_TAGGED_ZERO_r12", + [_PUSH_TAGGED_ZERO_r23] = "_PUSH_TAGGED_ZERO_r23", + [_PY_FRAME_EX] = "_PY_FRAME_EX", + [_PY_FRAME_EX_r31] = "_PY_FRAME_EX_r31", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", + [_PY_FRAME_GENERAL_r01] = "_PY_FRAME_GENERAL_r01", [_PY_FRAME_KW] = "_PY_FRAME_KW", + [_PY_FRAME_KW_r11] = "_PY_FRAME_KW_r11", + [_RECORD_3OS_GEN_FUNC] = "_RECORD_3OS_GEN_FUNC", + [_RECORD_4OS] = "_RECORD_4OS", + [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", + [_RECORD_CALLABLE] = "_RECORD_CALLABLE", + [_RECORD_CALLABLE_KW] = "_RECORD_CALLABLE_KW", + [_RECORD_CODE] = "_RECORD_CODE", + [_RECORD_NOS] = "_RECORD_NOS", + [_RECORD_NOS_GEN_FUNC] = "_RECORD_NOS_GEN_FUNC", + [_RECORD_NOS_TYPE] = "_RECORD_NOS_TYPE", + [_RECORD_TOS] = "_RECORD_TOS", + [_RECORD_TOS_TYPE] = "_RECORD_TOS_TYPE", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", + [_REPLACE_WITH_TRUE_r02] = "_REPLACE_WITH_TRUE_r02", + [_REPLACE_WITH_TRUE_r12] = "_REPLACE_WITH_TRUE_r12", + [_REPLACE_WITH_TRUE_r23] = "_REPLACE_WITH_TRUE_r23", [_RESUME_CHECK] = "_RESUME_CHECK", + [_RESUME_CHECK_r00] = "_RESUME_CHECK_r00", + [_RESUME_CHECK_r11] = "_RESUME_CHECK_r11", + [_RESUME_CHECK_r22] = "_RESUME_CHECK_r22", + [_RESUME_CHECK_r33] = "_RESUME_CHECK_r33", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", + [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", + [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", + [_RROT_3] = "_RROT_3", + [_RROT_3_r03] = "_RROT_3_r03", + [_RROT_3_r13] = "_RROT_3_r13", + [_RROT_3_r23] = "_RROT_3_r23", + [_RROT_3_r33] = "_RROT_3_r33", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", + [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", + [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", + [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", + [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", + [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", + [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", + [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", + [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", + [_SEND_VIRTUAL_TIER_TWO_r13] = "_SEND_VIRTUAL_TIER_TWO_r13", + [_SEND_VIRTUAL_TIER_TWO_r23] = "_SEND_VIRTUAL_TIER_TWO_r23", + [_SEND_VIRTUAL_TIER_TWO_r33] = "_SEND_VIRTUAL_TIER_TWO_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", + [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", + [_SET_ADD_r10] = "_SET_ADD_r10", [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", + [_SET_FUNCTION_ATTRIBUTE_r01] = "_SET_FUNCTION_ATTRIBUTE_r01", + [_SET_FUNCTION_ATTRIBUTE_r11] = "_SET_FUNCTION_ATTRIBUTE_r11", + [_SET_FUNCTION_ATTRIBUTE_r21] = "_SET_FUNCTION_ATTRIBUTE_r21", + [_SET_FUNCTION_ATTRIBUTE_r32] = "_SET_FUNCTION_ATTRIBUTE_r32", [_SET_IP] = "_SET_IP", + [_SET_IP_r00] = "_SET_IP_r00", + [_SET_IP_r11] = "_SET_IP_r11", + [_SET_IP_r22] = "_SET_IP_r22", + [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", + [_SET_UPDATE_r11] = "_SET_UPDATE_r11", + [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", + [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", + [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", + [_SPILL_OR_RELOAD_r03] = "_SPILL_OR_RELOAD_r03", + [_SPILL_OR_RELOAD_r10] = "_SPILL_OR_RELOAD_r10", + [_SPILL_OR_RELOAD_r12] = "_SPILL_OR_RELOAD_r12", + [_SPILL_OR_RELOAD_r13] = "_SPILL_OR_RELOAD_r13", + [_SPILL_OR_RELOAD_r20] = "_SPILL_OR_RELOAD_r20", + [_SPILL_OR_RELOAD_r21] = "_SPILL_OR_RELOAD_r21", + [_SPILL_OR_RELOAD_r23] = "_SPILL_OR_RELOAD_r23", + [_SPILL_OR_RELOAD_r30] = "_SPILL_OR_RELOAD_r30", + [_SPILL_OR_RELOAD_r31] = "_SPILL_OR_RELOAD_r31", + [_SPILL_OR_RELOAD_r32] = "_SPILL_OR_RELOAD_r32", [_START_EXECUTOR] = "_START_EXECUTOR", + [_START_EXECUTOR_r00] = "_START_EXECUTOR_r00", [_STORE_ATTR] = "_STORE_ATTR", + [_STORE_ATTR_r20] = "_STORE_ATTR_r20", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", + [_STORE_ATTR_INSTANCE_VALUE_r21] = "_STORE_ATTR_INSTANCE_VALUE_r21", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_STORE_ATTR_SLOT_r21] = "_STORE_ATTR_SLOT_r21", [_STORE_ATTR_WITH_HINT] = "_STORE_ATTR_WITH_HINT", + [_STORE_ATTR_WITH_HINT_r21] = "_STORE_ATTR_WITH_HINT_r21", [_STORE_DEREF] = "_STORE_DEREF", - [_STORE_FAST] = "_STORE_FAST", - [_STORE_FAST_0] = "_STORE_FAST_0", - [_STORE_FAST_1] = "_STORE_FAST_1", - [_STORE_FAST_2] = "_STORE_FAST_2", - [_STORE_FAST_3] = "_STORE_FAST_3", - [_STORE_FAST_4] = "_STORE_FAST_4", - [_STORE_FAST_5] = "_STORE_FAST_5", - [_STORE_FAST_6] = "_STORE_FAST_6", - [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_LOAD_FAST] = "_STORE_FAST_LOAD_FAST", - [_STORE_FAST_STORE_FAST] = "_STORE_FAST_STORE_FAST", + [_STORE_DEREF_r10] = "_STORE_DEREF_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", + [_STORE_GLOBAL_r10] = "_STORE_GLOBAL_r10", [_STORE_NAME] = "_STORE_NAME", + [_STORE_NAME_r10] = "_STORE_NAME_r10", [_STORE_SLICE] = "_STORE_SLICE", + [_STORE_SLICE_r30] = "_STORE_SLICE_r30", [_STORE_SUBSCR] = "_STORE_SUBSCR", + [_STORE_SUBSCR_r30] = "_STORE_SUBSCR_r30", [_STORE_SUBSCR_DICT] = "_STORE_SUBSCR_DICT", + [_STORE_SUBSCR_DICT_r31] = "_STORE_SUBSCR_DICT_r31", + [_STORE_SUBSCR_DICT_KNOWN_HASH] = "_STORE_SUBSCR_DICT_KNOWN_HASH", + [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = "_STORE_SUBSCR_DICT_KNOWN_HASH_r31", [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", + [_STORE_SUBSCR_LIST_INT_r32] = "_STORE_SUBSCR_LIST_INT_r32", [_SWAP] = "_SWAP", + [_SWAP_r11] = "_SWAP_r11", [_SWAP_2] = "_SWAP_2", + [_SWAP_2_r02] = "_SWAP_2_r02", + [_SWAP_2_r12] = "_SWAP_2_r12", + [_SWAP_2_r22] = "_SWAP_2_r22", + [_SWAP_2_r33] = "_SWAP_2_r33", [_SWAP_3] = "_SWAP_3", + [_SWAP_3_r03] = "_SWAP_3_r03", + [_SWAP_3_r13] = "_SWAP_3_r13", + [_SWAP_3_r23] = "_SWAP_3_r23", + [_SWAP_3_r33] = "_SWAP_3_r33", + [_SWAP_FAST] = "_SWAP_FAST", + [_SWAP_FAST_r01] = "_SWAP_FAST_r01", + [_SWAP_FAST_r11] = "_SWAP_FAST_r11", + [_SWAP_FAST_r22] = "_SWAP_FAST_r22", + [_SWAP_FAST_r33] = "_SWAP_FAST_r33", + [_SWAP_FAST_0] = "_SWAP_FAST_0", + [_SWAP_FAST_0_r01] = "_SWAP_FAST_0_r01", + [_SWAP_FAST_0_r11] = "_SWAP_FAST_0_r11", + [_SWAP_FAST_0_r22] = "_SWAP_FAST_0_r22", + [_SWAP_FAST_0_r33] = "_SWAP_FAST_0_r33", + [_SWAP_FAST_1] = "_SWAP_FAST_1", + [_SWAP_FAST_1_r01] = "_SWAP_FAST_1_r01", + [_SWAP_FAST_1_r11] = "_SWAP_FAST_1_r11", + [_SWAP_FAST_1_r22] = "_SWAP_FAST_1_r22", + [_SWAP_FAST_1_r33] = "_SWAP_FAST_1_r33", + [_SWAP_FAST_2] = "_SWAP_FAST_2", + [_SWAP_FAST_2_r01] = "_SWAP_FAST_2_r01", + [_SWAP_FAST_2_r11] = "_SWAP_FAST_2_r11", + [_SWAP_FAST_2_r22] = "_SWAP_FAST_2_r22", + [_SWAP_FAST_2_r33] = "_SWAP_FAST_2_r33", + [_SWAP_FAST_3] = "_SWAP_FAST_3", + [_SWAP_FAST_3_r01] = "_SWAP_FAST_3_r01", + [_SWAP_FAST_3_r11] = "_SWAP_FAST_3_r11", + [_SWAP_FAST_3_r22] = "_SWAP_FAST_3_r22", + [_SWAP_FAST_3_r33] = "_SWAP_FAST_3_r33", + [_SWAP_FAST_4] = "_SWAP_FAST_4", + [_SWAP_FAST_4_r01] = "_SWAP_FAST_4_r01", + [_SWAP_FAST_4_r11] = "_SWAP_FAST_4_r11", + [_SWAP_FAST_4_r22] = "_SWAP_FAST_4_r22", + [_SWAP_FAST_4_r33] = "_SWAP_FAST_4_r33", + [_SWAP_FAST_5] = "_SWAP_FAST_5", + [_SWAP_FAST_5_r01] = "_SWAP_FAST_5_r01", + [_SWAP_FAST_5_r11] = "_SWAP_FAST_5_r11", + [_SWAP_FAST_5_r22] = "_SWAP_FAST_5_r22", + [_SWAP_FAST_5_r33] = "_SWAP_FAST_5_r33", + [_SWAP_FAST_6] = "_SWAP_FAST_6", + [_SWAP_FAST_6_r01] = "_SWAP_FAST_6_r01", + [_SWAP_FAST_6_r11] = "_SWAP_FAST_6_r11", + [_SWAP_FAST_6_r22] = "_SWAP_FAST_6_r22", + [_SWAP_FAST_6_r33] = "_SWAP_FAST_6_r33", + [_SWAP_FAST_7] = "_SWAP_FAST_7", + [_SWAP_FAST_7_r01] = "_SWAP_FAST_7_r01", + [_SWAP_FAST_7_r11] = "_SWAP_FAST_7_r11", + [_SWAP_FAST_7_r22] = "_SWAP_FAST_7_r22", + [_SWAP_FAST_7_r33] = "_SWAP_FAST_7_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", + [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", + [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", + [_TIER2_RESUME_CHECK_r22] = "_TIER2_RESUME_CHECK_r22", + [_TIER2_RESUME_CHECK_r33] = "_TIER2_RESUME_CHECK_r33", [_TO_BOOL] = "_TO_BOOL", + [_TO_BOOL_r11] = "_TO_BOOL_r11", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", + [_TO_BOOL_BOOL_r01] = "_TO_BOOL_BOOL_r01", + [_TO_BOOL_BOOL_r11] = "_TO_BOOL_BOOL_r11", + [_TO_BOOL_BOOL_r22] = "_TO_BOOL_BOOL_r22", + [_TO_BOOL_BOOL_r33] = "_TO_BOOL_BOOL_r33", [_TO_BOOL_INT] = "_TO_BOOL_INT", + [_TO_BOOL_INT_r02] = "_TO_BOOL_INT_r02", + [_TO_BOOL_INT_r12] = "_TO_BOOL_INT_r12", + [_TO_BOOL_INT_r23] = "_TO_BOOL_INT_r23", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", + [_TO_BOOL_LIST_r02] = "_TO_BOOL_LIST_r02", + [_TO_BOOL_LIST_r12] = "_TO_BOOL_LIST_r12", + [_TO_BOOL_LIST_r23] = "_TO_BOOL_LIST_r23", [_TO_BOOL_NONE] = "_TO_BOOL_NONE", + [_TO_BOOL_NONE_r01] = "_TO_BOOL_NONE_r01", + [_TO_BOOL_NONE_r11] = "_TO_BOOL_NONE_r11", + [_TO_BOOL_NONE_r22] = "_TO_BOOL_NONE_r22", + [_TO_BOOL_NONE_r33] = "_TO_BOOL_NONE_r33", [_TO_BOOL_STR] = "_TO_BOOL_STR", + [_TO_BOOL_STR_r02] = "_TO_BOOL_STR_r02", + [_TO_BOOL_STR_r12] = "_TO_BOOL_STR_r12", + [_TO_BOOL_STR_r23] = "_TO_BOOL_STR_r23", [_UNARY_INVERT] = "_UNARY_INVERT", + [_UNARY_INVERT_r12] = "_UNARY_INVERT_r12", [_UNARY_NEGATIVE] = "_UNARY_NEGATIVE", + [_UNARY_NEGATIVE_r12] = "_UNARY_NEGATIVE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE] = "_UNARY_NEGATIVE_FLOAT_INPLACE", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r02] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r02", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r12] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r12", + [_UNARY_NEGATIVE_FLOAT_INPLACE_r23] = "_UNARY_NEGATIVE_FLOAT_INPLACE_r23", [_UNARY_NOT] = "_UNARY_NOT", + [_UNARY_NOT_r01] = "_UNARY_NOT_r01", + [_UNARY_NOT_r11] = "_UNARY_NOT_r11", + [_UNARY_NOT_r22] = "_UNARY_NOT_r22", + [_UNARY_NOT_r33] = "_UNARY_NOT_r33", [_UNPACK_EX] = "_UNPACK_EX", + [_UNPACK_EX_r10] = "_UNPACK_EX_r10", [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE_r10] = "_UNPACK_SEQUENCE_r10", [_UNPACK_SEQUENCE_LIST] = "_UNPACK_SEQUENCE_LIST", + [_UNPACK_SEQUENCE_LIST_r10] = "_UNPACK_SEQUENCE_LIST_r10", [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", + [_UNPACK_SEQUENCE_TUPLE_r10] = "_UNPACK_SEQUENCE_TUPLE_r10", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", + [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03", + [_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13] = "_UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10] = "_UNPACK_SEQUENCE_UNIQUE_TUPLE_r10", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12", + [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23] = "_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_WITH_EXCEPT_START_r33] = "_WITH_EXCEPT_START_r33", [_YIELD_VALUE] = "_YIELD_VALUE", + [_YIELD_VALUE_r11] = "_YIELD_VALUE_r11", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -731,10 +6195,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_FAST_AND_CLEAR: return 0; - case _LOAD_FAST_LOAD_FAST: - return 0; - case _LOAD_FAST_BORROW_LOAD_FAST_BORROW: - return 0; case _LOAD_CONST: return 0; case _LOAD_SMALL_INT_0: @@ -747,28 +6207,24 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_SMALL_INT: return 0; - case _STORE_FAST_0: + case _SWAP_FAST_0: return 1; - case _STORE_FAST_1: + case _SWAP_FAST_1: return 1; - case _STORE_FAST_2: + case _SWAP_FAST_2: return 1; - case _STORE_FAST_3: + case _SWAP_FAST_3: return 1; - case _STORE_FAST_4: + case _SWAP_FAST_4: return 1; - case _STORE_FAST_5: + case _SWAP_FAST_5: return 1; - case _STORE_FAST_6: + case _SWAP_FAST_6: return 1; - case _STORE_FAST_7: + case _SWAP_FAST_7: return 1; - case _STORE_FAST: + case _SWAP_FAST: return 1; - case _STORE_FAST_LOAD_FAST: - return 1; - case _STORE_FAST_STORE_FAST: - return 2; case _POP_TOP: return 1; case _POP_TOP_NOP: @@ -779,8 +6235,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _POP_TOP_UNICODE: return 1; - case _POP_TWO: - return 2; + case _POP_TOP_OPARG: + return oparg; case _PUSH_NULL: return 0; case _END_FOR: @@ -788,9 +6244,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _POP_ITER: return 2; case _END_SEND: - return 2; + return 3; case _UNARY_NEGATIVE: return 1; + case _UNARY_NEGATIVE_FLOAT_INPLACE: + return 1; case _UNARY_NOT: return 1; case _TO_BOOL: @@ -809,6 +6267,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _TO_BOOL_NONE: return 1; + case _GUARD_NOS_COMPACT_ASCII: + return 0; case _GUARD_NOS_UNICODE: return 0; case _GUARD_TOS_UNICODE: @@ -833,6 +6293,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_INT: return 2; + case _BINARY_OP_ADD_INT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE: + return 2; + case _BINARY_OP_ADD_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT: + return 2; case _GUARD_NOS_FLOAT: return 0; case _GUARD_TOS_FLOAT: @@ -843,16 +6315,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT: return 2; - case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_ADD_FLOAT_INPLACE: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE: return 2; - case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE: return 2; - case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: + case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT: return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _GUARD_BINARY_OP_EXTEND_LHS: + return 0; + case _GUARD_BINARY_OP_EXTEND_RHS: + return 0; case _GUARD_BINARY_OP_EXTEND: return 0; case _BINARY_OP_EXTEND: @@ -867,16 +6355,28 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBSCR_STR_INT: return 2; + case _BINARY_OP_SUBSCR_USTR_INT: + return 2; case _GUARD_NOS_TUPLE: return 0; case _GUARD_TOS_TUPLE: return 0; + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS: + return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; - case _GUARD_NOS_DICT: + case _GUARD_NOS_DICT_SUBSCRIPT: + return 0; + case _GUARD_NOS_DICT_STORE_SUBSCRIPT: + return 0; + case _GUARD_TOS_ANY_DICT: return 0; case _GUARD_TOS_DICT: return 0; + case _GUARD_TOS_FROZENDICT: + return 0; + case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH: + return 2; case _BINARY_OP_SUBSCR_DICT: return 2; case _BINARY_OP_SUBSCR_CHECK_FUNC: @@ -893,12 +6393,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 3; case _STORE_SUBSCR_DICT: return 3; + case _STORE_SUBSCR_DICT_KNOWN_HASH: + return 3; case _DELETE_SUBSCR: return 2; case _CALL_INTRINSIC_1: return 1; case _CALL_INTRINSIC_2: return 2; + case _MAKE_HEAP_SAFE: + return 0; case _RETURN_VALUE: return 1; case _GET_AITER: @@ -909,6 +6413,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _SEND_GEN_FRAME: return 1; + case _GUARD_TOS_IS_NONE: + return 0; + case _GUARD_NOS_NOT_NULL: + return 0; + case _SEND_VIRTUAL_TIER_TWO: + return 1; + case _GUARD_3OS_ASYNC_GEN_ASEND: + return 0; + case _SEND_ASYNC_GEN_TIER_TWO: + return 3; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -919,26 +6433,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _STORE_NAME: return 1; - case _DELETE_NAME: - return 0; case _UNPACK_SEQUENCE: return 1; case _UNPACK_SEQUENCE_TWO_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE: + return 1; + case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE: + return 1; case _UNPACK_SEQUENCE_TUPLE: return 1; + case _UNPACK_SEQUENCE_UNIQUE_TUPLE: + return 1; case _UNPACK_SEQUENCE_LIST: return 1; case _UNPACK_EX: return 1; case _STORE_ATTR: return 2; - case _DELETE_ATTR: - return 1; case _STORE_GLOBAL: return 1; - case _DELETE_GLOBAL: - return 0; case _LOAD_LOCALS: return 0; case _LOAD_NAME: @@ -995,13 +6509,19 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _LOAD_SUPER_ATTR_ATTR: return 3; + case _GUARD_NOS_TYPE_VERSION: + return 0; + case _GUARD_LOAD_SUPER_ATTR_METHOD: + return 0; case _LOAD_SUPER_ATTR_METHOD: return 3; case _LOAD_ATTR: return 1; case _GUARD_TYPE_VERSION: return 0; - case _GUARD_TYPE_VERSION_AND_LOCK: + case _GUARD_TYPE_VERSION_LOCKED: + return 0; + case _GUARD_TYPE: return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; @@ -1019,10 +6539,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_PROPERTY_FRAME: return 1; + case _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME: + return 1; case _GUARD_DORV_NO_DICT: return 0; case _STORE_ATTR_INSTANCE_VALUE: return 2; + case _LOCK_OBJECT: + return 0; case _STORE_ATTR_WITH_HINT: return 2; case _STORE_ATTR_SLOT: @@ -1041,6 +6565,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _GUARD_TOS_ANY_SET: return 0; + case _GUARD_TOS_SET: + return 0; + case _GUARD_TOS_FROZENSET: + return 0; case _CONTAINS_OP_SET: return 2; case _CONTAINS_OP_DICT: @@ -1067,10 +6595,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_ITER: return 1; - case _GET_YIELD_FROM_ITER: + case _GUARD_ITERATOR: + return 0; + case _GUARD_ITER_VIRTUAL: + return 0; + case _PUSH_TAGGED_ZERO: + return 0; + case _GET_ITER_TRAD: return 1; case _FOR_ITER_TIER_TWO: return 0; + case _GUARD_TYPE_ITER: + return 0; + case _ITER_NEXT_INLINE: + return 0; + case _GUARD_NOS_ITER_VIRTUAL: + return 0; + case _GUARD_TOS_NOT_NULL: + return 0; + case _FOR_ITER_VIRTUAL_TIER_TWO: + return 0; case _ITER_CHECK_LIST: return 0; case _GUARD_NOT_EXHAUSTED_LIST: @@ -1099,10 +6643,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _PUSH_EXC_INFO: return 1; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 0; - case _GUARD_KEYS_VERSION: - return 0; case _LOAD_ATTR_METHOD_WITH_VALUES: return 1; case _LOAD_ATTR_METHOD_NO_DICT: @@ -1159,8 +6699,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_NOS_NULL: return 0; - case _GUARD_NOS_NOT_NULL: - return 0; case _GUARD_THIRD_NULL: return 0; case _GUARD_CALLABLE_TYPE_1: @@ -1175,20 +6713,30 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_TUPLE_1: return 3; - case _CHECK_AND_ALLOCATE_OBJECT: + case _CHECK_OBJECT: + return 0; + case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: return 2 + oparg; case _EXIT_INIT_CHECK: return 1; + case _GUARD_CALLABLE_BUILTIN_CLASS: + return 0; case _CALL_BUILTIN_CLASS: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_O: + return 0; case _CALL_BUILTIN_O: return 2 + oparg; + case _GUARD_CALLABLE_BUILTIN_FAST: + return 0; case _CALL_BUILTIN_FAST: - return 2 + oparg; + return 0; + case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: + return 0; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_LEN: return 0; case _CALL_LEN: @@ -1201,14 +6749,32 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_LIST_APPEND: return 3; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O: + return 0; case _CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case _CHECK_RECURSION_LIMIT: + return 0; + case _CALL_METHOD_DESCRIPTOR_O_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: + return 0; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: + return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: return 2 + oparg; + case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE: + return 1 + oparg; + case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: + return 0; case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 0; + case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: + return 0; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: @@ -1225,6 +6791,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 3 + oparg; case _MAKE_CALLARGS_A_TUPLE: return 0; + case _CHECK_IS_PY_CALLABLE_EX: + return 0; + case _PY_FRAME_EX: + return 4; + case _CHECK_IS_NOT_PY_CALLABLE_EX: + return 0; + case _CALL_FUNCTION_EX_NON_PY_GENERAL: + return 4; case _MAKE_FUNCTION: return 1; case _SET_FUNCTION_ATTRIBUTE: @@ -1259,6 +6833,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_IS_FALSE_POP: return 1; + case _GUARD_BIT_IS_SET_POP_4: + return 1; + case _GUARD_BIT_IS_SET_POP_5: + return 1; + case _GUARD_BIT_IS_SET_POP_6: + return 1; + case _GUARD_BIT_IS_SET_POP_7: + return 1; + case _GUARD_BIT_IS_SET_POP: + return 1; + case _GUARD_BIT_IS_UNSET_POP_4: + return 1; + case _GUARD_BIT_IS_UNSET_POP_5: + return 1; + case _GUARD_BIT_IS_UNSET_POP_6: + return 1; + case _GUARD_BIT_IS_UNSET_POP_7: + return 1; + case _GUARD_BIT_IS_UNSET_POP: + return 1; case _GUARD_IS_NONE_POP: return 1; case _GUARD_IS_NOT_NONE_POP: @@ -1279,30 +6873,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_CONST_INLINE: return 0; - case _POP_TOP_LOAD_CONST_INLINE: - return 1; case _LOAD_CONST_INLINE_BORROW: return 0; - case _POP_CALL: - return 2; - case _POP_CALL_ONE: - return 3; - case _POP_CALL_TWO: - return 4; - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - return 1; - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: - return 3; - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: - return 4; - case _LOAD_CONST_UNDER_INLINE: - return 1; - case _LOAD_CONST_UNDER_INLINE_BORROW: - return 1; + case _RROT_3: + return 0; case _START_EXECUTOR: return 0; case _MAKE_WARM: @@ -1315,12 +6889,22 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _ERROR_POP_N: return 0; + case _SPILL_OR_RELOAD: + return 0; case _TIER2_RESUME_CHECK: return 0; case _COLD_EXIT: return 0; case _COLD_DYNAMIC_EXIT: return 0; + case _GUARD_CODE_VERSION__PUSH_FRAME: + return 0; + case _GUARD_CODE_VERSION_YIELD_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_VALUE: + return 0; + case _GUARD_CODE_VERSION_RETURN_GENERATOR: + return 0; case _GUARD_IP__PUSH_FRAME: return 0; case _GUARD_IP_YIELD_VALUE: @@ -1329,6 +6913,28 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_IP_RETURN_GENERATOR: return 0; + case _RECORD_TOS: + return 0; + case _RECORD_TOS_TYPE: + return 0; + case _RECORD_NOS: + return 0; + case _RECORD_NOS_TYPE: + return 0; + case _RECORD_NOS_GEN_FUNC: + return 0; + case _RECORD_3OS_GEN_FUNC: + return 0; + case _RECORD_4OS: + return 0; + case _RECORD_CALLABLE: + return 0; + case _RECORD_CALLABLE_KW: + return 0; + case _RECORD_BOUND_METHOD: + return 0; + case _RECORD_CODE: + return 0; default: return -1; } diff --git a/Include/longobject.h b/Include/longobject.h index 19f06977036d056..38673bc18785fab 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -166,6 +166,44 @@ PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + int8_t digit_endianness; +} PyLongLayout; + +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); + +typedef struct PyLongExport { + int64_t value; + uint8_t negative; + Py_ssize_t ndigits; + const void *digits; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; +} PyLongExport; + +PyAPI_FUNC(int) PyLong_Export( + PyObject *obj, + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + void **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); + #ifndef Py_LIMITED_API # define Py_CPYTHON_LONGOBJECT_H # include "cpython/longobject.h" diff --git a/Include/modsupport.h b/Include/modsupport.h index 094b9ff0e5ccf80..cb47ad8cd2727fd 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -132,7 +132,7 @@ PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); ) \ ///////////////////////////////////////////////////////// -#define _PyABIInfo_DEFAULT() { \ +#define _PyABIInfo_DEFAULT { \ 1, 0, \ PyABIInfo_DEFAULT_FLAGS, \ PY_VERSION_HEX, \ diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e83bc395aa46188..88c66672ff164a8 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -73,31 +73,6 @@ struct PyModuleDef_Slot { void *value; }; -#define Py_mod_create 1 -#define Py_mod_exec 2 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 -# define Py_mod_multiple_interpreters 3 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -# define Py_mod_gil 4 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -# define Py_mod_abi 5 -# define Py_mod_name 6 -# define Py_mod_doc 7 -# define Py_mod_state_size 8 -# define Py_mod_methods 9 -# define Py_mod_state_traverse 10 -# define Py_mod_state_clear 11 -# define Py_mod_state_free 12 -# define Py_mod_token 13 -#endif - - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 13 -#endif - #endif /* New in 3.5 */ /* for Py_mod_multiple_interpreters: */ @@ -113,16 +88,20 @@ struct PyModuleDef_Slot { # define Py_MOD_GIL_NOT_USED ((void *)1) #endif -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); +# endif #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec); -PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); -PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result); -PyAPI_FUNC(int) PyModule_GetToken(PyObject *, void **result); +PyAPI_FUNC(int) PyModule_Exec(PyObject *module); +PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *module, Py_ssize_t *result); +PyAPI_FUNC(int) PyModule_GetToken(PyObject *module, void **result); +PyAPI_FUNC(void*) PyModule_GetState_DuringGC(PyObject*); +PyAPI_FUNC(int) PyModule_GetToken_DuringGC(PyObject *module, void **result); #endif #ifndef _Py_OPAQUE_PYOBJECT diff --git a/Include/object.h b/Include/object.h index f17dcba4f476b05..20c2dab4401fef0 100644 --- a/Include/object.h +++ b/Include/object.h @@ -127,7 +127,7 @@ whose size is determined when the object is allocated. struct _object { _Py_ANONYMOUS union { #if SIZEOF_VOID_P > 4 - PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + int64_t ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ struct { # if PY_BIG_ENDIAN uint16_t ob_flags; @@ -140,12 +140,12 @@ struct _object { # endif }; #else - Py_ssize_t ob_refcnt; + Py_ssize_t ob_refcnt; // part of stable ABI; do not change #endif _Py_ALIGNED_DEF(_PyObject_MIN_ALIGNMENT, char) _aligner; }; - PyTypeObject *ob_type; + PyTypeObject *ob_type; // part of stable ABI; do not change }; #else // Objects that are not owned by any thread use a thread id (tid) of zero. @@ -173,7 +173,7 @@ struct _object { #ifndef _Py_OPAQUE_PYOBJECT struct PyVarObject { PyObject ob_base; - Py_ssize_t ob_size; /* Number of items in variable part */ + Py_ssize_t ob_size; // Number of items in variable part. Part of stable ABI }; #endif typedef struct PyVarObject PyVarObject; @@ -186,135 +186,72 @@ typedef struct PyVarObject PyVarObject; PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); #define Py_Is(x, y) ((x) == (y)) -#if defined(Py_GIL_DISABLED) && !defined(Py_LIMITED_API) -PyAPI_FUNC(uintptr_t) _Py_GetThreadLocal_Addr(void); +PyAPI_DATA(PyTypeObject) PyLong_Type; +PyAPI_DATA(PyTypeObject) PyBool_Type; -static inline uintptr_t -_Py_ThreadId(void) -{ - uintptr_t tid; -#if defined(_MSC_VER) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(_MSC_VER) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(_MSC_VER) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__MINGW32__) && defined(_M_X64) - tid = __readgsqword(48); -#elif defined(__MINGW32__) && defined(_M_IX86) - tid = __readfsdword(24); -#elif defined(__MINGW32__) && defined(_M_ARM64) - tid = __getReg(18); -#elif defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS -#elif defined(__MACH__) && defined(__x86_64__) - __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS -#elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS -#elif defined(__arm__) && __ARM_ARCH >= 7 - __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); -#elif defined(__aarch64__) && defined(__APPLE__) - __asm__ ("mrs %0, tpidrro_el0" : "=r" (tid)); -#elif defined(__aarch64__) - __asm__ ("mrs %0, tpidr_el0" : "=r" (tid)); -#elif defined(__powerpc64__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r13 is reserved for use as system thread ID by the Power 64-bit ABI. - register uintptr_t tp __asm__ ("r13"); - __asm__("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__powerpc__) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // r2 is reserved for use as system thread ID by the Power 32-bit ABI. - register uintptr_t tp __asm__ ("r2"); - __asm__ ("" : "=r" (tp)); - tid = tp; - #endif -#elif defined(__s390__) && defined(__GNUC__) - // Both GCC and Clang have supported __builtin_thread_pointer - // for s390 from long time ago. - tid = (uintptr_t)__builtin_thread_pointer(); -#elif defined(__riscv) - #if defined(__clang__) && _Py__has_builtin(__builtin_thread_pointer) - tid = (uintptr_t)__builtin_thread_pointer(); - #else - // tp is Thread Pointer provided by the RISC-V ABI. - __asm__ ("mv %0, tp" : "=r" (tid)); - #endif -#else - // Fallback to a portable implementation if we do not have a faster - // platform-specific implementation. - tid = _Py_GetThreadLocal_Addr(); +/* Definitions for the stable ABI */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) +PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); +#endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(Py_ssize_t) Py_SIZE(PyObject *ob); +PyAPI_FUNC(int) Py_IS_TYPE(PyObject *ob, PyTypeObject *type); +PyAPI_FUNC(void) Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size); #endif - return tid; -} -static inline Py_ALWAYS_INLINE int -_Py_IsOwnedByCurrentThread(PyObject *ob) +#ifndef _Py_OPAQUE_PYOBJECT + +static inline void +Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { -#ifdef _Py_THREAD_SANITIZER - return _Py_atomic_load_uintptr_relaxed(&ob->ob_tid) == _Py_ThreadId(); -#else - return ob->ob_tid == _Py_ThreadId(); -#endif + ob->ob_type = type; } -#endif -// Py_TYPE() implementation for the stable ABI -PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); - -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 - // Stable ABI implements Py_TYPE() as a function call - // on limited C API version 3.14 and newer. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 11) +// Non-limited API & limited API 3.11 & below: use static inline functions and +// use _PyObject_CAST so that users don't need their own casts +# define Py_TYPE(ob) _Py_TYPE_impl(_PyObject_CAST(ob)) +# define Py_SIZE(ob) _Py_SIZE_impl(_PyObject_CAST(ob)) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl(_PyObject_CAST(ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl(_PyVarObject_CAST(ob), (size)) +# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) +#elif Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 15) +// Limited API 3.11-3.14: use static inline functions, without casts +# define Py_SIZE(ob) _Py_SIZE_impl(ob) +# define Py_IS_TYPE(ob, type) _Py_IS_TYPE_impl((ob), (type)) +# define Py_SET_SIZE(ob, size) _Py_SET_SIZE_impl((ob), (size)) +# if Py_LIMITED_API+0 < _Py_PACK_VERSION(3, 14) +// Py_TYPE() is static inline only on Limited API 3.13 and below +# define Py_TYPE(ob) _Py_TYPE_impl(ob) +# endif #else - static inline PyTypeObject* _Py_TYPE(PyObject *ob) - { - return ob->ob_type; - } - #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 - # define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST(ob)) - #else - # define Py_TYPE(ob) _Py_TYPE(ob) - #endif +// Limited API 3.15+: use function calls #endif -PyAPI_DATA(PyTypeObject) PyLong_Type; -PyAPI_DATA(PyTypeObject) PyBool_Type; +static inline +PyTypeObject* _Py_TYPE_impl(PyObject *ob) +{ + return ob->ob_type; +} -#ifndef _Py_OPAQUE_PYOBJECT // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -static inline Py_ssize_t Py_SIZE(PyObject *ob) { +static inline Py_ssize_t +_Py_SIZE_impl(PyObject *ob) +{ assert(Py_TYPE(ob) != &PyLong_Type); assert(Py_TYPE(ob) != &PyBool_Type); return _PyVarObject_CAST(ob)->ob_size; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) -#endif -#endif // !defined(_Py_OPAQUE_PYOBJECT) -static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { +static inline int +_Py_IS_TYPE_impl(PyObject *ob, PyTypeObject *type) +{ return Py_TYPE(ob) == type; } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_IS_TYPE(ob, type) Py_IS_TYPE(_PyObject_CAST(ob), (type)) -#endif - -#ifndef _Py_OPAQUE_PYOBJECT -static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { - ob->ob_type = type; -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_TYPE(ob, type) Py_SET_TYPE(_PyObject_CAST(ob), type) -#endif - -static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { +static inline void +_Py_SET_SIZE_impl(PyVarObject *ob, Py_ssize_t size) +{ assert(Py_TYPE(_PyObject_CAST(ob)) != &PyLong_Type); assert(Py_TYPE(_PyObject_CAST(ob)) != &PyBool_Type); #ifdef Py_GIL_DISABLED @@ -323,9 +260,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { ob->ob_size = size; #endif } -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) -#endif + #endif // !defined(_Py_OPAQUE_PYOBJECT) @@ -371,6 +306,11 @@ typedef Py_hash_t (*hashfunc)(PyObject *); typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int); typedef PyObject *(*getiterfunc) (PyObject *); typedef PyObject *(*iternextfunc) (PyObject *); +typedef struct { + PyObject *object; + Py_ssize_t index; +} _PyObjectIndexPair; +typedef _PyObjectIndexPair (*_Py_iteritemfunc) (PyObject *, Py_ssize_t index); typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*initproc)(PyObject *, PyObject *, PyObject *); @@ -424,6 +364,9 @@ PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); #define Py_TP_USE_SPEC NULL #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_FromSlots(struct PySlot *slots); +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); @@ -638,8 +581,10 @@ given type object has a specified feature. #define _Py_IMMORTAL_FLAGS (1 << 0) #define _Py_LEGACY_ABI_CHECK_FLAG (1 << 1) /* see PyModuleDef_Init() */ #define _Py_STATICALLY_ALLOCATED_FLAG (1 << 2) -#if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) -#define _Py_TYPE_REVEALED_FLAG (1 << 3) +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) && defined(Py_DEBUG) +# define _Py_TYPE_REVEALED_FLAG (1 << 3) +# endif #endif #define Py_CONSTANT_NONE 0 @@ -840,8 +785,17 @@ PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, const void *token); +PyAPI_FUNC(void *) PyObject_GetTypeData_DuringGC(PyObject *obj, + PyTypeObject *cls); +PyAPI_FUNC(void *) PyType_GetModuleState_DuringGC(PyTypeObject *); +PyAPI_FUNC(int) PyType_GetBaseByToken_DuringGC(PyTypeObject *, + void *, PyTypeObject **); +PyAPI_FUNC(PyObject *) PyType_GetModule_DuringGC(PyTypeObject *); +PyAPI_FUNC(PyObject *) PyType_GetModuleByToken_DuringGC(PyTypeObject *type, + const void *token); #endif #ifdef __cplusplus diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35d..11342ae451b9f6c 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -26,111 +26,107 @@ extern "C" { #define FORMAT_WITH_SPEC 13 #define GET_AITER 14 #define GET_ANEXT 15 -#define GET_ITER 16 +#define GET_LEN 16 #define RESERVED 17 -#define GET_LEN 18 -#define GET_YIELD_FROM_ITER 19 -#define INTERPRETER_EXIT 20 -#define LOAD_BUILD_CLASS 21 -#define LOAD_LOCALS 22 -#define MAKE_FUNCTION 23 -#define MATCH_KEYS 24 -#define MATCH_MAPPING 25 -#define MATCH_SEQUENCE 26 -#define NOP 27 -#define NOT_TAKEN 28 -#define POP_EXCEPT 29 -#define POP_ITER 30 -#define POP_TOP 31 -#define PUSH_EXC_INFO 32 -#define PUSH_NULL 33 -#define RETURN_GENERATOR 34 -#define RETURN_VALUE 35 -#define SETUP_ANNOTATIONS 36 -#define STORE_SLICE 37 -#define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define INTERPRETER_EXIT 18 +#define LOAD_BUILD_CLASS 19 +#define LOAD_LOCALS 20 +#define MAKE_FUNCTION 21 +#define MATCH_KEYS 22 +#define MATCH_MAPPING 23 +#define MATCH_SEQUENCE 24 +#define NOP 25 +#define NOT_TAKEN 26 +#define POP_EXCEPT 27 +#define POP_ITER 28 +#define POP_TOP 29 +#define PUSH_EXC_INFO 30 +#define PUSH_NULL 31 +#define RETURN_GENERATOR 32 +#define RETURN_VALUE 33 +#define SETUP_ANNOTATIONS 34 +#define STORE_SLICE 35 +#define STORE_SUBSCR 36 +#define TO_BOOL 37 +#define UNARY_INVERT 38 +#define UNARY_NEGATIVE 39 +#define UNARY_NOT 40 +#define WITH_EXCEPT_START 41 +#define BINARY_OP 42 +#define BUILD_INTERPOLATION 43 +#define BUILD_LIST 44 +#define BUILD_MAP 45 +#define BUILD_SET 46 +#define BUILD_SLICE 47 +#define BUILD_STRING 48 +#define BUILD_TUPLE 49 +#define CALL 50 +#define CALL_INTRINSIC_1 51 +#define CALL_INTRINSIC_2 52 +#define CALL_KW 53 +#define COMPARE_OP 54 +#define CONTAINS_OP 55 +#define CONVERT_VALUE 56 +#define COPY 57 +#define COPY_FREE_VARS 58 +#define DELETE_DEREF 59 +#define DELETE_FAST 60 +#define DICT_MERGE 61 +#define DICT_UPDATE 62 +#define END_ASYNC_FOR 63 +#define EXTENDED_ARG 64 +#define FOR_ITER 65 +#define GET_AWAITABLE 66 +#define GET_ITER 67 +#define IMPORT_FROM 68 +#define IMPORT_NAME 69 +#define IS_OP 70 +#define JUMP_BACKWARD 71 +#define JUMP_BACKWARD_NO_INTERRUPT 72 +#define JUMP_FORWARD 73 +#define LIST_APPEND 74 +#define LIST_EXTEND 75 +#define LOAD_ATTR 76 +#define LOAD_COMMON_CONSTANT 77 +#define LOAD_CONST 78 +#define LOAD_DEREF 79 +#define LOAD_FAST 80 +#define LOAD_FAST_AND_CLEAR 81 +#define LOAD_FAST_BORROW 82 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 +#define LOAD_FAST_CHECK 84 +#define LOAD_FAST_LOAD_FAST 85 +#define LOAD_FROM_DICT_OR_DEREF 86 +#define LOAD_FROM_DICT_OR_GLOBALS 87 +#define LOAD_GLOBAL 88 +#define LOAD_NAME 89 +#define LOAD_SMALL_INT 90 +#define LOAD_SPECIAL 91 +#define LOAD_SUPER_ATTR 92 +#define MAKE_CELL 93 +#define MAP_ADD 94 +#define MATCH_CLASS 95 +#define POP_JUMP_IF_FALSE 96 +#define POP_JUMP_IF_NONE 97 +#define POP_JUMP_IF_NOT_NONE 98 +#define POP_JUMP_IF_TRUE 99 +#define RAISE_VARARGS 100 +#define RERAISE 101 +#define SEND 102 +#define SET_ADD 103 +#define SET_FUNCTION_ATTRIBUTE 104 +#define SET_UPDATE 105 +#define STORE_ATTR 106 +#define STORE_DEREF 107 +#define STORE_FAST 108 +#define STORE_FAST_LOAD_FAST 109 +#define STORE_FAST_STORE_FAST 110 +#define STORE_GLOBAL 111 +#define STORE_NAME 112 +#define SWAP 113 +#define UNPACK_EX 114 +#define UNPACK_SEQUENCE 115 +#define YIELD_VALUE 116 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -144,97 +140,107 @@ extern "C" { #define BINARY_OP_SUBSCR_LIST_SLICE 138 #define BINARY_OP_SUBSCR_STR_INT 139 #define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBTRACT_FLOAT 141 -#define BINARY_OP_SUBTRACT_INT 142 -#define CALL_ALLOC_AND_ENTER_INIT 143 -#define CALL_BOUND_METHOD_EXACT_ARGS 144 -#define CALL_BOUND_METHOD_GENERAL 145 -#define CALL_BUILTIN_CLASS 146 -#define CALL_BUILTIN_FAST 147 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 148 -#define CALL_BUILTIN_O 149 -#define CALL_ISINSTANCE 150 -#define CALL_KW_BOUND_METHOD 151 -#define CALL_KW_NON_PY 152 -#define CALL_KW_PY 153 -#define CALL_LEN 154 -#define CALL_LIST_APPEND 155 -#define CALL_METHOD_DESCRIPTOR_FAST 156 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 157 -#define CALL_METHOD_DESCRIPTOR_NOARGS 158 -#define CALL_METHOD_DESCRIPTOR_O 159 -#define CALL_NON_PY_GENERAL 160 -#define CALL_PY_EXACT_ARGS 161 -#define CALL_PY_GENERAL 162 -#define CALL_STR_1 163 -#define CALL_TUPLE_1 164 -#define CALL_TYPE_1 165 -#define COMPARE_OP_FLOAT 166 -#define COMPARE_OP_INT 167 -#define COMPARE_OP_STR 168 -#define CONTAINS_OP_DICT 169 -#define CONTAINS_OP_SET 170 -#define FOR_ITER_GEN 171 -#define FOR_ITER_LIST 172 -#define FOR_ITER_RANGE 173 -#define FOR_ITER_TUPLE 174 -#define JUMP_BACKWARD_JIT 175 -#define JUMP_BACKWARD_NO_JIT 176 -#define LOAD_ATTR_CLASS 177 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 178 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 179 -#define LOAD_ATTR_INSTANCE_VALUE 180 -#define LOAD_ATTR_METHOD_LAZY_DICT 181 -#define LOAD_ATTR_METHOD_NO_DICT 182 -#define LOAD_ATTR_METHOD_WITH_VALUES 183 -#define LOAD_ATTR_MODULE 184 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 185 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 186 -#define LOAD_ATTR_PROPERTY 187 -#define LOAD_ATTR_SLOT 188 -#define LOAD_ATTR_WITH_HINT 189 -#define LOAD_GLOBAL_BUILTIN 190 -#define LOAD_GLOBAL_MODULE 191 -#define LOAD_SUPER_ATTR_ATTR 192 -#define LOAD_SUPER_ATTR_METHOD 193 -#define RESUME_CHECK 194 -#define SEND_GEN 195 -#define STORE_ATTR_INSTANCE_VALUE 196 -#define STORE_ATTR_SLOT 197 -#define STORE_ATTR_WITH_HINT 198 -#define STORE_SUBSCR_DICT 199 -#define STORE_SUBSCR_LIST_INT 200 -#define TO_BOOL_ALWAYS_TRUE 201 -#define TO_BOOL_BOOL 202 -#define TO_BOOL_INT 203 -#define TO_BOOL_LIST 204 -#define TO_BOOL_NONE 205 -#define TO_BOOL_STR 206 -#define UNPACK_SEQUENCE_LIST 207 -#define UNPACK_SEQUENCE_TUPLE 208 -#define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define BINARY_OP_SUBSCR_USTR_INT 141 +#define BINARY_OP_SUBTRACT_FLOAT 142 +#define BINARY_OP_SUBTRACT_INT 143 +#define CALL_ALLOC_AND_ENTER_INIT 144 +#define CALL_BOUND_METHOD_EXACT_ARGS 145 +#define CALL_BOUND_METHOD_GENERAL 146 +#define CALL_BUILTIN_CLASS 147 +#define CALL_BUILTIN_FAST 148 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 +#define CALL_BUILTIN_O 150 +#define CALL_EX_NON_PY_GENERAL 151 +#define CALL_EX_PY 152 +#define CALL_ISINSTANCE 153 +#define CALL_KW_BOUND_METHOD 154 +#define CALL_KW_NON_PY 155 +#define CALL_KW_PY 156 +#define CALL_LEN 157 +#define CALL_LIST_APPEND 158 +#define CALL_METHOD_DESCRIPTOR_FAST 159 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 +#define CALL_METHOD_DESCRIPTOR_NOARGS 161 +#define CALL_METHOD_DESCRIPTOR_O 162 +#define CALL_NON_PY_GENERAL 163 +#define CALL_PY_EXACT_ARGS 164 +#define CALL_PY_GENERAL 165 +#define CALL_STR_1 166 +#define CALL_TUPLE_1 167 +#define CALL_TYPE_1 168 +#define COMPARE_OP_FLOAT 169 +#define COMPARE_OP_INT 170 +#define COMPARE_OP_STR 171 +#define CONTAINS_OP_DICT 172 +#define CONTAINS_OP_SET 173 +#define FOR_ITER_GEN 174 +#define FOR_ITER_LIST 175 +#define FOR_ITER_RANGE 176 +#define FOR_ITER_TUPLE 177 +#define FOR_ITER_VIRTUAL 178 +#define GET_ITER_SELF 179 +#define GET_ITER_VIRTUAL 180 +#define JUMP_BACKWARD_JIT 181 +#define JUMP_BACKWARD_NO_JIT 182 +#define LOAD_ATTR_CLASS 183 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 +#define LOAD_ATTR_INSTANCE_VALUE 186 +#define LOAD_ATTR_METHOD_LAZY_DICT 187 +#define LOAD_ATTR_METHOD_NO_DICT 188 +#define LOAD_ATTR_METHOD_WITH_VALUES 189 +#define LOAD_ATTR_MODULE 190 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 +#define LOAD_ATTR_PROPERTY 193 +#define LOAD_ATTR_SLOT 194 +#define LOAD_ATTR_WITH_HINT 195 +#define LOAD_GLOBAL_BUILTIN 196 +#define LOAD_GLOBAL_MODULE 197 +#define LOAD_SUPER_ATTR_ATTR 198 +#define LOAD_SUPER_ATTR_METHOD 199 +#define RESUME_CHECK 200 +#define RESUME_CHECK_JIT 201 +#define SEND_ASYNC_GEN 202 +#define SEND_GEN 203 +#define SEND_VIRTUAL 204 +#define STORE_ATTR_INSTANCE_VALUE 205 +#define STORE_ATTR_SLOT 206 +#define STORE_ATTR_WITH_HINT 207 +#define STORE_SUBSCR_DICT 208 +#define STORE_SUBSCR_LIST_INT 209 +#define TO_BOOL_ALWAYS_TRUE 210 +#define TO_BOOL_BOOL 211 +#define TO_BOOL_INT 212 +#define TO_BOOL_LIST 213 +#define TO_BOOL_NONE 214 +#define TO_BOOL_STR 215 +#define UNPACK_SEQUENCE_LIST 216 +#define UNPACK_SEQUENCE_TUPLE 217 +#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -247,9 +253,9 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 43 +#define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e3996ee86793dd0..d82d52584069413 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,13 +21,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 15 +#define PY_MINOR_VERSION 16 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.15.0a1+" +#define PY_VERSION "3.16.0a0" /*--end constants--*/ diff --git a/Include/py_curses.h b/Include/py_curses.h index 0948aabedd49939..a5f5f3d76075827 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,13 @@ #define NCURSES_OPAQUE 0 #endif +/* PDCurses exposes its ncurses-compatible mouse API, the one this module uses, + only when this is defined before the curses header is included below. + Ignored by other curses implementations. */ +#ifndef PDC_NCMOUSE +# define PDC_NCMOUSE +#endif + #if defined(HAVE_NCURSESW_NCURSES_H) # include #elif defined(HAVE_NCURSESW_CURSES_H) @@ -80,8 +87,18 @@ typedef struct PyCursesWindowObject { WINDOW *win; char *encoding; struct PyCursesWindowObject *orig; + PyObject *screen; /* the screen the window belongs to, or NULL, + kept alive for the lifetime of the window */ } PyCursesWindowObject; +typedef struct { + PyObject_HEAD + SCREEN *screen; /* NULL after the screen has been deleted */ + FILE *outfp; /* owned output stream, or NULL */ + FILE *infp; /* owned input stream, or NULL */ + PyObject *stdscr_win; /* the screen's standard window, or NULL */ +} PyCursesScreenObject; + #define PyCurses_CAPSULE_NAME "_curses._C_API" diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 000000000000000..21a6ab0c1ee6ea2 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,123 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + * + * This logic is currently partially duplicated in PC/pyconfig.h. + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pybuffer.h b/Include/pybuffer.h index ca1c6058d9052c5..6371b9b483777ba 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -67,27 +67,27 @@ PyAPI_FUNC(int) PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, error (i.e. the object does not have a buffer interface or it is not working). - If fort is 'F', then if the object is multi-dimensional, + If order is 'F', then if the object is multi-dimensional, then the data will be copied into the array in Fortran-style (first dimension varies the fastest). If - fort is 'C', then the data will be copied into the array - in C-style (last dimension varies the fastest). If fort + order is 'C', then the data will be copied into the array + in C-style (last dimension varies the fastest). If order is 'A', then it does not matter and the copy will be made in whatever way is more efficient. */ PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src); /* Copy the data from the src buffer to the buffer of destination. */ -PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char fort); +PyAPI_FUNC(int) PyBuffer_IsContiguous(const Py_buffer *view, char order); /*Fill the strides array with byte-strides of a contiguous - (Fortran-style if fort is 'F' or C-style otherwise) + (Fortran-style if order is 'F' or C-style otherwise) array of the given shape with the given number of bytes per element. */ PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, - char fort); + char order); /* Fills in a buffer-info structure correctly for an exporter that can only share a contiguous chunk of memory of diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 5d0028c116e2d86..cfabbc5fe8d5da0 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -91,6 +91,9 @@ PyAPI_DATA(PyObject *) PyExc_EOFError; PyAPI_DATA(PyObject *) PyExc_FloatingPointError; PyAPI_DATA(PyObject *) PyExc_OSError; PyAPI_DATA(PyObject *) PyExc_ImportError; +#if !defined(Py_LIMITED_API) +PyAPI_DATA(PyObject *) PyExc_ImportCycleError; +#endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 PyAPI_DATA(PyObject *) PyExc_ModuleNotFoundError; #endif diff --git a/Include/pyexpat.h b/Include/pyexpat.h index f523f8bb273983a..a676e16a7a457ea 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -62,6 +62,9 @@ struct PyExpat_CAPI XML_Parser parser, unsigned long long activationThresholdBytes); XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Include/pymacconfig.h b/Include/pymacconfig.h index 615abe103ca0388..9d63ddf8a716f43 100644 --- a/Include/pymacconfig.h +++ b/Include/pymacconfig.h @@ -18,7 +18,6 @@ #undef SIZEOF_UINTPTR_T #undef SIZEOF_PTHREAD_T #undef WORDS_BIGENDIAN -#undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 #undef DOUBLE_IS_BIG_ENDIAN_IEEE754 #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 #undef HAVE_GCC_ASM_FOR_X87 diff --git a/Include/pymacro.h b/Include/pymacro.h index 857cdf12db9bf26..7ecce44a0d2a428 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -116,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Include/pyport.h b/Include/pyport.h index e77b39026a59c1f..73a3e6cdaf09200 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,26 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -385,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX @@ -446,7 +415,9 @@ extern "C" { /* * Specify alignment on compilers that support it. */ -#if defined(__GNUC__) && __GNUC__ >= 3 +#ifdef Py_BUILD_CORE +// always use _Py_ALIGNED_DEF instead +#elif defined(__GNUC__) && __GNUC__ >= 3 #define Py_ALIGNED(x) __attribute__((aligned(x))) #else #define Py_ALIGNED(x) @@ -504,13 +475,20 @@ extern "C" { * Thread support is stubbed and any attempt to create a new thread fails. */ #if (!defined(HAVE_PTHREAD_STUBS) && \ + !defined(__wasi__) && \ (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) # define Py_CAN_START_THREADS 1 #endif -#ifdef WITH_THREAD -// HAVE_THREAD_LOCAL is just defined here for compatibility's sake + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. # define HAVE_THREAD_LOCAL 1 +#endif + +#ifdef WITH_THREAD # ifdef thread_local # define _Py_thread_local thread_local # elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) @@ -560,8 +538,11 @@ extern "C" { // // Example: _Py_TYPEOF(x) x_copy = (x); // -// The macro is only defined if GCC or clang compiler is used. -#if defined(__GNUC__) || defined(__clang__) +// On C23, use typeof(). Otherwise, the macro is only defined +// if GCC or clang compiler is used. +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L +# define _Py_TYPEOF(expr) typeof(expr) +#elif defined(__GNUC__) || defined(__clang__) # define _Py_TYPEOF(expr) __typeof__(expr) #endif @@ -572,6 +553,7 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) # endif # endif # if __has_feature(address_sanitizer) @@ -610,6 +592,9 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) @@ -677,4 +662,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e67..8dad748238f4f39 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,29 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +/* PEP 788 -- Protection against interpreter finalization */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) + +typedef struct PyInterpreterGuard PyInterpreterGuard; +typedef struct PyInterpreterView PyInterpreterView; + +typedef void PyThreadStateToken; + +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterGuard_Close(PyInterpreterGuard *guard); +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromView(PyInterpreterView *view); + +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterView_Close(PyInterpreterView *view); +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromMain(void); + +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_Ensure(PyInterpreterGuard *guard); +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_EnsureFromView(PyInterpreterView *view); +PyAPI_FUNC(void) PyThreadState_Release(PyThreadStateToken *tstate); + +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd1..1d0b4e3e85ef74a 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -14,6 +14,7 @@ typedef struct PyModuleDef_Slot PyModuleDef_Slot; typedef struct PyMethodDef PyMethodDef; typedef struct PyGetSetDef PyGetSetDef; typedef struct PyMemberDef PyMemberDef; +typedef struct PySlot PySlot; typedef struct _object PyObject; typedef struct _longobject PyLongObject; diff --git a/Include/refcount.h b/Include/refcount.h index 51346c7e519321d..80fe7ff70a11e87 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -5,6 +5,7 @@ extern "C" { #endif +#if !defined(_Py_OPAQUE_PYOBJECT) /* Immortalization: @@ -90,14 +91,16 @@ check by comparing the reference count field to the minimum immortality refcount # define _Py_REF_SHARED(refcnt, flags) \ (((refcnt) << _Py_REF_SHARED_SHIFT) + (flags)) #endif // Py_GIL_DISABLED - +#endif // _Py_OPAQUE_PYOBJECT // Py_REFCNT() implementation for the stable ABI PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 // Stable ABI implements Py_REFCNT() as a function call - // on limited C API version 3.14 and newer. + // on limited C API version 3.14 and newer, and on abi3t. +#elif defined(_Py_OPAQUE_PYOBJECT) + // Py_REFCNT() is also a function call in abi3t. #else static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { #if !defined(Py_GIL_DISABLED) @@ -126,7 +129,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) return (_Py_atomic_load_uint32_relaxed(&op->ob_ref_local) == _Py_IMMORTAL_REFCNT_LOCAL); #elif SIZEOF_VOID_P > 4 - return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; + return _Py_CAST(int32_t, op->ob_refcnt) < 0; #else return op->ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT; #endif @@ -150,9 +153,10 @@ PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { assert(refcnt >= 0); -#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_SET_REFCNT() as a function call - // on limited C API version 3.13 and newer. + // on limited C API version 3.13 and newer, and abi3t. _Py_SetRefcnt(ob, refcnt); #else // This immortal check is for code that is unaware of immortal objects. @@ -164,7 +168,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { } #ifndef Py_GIL_DISABLED #if SIZEOF_VOID_P > 4 - ob->ob_refcnt = (PY_UINT32_T)refcnt; + ob->ob_refcnt = (uint32_t)refcnt; #else ob->ob_refcnt = refcnt; #endif @@ -191,7 +195,7 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); } #endif // Py_GIL_DISABLED -#endif // Py_LIMITED_API+0 < 0x030d0000 +#endif // Py_LIMITED_API } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_REFCNT(ob, refcnt) Py_SET_REFCNT(_PyObject_CAST(ob), (refcnt)) @@ -250,10 +254,11 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, and on Python built in debug mode. _Py_IncRef() - // was added to Python 3.10.0a7, use Py_IncRef() on older Python versions. + // version 3.12 and newer, abi3t, and on Python built in debug mode. + // _Py_IncRef() was added to Python 3.10.0a7, use Py_IncRef() on older versions. // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. # if Py_LIMITED_API+0 >= 0x030a00A7 _Py_IncRef(op); @@ -278,7 +283,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt; + uint32_t cur_refcnt = op->ob_refcnt; if (cur_refcnt >= _Py_IMMORTAL_INITIAL_REFCNT) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); @@ -305,8 +310,8 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) # define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op)) #endif - -#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +#if !defined(Py_LIMITED_API) +#if defined(Py_GIL_DISABLED) // Implements Py_DECREF on objects not owned by the current thread. PyAPI_FUNC(void) _Py_DecRefShared(PyObject *); PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); @@ -316,12 +321,14 @@ PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int); // zero. Otherwise, the thread gives up ownership and merges the reference // count fields. PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); -#endif +#endif // Py_GIL_DISABLED +#endif // Py_LIMITED_API -#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) +#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ + || defined(_Py_OPAQUE_PYOBJECT) // Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was -// added to Python 3.10.0a7, use Py_DecRef() on older Python versions. +// version 3.12 and newer, abi3t, and on Python built in debug mode. +// _Py_DecRef() was added to Python 3.10.0a7, use Py_DecRef() on older versions. // Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't. static inline void Py_DECREF(PyObject *op) { # if Py_LIMITED_API+0 >= 0x030a00A7 @@ -387,7 +394,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #if SIZEOF_VOID_P > 4 /* If an object has been freed, it will have a negative full refcnt * If it has not it been freed, will have a very large refcnt */ - if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) { + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((uint32_t)-1) - (1<<20))) { #else if (op->ob_refcnt <= 0) { #endif diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 00c70a6e911b417..9d6a16da95fe2f5 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,9 +45,9 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, #endif #ifndef Py_LIMITED_API -# define Py_CPYTHON_SLICEOBJECT_H +# define _Py_CPYTHON_SLICEOBJECT_H # include "cpython/sliceobject.h" -# undef Py_CPYTHON_SLICEOBJECT_H +# undef _Py_CPYTHON_SLICEOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/slots.h b/Include/slots.h new file mode 100644 index 000000000000000..4bf7bda0208a8df --- /dev/null +++ b/Include/slots.h @@ -0,0 +1,56 @@ +#ifndef _Py_HAVE_SLOTS_H +#define _Py_HAVE_SLOTS_H + +typedef void (*_Py_funcptr_t)(void); + +struct PySlot { + uint16_t sl_id; + uint16_t sl_flags; + _Py_ANONYMOUS union { + uint32_t sl_reserved; // must be 0 + }; + _Py_ANONYMOUS union { + void *sl_ptr; + _Py_funcptr_t sl_func; + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; +}; + +#define PySlot_OPTIONAL 0x0001 +#define PySlot_STATIC 0x0002 +#define PySlot_INTPTR 0x0004 + +#define Py_slot_invalid 0xffff + +#define PySlot_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_INTPTR, .sl_ptr=(void*)(VALUE)} + +#define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=(NAME), .sl_func=(_Py_funcptr_t)(VALUE)} + +#define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=(NAME), .sl_size=(Py_ssize_t)(VALUE)} + +#define PySlot_INT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_int64=(int64_t)(VALUE)} + +#define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_uint64=(uint64_t)(VALUE)} + +#define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + +#define PySlot_END {0} + + +// Macros without designated initializers (for C++11 and below): + +#define PySlot_PTR(NAME, VALUE) \ + {(NAME), PySlot_INTPTR, {0}, {(void*)(VALUE)}} + +#define PySlot_PTR_STATIC(NAME, VALUE) \ + {(NAME), PySlot_INTPTR | PySlot_STATIC, {0}, {(void*)(VALUE)}} + +#endif // _Py_HAVE_SLOTS_H diff --git a/Include/slots_generated.h b/Include/slots_generated.h new file mode 100644 index 000000000000000..42edd3ad4c69ffd --- /dev/null +++ b/Include/slots_generated.h @@ -0,0 +1,121 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_SLOTS_GENERATED_H +#define _PY_HAVE_SLOTS_GENERATED_H + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW +#else +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD +#endif + +#define Py_slot_end 0 +#define Py_mp_subscript 5 +#define Py_nb_absolute 6 +#define Py_nb_add 7 +#define Py_nb_and 8 +#define Py_nb_bool 9 +#define Py_nb_divmod 10 +#define Py_nb_float 11 +#define Py_nb_floor_divide 12 +#define Py_nb_index 13 +#define Py_nb_inplace_add 14 +#define Py_nb_inplace_and 15 +#define Py_nb_inplace_floor_divide 16 +#define Py_nb_inplace_lshift 17 +#define Py_nb_inplace_multiply 18 +#define Py_nb_inplace_or 19 +#define Py_nb_inplace_power 20 +#define Py_nb_inplace_remainder 21 +#define Py_nb_inplace_rshift 22 +#define Py_nb_inplace_subtract 23 +#define Py_nb_inplace_true_divide 24 +#define Py_nb_inplace_xor 25 +#define Py_nb_int 26 +#define Py_nb_invert 27 +#define Py_nb_lshift 28 +#define Py_nb_multiply 29 +#define Py_nb_negative 30 +#define Py_nb_or 31 +#define Py_nb_positive 32 +#define Py_nb_power 33 +#define Py_nb_remainder 34 +#define Py_nb_rshift 35 +#define Py_nb_subtract 36 +#define Py_nb_true_divide 37 +#define Py_nb_xor 38 +#define Py_sq_ass_item 39 +#define Py_sq_concat 40 +#define Py_sq_contains 41 +#define Py_sq_inplace_concat 42 +#define Py_sq_inplace_repeat 43 +#define Py_sq_item 44 +#define Py_sq_length 45 +#define Py_sq_repeat 46 +#define Py_tp_alloc 47 +#define Py_tp_base 48 +#define Py_tp_bases 49 +#define Py_tp_call 50 +#define Py_tp_clear 51 +#define Py_tp_dealloc 52 +#define Py_tp_del 53 +#define Py_tp_descr_get 54 +#define Py_tp_descr_set 55 +#define Py_tp_doc 56 +#define Py_tp_getattr 57 +#define Py_tp_getattro 58 +#define Py_tp_hash 59 +#define Py_tp_init 60 +#define Py_tp_is_gc 61 +#define Py_tp_iter 62 +#define Py_tp_iternext 63 +#define Py_tp_methods 64 +#define Py_tp_new 65 +#define Py_tp_repr 66 +#define Py_tp_richcompare 67 +#define Py_tp_setattr 68 +#define Py_tp_setattro 69 +#define Py_tp_str 70 +#define Py_tp_traverse 71 +#define Py_tp_members 72 +#define Py_tp_getset 73 +#define Py_tp_free 74 +#define Py_nb_matrix_multiply 75 +#define Py_nb_inplace_matrix_multiply 76 +#define Py_am_await 77 +#define Py_am_aiter 78 +#define Py_am_anext 79 +#define Py_tp_finalize 80 +#define Py_am_send 81 +#define Py_tp_vectorcall 82 +#define Py_tp_token 83 +#define Py_mod_create _Py_SLOT_COMPAT_VALUE(1, 84) +#define Py_mod_exec _Py_SLOT_COMPAT_VALUE(2, 85) +#define Py_mod_multiple_interpreters _Py_SLOT_COMPAT_VALUE(3, 86) +#define Py_mod_gil _Py_SLOT_COMPAT_VALUE(4, 87) +#define Py_bf_getbuffer _Py_SLOT_COMPAT_VALUE(1, 88) +#define Py_bf_releasebuffer _Py_SLOT_COMPAT_VALUE(2, 89) +#define Py_mp_ass_subscript _Py_SLOT_COMPAT_VALUE(3, 90) +#define Py_mp_length _Py_SLOT_COMPAT_VALUE(4, 91) +#define Py_slot_subslots 92 +#define Py_tp_slots 93 +#define Py_mod_slots 94 +#define Py_tp_name 95 +#define Py_tp_basicsize 96 +#define Py_tp_extra_basicsize 97 +#define Py_tp_itemsize 98 +#define Py_tp_flags 99 +#define Py_mod_name 100 +#define Py_mod_doc 101 +#define Py_mod_state_size 102 +#define Py_mod_methods 103 +#define Py_mod_state_traverse 104 +#define Py_mod_state_clear 105 +#define Py_mod_state_free 106 +#define Py_tp_metaclass 107 +#define Py_tp_module 108 +#define Py_mod_abi 109 +#define Py_mod_token 110 + +#define _Py_slot_COUNT 111 +#endif /* _PY_HAVE_SLOTS_GENERATED_H */ diff --git a/Include/structseq.h b/Include/structseq.h index e52d6188030af9d..e5da785f13d46b8 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -29,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -# define Py_CPYTHON_STRUCTSEQ_H +# define _Py_CPYTHON_STRUCTSEQ_H # include "cpython/structseq.h" -# undef Py_CPYTHON_STRUCTSEQ_H +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Include/typeslots.h b/Include/typeslots.h deleted file mode 100644 index a7f3017ec02e92f..000000000000000 --- a/Include/typeslots.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Do not renumber the file; these numbers are part of the stable ABI. */ -#define Py_bf_getbuffer 1 -#define Py_bf_releasebuffer 2 -#define Py_mp_ass_subscript 3 -#define Py_mp_length 4 -#define Py_mp_subscript 5 -#define Py_nb_absolute 6 -#define Py_nb_add 7 -#define Py_nb_and 8 -#define Py_nb_bool 9 -#define Py_nb_divmod 10 -#define Py_nb_float 11 -#define Py_nb_floor_divide 12 -#define Py_nb_index 13 -#define Py_nb_inplace_add 14 -#define Py_nb_inplace_and 15 -#define Py_nb_inplace_floor_divide 16 -#define Py_nb_inplace_lshift 17 -#define Py_nb_inplace_multiply 18 -#define Py_nb_inplace_or 19 -#define Py_nb_inplace_power 20 -#define Py_nb_inplace_remainder 21 -#define Py_nb_inplace_rshift 22 -#define Py_nb_inplace_subtract 23 -#define Py_nb_inplace_true_divide 24 -#define Py_nb_inplace_xor 25 -#define Py_nb_int 26 -#define Py_nb_invert 27 -#define Py_nb_lshift 28 -#define Py_nb_multiply 29 -#define Py_nb_negative 30 -#define Py_nb_or 31 -#define Py_nb_positive 32 -#define Py_nb_power 33 -#define Py_nb_remainder 34 -#define Py_nb_rshift 35 -#define Py_nb_subtract 36 -#define Py_nb_true_divide 37 -#define Py_nb_xor 38 -#define Py_sq_ass_item 39 -#define Py_sq_concat 40 -#define Py_sq_contains 41 -#define Py_sq_inplace_concat 42 -#define Py_sq_inplace_repeat 43 -#define Py_sq_item 44 -#define Py_sq_length 45 -#define Py_sq_repeat 46 -#define Py_tp_alloc 47 -#define Py_tp_base 48 -#define Py_tp_bases 49 -#define Py_tp_call 50 -#define Py_tp_clear 51 -#define Py_tp_dealloc 52 -#define Py_tp_del 53 -#define Py_tp_descr_get 54 -#define Py_tp_descr_set 55 -#define Py_tp_doc 56 -#define Py_tp_getattr 57 -#define Py_tp_getattro 58 -#define Py_tp_hash 59 -#define Py_tp_init 60 -#define Py_tp_is_gc 61 -#define Py_tp_iter 62 -#define Py_tp_iternext 63 -#define Py_tp_methods 64 -#define Py_tp_new 65 -#define Py_tp_repr 66 -#define Py_tp_richcompare 67 -#define Py_tp_setattr 68 -#define Py_tp_setattro 69 -#define Py_tp_str 70 -#define Py_tp_traverse 71 -#define Py_tp_members 72 -#define Py_tp_getset 73 -#define Py_tp_free 74 -#define Py_nb_matrix_multiply 75 -#define Py_nb_inplace_matrix_multiply 76 -#define Py_am_await 77 -#define Py_am_aiter 78 -#define Py_am_anext 79 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -#define Py_tp_finalize 80 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 -/* New in 3.10 */ -#define Py_am_send 81 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_vectorcall 82 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_token 83 -#endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b72d581ec25804b..29f1d1b01c161f4 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,13 +64,9 @@ Copyright (c) Corporation for National Research Initiatives. #error Must define SIZEOF_WCHAR_T #endif +/* Soft-deprecated defines */ #define Py_UNICODE_SIZE SIZEOF_WCHAR_T - -/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. - Otherwise, Unicode strings are stored as UCS-2 (with limited support - for UTF-16) */ - -#if Py_UNICODE_SIZE >= 4 +#if SIZEOF_WCHAR_T >= 4 #define Py_UNICODE_WIDE #endif diff --git a/InternalDocs/README.md b/InternalDocs/README.md index a6e2df5ae4a9c39..3e8ab4423157534 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -11,6 +11,11 @@ it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). +General Resources +--- + +- [Source Code Structure](structure.md) + Compiling Python Source Code --- @@ -29,13 +34,13 @@ Runtime Objects - [Frames](frames.md) -- [String Interning](string_interning.md) - Program Execution --- - [The Bytecode Interpreter](interpreter.md) +- [Stack references (_PyStackRef)](stackrefs.md) + - [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) @@ -46,6 +51,14 @@ Program Execution - [Stack protection](stack_protection.md) +Built-in Types +--- + +- [String Interning](string_interning.md) + +- [List sort algorithm](../Objects/listsort.txt) + + Modules --- diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index a91a7043c1b8d41..cccbe71588622cf 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -70,14 +70,6 @@ The `co_linetable` bytes object of code objects contains a compact representation of the source code positions of instructions, which are returned by the `co_positions()` iterator. -> [!NOTE] -> `co_linetable` is not to be confused with `co_lnotab`. -> For backwards compatibility, `co_lnotab` exposes the format -> as it existed in Python 3.10 and lower: this older format -> stores only the start line for each instruction. -> It is lazily created from `co_linetable` when accessed. -> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. - `co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 742af5efcf5ce50..9ed4d0eb65a0bdd 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -359,19 +359,19 @@ in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instruction These are similar to bytecode, but in some cases they are more abstract, and are resolved later into actual bytecode. The construction of this instruction sequence is handled by several functions that break the task down by various AST node types. -The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +The functions are all named `codegen_visit_{xx}` where *xx* is the name of the node type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` and `{xx}_ty` where *xx* is the AST node type. Typically these functions consist of a large 'switch' statement, branching based on the kind of node type passed to it. Simple things are handled inline in the 'switch' statement with more complex transformations farmed out to other -functions named `compiler_{xx}` with *xx* being a descriptive name of what is +functions named `codegen_{xx}` with *xx* being a descriptive name of what is being handled. When transforming an arbitrary AST node, use the `VISIT()` macro. -The appropriate `compiler_visit_{xx}` function is called, based on the value +The appropriate `codegen_visit_{xx}` function is called, based on the value passed in for (so `VISIT({c}, expr, {node})` calls -`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +`codegen_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, but is called on AST node sequences (those values that were created as arguments to a node that used the '*' modifier). @@ -414,8 +414,8 @@ which is added at the end of a function is not associated with any line in the source code. There are several helper functions that will emit pseudo-instructions -and are named `compiler_{xx}()` where *xx* is what the function helps -with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +and are named `codegen_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `codegen_nameop()`. This function looks up the scope of a variable and, based on the expression context, emits the proper opcode to load, store, or delete the variable. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 804d7436018a10e..475ca75e28e1a5d 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,59 @@ The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. +### Base frame + +Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame" +that serves as a sentinel at the bottom of the frame stack. This frame is allocated +in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized +when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`. + +External profilers and sampling tools can validate that they have successfully unwound +the complete call stack by checking that the frame chain terminates at the base frame. +The `PyThreadState.base_frame` pointer provides the expected address to compare against. +If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a +race condition) and should be discarded. + +The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because +`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the +public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access +the address without needing internal headers. + +See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c). + +#### How profilers should use the base frame + +External profilers should read `tstate->base_frame` before walking the stack, then +walk from `tstate->current_frame` following `frame->previous` pointers until reaching +a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the +last frame address matches `base_frame`. If not, discard the sample as incomplete +since the frame chain may have been in an inconsistent state due to concurrent updates. + + +### Remote Profiling Frame Cache + +The `last_profiled_frame` and `last_profiled_frame_seq` fields in +`PyThreadState` support an optimization for remote profilers that sample call +stacks from external processes. When a remote profiler reads the call stack, it +writes the current frame address to `last_profiled_frame`. The eval loop then +keeps this pointer valid by updating it to the parent frame whenever a frame +returns (in `_PyEval_FrameClearAndPop`) and increments the sequence. + +This creates a "high-water mark" that always points to a frame still on the stack. +On subsequent samples, the profiler can walk from `current_frame` until it reaches +`last_profiled_frame`, then validate the pointer and sequence before using cached +callers. This prevents a later frame that reuses the same `_PyInterpreterFrame` +address from being mistaken for the sampled frame. The cache significantly +reduces the amount of remote memory reads needed when call stacks are deep and +stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only advances the +pointer and sequence when `last_profiled_frame` is non-NULL AND matches the +frame being popped. This prevents transient frames (called and returned between +profiler samples) from corrupting the cache anchor, while avoiding any overhead +when profiling is inactive. + + ### The Instruction Pointer `_PyInterpreterFrame` has two fields which are used to maintain the instruction diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index a7d872f3ec43921..0ef45ff8e02bc5f 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ As is explained later in the [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -153,7 +153,11 @@ pointer-sized (that is, eight bytes on a 64-bit platform). The garbage collector also temporarily repurposes the `ob_tid` (thread ID) and `ob_ref_local` (local reference count) fields for other purposes during -collections. +collections. The `ob_tid` field is later restored from the containing +mimalloc segment data structure. In some cases, such as when the original +allocating thread exits, this can result in a different `ob_tid` value. +Code should not rely on `ob_tid` being stable across operations that may +trigger garbage collection. C APIs @@ -199,22 +203,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -356,11 +360,12 @@ follows these steps in order: the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -368,76 +373,29 @@ This has proven to be very close to the reality of many Python programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -446,7 +404,7 @@ function: ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -459,84 +417,61 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -767,6 +702,14 @@ tuples can be untracked. A tuple can be untracked if all of its contents are already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 38e9f6fced60880..7fc41a807dd5667 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -226,10 +226,11 @@ Up through 3.10, the call stack was implemented as a singly-linked list of heap allocation for the stack frame. Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal -`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator -function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a -frame structure. Usually a frame allocation is just a pointer bump, which improves -memory locality. +`_PyInterpreterFrame` structure is used. Most frames are allocated contiguously in a +per-thread stack (see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)), +which improves memory locality and reduces overhead. +If the current `datastack_chunk` has enough space (`_PyThreadState_HasStackSpace`) +then the lightweight `_PyFrame_PushUnchecked` can be used instead of `_PyThreadState_PushFrame`. Sometimes an actual `PyFrameObject` is needed, such as when Python code calls `sys._getframe()` or an extension module calls @@ -506,6 +507,38 @@ After the last `DEOPT_IF` has passed, a hit should be recorded with After an optimization has been deferred in the adaptive instruction, that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. +## Interpreter types +There are three different types of interpreters to choose from based on compiler support: + + * traditional switch-case interpreter + + Supported by all compilers covered in PEP 7. + + * computed-gotos interpreter + + Enabled using configure option `--with-computed-gotos` and used by default on supported compilers. + It uses [Labels as Values](https://gcc.gnu.org/onlinedocs/gcc/Labels-as-Values.html) + for more efficient dispatching. + + * tail-calling interpreter + + Enabled using configure option `--with-tail-call-interp` (or `--tail-call-interp` for build.bat on Windows). + It uses [tail calls](https://clang.llvm.org/docs/AttributeReference.html#musttail) and the + [preserve_none](https://clang.llvm.org/docs/AttributeReference.html#preserve-none) + calling convention between the small C functions that implement individual Python opcodes. + + Not all compilers support these and if they do not all targets might be supported (for example, + MSVC currently only supports x64 and only in optimized builds). + + In addition, compilers must do [escape analysis](https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-musttail) + of the lifetimes of automatic variables, function parameters, and temporaries to ensure proper tail-calls. They + emit a compile error in case of a violation or detection failure. The ability to detect this varies depending on the compiler and + also on the optimization level. Following techniques are particularly helpful to the MSVC compiler in this regard + * [Introducing additional scopes](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Python/bytecodes.c#L2526) + * [extracting problematic code paths into a separate function](https://github.com/python/cpython/pull/143068/files#diff-729a985b0cb8b431cb291f1edb561bbbfea22e3f8c262451cd83328a0936a342R3724) + * [returning a pointer instead of taking it as an output parameter](https://github.com/python/cpython/blob/3908593039bde9d4b591ab09919003ee57418d64/Include/internal/pycore_ceval.h#L489-L492) + + Using `restrict` is another (currently unused) remedy. Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index 1740b22b85f77b9..1345f2db8b34db9 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -11,24 +11,54 @@ and this enables optimizations that span multiple instructions. Historically, the adaptive interpreter was referred to as `tier 1` and the JIT as `tier 2`. You will see remnants of this in the code. -## The Optimizer and Executors +## The Trace Recorder and Executors -The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` -instruction determines that it is "hot" because the counter in its +There are two interpreters in this section: + 1. Adaptive interpreter (the default behavior) + 2. Trace recording interpreter (enabled on JIT builds) + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` or +`RESUME` instruction determines that it is "hot" because the counter in its [inline cache](interpreter.md#inline-cache-entries) indicates that it executed more than some threshold number of times (see [`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). -It then calls the function `_PyOptimizer_Optimize()` in +It then calls the function `_PyJit_TryInitializeTracing` in [`Python/optimizer.c`](../Python/optimizer.c), passing it the current -[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` -constructs an object of type -[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h) which implements -an optimized version of the instruction trace beginning at this jump. - -The optimizer determines where the trace ends, and the executor is set up +[frame](frames.md), instruction pointer and state. +The interpreter then switches into "tracing mode" via the macro +`ENTER_TRACING()`. On platforms that support computed goto and tail-calling +interpreters, the dispatch table is swapped out, while other platforms that do +not support either use a single flag in the opcode. +Execution between the normal interpreter and tracing interpreter are +interleaved via this dispatch mechanism. This means that while logically +there are two interpreters, the implementation appears to be a single +interpreter. + +During tracing mode, after each interpreter instruction's `DISPATCH()`, +the interpreter jumps to the `TRACE_RECORD` instruction. This instruction +records the previous instruction executed and also any live values of the next +operation it may require. It then translates the previous instruction to +a sequence of micro-ops using `_PyJit_translate_single_bytecode_to_trace`. +To ensure that the adaptive interpreter instructions +and cache entries are up-to-date, the trace recording interpreter always resets +the adaptive counters of adaptive instructions it sees. +This forces a re-specialization of any new instruction should an instruction +deoptimize. Thus, feeding the trace recorder up-to-date information. +Finally, the `TRACE_RECORD` instruction decides when to stop tracing +using various heuristics. + +Once trace recording concludes, `LEAVE_TRACING()` swaps out the dispatch +table/the opcode flag set earlier by `ENTER_TRACING()` is unset. +`stop_tracing_and_jit()` then calls `_PyOptimizer_Optimize()` which optimizes +the trace and constructs an +[`_PyExecutorObject`](../Include/internal/pycore_optimizer.h). + +JIT execution is set up to either return to the adaptive interpreter and resume execution, or transfer control to another executor (see `_PyExitData` in -Include/internal/pycore_optimizer.h). +Include/internal/pycore_optimizer.h). When resuming to the adaptive interpreter, +a "side exit", generated by an `EXIT_IF` may trigger recording of another trace. +While a "deopt", generated by a `DEOPT_IF`, does not trigger recording. The executor is stored on the [`code object`](code_objects.md) of the frame, in the `co_executors` field which is an array of executors. The start @@ -40,12 +70,7 @@ executor in `co_executors`. The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in [`Python/optimizer.c`](../Python/optimizer.c) as `_PyOptimizer_Optimize`. -It translates an instruction trace into a sequence of micro-ops by replacing -each bytecode by an equivalent sequence of micro-ops (see -`_PyOpcode_macro_expansion` in -[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) -which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). -The micro-op sequence is then optimized by +It takes a micro-op sequence from the trace recorder and optimizes with `_Py_uop_analyze_and_optimize` in [`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) and an instance of `_PyUOpExecutor_Type` is created to contain it. @@ -78,8 +103,8 @@ and execution returns to the adaptive interpreter. ## Invalidating Executors In addition to being stored on the code object, each executor is also -inserted into a list of all executors, which is stored in the interpreter -state's `executor_list_head` field. This list is used when it is necessary +inserted into contiguous arrays (`executor_blooms` and `executor_ptrs`) +stored in the interpreter state. These arrays are used when it is necessary to invalidate executors because values they used in their construction may have changed. diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 1d0ffe6d40d078a..1bb4cdea5439f17 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -819,6 +819,13 @@ directory on the CPython repository and manually call the parser generator by ex $ python -m pegen python ``` +> [!CAUTION] +> Python's grammar (the `Grammar/python.gram` file) is written for the +> C backend. To experiment, you will need to write a grammar +> without C-specific parts like actions and the trailer. +> See [#133560](https://github.com/python/cpython/issues/133560) +> and [#96424](https://github.com/python/cpython/issues/96424) for more information. + This will generate a file called `parse.py` in the same directory that you can use to parse some input: diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md new file mode 100644 index 000000000000000..7e4592a0d897055 --- /dev/null +++ b/InternalDocs/profiling_binary_format.md @@ -0,0 +1,541 @@ +# Profiling Binary Format + +The profiling module includes a binary file format for storing sampling +profiler data. This document describes the format's structure and the +design decisions behind it. + +The implementation is in +[`Modules/_remote_debugging/binary_io_writer.c`](../Modules/_remote_debugging/binary_io_writer.c) +and [`Modules/_remote_debugging/binary_io_reader.c`](../Modules/_remote_debugging/binary_io_reader.c), +with declarations in +[`Modules/_remote_debugging/binary_io.h`](../Modules/_remote_debugging/binary_io.h). + +## Overview + +The sampling profiler can generate enormous amounts of data. A typical +profiling session sampling at 1000 Hz for 60 seconds produces 60,000 samples. +Each sample contains a full call stack, often 20-50 frames deep, and each +frame includes a filename, function name, and line number. In a text-based +format like collapsed stacks, this would mean repeating the same long file +paths and function names thousands of times. + +The binary format addresses this through two key strategies: + +1. **Deduplication**: Strings and frames are stored once in lookup tables, + then referenced by small integer indices. A 100-character file path that + appears in 50,000 samples is stored once, not 50,000 times. + +2. **Compact encoding**: Variable-length integers (varints) encode small + values in fewer bytes. Since most indices are small (under 128), they + typically need only one byte instead of four. + +Together with optional zstd compression, these techniques reduce file sizes +by 10-50x compared to text formats while also enabling faster I/O. + +## File Layout + +The file consists of five sections: + +``` ++------------------+ Offset 0 +| Header | 64 bytes (fixed) ++------------------+ Offset 64 +| | +| Sample Data | Variable size (optionally compressed) +| | ++------------------+ string_table_offset +| String Table | Variable size ++------------------+ frame_table_offset +| Frame Table | Variable size ++------------------+ file_size - 32 +| Footer | 32 bytes (fixed) ++------------------+ file_size +``` + +The layout is designed for streaming writes during profiling. The profiler +cannot know in advance how many unique strings or frames will be encountered, +so these tables must be built incrementally and written at the end. + +The header comes first so readers can quickly validate the file and locate +the metadata tables. The sample data follows immediately, allowing the writer +to stream samples directly to disk (or through a compression stream) without +buffering the entire dataset in memory. + +The string and frame tables are placed after sample data because they grow +as new unique entries are discovered during profiling. By deferring their +output until finalization, the writer avoids the complexity of reserving +space or rewriting portions of the file. + +The footer at the end contains counts needed to allocate arrays before +parsing the tables. Placing it at a fixed offset from the end (rather than +at a variable offset recorded in the header) means readers can locate it +with a single seek to `file_size - 32`, without first reading the header. + +## Header + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | Magic number (0x54414348 = "TACH") | +| 4 | 4 | uint32 | Format version | +| 8 | 4 | bytes | Python version (major, minor, micro, | +| | | | reserved) | +| 12 | 8 | uint64 | Start timestamp (microseconds) | +| 20 | 8 | uint64 | Sample interval (microseconds) | +| 28 | 4 | uint32 | Total sample count | +| 32 | 4 | uint32 | Thread count | +| 36 | 8 | uint64 | String table offset | +| 44 | 8 | uint64 | Frame table offset | +| 52 | 4 | uint32 | Compression type (0=none, 1=zstd) | +| 56 | 8 | bytes | Reserved (zero-filled) | ++--------+------+---------+----------------------------------------+ +``` + +The magic number `0x54414348` ("TACH" for Tachyon) identifies the file format +and also serves as an **endianness marker**. When read on a system with +different byte order than the writer, it appears as `0x48434154`. The reader +uses this to detect cross-endian files and automatically byte-swap all +multi-byte integer fields. + +The Python version field records the major, minor, and micro version numbers +of the Python interpreter that generated the file. This allows analysis tools +to detect version mismatches when replaying data collected on a different +Python version, which may have different internal structures or behaviors. + +The header is written as zeros initially, then overwritten with actual values +during finalization. This requires the output stream to be seekable, which +is acceptable since the format targets regular files rather than pipes or +network streams. + +## Sample Data + +Sample data begins at offset 64 and extends to `string_table_offset`. Samples +use delta compression to minimize redundancy when consecutive samples from the +same thread have identical or similar call stacks. + +### Stack Encoding Types + +Each sample record begins with thread identification, then an encoding byte: + +| Code | Name | Description | +|------|------|-------------| +| 0x00 | REPEAT | RLE: identical stack repeated N times | +| 0x01 | FULL | Complete stack (first sample or no match) | +| 0x02 | SUFFIX | Shares N frames from bottom of previous stack | +| 0x03 | POP_PUSH | Remove M frames from top, add N new frames | + +### Record Formats + +**REPEAT (0x00) - Run-Length Encoded Identical Stacks:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x00 (REPEAT) | +| count | varint | Number of samples in this RLE group | +| samples | varies | Interleaved: [delta: varint, status: 1]| +| | | repeated count times | ++-----------------+-----------+----------------------------------------+ +``` +The stack is inherited from this thread's previous sample. Each sample in the +group gets its own timestamp delta and status byte, stored as interleaved pairs +(delta1, status1, delta2, status2, ...) rather than separate arrays. + +**FULL (0x01) - Complete Stack:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x01 (FULL) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| stack_depth | varint | Number of frames in call stack | +| frame_indices | varint[] | Array of frame table indices | ++-----------------+-----------+----------------------------------------+ +``` +Used for the first sample from a thread, or when delta encoding would not +provide savings. + +**SUFFIX (0x02) - Shared Suffix Match:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x02 (SUFFIX) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| shared_count | varint | Frames shared from bottom of prev stack| +| new_count | varint | New frames at top of stack | +| new_frames | varint[] | Array of new_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when a function call added frames to the top of the stack. The shared +frames from the previous stack are kept, and new frames are prepended. + +**POP_PUSH (0x03) - Pop and Push:** +``` ++-----------------+-----------+----------------------------------------+ +| thread_id | 8 bytes | Thread identifier (uint64, fixed) | +| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | +| encoding | 1 byte | 0x03 (POP_PUSH) | +| timestamp_delta | varint | Microseconds since thread's last sample| +| status | 1 byte | Thread state flags | +| pop_count | varint | Frames to remove from top of prev stack| +| push_count | varint | New frames to add at top | +| new_frames | varint[] | Array of push_count frame indices | ++-----------------+-----------+----------------------------------------+ +``` +Used when the code path changed: some frames were popped (function returns) +and new frames were pushed (different function calls). + +### Thread and Interpreter Identification + +Thread IDs are 64-bit values that can be large (memory addresses on some +platforms) and vary unpredictably. Using a fixed 8-byte encoding avoids +the overhead of varint encoding for large values and simplifies parsing +since the reader knows exactly where each field begins. + +The interpreter ID identifies which Python sub-interpreter the thread +belongs to, allowing analysis tools to separate activity across interpreters +in processes using multiple sub-interpreters. + +### Status Byte + +The status byte is a bitfield encoding thread state at sample time: + +| Bit | Flag | Meaning | +|-----|-----------------------|--------------------------------------------| +| 0 | THREAD_STATUS_HAS_GIL | Thread holds the GIL (Global Interpreter Lock) | +| 1 | THREAD_STATUS_ON_CPU | Thread is actively running on a CPU core | +| 2 | THREAD_STATUS_UNKNOWN | Thread state could not be determined | +| 3 | THREAD_STATUS_GIL_REQUESTED | Thread is waiting to acquire the GIL | +| 4 | THREAD_STATUS_HAS_EXCEPTION | Thread has a pending exception | + +Multiple flags can be set simultaneously (e.g., a thread can hold the GIL +while also running on CPU). Analysis tools use these to filter samples or +visualize thread states over time. + +### Timestamp Delta Encoding + +Timestamps use delta encoding rather than absolute values. Absolute +timestamps in microseconds require 8 bytes each, but consecutive samples +from the same thread are typically separated by the sampling interval +(e.g., 1000 microseconds), so the delta between them is small and fits +in 1-2 varint bytes. The writer tracks the previous timestamp for each +thread separately. The first sample from a thread encodes its delta from +the profiling start time; subsequent samples encode the delta from that +thread's previous sample. This per-thread tracking is necessary because +samples are interleaved across threads in arrival order, not grouped by +thread. + +For REPEAT (RLE) records, timestamp deltas and status bytes are stored as +interleaved pairs (delta, status, delta, status, ...) - one pair per +repeated sample - allowing efficient batching while preserving the exact +timing and state of each sample. + +### Frame Indexing + +Each frame in a call stack is represented by an index into the frame table +rather than inline data. This provides massive space savings because call +stacks are highly repetitive: the same function appears in many samples +(hot functions), call stacks often share common prefixes (main -> app -> +handler -> ...), and recursive functions create repeated frame sequences. +A frame index is typically 1-2 varint bytes. Inline frame data would be +20-200+ bytes (two strings plus a line number). For a profile with 100,000 +samples averaging 30 frames each, this reduces frame data from potentially +gigabytes to tens of megabytes. + +Frame indices are written innermost-first (the currently executing frame +has index 0 in the array). This ordering works well with delta compression: +function calls typically add frames at the top (index 0), while shared +frames remain at the bottom. + +## String Table + +The string table stores deduplicated UTF-8 strings (filenames and function +names). It begins at `string_table_offset` and contains entries in order of +their assignment during writing: + +``` ++----------------+ +| length: varint | +| data: bytes | ++----------------+ (repeated for each string) +``` + +Strings are stored in the order they were first encountered during writing. +The first unique filename gets index 0, the second gets index 1, and so on. +Length-prefixing (rather than null-termination) allows strings containing +null bytes and enables readers to allocate exact-sized buffers. The varint +length encoding means short strings (under 128 bytes) need only one length +byte. + +## Frame Table + +The frame table stores deduplicated frame entries with full source position +information and bytecode opcode: + +``` ++----------------------------+ +| filename_idx: varint | +| funcname_idx: varint | +| lineno: svarint | +| end_lineno_delta: svarint | +| column: svarint | +| end_column_delta: svarint | +| opcode: u8 | ++----------------------------+ (repeated for each frame) +``` + +### Field Definitions + +| Field | Type | Description | +|------------------|---------------|----------------------------------------------------------| +| filename_idx | varint | Index into string table for file name | +| funcname_idx | varint | Index into string table for function name | +| lineno | zigzag varint | Start line number (-1 for synthetic frames) | +| end_lineno_delta | zigzag varint | Delta from lineno (end_lineno = lineno + delta) | +| column | zigzag varint | Start column offset in UTF-8 bytes (-1 if not available) | +| end_column_delta | zigzag varint | Delta from column (end_column = column + delta) | +| opcode | u8 | Python bytecode opcode (0-254) or 255 for None | + +### Delta Encoding + +Position end values use delta encoding for efficiency: + +- `end_lineno = lineno + end_lineno_delta` +- `end_column = column + end_column_delta` + +Typical values: +- `end_lineno_delta`: Usually 0 (single-line expressions) → encodes to 1 byte +- `end_column_delta`: Usually 5-20 (expression width) → encodes to 1 byte + +This saves ~1-2 bytes per frame compared to absolute encoding. When the base +value (lineno or column) is -1 (not available), the delta is stored as 0 and +the reconstructed value is -1. + +### Sentinel Values + +- `opcode = 255`: No opcode captured +- `lineno = -1`: Synthetic frame (no source location) +- `column = -1`: Column offset not available + +### Deduplication + +Each unique (filename, funcname, lineno, end_lineno, column, end_column, +opcode) combination gets one entry. This enables instruction-level profiling +where multiple bytecode instructions on the same line can be distinguished. + +Strings and frames are deduplicated separately because they have different +cardinalities and reference patterns. A codebase might have hundreds of +unique source files but thousands of unique functions. Many functions share +the same filename, so storing the filename index in each frame entry (rather +than the full string) provides an additional layer of deduplication. A frame +entry is typically 7-9 bytes rather than two full strings plus location data. + +### Size Analysis + +Typical frame size with delta encoding: +- file_idx: 1-2 bytes +- func_idx: 1-2 bytes +- lineno: 1-2 bytes +- end_lineno_delta: 1 byte (usually 0) +- column: 1 byte (usually < 64) +- end_column_delta: 1 byte (usually < 64) +- opcode: 1 byte + +**Total: ~7-9 bytes per frame** + +Line numbers and columns use signed varint (zigzag encoding) to handle +sentinel values efficiently. Synthetic frames—generated frames that don't +correspond directly to Python source code, such as C extension boundaries or +internal interpreter frames—use -1 to indicate the absence of a source +location. Zigzag encoding ensures these small negative values encode +efficiently (−1 becomes 1, which is one byte) rather than requiring the +maximum varint length. + +## Footer + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 4 | uint32 | String count | +| 4 | 4 | uint32 | Frame count | +| 8 | 8 | uint64 | Total file size | +| 16 | 16 | bytes | Checksum (reserved, currently zeros) | ++--------+------+---------+----------------------------------------+ +``` + +The string and frame counts allow readers to pre-allocate arrays of the +correct size before parsing the tables. Without these counts, readers would +need to either scan the tables twice (once to count, once to parse) or use +dynamically-growing arrays. + +The file size field provides a consistency check: if the actual file size +does not match, the file may be truncated or corrupted. + +The checksum field is reserved for future use. A checksum would allow +detection of corruption but adds complexity and computation cost. The +current implementation leaves this as zeros. + +## Variable-Length Integer Encoding + +The format uses LEB128 (Little Endian Base 128) for unsigned integers and +zigzag + LEB128 for signed integers. These encodings are widely used +(Protocol Buffers, DWARF debug info, WebAssembly) and well-understood. + +### Unsigned Varint (LEB128) + +Each byte stores 7 bits of data. The high bit indicates whether more bytes +follow: + +``` +Value Encoded bytes +0-127 [0xxxxxxx] (1 byte) +128-16383 [1xxxxxxx] [0xxxxxxx] (2 bytes) +16384+ [1xxxxxxx] [1xxxxxxx] ... (3+ bytes) +``` + +Most indices in profiling data are small. A profile with 1000 unique frames +needs at most 2 bytes per frame index. The common case (indices under 128) +needs only 1 byte. + +### Signed Varint (Zigzag) + +Standard LEB128 encodes −1 as a very large unsigned value, requiring many +bytes. Zigzag encoding interleaves positive and negative values: + +``` + 0 -> 0 -1 -> 1 1 -> 2 -2 -> 3 2 -> 4 +``` + +This ensures small-magnitude values (whether positive or negative) encode +in few bytes. + +## Compression + +When compression is enabled, the sample data region contains a zstd stream. +The string table, frame table, and footer remain uncompressed so readers can +access metadata without decompressing the entire file. A tool that only needs +to report "this file contains 50,000 samples of 3 threads" can read the header +and footer without touching the compressed sample data. This also simplifies +the format: the header's offset fields point directly to the tables rather +than to positions within a decompressed stream. + +Zstd provides an excellent balance of compression ratio and speed. Profiling +data compresses very well (often 5-10x) due to repetitive patterns: the same +small set of frame indices appears repeatedly, and delta-encoded timestamps +cluster around the sampling interval. Zstd's streaming API allows compression +without buffering the entire dataset. The writer feeds sample data through +the compressor incrementally, flushing compressed chunks to disk as they +become available. + +Level 5 compression is used as a default. Lower levels (1-3) are faster but +compress less; higher levels (6+) compress more but slow down writing. Level +5 provides good compression with minimal impact on profiling overhead. + +## Reading and Writing + +### Writing + +1. Open the output file and write 64 zero bytes as a placeholder header +2. Initialize empty string and frame dictionaries for deduplication +3. For each sample: + - Intern any new strings, assigning sequential indices + - Intern any new frames, assigning sequential indices + - Encode the sample record and write to the buffer + - Flush the buffer through compression (if enabled) when full +4. Flush remaining buffered data and finalize compression +5. Write the string table (length-prefixed strings in index order) +6. Write the frame table (varint-encoded entries in index order) +7. Write the footer with final counts +8. Seek to offset 0 and write the header with actual values + +The writer maintains two dictionaries: one mapping strings to indices, one +mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These +enable O(1) lookup during interning. + +### Reading + +1. Read the header magic number to detect endianness (set `needs_swap` flag + if the magic appears byte-swapped) +2. Validate version and read remaining header fields (byte-swapping if needed) +3. Seek to end − 32 and read the footer (byte-swapping counts if needed) +4. Allocate string array of `string_count` elements +5. Parse the string table, populating the array +6. Allocate frame array of `frame_count * 3` uint32 elements +7. Parse the frame table, populating the array +8. If compressed, decompress the sample data region +9. Iterate through samples, resolving indices to strings/frames + (byte-swapping thread_id and interpreter_id if needed) + +The reader builds lookup arrays rather than dictionaries since it only needs +index-to-value mapping, not value-to-index. + +## Platform Considerations + +### Byte Ordering and Cross-Platform Portability + +The binary format uses **native byte order** for all multi-byte integer +fields when writing. However, the reader supports **cross-endian reading**: +files written on a little-endian system (x86, ARM) can be read on a +big-endian system (s390x, PowerPC), and vice versa. + +The magic number doubles as an endianness marker. When read on a system with +different byte order, it appears byte-swapped (`0x48434154` instead of +`0x54414348`). The reader detects this and automatically byte-swaps all +fixed-width integer fields during parsing. + +Writers must use `memcpy()` from properly-sized integer types when writing +fixed-width integer fields. When the source variable's type differs from the +field width (e.g., `size_t` written as 4 bytes), explicit casting to the +correct type (e.g., `uint32_t`) is required before `memcpy()`. On big-endian +systems, copying from an oversized type would copy the wrong bytes—high-order +zeros instead of the actual value. + +The reader tracks whether byte-swapping is needed via a `needs_swap` flag set +during header parsing. All fixed-width fields in the header, footer, and +sample data are conditionally byte-swapped using Python's internal byte-swap +functions (`_Py_bswap32`, `_Py_bswap64` from `pycore_bitutils.h`). + +Variable-length integers (varints) are byte-order independent since they +encode values one byte at a time using the LEB128 scheme, so they require +no special handling for cross-endian reading. + +### Memory-Mapped I/O + +On Unix systems (Linux, macOS), the reader uses `mmap()` to map the file +into the process address space. The kernel handles paging data in and out +as needed, no explicit read() calls or buffer management are required, +multiple readers can share the same physical pages, and sequential access +patterns benefit from kernel read-ahead. + +The implementation uses `madvise()` to hint the access pattern to the kernel: +`MADV_SEQUENTIAL` indicates the file will be read linearly, enabling +aggressive read-ahead. `MADV_WILLNEED` requests pre-faulting of pages. +On Linux, `MAP_POPULATE` pre-faults all pages at mmap time rather than on +first access, moving page fault overhead from the parsing loop to the +initial mapping for more predictable performance. For large files (over +32 MB), `MADV_HUGEPAGE` requests transparent huge pages (2 MB instead of +4 KB) to reduce TLB pressure when accessing large amounts of data. + +On Windows, the implementation falls back to standard file I/O with full +file buffering. Profiling data files are typically small enough (tens to +hundreds of megabytes) that this is acceptable. + +The writer uses a 512 KB buffer to batch small writes. Each sample record +is typically tens of bytes; writing these individually would incur excessive +syscall overhead. The buffer accumulates data until full, then flushes in +one write() call (or feeds through the compression stream). + +## Future Considerations + +The format reserves space for future extensions. The 12 reserved bytes in +the header could hold additional metadata. The 16-byte checksum field in +the footer is currently unused. The version field allows incompatible +changes with graceful rejection. New compression types could be added +(compression_type > 1). + +Any changes that alter the meaning of existing fields or the parsing logic +should increment the version number to prevent older readers from +misinterpreting new files. diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index 1c4a79a7b44436a..d511396fdab6452 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list: To reduce memory contention from frequent updates to the global `wr_seq`, its advancement is sometimes deferred. Instead of incrementing `wr_seq` on every -reclamation request, each thread tracks its number of deferrals locally. Once -the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the -thread advances the global `wr_seq` and resets its local count. - -When an object is added to the deferred-free list, its qsbr_goal is set to -`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe -to defer the global counter advancement. This optimization improves runtime -speed but may increase peak memory usage by slightly delaying when memory can -be reclaimed. +reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value +the counter *would* take on its next advance) without actually advancing the +global counter. This is safe because the goal still corresponds to a future +sequence value that no thread has yet observed as quiescent. + +Whether to actually advance `wr_seq` is decided per request, based on how +much memory and how many items the calling thread has already deferred since +its last advance: + +* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a + count (`deferred_count`) and an estimate of the held memory + (`deferred_memory`). The global `wr_seq` is advanced when the freed block + is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred + memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` + (127, sized so a chunk of work items is processed before it overflows). + Crossing any of these thresholds also sets a per-thread `should_process` + flag, signalling that the deferred-free list should be drained. + +* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` + and advances `wr_seq` when either the individual page or the accumulated + page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing + promptly here matters because a held page cannot be reused for a different + size class or by a different thread. + +Processing of the deferred-free list normally happens from the eval breaker +(rather than from inside `_PyMem_FreeDelayed`), which gives the global +`rd_seq` a better chance to have advanced far enough that items can actually +be freed. `_PyMem_ProcessDelayed` is still called from the free path as a +safety valve when a work-item chunk fills up. + +This optimization improves runtime speed but may increase peak memory usage +by slightly delaying when memory can be reclaimed; the size-based thresholds +above bound that extra memory. ## Limitations diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md index fa025bd930f74ed..14802e57d095f4d 100644 --- a/InternalDocs/stack_protection.md +++ b/InternalDocs/stack_protection.md @@ -38,12 +38,19 @@ Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecurs ```python kb_used = (stack_top - stack_pointer)>>10 -if stack_pointer < hard_limit: +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") elif stack_pointer < soft_limit: raise RecursionError(f"Stack overflow (used {kb_used} kB)") ``` +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + ### Diagnosing and fixing stack overflows For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. diff --git a/InternalDocs/stackrefs.md b/InternalDocs/stackrefs.md new file mode 100644 index 000000000000000..5774be9c56d3635 --- /dev/null +++ b/InternalDocs/stackrefs.md @@ -0,0 +1,79 @@ +# Stack references (`_PyStackRef`) + +Stack references are the interpreter's tagged representation of values on the evaluation stack. +They carry metadata to track ownership and support optimizations such as tagged small ints. + +## Shape and tagging + +- A `_PyStackRef` is a tagged pointer-sized value (see `Include/internal/pycore_stackref.h`). +- Tag bits distinguish three cases: + - `Py_TAG_REFCNT` unset - reference count lives on the pointed-to object. + - `Py_TAG_REFCNT` set - ownership is "borrowed" (no refcount to drop on close) or the object is immortal. + - `Py_INT_TAG` set - tagged small integer stored directly in the stackref (no heap allocation). +- Special constants: `PyStackRef_NULL`, `PyStackRef_ERROR`, and embedded `None`/`True`/`False`. + +In GIL builds, most objects carry their refcount; tagged borrowed refs skip decref on close. In free +threading builds, the tag is also used to mark deferred refcounted objects so the GC can see them and +to avoid refcount contention on commonly shared objects. + +## Converting to and from PyObject* + +Three conversions control ownership: + +- `PyStackRef_FromPyObjectNew(obj)` - create a new reference (INCREF if mortal). +- `PyStackRef_FromPyObjectSteal(obj)` - take over ownership without changing the count unless the + object is immortal. +- `PyStackRef_FromPyObjectBorrow(obj)` - create a borrowed stackref (never decref on close). + +The `obj` argument must not be `NULL`. + +Going back to `PyObject*` mirrors this: + +- `PyStackRef_AsPyObjectBorrow(ref)` - borrow the underlying pointer +- `PyStackRef_AsPyObjectSteal(ref)` - transfer ownership from the stackref; if ref is borrowed or + deferred, this creates a new owning `PyObject*` reference. +- `PyStackRef_AsPyObjectNew(ref)` - create a new owning reference + +Only `PyStackRef_AsPyObjectBorrow` allows ref to be `PyStackRef_NULL`. + +## Operations on stackrefs + +The interpreter treats `_PyStackRef` as the unit of stack storage. Ownership must be managed with +the stackref primitives: + +- `PyStackRef_DUP` - like `Py_NewRef` for stackrefs; preserves the original. +- `PyStackRef_Borrow` - create a borrowed stackref from another stackref. +- `PyStackRef_CLOSE` / `PyStackRef_XCLOSE` - like `Py_DECREF`; invalidates the stackref. +- `PyStackRef_CLEAR` - like `Py_CLEAR`; closes and sets the stackref to `PyStackRef_NULL` +- `PyStackRef_MakeHeapSafe` - converts borrowed reference to owning reference + +Borrow tracking (for debug builds with `Py_STACKREF_DEBUG`) records who you borrowed from and reports +double-close, leaked borrows, or use-after-close via fatal errors. + +## Borrow-friendly opcodes + +The interpreter can push borrowed references directly. For example, `LOAD_FAST_BORROW` loads a local +variable as a borrowed `_PyStackRef`, avoiding both INCREF and DECREF for the temporary lifetime on +the evaluation stack. + +## Tagged integers on the stack + +Small ints can be stored inline with `Py_INT_TAG`, so no heap object is involved. Helpers like +`PyStackRef_TagInt`, `PyStackRef_UntagInt`, and `PyStackRef_IncrementTaggedIntNoOverflow` operate on +these values. Type checks use `PyStackRef_IsTaggedInt` and `PyStackRef_LongCheck`. + +## Free threading considerations + +Objects that support deferred reference counting can be pushed to the evaluation +stack and stored in local variables without directly incrementing the reference +count because they are only freed during cyclic garbage collection. This avoids +reference count contention on commonly shared objects such as methods and types. The GC +scans each thread's locals and evaluation stack to keep objects that use +deferred reference counting alive. + +## Debugging support + +`Py_STACKREF_DEBUG` builds replace the inline tags with table-backed IDs so the runtime can track +creation sites, borrows, closes, and leaks. Enabling `Py_STACKREF_CLOSE_DEBUG` additionally records +double closes. The tables live on `PyInterpreterState` and are initialized in `pystate.c`; helper +routines reside in `Python/stackrefs.c`. diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 26a5197c6e70f38..829a27654a37d34 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -16,8 +16,8 @@ dynamic interning. The 256 possible one-character latin-1 strings, which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, -`_PyRuntime.static_objects.strings.ascii` and -`_PyRuntime.static_objects.strings.latin1`. +`_PyRuntime.static_objects.singletons.strings.ascii` and +`_PyRuntime.static_objects.singletons.strings.latin1`. Longer singleton strings are marked in C source with `_Py_ID` (if the string is a valid C identifier fragment) or `_Py_STR` (if it needs a separate @@ -52,15 +52,9 @@ The key and value of each entry in this dict reference the same object. ## Immortality and reference counting -Invariant: Every immortal string is interned. +In the GIL-enabled build interned strings may be mortal or immortal. In the +free-threaded build, interned strings are always immortal. -In practice, this means that you must not use `_Py_SetImmortal` on -a string. (If you know it's already immortal, don't immortalize it; -if you know it's not interned you might be immortalizing a redundant copy; -if it's interned and mortal it needs extra processing in -`_PyUnicode_InternImmortal`.) - -The converse is not true: interned strings can be mortal. For mortal interned strings: - the 2 references from the interned dict (key & value) are excluded from diff --git a/InternalDocs/structure.md b/InternalDocs/structure.md new file mode 100644 index 000000000000000..75c8476aa0ad989 --- /dev/null +++ b/InternalDocs/structure.md @@ -0,0 +1,40 @@ +# CPython source code + +This section gives an overview of CPython's code structure and provides +a summary of file locations for modules and built-ins. + + +## Source code layout + +For a Python module, the typical layout is: + +* `Lib/.py` +* `Modules/_.c` (if there's also a C accelerator module) +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For an extension module, the typical layout is: + +* `Modules/module.c` +* `Lib/test/test_.py` +* `Doc/library/.rst` + +For builtin types, the typical layout is: + +* `Objects/object.c` +* `Lib/test/test_.py` +* [`Doc/library/stdtypes.rst`](../Doc/library/stdtypes.rst) + +For builtin functions, the typical layout is: + +* [`Python/bltinmodule.c`](../Python/bltinmodule.c) +* [`Lib/test/test_builtin.py`](../Lib/test/test_builtin.py) +* [`Doc/library/functions.rst`](../Doc/library/functions.rst) + +Some exceptions to these layouts are: + +* built-in type `int` is at [`Objects/longobject.c`](../Objects/longobject.c) +* built-in type `str` is at [`Objects/unicodeobject.c`](../Objects/unicodeobject.c) +* built-in module `sys` is at [`Python/sysmodule.c`](../Python/sysmodule.c) +* built-in module `marshal` is at [`Python/marshal.c`](../Python/marshal.c) +* Windows-only module `winreg` is at [`PC/winreg.c`](../PC/winreg.c) diff --git a/Lib/.ruff.toml b/Lib/.ruff.toml new file mode 100644 index 000000000000000..c9ae9b95641b318 --- /dev/null +++ b/Lib/.ruff.toml @@ -0,0 +1,21 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +# Unlike Tools/, stdlib can use newer syntax than PYTHON_FOR_REGEN +target-version = "py315" + +[lint] +select = [ + "F401", # Unused import +] + +[lint.per-file-ignores] +"ctypes/__init__.py" = ["F401"] # Re-exports from _ctypes +"ensurepip/__init__.py" = ["F401"] # `import zlib` availability check +"idlelib/idle_test/htest.py" = ["F401"] # Import for Windows DPI side effect +"idlelib/idle_test/test_iomenu.py" = ["F401"] # Imports checked for existence +"importlib/_abc.py" = ["F401"] # Bootstrap-sensitive _bootstrap import +"importlib/machinery.py" = ["F401"] # NamespacePath re-export +"importlib/metadata/__init__.py" = ["F401"] # Synced from importlib_metadata +"profiling/sampling/sample.py" = ["F401"] # Re-exports PROFILING_MODE_* constants +"ssl.py" = ["F401"] # Re-exports from _ssl +"warnings.py" = ["F401"] # Re-exports from _py_warnings diff --git a/Lib/_android_support.py b/Lib/_android_support.py index a439d03a144dd22..320dab52acdc0bd 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -168,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 1c8741b5a554833..916bb25d74dee9b 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -239,11 +239,11 @@ def visit_NamedExpr(self, node): self.traverse(node.value) def visit_Import(self, node): - self.fill("import ") + self.fill("lazy import " if node.is_lazy else "import ") self.interleave(lambda: self.write(", "), self.traverse, node.names) def visit_ImportFrom(self, node): - self.fill("from ") + self.fill("lazy from " if node.is_lazy else "from ") self.write("." * (node.level or 0)) if node.module: self.write(node.module) @@ -738,9 +738,13 @@ def visit_SetComp(self, node): def visit_DictComp(self, node): with self.delimit("{", "}"): - self.traverse(node.key) - self.write(": ") - self.traverse(node.value) + if node.value: + self.traverse(node.key) + self.write(": ") + self.traverse(node.value) + else: + self.write("**") + self.traverse(node.key) for gen in node.generators: self.traverse(gen) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 60b471317ce97ca..24d08605429d340 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -67,7 +67,6 @@ def _f(): pass # are not included on this list. bytes_iterator = type(iter(b'')) bytearray_iterator = type(iter(bytearray())) -#callable_iterator = ??? dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) @@ -319,7 +318,6 @@ def __subclasshook__(cls, C): Iterator.register(bytes_iterator) Iterator.register(bytearray_iterator) -#Iterator.register(callable_iterator) Iterator.register(dict_keyiterator) Iterator.register(dict_valueiterator) Iterator.register(dict_itemiterator) @@ -461,8 +459,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -823,6 +821,7 @@ def __eq__(self, other): __reversed__ = None +Mapping.register(frozendict) Mapping.register(mappingproxy) Mapping.register(framelocalsproxy) @@ -927,8 +926,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -962,9 +962,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1029,8 +1032,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1138,15 +1141,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 57b712bc068d4e1..27eb7f13baca971 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,22 +1,19 @@ -import io +import builtins import os import sys from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field, Field +lazy from typing import IO, Literal, Self, ClassVar COLORIZE = True - -# types -if False: - from typing import IO, Self, ClassVar - _theme: Theme +_theme: Theme +type BackgroundStyle = Literal["dark", "light"] class ANSIColors: RESET = "\x1b[0m" - BLACK = "\x1b[30m" BLUE = "\x1b[34m" CYAN = "\x1b[36m" @@ -75,6 +72,19 @@ class ANSIColors: setattr(NoColors, attr, "") +class CursesColors: + """Curses color constants for terminal UI theming.""" + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = -1 + + # # Experimental theming support (see gh-133346) # @@ -169,12 +179,33 @@ class Argparse(ThemeSection): short_option: str = ANSIColors.BOLD_GREEN label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN + default: str = ANSIColors.GREY + interpolated_value: str = ANSIColors.YELLOW reset: str = ANSIColors.RESET error: str = ANSIColors.BOLD_MAGENTA warning: str = ANSIColors.BOLD_YELLOW message: str = ANSIColors.MAGENTA +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Calendar(ThemeSection): + header: str = ANSIColors.BOLD + highlight: str = ANSIColors.BLACK + ANSIColors.BACKGROUND_YELLOW + weekday: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Difflib(ThemeSection): """A 'git diff'-like theme for `difflib.unified_diff`.""" @@ -186,6 +217,188 @@ class Difflib(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class FancyCompleter(ThemeSection): + # functions and methods + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN + + # numbers + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW + + # others + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN + + +@dataclass(frozen=True, kw_only=True) +class HttpServer(ThemeSection): + error: str = ANSIColors.YELLOW + path: str = ANSIColors.CYAN + serving: str = ANSIColors.GREEN + size: str = ANSIColors.GREY + status_informational: str = ANSIColors.RESET + status_ok: str = ANSIColors.GREEN + status_redirect: str = ANSIColors.INTENSE_CYAN + status_client_error: str = ANSIColors.YELLOW + status_server_error: str = ANSIColors.RED + timestamp: str = ANSIColors.GREY + url: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class LiveProfiler(ThemeSection): + """Theme section for the live profiling TUI (Tachyon profiler). + + Colors use CursesColors constants (BLACK, RED, GREEN, YELLOW, + BLUE, MAGENTA, CYAN, WHITE, DEFAULT). + """ + # Header colors + title_fg: int = CursesColors.CYAN + title_bg: int = CursesColors.DEFAULT + + # Status display colors + pid_fg: int = CursesColors.CYAN + uptime_fg: int = CursesColors.GREEN + time_fg: int = CursesColors.YELLOW + interval_fg: int = CursesColors.MAGENTA + + # Thread view colors + thread_all_fg: int = CursesColors.GREEN + thread_single_fg: int = CursesColors.MAGENTA + + # Progress bar colors + bar_good_fg: int = CursesColors.GREEN + bar_bad_fg: int = CursesColors.RED + + # Stats colors + on_gil_fg: int = CursesColors.GREEN + off_gil_fg: int = CursesColors.RED + waiting_gil_fg: int = CursesColors.YELLOW + gc_fg: int = CursesColors.MAGENTA + + # Function display colors + func_total_fg: int = CursesColors.CYAN + func_exec_fg: int = CursesColors.GREEN + func_stack_fg: int = CursesColors.YELLOW + func_shown_fg: int = CursesColors.MAGENTA + + # Table header colors (for sorted column highlight) + sorted_header_fg: int = CursesColors.BLACK + sorted_header_bg: int = CursesColors.CYAN + + # Normal header colors (non-sorted columns) - use reverse video style + normal_header_fg: int = CursesColors.BLACK + normal_header_bg: int = CursesColors.WHITE + + # Data row colors + samples_fg: int = CursesColors.CYAN + file_fg: int = CursesColors.GREEN + func_fg: int = CursesColors.YELLOW + + # Trend indicator colors + trend_up_fg: int = CursesColors.GREEN + trend_down_fg: int = CursesColors.RED + + # Medal colors for top functions + medal_gold_fg: int = CursesColors.RED + medal_silver_fg: int = CursesColors.YELLOW + medal_bronze_fg: int = CursesColors.GREEN + + # Background style: 'dark' or 'light' + background_style: BackgroundStyle = "dark" + + +LiveProfilerLight = LiveProfiler( + # Header colors + title_fg=CursesColors.BLUE, # Blue is more readable than cyan on light bg + + # Status display colors - darker colors for light backgrounds + pid_fg=CursesColors.BLUE, + uptime_fg=CursesColors.BLACK, + time_fg=CursesColors.BLACK, + interval_fg=CursesColors.BLUE, + + # Thread view colors + thread_all_fg=CursesColors.BLACK, + thread_single_fg=CursesColors.BLUE, + + # Stats colors + waiting_gil_fg=CursesColors.RED, + gc_fg=CursesColors.BLUE, + + # Function display colors + func_total_fg=CursesColors.BLUE, + func_exec_fg=CursesColors.BLACK, + func_stack_fg=CursesColors.BLACK, + func_shown_fg=CursesColors.BLUE, + + # Table header colors (for sorted column highlight) + sorted_header_fg=CursesColors.WHITE, + sorted_header_bg=CursesColors.BLUE, + + # Normal header colors (non-sorted columns) + normal_header_fg=CursesColors.WHITE, + normal_header_bg=CursesColors.BLACK, + + # Data row colors - use dark colors readable on white + samples_fg=CursesColors.BLACK, + file_fg=CursesColors.BLACK, + func_fg=CursesColors.BLUE, # Blue is more readable than magenta on light bg + + # Medal colors for top functions + medal_silver_fg=CursesColors.BLUE, + + # Background style + background_style="light", +) + + +@dataclass(frozen=True, kw_only=True) +class ProfilerDump(ThemeSection): + header: str = ANSIColors.BOLD_BLUE + interpreter: str = ANSIColors.GREY + thread: str = ANSIColors.BOLD_CYAN + status: str = ANSIColors.YELLOW + frame_index: str = ANSIColors.GREY + frame: str = ANSIColors.BOLD_GREEN + filename: str = ANSIColors.CYAN + line_no: str = ANSIColors.YELLOW + source: str = ANSIColors.WHITE + source_highlight: str = ANSIColors.BOLD_YELLOW + opcode: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA @@ -201,10 +414,31 @@ class Syntax(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class Timeit(ThemeSection): + timing: str = ANSIColors.CYAN + best: str = ANSIColors.BOLD_GREEN + per_loop: str = ANSIColors.GREEN + punctuation: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + warning_worst: str = ANSIColors.MAGENTA + warning_best: str = ANSIColors.GREEN + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA message: str = ANSIColors.MAGENTA + note: str = ANSIColors.CYAN filename: str = ANSIColors.MAGENTA line_no: str = ANSIColors.MAGENTA frame: str = ANSIColors.MAGENTA @@ -230,8 +464,17 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) + calendar: Calendar = field(default_factory=Calendar) difflib: Difflib = field(default_factory=Difflib) + fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) + http_server: HttpServer = field(default_factory=HttpServer) + live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) + profiler_dump: ProfilerDump = field(default_factory=ProfilerDump) syntax: Syntax = field(default_factory=Syntax) + timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -239,8 +482,17 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, + calendar: Calendar | None = None, difflib: Difflib | None = None, + fancycompleter: FancyCompleter | None = None, + http_server: HttpServer | None = None, + live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, + profiler_dump: ProfilerDump | None = None, syntax: Syntax | None = None, + timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -251,8 +503,17 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, + calendar=calendar or self.calendar, difflib=difflib or self.difflib, + fancycompleter=fancycompleter or self.fancycompleter, + http_server=http_server or self.http_server, + live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, + profiler_dump=profiler_dump or self.profiler_dump, syntax=syntax or self.syntax, + timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -267,8 +528,17 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), + calendar=Calendar.no_colors(), difflib=Difflib.no_colors(), + fancycompleter=FancyCompleter.no_colors(), + http_server=HttpServer.no_colors(), + live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), + profiler_dump=ProfilerDump.no_colors(), syntax=Syntax.no_colors(), + timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) @@ -330,13 +600,16 @@ def _safe_getenv(k: str, fallback: str | None = None) -> str | None: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() default_theme = Theme() theme_no_color = default_theme.no_colors() +# Convenience theme with light profiler colors (for white/light terminal backgrounds) +light_profiler_theme = default_theme.copy_with(live_profiler=LiveProfilerLight) + def get_theme( *, diff --git a/Lib/_compat_pickle.py b/Lib/_compat_pickle.py index a981326432429b3..928db663b446ba2 100644 --- a/Lib/_compat_pickle.py +++ b/Lib/_compat_pickle.py @@ -240,6 +240,7 @@ REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ImportCycleError', 'ModuleNotFoundError', ) diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a329489..df92eae151d2488 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -2,364 +2,383 @@ # from: # Python/bytecodes.c # Do not edit! -_specializations = { - "RESUME": [ - "RESUME_CHECK", - ], - "TO_BOOL": [ - "TO_BOOL_ALWAYS_TRUE", - "TO_BOOL_BOOL", - "TO_BOOL_INT", - "TO_BOOL_LIST", - "TO_BOOL_NONE", - "TO_BOOL_STR", - ], - "BINARY_OP": [ - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_ADD_INT", - "BINARY_OP_SUBTRACT_INT", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_SUBSCR_LIST_INT", - "BINARY_OP_SUBSCR_LIST_SLICE", - "BINARY_OP_SUBSCR_TUPLE_INT", - "BINARY_OP_SUBSCR_STR_INT", - "BINARY_OP_SUBSCR_DICT", - "BINARY_OP_SUBSCR_GETITEM", - "BINARY_OP_EXTEND", - "BINARY_OP_INPLACE_ADD_UNICODE", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "SEND": [ - "SEND_GEN", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_TWO_TUPLE", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_LIST", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_MODULE", - "LOAD_GLOBAL_BUILTIN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_WITH_HINT", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_CLASS", - "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_METHOD_WITH_VALUES", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "CONTAINS_OP": [ - "CONTAINS_OP_SET", - "CONTAINS_OP_DICT", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_NO_JIT", - "JUMP_BACKWARD_JIT", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "CALL": [ - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_PY_EXACT_ARGS", - "CALL_TYPE_1", - "CALL_STR_1", - "CALL_TUPLE_1", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_O", - "CALL_BUILTIN_FAST", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_LEN", - "CALL_ISINSTANCE", - "CALL_LIST_APPEND", - "CALL_METHOD_DESCRIPTOR_O", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_NOARGS", - "CALL_METHOD_DESCRIPTOR_FAST", - "CALL_ALLOC_AND_ENTER_INIT", - "CALL_PY_GENERAL", - "CALL_BOUND_METHOD_GENERAL", - "CALL_NON_PY_GENERAL", - ], - "CALL_KW": [ - "CALL_KW_BOUND_METHOD", - "CALL_KW_PY", - "CALL_KW_NON_PY", - ], -} +_specializations = frozendict( + RESUME=( + "RESUME_CHECK", + "RESUME_CHECK_JIT", + ), + TO_BOOL=( + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ), + BINARY_OP=( + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_SUBSCR_LIST_INT", + "BINARY_OP_SUBSCR_LIST_SLICE", + "BINARY_OP_SUBSCR_TUPLE_INT", + "BINARY_OP_SUBSCR_STR_INT", + "BINARY_OP_SUBSCR_USTR_INT", + "BINARY_OP_SUBSCR_DICT", + "BINARY_OP_SUBSCR_GETITEM", + "BINARY_OP_INPLACE_ADD_UNICODE", + "BINARY_OP_EXTEND", + ), + STORE_SUBSCR=( + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ), + SEND=( + "SEND_GEN", + "SEND_VIRTUAL", + "SEND_ASYNC_GEN", + ), + UNPACK_SEQUENCE=( + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ), + STORE_ATTR=( + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ), + LOAD_GLOBAL=( + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ), + LOAD_SUPER_ATTR=( + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ), + LOAD_ATTR=( + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ), + COMPARE_OP=( + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ), + CONTAINS_OP=( + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ), + JUMP_BACKWARD=( + "JUMP_BACKWARD_NO_JIT", + "JUMP_BACKWARD_JIT", + ), + GET_ITER=( + "GET_ITER_SELF", + "GET_ITER_VIRTUAL", + ), + FOR_ITER=( + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + "FOR_ITER_VIRTUAL", + ), + CALL=( + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ), + CALL_KW=( + "CALL_KW_BOUND_METHOD", + "CALL_KW_PY", + "CALL_KW_NON_PY", + ), + CALL_FUNCTION_EX=( + "CALL_EX_PY", + "CALL_EX_NON_PY_GENERAL", + ), +) -_specialized_opmap = { - 'BINARY_OP_ADD_FLOAT': 129, - 'BINARY_OP_ADD_INT': 130, - 'BINARY_OP_ADD_UNICODE': 131, - 'BINARY_OP_EXTEND': 132, - 'BINARY_OP_INPLACE_ADD_UNICODE': 3, - 'BINARY_OP_MULTIPLY_FLOAT': 133, - 'BINARY_OP_MULTIPLY_INT': 134, - 'BINARY_OP_SUBSCR_DICT': 135, - 'BINARY_OP_SUBSCR_GETITEM': 136, - 'BINARY_OP_SUBSCR_LIST_INT': 137, - 'BINARY_OP_SUBSCR_LIST_SLICE': 138, - 'BINARY_OP_SUBSCR_STR_INT': 139, - 'BINARY_OP_SUBSCR_TUPLE_INT': 140, - 'BINARY_OP_SUBTRACT_FLOAT': 141, - 'BINARY_OP_SUBTRACT_INT': 142, - 'CALL_ALLOC_AND_ENTER_INIT': 143, - 'CALL_BOUND_METHOD_EXACT_ARGS': 144, - 'CALL_BOUND_METHOD_GENERAL': 145, - 'CALL_BUILTIN_CLASS': 146, - 'CALL_BUILTIN_FAST': 147, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148, - 'CALL_BUILTIN_O': 149, - 'CALL_ISINSTANCE': 150, - 'CALL_KW_BOUND_METHOD': 151, - 'CALL_KW_NON_PY': 152, - 'CALL_KW_PY': 153, - 'CALL_LEN': 154, - 'CALL_LIST_APPEND': 155, - 'CALL_METHOD_DESCRIPTOR_FAST': 156, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 158, - 'CALL_METHOD_DESCRIPTOR_O': 159, - 'CALL_NON_PY_GENERAL': 160, - 'CALL_PY_EXACT_ARGS': 161, - 'CALL_PY_GENERAL': 162, - 'CALL_STR_1': 163, - 'CALL_TUPLE_1': 164, - 'CALL_TYPE_1': 165, - 'COMPARE_OP_FLOAT': 166, - 'COMPARE_OP_INT': 167, - 'COMPARE_OP_STR': 168, - 'CONTAINS_OP_DICT': 169, - 'CONTAINS_OP_SET': 170, - 'FOR_ITER_GEN': 171, - 'FOR_ITER_LIST': 172, - 'FOR_ITER_RANGE': 173, - 'FOR_ITER_TUPLE': 174, - 'JUMP_BACKWARD_JIT': 175, - 'JUMP_BACKWARD_NO_JIT': 176, - 'LOAD_ATTR_CLASS': 177, - 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179, - 'LOAD_ATTR_INSTANCE_VALUE': 180, - 'LOAD_ATTR_METHOD_LAZY_DICT': 181, - 'LOAD_ATTR_METHOD_NO_DICT': 182, - 'LOAD_ATTR_METHOD_WITH_VALUES': 183, - 'LOAD_ATTR_MODULE': 184, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186, - 'LOAD_ATTR_PROPERTY': 187, - 'LOAD_ATTR_SLOT': 188, - 'LOAD_ATTR_WITH_HINT': 189, - 'LOAD_GLOBAL_BUILTIN': 190, - 'LOAD_GLOBAL_MODULE': 191, - 'LOAD_SUPER_ATTR_ATTR': 192, - 'LOAD_SUPER_ATTR_METHOD': 193, - 'RESUME_CHECK': 194, - 'SEND_GEN': 195, - 'STORE_ATTR_INSTANCE_VALUE': 196, - 'STORE_ATTR_SLOT': 197, - 'STORE_ATTR_WITH_HINT': 198, - 'STORE_SUBSCR_DICT': 199, - 'STORE_SUBSCR_LIST_INT': 200, - 'TO_BOOL_ALWAYS_TRUE': 201, - 'TO_BOOL_BOOL': 202, - 'TO_BOOL_INT': 203, - 'TO_BOOL_LIST': 204, - 'TO_BOOL_NONE': 205, - 'TO_BOOL_STR': 206, - 'UNPACK_SEQUENCE_LIST': 207, - 'UNPACK_SEQUENCE_TUPLE': 208, - 'UNPACK_SEQUENCE_TWO_TUPLE': 209, -} +_specialized_opmap = frozendict( + BINARY_OP_ADD_FLOAT=129, + BINARY_OP_ADD_INT=130, + BINARY_OP_ADD_UNICODE=131, + BINARY_OP_EXTEND=132, + BINARY_OP_INPLACE_ADD_UNICODE=3, + BINARY_OP_MULTIPLY_FLOAT=133, + BINARY_OP_MULTIPLY_INT=134, + BINARY_OP_SUBSCR_DICT=135, + BINARY_OP_SUBSCR_GETITEM=136, + BINARY_OP_SUBSCR_LIST_INT=137, + BINARY_OP_SUBSCR_LIST_SLICE=138, + BINARY_OP_SUBSCR_STR_INT=139, + BINARY_OP_SUBSCR_TUPLE_INT=140, + BINARY_OP_SUBSCR_USTR_INT=141, + BINARY_OP_SUBTRACT_FLOAT=142, + BINARY_OP_SUBTRACT_INT=143, + CALL_ALLOC_AND_ENTER_INIT=144, + CALL_BOUND_METHOD_EXACT_ARGS=145, + CALL_BOUND_METHOD_GENERAL=146, + CALL_BUILTIN_CLASS=147, + CALL_BUILTIN_FAST=148, + CALL_BUILTIN_FAST_WITH_KEYWORDS=149, + CALL_BUILTIN_O=150, + CALL_EX_NON_PY_GENERAL=151, + CALL_EX_PY=152, + CALL_ISINSTANCE=153, + CALL_KW_BOUND_METHOD=154, + CALL_KW_NON_PY=155, + CALL_KW_PY=156, + CALL_LEN=157, + CALL_LIST_APPEND=158, + CALL_METHOD_DESCRIPTOR_FAST=159, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, + CALL_METHOD_DESCRIPTOR_NOARGS=161, + CALL_METHOD_DESCRIPTOR_O=162, + CALL_NON_PY_GENERAL=163, + CALL_PY_EXACT_ARGS=164, + CALL_PY_GENERAL=165, + CALL_STR_1=166, + CALL_TUPLE_1=167, + CALL_TYPE_1=168, + COMPARE_OP_FLOAT=169, + COMPARE_OP_INT=170, + COMPARE_OP_STR=171, + CONTAINS_OP_DICT=172, + CONTAINS_OP_SET=173, + FOR_ITER_GEN=174, + FOR_ITER_LIST=175, + FOR_ITER_RANGE=176, + FOR_ITER_TUPLE=177, + FOR_ITER_VIRTUAL=178, + GET_ITER_SELF=179, + GET_ITER_VIRTUAL=180, + JUMP_BACKWARD_JIT=181, + JUMP_BACKWARD_NO_JIT=182, + LOAD_ATTR_CLASS=183, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, + LOAD_ATTR_INSTANCE_VALUE=186, + LOAD_ATTR_METHOD_LAZY_DICT=187, + LOAD_ATTR_METHOD_NO_DICT=188, + LOAD_ATTR_METHOD_WITH_VALUES=189, + LOAD_ATTR_MODULE=190, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, + LOAD_ATTR_PROPERTY=193, + LOAD_ATTR_SLOT=194, + LOAD_ATTR_WITH_HINT=195, + LOAD_GLOBAL_BUILTIN=196, + LOAD_GLOBAL_MODULE=197, + LOAD_SUPER_ATTR_ATTR=198, + LOAD_SUPER_ATTR_METHOD=199, + RESUME_CHECK=200, + RESUME_CHECK_JIT=201, + SEND_ASYNC_GEN=202, + SEND_GEN=203, + SEND_VIRTUAL=204, + STORE_ATTR_INSTANCE_VALUE=205, + STORE_ATTR_SLOT=206, + STORE_ATTR_WITH_HINT=207, + STORE_SUBSCR_DICT=208, + STORE_SUBSCR_LIST_INT=209, + TO_BOOL_ALWAYS_TRUE=210, + TO_BOOL_BOOL=211, + TO_BOOL_INT=212, + TO_BOOL_LIST=213, + TO_BOOL_NONE=214, + TO_BOOL_STR=215, + UNPACK_SEQUENCE_LIST=216, + UNPACK_SEQUENCE_TUPLE=217, + UNPACK_SEQUENCE_TWO_TUPLE=218, +) -opmap = { - 'CACHE': 0, - 'RESERVED': 17, - 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, - 'BINARY_SLICE': 1, - 'BUILD_TEMPLATE': 2, - 'CALL_FUNCTION_EX': 4, - 'CHECK_EG_MATCH': 5, - 'CHECK_EXC_MATCH': 6, - 'CLEANUP_THROW': 7, - 'DELETE_SUBSCR': 8, - 'END_FOR': 9, - 'END_SEND': 10, - 'EXIT_INIT_CHECK': 11, - 'FORMAT_SIMPLE': 12, - 'FORMAT_WITH_SPEC': 13, - 'GET_AITER': 14, - 'GET_ANEXT': 15, - 'GET_ITER': 16, - 'GET_LEN': 18, - 'GET_YIELD_FROM_ITER': 19, - 'INTERPRETER_EXIT': 20, - 'LOAD_BUILD_CLASS': 21, - 'LOAD_LOCALS': 22, - 'MAKE_FUNCTION': 23, - 'MATCH_KEYS': 24, - 'MATCH_MAPPING': 25, - 'MATCH_SEQUENCE': 26, - 'NOP': 27, - 'NOT_TAKEN': 28, - 'POP_EXCEPT': 29, - 'POP_ITER': 30, - 'POP_TOP': 31, - 'PUSH_EXC_INFO': 32, - 'PUSH_NULL': 33, - 'RETURN_GENERATOR': 34, - 'RETURN_VALUE': 35, - 'SETUP_ANNOTATIONS': 36, - 'STORE_SLICE': 37, - 'STORE_SUBSCR': 38, - 'TO_BOOL': 39, - 'UNARY_INVERT': 40, - 'UNARY_NEGATIVE': 41, - 'UNARY_NOT': 42, - 'WITH_EXCEPT_START': 43, - 'BINARY_OP': 44, - 'BUILD_INTERPOLATION': 45, - 'BUILD_LIST': 46, - 'BUILD_MAP': 47, - 'BUILD_SET': 48, - 'BUILD_SLICE': 49, - 'BUILD_STRING': 50, - 'BUILD_TUPLE': 51, - 'CALL': 52, - 'CALL_INTRINSIC_1': 53, - 'CALL_INTRINSIC_2': 54, - 'CALL_KW': 55, - 'COMPARE_OP': 56, - 'CONTAINS_OP': 57, - 'CONVERT_VALUE': 58, - 'COPY': 59, - 'COPY_FREE_VARS': 60, - 'DELETE_ATTR': 61, - 'DELETE_DEREF': 62, - 'DELETE_FAST': 63, - 'DELETE_GLOBAL': 64, - 'DELETE_NAME': 65, - 'DICT_MERGE': 66, - 'DICT_UPDATE': 67, - 'END_ASYNC_FOR': 68, - 'EXTENDED_ARG': 69, - 'FOR_ITER': 70, - 'GET_AWAITABLE': 71, - 'IMPORT_FROM': 72, - 'IMPORT_NAME': 73, - 'IS_OP': 74, - 'JUMP_BACKWARD': 75, - 'JUMP_BACKWARD_NO_INTERRUPT': 76, - 'JUMP_FORWARD': 77, - 'LIST_APPEND': 78, - 'LIST_EXTEND': 79, - 'LOAD_ATTR': 80, - 'LOAD_COMMON_CONSTANT': 81, - 'LOAD_CONST': 82, - 'LOAD_DEREF': 83, - 'LOAD_FAST': 84, - 'LOAD_FAST_AND_CLEAR': 85, - 'LOAD_FAST_BORROW': 86, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, - 'LOAD_FAST_CHECK': 88, - 'LOAD_FAST_LOAD_FAST': 89, - 'LOAD_FROM_DICT_OR_DEREF': 90, - 'LOAD_FROM_DICT_OR_GLOBALS': 91, - 'LOAD_GLOBAL': 92, - 'LOAD_NAME': 93, - 'LOAD_SMALL_INT': 94, - 'LOAD_SPECIAL': 95, - 'LOAD_SUPER_ATTR': 96, - 'MAKE_CELL': 97, - 'MAP_ADD': 98, - 'MATCH_CLASS': 99, - 'POP_JUMP_IF_FALSE': 100, - 'POP_JUMP_IF_NONE': 101, - 'POP_JUMP_IF_NOT_NONE': 102, - 'POP_JUMP_IF_TRUE': 103, - 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, - 'ANNOTATIONS_PLACEHOLDER': 256, - 'JUMP': 257, - 'JUMP_IF_FALSE': 258, - 'JUMP_IF_TRUE': 259, - 'JUMP_NO_INTERRUPT': 260, - 'LOAD_CLOSURE': 261, - 'POP_BLOCK': 262, - 'SETUP_CLEANUP': 263, - 'SETUP_FINALLY': 264, - 'SETUP_WITH': 265, - 'STORE_FAST_MAYBE_NULL': 266, -} +opmap = frozendict( + CACHE=0, + RESERVED=17, + RESUME=128, + INSTRUMENTED_LINE=253, + ENTER_EXECUTOR=254, + TRACE_RECORD=255, + BINARY_SLICE=1, + BUILD_TEMPLATE=2, + CALL_FUNCTION_EX=4, + CHECK_EG_MATCH=5, + CHECK_EXC_MATCH=6, + CLEANUP_THROW=7, + DELETE_SUBSCR=8, + END_FOR=9, + END_SEND=10, + EXIT_INIT_CHECK=11, + FORMAT_SIMPLE=12, + FORMAT_WITH_SPEC=13, + GET_AITER=14, + GET_ANEXT=15, + GET_LEN=16, + INTERPRETER_EXIT=18, + LOAD_BUILD_CLASS=19, + LOAD_LOCALS=20, + MAKE_FUNCTION=21, + MATCH_KEYS=22, + MATCH_MAPPING=23, + MATCH_SEQUENCE=24, + NOP=25, + NOT_TAKEN=26, + POP_EXCEPT=27, + POP_ITER=28, + POP_TOP=29, + PUSH_EXC_INFO=30, + PUSH_NULL=31, + RETURN_GENERATOR=32, + RETURN_VALUE=33, + SETUP_ANNOTATIONS=34, + STORE_SLICE=35, + STORE_SUBSCR=36, + TO_BOOL=37, + UNARY_INVERT=38, + UNARY_NEGATIVE=39, + UNARY_NOT=40, + WITH_EXCEPT_START=41, + BINARY_OP=42, + BUILD_INTERPOLATION=43, + BUILD_LIST=44, + BUILD_MAP=45, + BUILD_SET=46, + BUILD_SLICE=47, + BUILD_STRING=48, + BUILD_TUPLE=49, + CALL=50, + CALL_INTRINSIC_1=51, + CALL_INTRINSIC_2=52, + CALL_KW=53, + COMPARE_OP=54, + CONTAINS_OP=55, + CONVERT_VALUE=56, + COPY=57, + COPY_FREE_VARS=58, + DELETE_DEREF=59, + DELETE_FAST=60, + DICT_MERGE=61, + DICT_UPDATE=62, + END_ASYNC_FOR=63, + EXTENDED_ARG=64, + FOR_ITER=65, + GET_AWAITABLE=66, + GET_ITER=67, + IMPORT_FROM=68, + IMPORT_NAME=69, + IS_OP=70, + JUMP_BACKWARD=71, + JUMP_BACKWARD_NO_INTERRUPT=72, + JUMP_FORWARD=73, + LIST_APPEND=74, + LIST_EXTEND=75, + LOAD_ATTR=76, + LOAD_COMMON_CONSTANT=77, + LOAD_CONST=78, + LOAD_DEREF=79, + LOAD_FAST=80, + LOAD_FAST_AND_CLEAR=81, + LOAD_FAST_BORROW=82, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, + LOAD_FAST_CHECK=84, + LOAD_FAST_LOAD_FAST=85, + LOAD_FROM_DICT_OR_DEREF=86, + LOAD_FROM_DICT_OR_GLOBALS=87, + LOAD_GLOBAL=88, + LOAD_NAME=89, + LOAD_SMALL_INT=90, + LOAD_SPECIAL=91, + LOAD_SUPER_ATTR=92, + MAKE_CELL=93, + MAP_ADD=94, + MATCH_CLASS=95, + POP_JUMP_IF_FALSE=96, + POP_JUMP_IF_NONE=97, + POP_JUMP_IF_NOT_NONE=98, + POP_JUMP_IF_TRUE=99, + RAISE_VARARGS=100, + RERAISE=101, + SEND=102, + SET_ADD=103, + SET_FUNCTION_ATTRIBUTE=104, + SET_UPDATE=105, + STORE_ATTR=106, + STORE_DEREF=107, + STORE_FAST=108, + STORE_FAST_LOAD_FAST=109, + STORE_FAST_STORE_FAST=110, + STORE_GLOBAL=111, + STORE_NAME=112, + SWAP=113, + UNPACK_EX=114, + UNPACK_SEQUENCE=115, + YIELD_VALUE=116, + INSTRUMENTED_END_FOR=233, + INSTRUMENTED_POP_ITER=234, + INSTRUMENTED_END_SEND=235, + INSTRUMENTED_FOR_ITER=236, + INSTRUMENTED_INSTRUCTION=237, + INSTRUMENTED_JUMP_FORWARD=238, + INSTRUMENTED_NOT_TAKEN=239, + INSTRUMENTED_POP_JUMP_IF_TRUE=240, + INSTRUMENTED_POP_JUMP_IF_FALSE=241, + INSTRUMENTED_POP_JUMP_IF_NONE=242, + INSTRUMENTED_POP_JUMP_IF_NOT_NONE=243, + INSTRUMENTED_RESUME=244, + INSTRUMENTED_RETURN_VALUE=245, + INSTRUMENTED_YIELD_VALUE=246, + INSTRUMENTED_END_ASYNC_FOR=247, + INSTRUMENTED_LOAD_SUPER_ATTR=248, + INSTRUMENTED_CALL=249, + INSTRUMENTED_CALL_KW=250, + INSTRUMENTED_CALL_FUNCTION_EX=251, + INSTRUMENTED_JUMP_BACKWARD=252, + ANNOTATIONS_PLACEHOLDER=256, + JUMP=257, + JUMP_IF_FALSE=258, + JUMP_IF_TRUE=259, + JUMP_NO_INTERRUPT=260, + LOAD_CLOSURE=261, + POP_BLOCK=262, + SETUP_CLEANUP=263, + SETUP_FINALLY=264, + SETUP_WITH=265, + STORE_FAST_MAYBE_NULL=266, +) -HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +HAVE_ARGUMENT = 41 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 0cb064fcd791be0..29b89e311cb1fe7 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -17,7 +17,8 @@ _UNIVERSAL_CONFIG_VARS = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS', 'BASECFLAGS', 'BLDSHARED', 'LDSHARED', 'CC', 'CXX', 'PY_CFLAGS', 'PY_LDFLAGS', 'PY_CPPFLAGS', - 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS') + 'PY_CORE_CFLAGS', 'PY_CORE_LDFLAGS', + 'PY_CORE_EXE_LDFLAGS') # configuration variables that may contain compiler calls _COMPILER_CONFIG_VARS = ('BLDSHARED', 'LDSHARED', 'CC', 'CXX') diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 67c74fdd2d0b421..ab09913de6812dd 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -563,7 +563,6 @@ def warn_explicit(message, category, filename, lineno, else: text = message message = category(message) - modules = None key = (text, category, lineno) with _wm._lock: if registry is None: @@ -621,17 +620,18 @@ def warn_explicit(message, category, filename, lineno, linecache.getlines(filename, module_globals) # Print message and context - msg = _wm.WarningMessage(message, category, filename, lineno, source=source) + msg = _wm.WarningMessage(message, category, filename, lineno, + module=module, source=source) _wm._showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line", "source") + "line", "source", "module") def __init__(self, message, category, filename, lineno, file=None, - line=None, source=None): + line=None, source=None, module=None): self.message = message self.category = category self.filename = filename @@ -639,12 +639,14 @@ def __init__(self, message, category, filename, lineno, file=None, self.file = file self.line = line self.source = source + self.module = module self._category_name = category.__name__ if category else None def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) + return ("{message : %r, category : %r, module : %r, " + "filename : %r, lineno : %s, line : %r}" % ( + self.message, self._category_name, self.module, + self.filename, self.lineno, self.line)) def __repr__(self): return f'<{type(self).__qualname__} {self}>' @@ -704,8 +706,8 @@ def __enter__(self): context = None self._filters = self._module.filters self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._showwarning = self._module.showwarning self._module._filters_mutated_lock_held() if self._record: if _use_context: @@ -713,9 +715,9 @@ def __enter__(self): else: log = [] self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig else: log = None if self._filter is not None: @@ -730,8 +732,8 @@ def __exit__(self, *exc_info): self._module._warnings_context.set(self._saved_context) else: self._module.filters = self._filters - self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module.showwarning = self._showwarning self._module._filters_mutated_lock_held() diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b6d68f2372850a7..c47f4e671b39def 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -272,12 +272,12 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC') and - object.year < 1000 and _need_normalize_century()): + elif (ch in 'YGFC' and timetuple[0] < 1000 and + _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) else: - year = object.year + year = timetuple[0] if ch == 'C': push('{:02}'.format(year // 100)) else: @@ -358,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr): def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") year = int(dtstr[0:4]) has_sep = dtstr[4] == '-' @@ -1987,7 +1988,7 @@ def fromisoformat(cls, date_string): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ef889ea0cc834c1..8c0afd14d616e85 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -107,8 +107,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -225,11 +225,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -322,8 +323,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -2901,8 +2903,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2973,7 +2975,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -4110,9 +4113,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4168,16 +4171,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4240,8 +4244,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4268,7 +4272,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4926,8 +4931,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5203,19 +5208,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5269,13 +5274,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5309,9 +5314,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5369,8 +5374,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5442,8 +5447,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 423178e87a86844..1118b54633b7cc1 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -10,7 +10,7 @@ import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock -if sys.platform in {'win32', 'cygwin'}: +if sys.platform == 'win32': from msvcrt import setmode as _setmode else: _setmode = None @@ -83,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -111,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -147,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -164,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -351,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -367,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -492,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -546,7 +550,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -620,7 +624,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -628,7 +632,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -941,7 +945,7 @@ def read(self, size=-1): newpos = min(len(self._buffer), self._pos + size) b = self._buffer[self._pos : newpos] self._pos = newpos - return bytes(b) + return b.take_bytes() def read1(self, size=-1): """This is the same as read. @@ -949,23 +953,24 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 + if n == 0: + return 0 - with self._lock: - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n - return n + with self._lock: + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n + return n def seek(self, pos, whence=0): if self.closed: @@ -995,6 +1000,13 @@ def tell(self): raise ValueError("tell on closed file") return self._pos + def peek(self, size=0): + if self.closed: + raise ValueError("peek on closed file") + if size < 1: + return self._buffer[self._pos:self._pos + io.DEFAULT_BUFFER_SIZE] + return self._buffer[self._pos:self._pos + size] + def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") @@ -1504,17 +1516,22 @@ class FileIO(RawIOBase): _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. @@ -1738,7 +1755,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" @@ -1753,8 +1770,8 @@ def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1766,11 +1783,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1803,8 +1821,8 @@ def truncate(self, size=None): def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: self._stat_atopen = None @@ -1902,8 +1920,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1917,7 +1935,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index cf59e007f4df80a..17bf5cdc819542d 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -3,6 +3,7 @@ import importlib import os import pkgutil +import re import sys import token import tokenize @@ -12,21 +13,36 @@ from dataclasses import dataclass from itertools import chain from tokenize import TokenInfo +from .fancycompleter import safe_getattr TYPE_CHECKING = False if TYPE_CHECKING: + from types import ModuleType from typing import Any, Iterable, Iterator, Mapping + from .types import CompletionAction HARDCODED_SUBMODULES = { # Standard library submodules that are not detected by pkgutil.iter_modules # but can be imported, so should be proposed in completion "collections": ["abc"], + "math": ["integer"], "os": ["path"], "xml.parsers.expat": ["errors", "model"], } +AUTO_IMPORT_DENYLIST = { + # Standard library modules/submodules that have import side effects + # and must not be automatically imported to complete attributes + re.compile(r"antigravity"), # Calls webbrowser.open + re.compile(r"idlelib\..+"), # May open IDLE GUI + re.compile(r"test\..+"), # Various side-effects + re.compile(r"this"), # Prints to stdout + re.compile(r"_ios_support"), # Spawns a subprocess + re.compile(r".+\.__main__"), # Should not be imported +} + def make_default_module_completer() -> ModuleCompleter: # Inside pyrepl, __package__ is set to None by default @@ -52,37 +68,73 @@ class ModuleCompleter: def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] + self._failed_imports: set[str] = set() self._curr_sys_path: list[str] = sys.path[:] self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str] | None: - """Return the next possible import completions for 'line'.""" + def get_completions( + self, line: str, *, include_values: bool = True + ) -> tuple[list[str], list[Any], CompletionAction | None] | None: + """Return the next possible import completions for 'line'. + + For attributes completion, if the module to complete from is not + imported, also return an action (prompt + callback to run if the + user press TAB again) to import the module. + + If *include_values* is false, the returned values list is empty and + attribute values are not resolved. + """ result = ImportParser(line).parse() if not result: return None try: - return self.complete(*result) + return self.complete(*result, include_values=include_values) except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [] - - def complete(self, from_name: str | None, name: str | None) -> list[str]: + return [], [], None + + def complete( + self, + from_name: str | None, + name: str | None, + *, + include_values: bool = True, + ) -> tuple[list[str], list[Any], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules] + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None # from x.y import z - return self.find_modules(from_name, name) + submodules = self.find_modules(from_name, name) + attr_names, attr_module, action = self._find_attributes(from_name, name) + all_names = sorted({*submodules, *attr_names}) + if not include_values: + return all_names, [], action + + # Build values list matching the sorted order: + # submodules use `sys` as a dummy value so they get the 'module' color, + # attributes use their actual value. + attr_map = {} + if attr_module is not None: + attr_map = {n: safe_getattr(attr_module, n) for n in attr_names} + all_values = [attr_map[n] if n in attr_map else sys for n in all_names] + return all_names, all_values, action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -100,14 +152,28 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if self.is_suggestion_match(module.name, prefix)] return sorted(builtin_modules + third_party_modules) - if path.startswith('.'): - # Convert relative path to absolute path - package = self.namespace.get('__package__', '') - path = self.resolve_relative_name(path, package) # type: ignore[assignment] - if path is None: - return [] + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those whose name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + def _safe_find_spec(mod: pkgutil.ModuleInfo) -> bool: + try: + return mod.module_finder.find_spec(mod.name, None) == spec + except Exception: + return False + modules = [mod for mod in modules + if mod.name == spec.name + and _safe_find_spec(mod)] + else: + modules = [] + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules @@ -129,6 +195,44 @@ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: return (isinstance(module_info.module_finder, FileFinder) and module_info.module_finder.path == self._stdlib_path) + def find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], list[Any], CompletionAction | None]: + """Find all attributes of module 'path' that start with 'prefix'.""" + attributes, module, action = self._find_attributes(path, prefix) + if module is not None: + values = [safe_getattr(module, attr) for attr in attributes] + else: + values = [] + return attributes, values, action + + def _find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], ModuleType | None, CompletionAction | None]: + path = self._resolve_relative_path(path) # type: ignore[assignment] + if path is None: + return [], None, None + + imported_module = sys.modules.get(path) + if not imported_module: + if path in self._failed_imports: # Do not propose to import again + return [], None, None + imported_module = self._maybe_import_module(path) + if not imported_module: + return [], None, self._get_import_completion_action(path) + try: + module_attributes = dir(imported_module) + except Exception: + module_attributes = [] + # Filter out invalid attribute names, such as dashes that cannot be + # imported with 'import'. + names = [ + attr_name for attr_name in module_attributes + if (self.is_suggestion_match(attr_name, prefix) + and attr_name.isidentifier()) + ] + return names, imported_module, None + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: return module_name.startswith(prefix) @@ -173,6 +277,13 @@ def format_completion(self, path: str, module: str) -> str: return f'{path}{module}' return f'{path}.{module}' + def _resolve_relative_path(self, path: str) -> str | None: + """Resolve a relative import path to absolute. Returns None if unresolvable.""" + if path.startswith('.'): + package = self.namespace.get('__package__', '') + return self.resolve_relative_name(path, package) + return path + def resolve_relative_name(self, name: str, package: str) -> str | None: """Resolve a relative module name to an absolute name. @@ -196,10 +307,40 @@ def global_cache(self) -> list[pkgutil.ModuleInfo]: """Global module cache""" if not self._global_cache or self._curr_sys_path != sys.path: self._curr_sys_path = sys.path[:] - # print('getting packages') self._global_cache = list(pkgutil.iter_modules()) + self._failed_imports.clear() # retry on sys.path change return self._global_cache + def _maybe_import_module(self, fqname: str) -> ModuleType | None: + if any(pattern.fullmatch(fqname) for pattern in AUTO_IMPORT_DENYLIST): + # Special-cased modules with known import side-effects + return None + root = fqname.split(".")[0] + mod_info = next((m for m in self.global_cache if m.name == root), None) + if not mod_info or not self._is_stdlib_module(mod_info): + # Only import stdlib modules (no risk of import side-effects) + return None + try: + return importlib.import_module(fqname) + except Exception: + sys.modules.pop(fqname, None) # Clean half-imported module + return None + + def _get_import_completion_action(self, path: str) -> CompletionAction: + prompt = ("[ module not imported, press again to import it " + "and propose attributes ]") + + def _do_import() -> str | None: + try: + importlib.import_module(path) + return None + except Exception as exc: + sys.modules.pop(path, None) # Clean half-imported module + self._failed_imports.add(path) + return f"[ error during import: {exc} ]" + + return (prompt, _do_import) + class ImportParser: """ diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 10127e58897a580..e79fbfa6bb0b386 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -22,6 +22,7 @@ from __future__ import annotations import os import time +from typing import TYPE_CHECKING # Categories of actions: # killing @@ -32,10 +33,11 @@ # finishing # [completion] +from .render import RenderedScreen from .trace import trace # types -if False: +if TYPE_CHECKING: from .historical_reader import HistoricalReader @@ -74,7 +76,7 @@ def kill_range(self, start: int, end: int) -> None: else: r.kill_ring.append(text) r.pos = start - r.dirty = True + r.invalidate_buffer(start) class YankCommand(Command): @@ -125,24 +127,27 @@ def do(self) -> None: r.arg = 10 * r.arg - d else: r.arg = 10 * r.arg + d - r.dirty = True + r.invalidate_prompt() class clear_screen(Command): def do(self) -> None: r = self.reader + trace("command.clear_screen") r.console.clear() - r.dirty = True + r.invalidate_full() class refresh(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.refresh") + self.reader.invalidate_full() class repaint(Command): def do(self) -> None: - self.reader.dirty = True + trace("command.repaint") + self.reader.invalidate_full() self.reader.console.repaint() @@ -208,9 +213,10 @@ def do(self) -> None: repl = len(r.kill_ring[-1]) r.kill_ring.insert(0, r.kill_ring.pop()) t = r.kill_ring[-1] + start = r.pos - repl b[r.pos - repl : r.pos] = t r.pos = r.pos - repl + len(t) - r.dirty = True + r.invalidate_buffer(start) class interrupt(FinishCommand): @@ -242,8 +248,9 @@ def do(self) -> None: r.console.prepare() r.pos = p # r.posxy = 0, 0 # XXX this is invalid - r.dirty = True - r.console.screen = [] + r.invalidate_full() + trace("command.suspend sync_rendered_screen") + r.console.sync_rendered_screen(RenderedScreen.empty(), r.console.posxy) class up(MotionCommand): @@ -369,6 +376,7 @@ class self_insert(EditCommand): def do(self) -> None: r = self.reader text = self.event * r.get_arg() + start = r.pos r.insert(text) if r.paste_mode: data = "" @@ -376,7 +384,7 @@ def do(self) -> None: data += ev.data if data: r.insert(data) - r.last_refresh_cache.invalidated = True + r.invalidate_buffer(start) class insert_nl(EditCommand): @@ -400,20 +408,23 @@ def do(self) -> None: del b[s] b.insert(t, c) r.pos = t - r.dirty = True + r.invalidate_buffer(s) class backspace(EditCommand): def do(self) -> None: r = self.reader b = r.buffer + changed_from: int | None = None for i in range(r.get_arg()): if r.pos > 0: r.pos -= 1 del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("can't backspace at start") + if changed_from is not None: + r.invalidate_buffer(changed_from) class delete(EditCommand): @@ -431,12 +442,15 @@ def do(self) -> None: r.console.finish() raise EOFError + changed_from: int | None = None for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] - r.dirty = True + changed_from = r.pos if changed_from is None else min(changed_from, r.pos) else: self.reader.error("end of buffer") + if changed_from is not None: + r.invalidate_buffer(changed_from) class accept(FinishCommand): @@ -450,6 +464,7 @@ def do(self) -> None: with self.reader.suspend(): self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment] + self.reader.invalidate_prompt() class invalid_key(Command): @@ -470,22 +485,24 @@ def do(self) -> None: from .pager import get_pager from site import gethistoryfile + # After the pager exits, the screen state is unknown (Unix may + # restore via alternate screen, Windows shows pager output). + # Clear and force a full redraw at the end for consistency. + self.reader.console.clear() + history = os.linesep.join(self.reader.history[:]) self.reader.console.restore() pager = get_pager() pager(history, gethistoryfile()) self.reader.console.prepare() - # We need to copy over the state so that it's consistent between - # console and reader, and console does not overwrite/append stuff - self.reader.console.screen = self.reader.screen.copy() - self.reader.console.posxy = self.reader.cxy + self.reader.invalidate_full() class paste_mode(Command): def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode - self.reader.dirty = True + self.reader.invalidate_prompt() class perform_bracketed_paste(Command): @@ -502,4 +519,3 @@ def do(self) -> None: s=time.time() - start, ) self.reader.insert(data.replace(done, "")) - self.reader.last_refresh_cache.invalidated = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index 9d2d43be5144e8c..7eb3ebac84124b5 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -21,16 +21,18 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import TYPE_CHECKING import re from . import commands, console, reader +from .render import RenderLine, ScreenOverlay from .reader import Reader # types Command = commands.Command -if False: - from .types import KeySpec, CommandName +if TYPE_CHECKING: + from .types import CompletionAction, Keymap def prefix(wordlist: list[str], j: int = 0) -> str: @@ -168,39 +170,68 @@ def do(self) -> None: r: CompletingReader r = self.reader # type: ignore[assignment] last_is_completer = r.last_command_is(self.__class__) + if r.cmpltn_action: + if last_is_completer: # double-tab: execute action + msg = r.cmpltn_action[1]() + r.cmpltn_action = None # consumed + if msg: + r.msg = msg + r.cmpltn_message_visible = True + r.invalidate_message() + else: # other input since last tab: cancel action + r.cmpltn_action = None + immutable_completions = r.assume_immutable_completions completions_unchangable = last_is_completer and immutable_completions stem = r.get_stem() if not completions_unchangable: - r.cmpltn_menu_choices = r.get_completions(stem) + r.cmpltn_menu_choices, r.cmpltn_action = r.get_completions(stem) completions = r.cmpltn_menu_choices if not completions: - r.error("no matches") + if not r.cmpltn_action: + r.error("no matches") elif len(completions) == 1: - if completions_unchangable and len(completions[0]) == len(stem): + completion = stripcolor(completions[0]) + if completions_unchangable and len(completion) == len(stem): r.msg = "[ sole completion ]" - r.dirty = True - r.insert(completions[0][len(stem):]) + r.cmpltn_message_visible = True + r.invalidate_message() + r.insert(completion[len(stem):]) else: - p = prefix(completions, len(stem)) + clean_completions = [stripcolor(word) for word in completions] + p = prefix(clean_completions, len(stem)) if p: r.insert(p) if last_is_completer: r.cmpltn_menu_visible = True - r.cmpltn_message_visible = False r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, r.cmpltn_menu_end, r.use_brackets, r.sort_in_column) - r.dirty = True + if r.msg: + r.msg = "" + r.cmpltn_message_visible = False + r.invalidate_message() + r.invalidate_overlay() elif not r.cmpltn_menu_visible: - r.cmpltn_message_visible = True - if stem + p in completions: + if stem + p in clean_completions: r.msg = "[ complete but not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() else: r.msg = "[ not unique ]" - r.dirty = True + r.cmpltn_message_visible = True + r.invalidate_message() + + if r.cmpltn_action: + if r.msg and r.cmpltn_message_visible: + # There is already a message (eg. [ not unique ]) that + # would conflict for next tab: cancel action + r.cmpltn_action = None + else: + r.msg = r.cmpltn_action[0] + r.cmpltn_message_visible = True + r.invalidate_message() class self_insert(commands.self_insert): @@ -215,11 +246,12 @@ def do(self) -> None: r.cmpltn_reset() else: completions = [w for w in r.cmpltn_menu_choices - if w.startswith(stem)] + if stripcolor(w).startswith(stem)] if completions: r.cmpltn_menu, r.cmpltn_menu_end = build_menu( r.console, completions, 0, r.use_brackets, r.sort_in_column) + r.invalidate_overlay() else: r.cmpltn_reset() @@ -240,6 +272,7 @@ class CompletingReader(Reader): cmpltn_message_visible: bool = field(init=False) cmpltn_menu_end: int = field(init=False) cmpltn_menu_choices: list[str] = field(init=False) + cmpltn_action: CompletionAction | None = field(init=False) def __post_init__(self) -> None: super().__post_init__() @@ -248,7 +281,7 @@ def __post_init__(self) -> None: self.commands[c.__name__] = c self.commands[c.__name__.replace('_', '-')] = c - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + def collect_keymap(self) -> Keymap: return super().collect_keymap() + ( (r'\t', 'complete'),) @@ -257,30 +290,30 @@ def after_command(self, cmd: Command) -> None: if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() - def calc_screen(self) -> list[str]: - screen = super().calc_screen() - if self.cmpltn_menu_visible: - # We display the completions menu below the current prompt - ly = self.lxy[1] + 1 - screen[ly:ly] = self.cmpltn_menu - # If we're not in the middle of multiline edit, don't append to screeninfo - # since that screws up the position calculation in pos2xy function. - # This is a hack to prevent the cursor jumping - # into the completions menu when pressing left or down arrow. - if self.pos != len(self.buffer): - self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) - return screen + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + if not self.cmpltn_menu_visible: + return () + return ( + ScreenOverlay( + self.lxy[1] + 1, + tuple(RenderLine.from_rendered_text(line) for line in self.cmpltn_menu), + insert=True, + ), + ) def finish(self) -> None: super().finish() self.cmpltn_reset() def cmpltn_reset(self) -> None: + if getattr(self, "cmpltn_menu_visible", False): + self.invalidate_overlay() self.cmpltn_menu = [] self.cmpltn_menu_visible = False self.cmpltn_message_visible = False self.cmpltn_menu_end = 0 self.cmpltn_menu_choices = [] + self.cmpltn_action = None def get_stem(self) -> str: st = self.syntax_table @@ -291,8 +324,8 @@ def get_stem(self) -> str: p -= 1 return ''.join(b[p+1:self.pos]) - def get_completions(self, stem: str) -> list[str]: - return [] + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: + return [], None def get_line(self) -> str: """Return the current line until the cursor position.""" diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index e0535d503963164..2a53d5ff581fa2c 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -19,23 +19,25 @@ from __future__ import annotations +import os import _colorize from abc import ABC, abstractmethod import ast import code import linecache -from dataclasses import dataclass, field -import os.path +from dataclasses import dataclass import re import sys +from typing import TYPE_CHECKING - -TYPE_CHECKING = False +from .render import RenderedScreen +from .trace import trace if TYPE_CHECKING: - from typing import IO - from typing import Callable + from typing import Callable, IO + + from .types import CursorXY @dataclass @@ -47,10 +49,17 @@ class Event: @dataclass class Console(ABC): - posxy: tuple[int, int] - screen: list[str] = field(default_factory=list) + posxy: CursorXY = (0, 0) height: int = 25 width: int = 80 + _redraw_debug_palette: tuple[str, ...] = ( + "\x1b[41m", + "\x1b[42m", + "\x1b[43m", + "\x1b[44m", + "\x1b[45m", + "\x1b[46m", + ) def __init__( self, @@ -71,8 +80,52 @@ def __init__( else: self.output_fd = f_out.fileno() + self.posxy = (0, 0) + self.height = 25 + self.width = 80 + self._rendered_screen = RenderedScreen.empty() + self._redraw_visual_cycle = 0 + + @property + def screen(self) -> list[str]: + return list(self._rendered_screen.screen_lines) + + def sync_rendered_screen( + self, + rendered_screen: RenderedScreen, + posxy: CursorXY | None = None, + ) -> None: + if posxy is None: + posxy = rendered_screen.cursor + self.posxy = posxy + self._rendered_screen = rendered_screen + trace( + "console.sync_rendered_screen lines={lines} cursor={cursor}", + lines=len(rendered_screen.composed_lines), + cursor=posxy, + ) + + def invalidate_render_state(self) -> None: + self._rendered_screen = RenderedScreen.empty() + trace("console.invalidate_render_state") + + def begin_redraw_visualization(self) -> str | None: + if "PYREPL_VISUALIZE_REDRAWS" not in os.environ: + return None + + palette = self._redraw_debug_palette + cycle = self._redraw_visual_cycle + style = palette[cycle % len(palette)] + self._redraw_visual_cycle = cycle + 1 + trace( + "console.begin_redraw_visualization cycle={cycle} style={style!r}", + cycle=cycle, + style=style, + ) + return style + @abstractmethod - def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + def refresh(self, rendered_screen: RenderedScreen) -> None: ... @abstractmethod def prepare(self) -> None: ... diff --git a/Lib/_pyrepl/content.py b/Lib/_pyrepl/content.py new file mode 100644 index 000000000000000..3cb22fee84b7406 --- /dev/null +++ b/Lib/_pyrepl/content.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from .utils import ColorSpan, StyleRef, THEME, iter_display_chars, unbracket, wlen + + +@dataclass(frozen=True, slots=True) +class ContentFragment: + """A single display character with its visual width and style. + + The body of ``>>> def greet`` becomes one fragment per character:: + + d e f g r e e t + ╰──┴──╯ ╰──┴──┴──┴──╯ + keyword (unstyled) + + e.g. ``ContentFragment("d", 1, StyleRef(tag="keyword"))``. + """ + + text: str + width: int + style: StyleRef = StyleRef() + + +@dataclass(frozen=True, slots=True) +class PromptContent: + """The prompt split into leading full-width lines and an inline portion. + + For the common ``">>> "`` prompt (no newlines):: + + >>> def greet(name): + ╰─╯ + text=">>> ", width=4, leading_lines=() + + If ``sys.ps1`` contains newlines, e.g. ``"Python 3.13\\n>>> "``:: + + Python 3.13 ← leading_lines[0] + >>> def greet(name): + ╰─╯ + text=">>> ", width=4 + """ + + leading_lines: tuple[ContentFragment, ...] + text: str + width: int + + +@dataclass(frozen=True, slots=True) +class SourceLine: + """One logical line from the editor buffer, before styling. + + Given this two-line input in the REPL:: + + >>> def greet(name): + ... return name + ▲ cursor + + The buffer ``"def greet(name):\\n return name"`` yields:: + + SourceLine(lineno=0, text="def greet(name):", + start_offset=0, has_newline=True) + SourceLine(lineno=1, text=" return name", + start_offset=17, cursor_index=14) + """ + + lineno: int + text: str + start_offset: int + has_newline: bool + cursor_index: int | None = None + + @property + def cursor_on_line(self) -> bool: + return self.cursor_index is not None + + +@dataclass(frozen=True, slots=True) +class ContentLine: + """A logical line paired with its prompt and styled body. + + For ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + prompt body: one ContentFragment per character + """ + + source: SourceLine + prompt: PromptContent + body: tuple[ContentFragment, ...] + + +def process_prompt(prompt: str) -> PromptContent: + r"""Return prompt content with width measured without zero-width markup.""" + + prompt_text = unbracket(prompt, including_content=False) + visible_prompt = unbracket(prompt, including_content=True) + leading_lines: list[ContentFragment] = [] + + while "\n" in prompt_text: + leading_text, _, prompt_text = prompt_text.partition("\n") + visible_leading, _, visible_prompt = visible_prompt.partition("\n") + leading_lines.append(ContentFragment(leading_text, wlen(visible_leading))) + + return PromptContent(tuple(leading_lines), prompt_text, wlen(visible_prompt)) + + +def build_body_fragments( + buffer: str, + colors: list[ColorSpan] | None, + start_index: int, +) -> tuple[ContentFragment, ...]: + """Convert a line's text into styled content fragments.""" + # Two separate loops to avoid the THEME() call in the common uncolored path. + if colors is None: + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) + + theme = THEME() + return tuple( + ContentFragment( + styled_char.text, + styled_char.width, + StyleRef.from_tag(styled_char.tag, theme[styled_char.tag]) + if styled_char.tag + else StyleRef(), + ) + for styled_char in iter_display_chars(buffer, colors, start_index) + ) diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py new file mode 100644 index 000000000000000..ac4f0afdbc721c3 --- /dev/null +++ b/Lib/_pyrepl/fancycompleter.py @@ -0,0 +1,213 @@ +# Copyright 2010-2025 Antonio Cuni +# Daniel Hahler +# +# All Rights Reserved +"""Colorful tab completion for Python prompt""" +from __future__ import annotations + +from _colorize import ANSIColors, get_colors, get_theme +import rlcompleter +import keyword +import types + +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + from _colorize import Theme + + +def safe_getattr(obj, name): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(obj), name, None), property): + return None + if (isinstance(obj, types.ModuleType) + and isinstance(obj.__dict__.get(name), types.LazyImportType) + ): + return obj.__dict__.get(name) + return getattr(obj, name, None) + + +def colorize_matches(names: list[str], values: list[Any], theme: Theme) -> list[str]: + return [ + _color_for_obj(name, obj, theme) + for name, obj in zip(names, values) + ] + +def _color_for_obj(name: str, value: Any, theme: Theme) -> str: + t = type(value) + color = _color_by_type(t, theme) + return f"{color}{name}{ANSIColors.RESET}" + + +def _color_by_type(t, theme): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(theme.fancycompleter, typename, ANSIColors.RESET) + + +class Completer(rlcompleter.Completer): + """ + When doing something like a.b., keep the full a.b.attr completion + stem so readline-style completion can keep refining the menu as you type. + + Optionally, display the various completions in different colors + depending on the type. + """ + def __init__( + self, + namespace=None, + *, + use_colors='auto', + consider_getitems=True, + ): + from _pyrepl import readline + rlcompleter.Completer.__init__(self, namespace) + if use_colors == 'auto': + # use colors only if we can + use_colors = get_colors().RED != "" + self.use_colors = use_colors + self.consider_getitems = consider_getitems + + if self.use_colors: + # In GNU readline, this prevents escaping of ANSI control + # characters in completion results. pyrepl's parse_and_bind() + # is a no-op, but pyrepl handles ANSI sequences natively + # via real_len()/stripcolor(). + readline.parse_and_bind('set dont-escape-ctrl-chars on') + self.theme = get_theme() + else: + self.theme = None + + if self.consider_getitems: + delims = readline.get_completer_delims() + delims = delims.replace('[', '') + delims = delims.replace(']', '') + readline.set_completer_delims(delims) + + def complete(self, text, state): + # if you press at the beginning of a line, insert an actual + # \t. Else, trigger completion. + if text == "": + return ('\t', None)[state] + else: + return rlcompleter.Completer.complete(self, text, state) + + def _callable_postfix(self, val, word): + # disable automatic insertion of '(' for global callables + return word + + def _callable_attr_postfix(self, val, word): + return rlcompleter.Completer._callable_postfix(self, val, word) + + def global_matches(self, text): + names = rlcompleter.Completer.global_matches(self, text) + prefix = commonprefix(names) + if prefix and prefix != text: + return [prefix] + + names.sort() + values = [] + for name in names: + clean_name = name.rstrip(': ') + if keyword.iskeyword(clean_name) or keyword.issoftkeyword(clean_name): + values.append(None) + else: + try: + values.append(eval(name, self.namespace)) + except Exception: + values.append(None) + if self.use_colors and names: + return self.colorize_matches(names, values) + return names + + def attr_matches(self, text): + try: + expr, attr, names, values = self._attr_matches(text) + except ValueError: + return [] + + if not names: + return [] + + if len(names) == 1: + # No coloring: when returning a single completion, readline + # inserts it directly into the prompt, so ANSI codes would + # appear as literal characters. + return [self._callable_attr_postfix(values[0], f'{expr}.{names[0]}')] + + prefix = commonprefix(names) + if prefix and prefix != attr: + return [f'{expr}.{prefix}'] # autocomplete prefix + + names = [f'{expr}.{name}' for name in names] + if self.use_colors: + return self.colorize_matches(names, values) + return names + + def _attr_matches(self, text): + expr, attr = text.rsplit('.', 1) + if '(' in expr or ')' in expr: # don't call functions + return expr, attr, [], [] + try: + thisobject = eval(expr, self.namespace) + except Exception: + return expr, attr, [], [] + + # get the content of the object, except __builtins__ + words = set(dir(thisobject)) - {'__builtins__'} + + if hasattr(thisobject, '__class__'): + words.add('__class__') + words.update(rlcompleter.get_class_members(thisobject.__class__)) + names = [] + values = [] + n = len(attr) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + + # sort the words now to make sure to return completions in + # alphabetical order. It's easier to do it now, else we would need to + # sort 'names' later but make sure that 'values' in kept in sync, + # which is annoying. + words = sorted(words) + while True: + for word in words: + if ( + word[:n] == attr + and not (noprefix and word[:n+1] == noprefix) + ): + value = safe_getattr(thisobject, word) + names.append(word) + values.append(value) + if names or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + + return expr, attr, names, values + + def colorize_matches(self, names, values): + return colorize_matches(names, values, self.theme) + + +def commonprefix(names): + """Return the common prefix of all 'names'""" + if not names: + return '' + s1 = min(names) + s2 = max(names) + for i, c in enumerate(s1): + if c != s2[i]: + return s1[:i] + return s1 diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index c4b95fa2e81ee60..09b969d80bc231f 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -90,7 +90,7 @@ def do(self) -> None: if r.get_unicode() != r.history[r.historyi]: r.buffer = list(r.history[r.historyi]) r.pos = len(r.buffer) - r.dirty = True + r.invalidate_buffer(0) class first_history(commands.Command): @@ -130,10 +130,11 @@ def do(self) -> None: o = len(r.yank_arg_yanked) else: o = 0 + start = r.pos - o b[r.pos - o : r.pos] = list(w) r.yank_arg_yanked = w r.pos += len(w) - o - r.dirty = True + r.invalidate_buffer(start) class forward_history_isearch(commands.Command): @@ -142,7 +143,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_FORWARDS r.isearch_start = r.historyi, r.pos r.isearch_term = "" - r.dirty = True + r.invalidate_prompt() r.push_input_trans(r.isearch_trans) @@ -150,7 +151,7 @@ class reverse_history_isearch(commands.Command): def do(self) -> None: r = self.reader r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS - r.dirty = True + r.invalidate_prompt() r.isearch_term = "" r.push_input_trans(r.isearch_trans) r.isearch_start = r.historyi, r.pos @@ -163,7 +164,7 @@ def do(self) -> None: r.pop_input_trans() r.select_item(r.isearch_start[0]) r.pos = r.isearch_start[1] - r.dirty = True + r.invalidate_prompt() class isearch_add_character(commands.Command): @@ -171,7 +172,7 @@ def do(self) -> None: r = self.reader b = r.buffer r.isearch_term += self.event[-1] - r.dirty = True + r.invalidate_prompt() p = r.pos + len(r.isearch_term) - 1 if b[p : p + 1] != [r.isearch_term[-1]]: r.isearch_next() @@ -182,7 +183,7 @@ def do(self) -> None: r = self.reader if len(r.isearch_term) > 0: r.isearch_term = r.isearch_term[:-1] - r.dirty = True + r.invalidate_prompt() else: r.error("nothing to rubout") @@ -207,7 +208,7 @@ def do(self) -> None: r.isearch_direction = ISEARCH_DIRECTION_NONE r.console.forgetinput() r.pop_input_trans() - r.dirty = True + r.invalidate_prompt() @dataclass @@ -241,7 +242,6 @@ def __post_init__(self) -> None: isearch_end, isearch_add_character, isearch_cancel, - isearch_add_character, isearch_backspace, isearch_forwards, isearch_backwards, @@ -279,8 +279,7 @@ def select_item(self, i: int) -> None: self.buffer = list(buf) self.historyi = i self.pos = len(self.buffer) - self.dirty = True - self.last_refresh_cache.invalidated = True + self.invalidate_buffer(0) def get_item(self, i: int) -> str: if i != len(self.history): @@ -358,7 +357,7 @@ def search_next(self, *, forwards: bool) -> None: if forwards and not match_prefix: self.pos = 0 self.buffer = [] - self.dirty = True + self.invalidate_buffer(0) else: self.error("not found") return diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 21c24eb5cde3e36..2d65246c700f274 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -38,10 +38,11 @@ from abc import ABC, abstractmethod import unicodedata from collections import deque +from typing import TYPE_CHECKING # types -if False: +if TYPE_CHECKING: from .types import EventTuple diff --git a/Lib/_pyrepl/layout.py b/Lib/_pyrepl/layout.py new file mode 100644 index 000000000000000..6d854d1142dd9f1 --- /dev/null +++ b/Lib/_pyrepl/layout.py @@ -0,0 +1,268 @@ +"""Wrap content lines to the terminal width before rendering.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Self + +from .content import ContentFragment, ContentLine +from .types import CursorXY, ScreenInfoRow + + +@dataclass(frozen=True, slots=True) +class LayoutRow: + """Metadata for one physical screen row. + + For the row ``>>> def greet(name):``:: + + >>> def greet(name): + ╰─╯ ╰──────────────╯ + 4 char_widths=(1,1,1,…) ← 16 entries + buffer_advance=17 ← includes the newline + """ + + prompt_width: int + char_widths: tuple[int, ...] + suffix_width: int = 0 + buffer_advance: int = 0 + + @property + def width(self) -> int: + return self.prompt_width + sum(self.char_widths) + self.suffix_width + + @property + def screeninfo(self) -> ScreenInfoRow: + widths = list(self.char_widths) + if self.suffix_width: + widths.append(self.suffix_width) + return self.prompt_width, widths + + +@dataclass(frozen=True, slots=True) +class LayoutMap: + """Mapping between buffer positions and screen coordinates. + + Single source of truth for cursor placement. Given:: + + >>> def greet(name): ← row 0, buffer_advance=17 + ... return name ← row 1, buffer_advance=15 + ▲cursor + + ``pos_to_xy(31)`` → ``(18, 1)``: prompt width 4 + 14 body chars. + """ + rows: tuple[LayoutRow, ...] + + @classmethod + def empty(cls) -> Self: + return cls((LayoutRow(0, ()),)) + + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return [row.screeninfo for row in self.rows] + + def max_column(self, y: int) -> int: + return self.rows[y].width + + def max_row(self) -> int: + return len(self.rows) - 1 + + def pos_to_xy(self, pos: int) -> CursorXY: + if not self.rows: + return 0, 0 + + remaining = pos + for y, row in enumerate(self.rows): + if remaining <= len(row.char_widths): + # Prompt-only leading rows are terminal scenery, not real + # buffer positions. Treating them as real just manufactures + # bugs. + if remaining == 0 and not row.char_widths and row.buffer_advance == 0 and y < len(self.rows) - 1: + continue + x = row.prompt_width + for width in row.char_widths[:remaining]: + x += width + return x, y + remaining -= row.buffer_advance + last_row = self.rows[-1] + return last_row.width - last_row.suffix_width, len(self.rows) - 1 + + def xy_to_pos(self, x: int, y: int) -> int: + if not self.rows: + return 0 + + pos = 0 + for row in self.rows[:y]: + pos += row.buffer_advance + + row = self.rows[y] + cur_x = row.prompt_width + char_widths = row.char_widths + i = 0 + for i, width in enumerate(char_widths): + if cur_x >= x: + # Include trailing zero-width (combining) chars at this position + for trailing_width in char_widths[i:]: + if trailing_width == 0: + pos += 1 + else: + break + return pos + if width == 0: + pos += 1 + continue + cur_x += width + pos += 1 + return pos + + +@dataclass(frozen=True, slots=True) +class WrappedRow: + """One physical screen row after wrapping, ready for rendering. + + When a line overflows the terminal width, it splits into + multiple rows with a ``\\`` continuation marker:: + + >>> x = "a very long li\\ ← suffix="\\", suffix_width=1 + ne that wraps" ← prompt_text="" (continuation) + """ + prompt_text: str = "" + prompt_width: int = 0 + fragments: tuple[ContentFragment, ...] = () + layout_widths: tuple[int, ...] = () + suffix: str = "" + suffix_width: int = 0 + buffer_advance: int = 0 + + +@dataclass(frozen=True, slots=True) +class LayoutResult: + wrapped_rows: tuple[WrappedRow, ...] + layout_map: LayoutMap + line_end_offsets: tuple[int, ...] + + +def layout_content_lines( + lines: tuple[ContentLine, ...], + width: int, + start_offset: int, +) -> LayoutResult: + """Wrap content lines to fit *width* columns. + + A short line passes through as one ``WrappedRow``; a long line is + split at the column boundary with ``\\`` markers:: + + >>> short = 1 ← one WrappedRow + >>> x = "a long stri\\ ← two WrappedRows, first has suffix="\\" + ng" + """ + if width <= 0: + return LayoutResult((), LayoutMap(()), ()) + + offset = start_offset + wrapped_rows: list[WrappedRow] = [] + layout_rows: list[LayoutRow] = [] + line_end_offsets: list[int] = [] + + for line in lines: + newline_advance = int(line.source.has_newline) + for leading in line.prompt.leading_lines: + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + fragments=(leading,), + ) + ) + layout_rows.append(LayoutRow(0, (), buffer_advance=0)) + + prompt_text = line.prompt.text + prompt_width = line.prompt.width + body = tuple(line.body) + body_widths = tuple(fragment.width for fragment in body) + + # Fast path: line fits on one row. + if not body_widths or (sum(body_widths) + prompt_width) < width: + offset += len(body) + newline_advance + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=prompt_text, + prompt_width=prompt_width, + fragments=body, + layout_widths=body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + layout_rows.append( + LayoutRow( + prompt_width, + body_widths, + buffer_advance=len(body) + newline_advance, + ) + ) + continue + + # Slow path: line needs wrapping. + current_prompt = prompt_text + current_prompt_width = prompt_width + start = 0 + total = len(body) + while True: + # Find how many characters fit on this row. + index_to_wrap_before = 0 + column = 0 + for char_width in body_widths[start:]: + if column + char_width + current_prompt_width >= width: + break + index_to_wrap_before += 1 + column += char_width + + if index_to_wrap_before == 0 and start < total: + index_to_wrap_before = 1 # force progress + + at_line_end = (start + index_to_wrap_before) >= total + if at_line_end: + offset += index_to_wrap_before + newline_advance + suffix = "" + suffix_width = 0 + buffer_advance = index_to_wrap_before + newline_advance + else: + offset += index_to_wrap_before + suffix = "\\" + suffix_width = 1 + buffer_advance = index_to_wrap_before + + end = start + index_to_wrap_before + row_fragments = body[start:end] + row_widths = body_widths[start:end] + line_end_offsets.append(offset) + wrapped_rows.append( + WrappedRow( + prompt_text=current_prompt, + prompt_width=current_prompt_width, + fragments=row_fragments, + layout_widths=row_widths, + suffix=suffix, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + layout_rows.append( + LayoutRow( + current_prompt_width, + row_widths, + suffix_width=suffix_width, + buffer_advance=buffer_advance, + ) + ) + + start = end + current_prompt = "" + current_prompt_width = 0 + if at_line_end: + break + + return LayoutResult( + tuple(wrapped_rows), + LayoutMap(tuple(layout_rows)), + tuple(line_end_offsets), + ) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 1fddc63e3ee3ad0..92afaa5933a2250 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -138,7 +138,7 @@ def pipe_pager(text: str, cmd: str, title: str = '') -> None: '.' '?e (END):?pB %pB\\%..' ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + env['LESS'] = '-RcmPm{0}$PM{0}$'.format(prompt_string) proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, errors='backslashreplace', env=env) assert proc.stdin is not None diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0ebd9162eca4bba..832e67b534f2969 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -25,16 +25,41 @@ import _colorize from contextlib import contextmanager -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace +from typing import Self from . import commands, console, input -from .utils import wlen, unbracket, disp_str, gen_colors, THEME +from .content import ( + ContentFragment, + ContentLine, + SourceLine, + build_body_fragments, + process_prompt as build_prompt_content, +) +from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines +from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay +from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors from .trace import trace # types Command = commands.Command -from .types import Callback, SimpleContextManager, KeySpec, CommandName +from collections.abc import Callable, Iterator +from .types import ( + Callback, + CommandName, + CursorXY, + Dimensions, + EventData, + KeySpec, + Keymap, + ScreenInfoRow, + SimpleContextManager, +) + +type CommandClass = type[Command] +type CommandInput = tuple[CommandName | CommandClass, EventData] +type PromptCellCacheKey = tuple[str, bool] # syntax classes @@ -52,8 +77,8 @@ def make_default_syntax_table() -> dict[str, int]: return st -def make_default_commands() -> dict[CommandName, type[Command]]: - result: dict[CommandName, type[Command]] = {} +def make_default_commands() -> dict[CommandName, CommandClass]: + result: dict[CommandName, CommandClass] = {} for v in vars(commands).values(): if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): result[v.__name__] = v @@ -61,7 +86,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]: return result -default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( +default_keymap: Keymap = tuple( [ (r"\C-a", "beginning-of-line"), (r"\C-b", "left"), @@ -131,6 +156,77 @@ def make_default_commands() -> dict[CommandName, type[Command]]: ) +@dataclass(frozen=True, slots=True) +class RefreshInvalidation: + """Which parts of the screen need to be recomputed on the next refresh.""" + + cursor_only: bool = False + buffer_from_pos: int | None = None + prompt: bool = False + layout: bool = False + theme: bool = False + message: bool = False + overlay: bool = False + full: bool = False + + @classmethod + def empty(cls) -> Self: + return cls() + + @property + def needs_screen_refresh(self) -> bool: + return any( + ( + self.buffer_from_pos is not None, + self.prompt, + self.layout, + self.theme, + self.message, + self.overlay, + self.full, + ) + ) + + @property + def is_cursor_only(self) -> bool: + return self.cursor_only and not self.needs_screen_refresh + + @property + def buffer_rebuild_from_pos(self) -> int | None: + if self.full or self.prompt or self.layout or self.theme: + return 0 + return self.buffer_from_pos + + def with_cursor(self) -> Self: + if self.needs_screen_refresh: + return self + return replace(self, cursor_only=True) + + def with_buffer(self, from_pos: int) -> Self: + current = from_pos + if self.buffer_from_pos is not None: + current = min(current, self.buffer_from_pos) + return replace(self, cursor_only=False, buffer_from_pos=current) + + def with_prompt(self) -> Self: + return replace(self, cursor_only=False, prompt=True) + + def with_layout(self) -> Self: + return replace(self, cursor_only=False, layout=True) + + def with_theme(self) -> Self: + return replace(self, cursor_only=False, theme=True) + + def with_message(self) -> Self: + return replace(self, cursor_only=False, message=True) + + def with_overlay(self) -> Self: + return replace(self, cursor_only=False, overlay=True) + + def with_full(self) -> Self: + return replace(self, cursor_only=False, full=True) + + @dataclass(slots=True) class Reader: """The Reader class implements the bare bones of a command reader, @@ -148,10 +244,9 @@ class Reader: * pos: A 0-based index into 'buffer' for where the insertion point is. - * screeninfo: - A list of screen position tuples. Each list element is a tuple - representing information on visible line length for a given line. - Allows for efficient skipping of color escape sequences. + * layout: + A mapping between buffer positions and rendered rows/columns. + It is the internal source of truth for cursor placement. * cxy, lxy: the position of the insertion point in screen ... * syntax_table: @@ -162,8 +257,6 @@ class Reader: * arg: The emacs-style prefix argument. It will be None if no such argument has been provided. - * dirty: - True if we need to refresh the display. * kill_ring: The emacs-style kill-ring; manipulated with yank & yank-pop * ps1, ps2, ps3, ps4: @@ -198,66 +291,89 @@ class Reader: kill_ring: list[list[str]] = field(default_factory=list) msg: str = "" arg: int | None = None - dirty: bool = False finished: bool = False paste_mode: bool = False - commands: dict[str, type[Command]] = field(default_factory=make_default_commands) - last_command: type[Command] | None = None + commands: dict[CommandName, CommandClass] = field(default_factory=make_default_commands) + last_command: CommandClass | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) - keymap: tuple[tuple[str, str], ...] = () + keymap: Keymap = () input_trans: input.KeymapTranslator = field(init=False) input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) - cxy: tuple[int, int] = field(init=False) - lxy: tuple[int, int] = field(init=False) - scheduled_commands: list[str] = field(default_factory=list) + rendered_screen: RenderedScreen = field(init=False) + layout: LayoutMap = field(init=False) + cxy: CursorXY = field(init=False) + lxy: CursorXY = field(init=False) + scheduled_commands: list[CommandName] = field(default_factory=list) can_colorize: bool = False + gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors threading_hook: Callback | None = None + invalidation: RefreshInvalidation = field(init=False) ## cached metadata to speed up screen refreshes @dataclass class RefreshCache: - screen: list[str] = field(default_factory=list) - screeninfo: list[tuple[int, list[int]]] = field(init=False) + """Previously computed render/layout data for incremental refresh.""" + + render_lines: list[RenderLine] = field(default_factory=list) + layout_rows: list[LayoutRow] = field(default_factory=list) line_end_offsets: list[int] = field(default_factory=list) - pos: int = field(init=False) - cxy: tuple[int, int] = field(init=False) - dimensions: tuple[int, int] = field(init=False) - invalidated: bool = False + pos: int = 0 + dimensions: Dimensions = (0, 0) def update_cache(self, reader: Reader, - screen: list[str], - screeninfo: list[tuple[int, list[int]]], + render_lines: list[RenderLine], + layout_rows: list[LayoutRow], + line_end_offsets: list[int], ) -> None: - self.screen = screen.copy() - self.screeninfo = screeninfo.copy() + self.render_lines = render_lines.copy() + self.layout_rows = layout_rows.copy() + self.line_end_offsets = line_end_offsets.copy() self.pos = reader.pos - self.cxy = reader.cxy self.dimensions = reader.console.width, reader.console.height - self.invalidated = False def valid(self, reader: Reader) -> bool: - if self.invalidated: - return False dimensions = reader.console.width, reader.console.height dimensions_changed = dimensions != self.dimensions return not dimensions_changed - def get_cached_location(self, reader: Reader) -> tuple[int, int]: - if self.invalidated: - raise ValueError("Cache is invalidated") - offset = 0 - earliest_common_pos = min(reader.pos, self.pos) + def get_cached_location( + self, + reader: Reader, + buffer_from_pos: int | None = None, + *, + reuse_full: bool = False, + ) -> tuple[int, int]: + """Return (buffer_offset, num_reusable_lines) for incremental refresh. + + Three paths: + - reuse_full (overlay/message-only): reuse all cached lines. + - buffer_from_pos=None (full rebuild): rewind to common cursor pos. + - explicit buffer_from_pos: reuse lines before that position. + """ + if reuse_full: + if self.line_end_offsets: + last_offset = self.line_end_offsets[-1] + if last_offset >= len(reader.buffer): + return last_offset, len(self.line_end_offsets) + return 0, 0 + if buffer_from_pos is None: + buffer_from_pos = min(reader.pos, self.pos) num_common_lines = len(self.line_end_offsets) while num_common_lines > 0: - offset = self.line_end_offsets[num_common_lines - 1] - if earliest_common_pos > offset: + candidate = self.line_end_offsets[num_common_lines - 1] + if buffer_from_pos > candidate: break num_common_lines -= 1 - else: - offset = 0 + # Prompt-only leading rows consume no buffer content. Reusing them + # in isolation causes the next incremental rebuild to emit them a + # second time. + while ( + num_common_lines > 0 + and self.layout_rows[num_common_lines - 1].buffer_advance == 0 + ): + num_common_lines -= 1 + offset = self.line_end_offsets[num_common_lines - 1] if num_common_lines else 0 return offset, num_common_lines last_refresh_cache: RefreshCache = field(default_factory=RefreshCache) @@ -270,123 +386,267 @@ def __post_init__(self) -> None: self.input_trans = input.KeymapTranslator( self.keymap, invalid_cls="invalid-key", character_cls="self-insert" ) - self.screeninfo = [(0, [])] + self.layout = LayoutMap.empty() self.cxy = self.pos2xy() self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() self.can_colorize = _colorize.can_colorize() + self.invalidation = RefreshInvalidation.empty() - self.last_refresh_cache.screeninfo = self.screeninfo + self.last_refresh_cache.layout_rows = list(self.layout.rows) self.last_refresh_cache.pos = self.pos - self.last_refresh_cache.cxy = self.cxy + self.last_refresh_cache.dimensions = (0, 0) - def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: - return default_keymap + @property + def screen(self) -> list[str]: + return list(self.rendered_screen.screen_lines) - def calc_screen(self) -> list[str]: - """Translate changes in self.buffer into changes in self.console.screen.""" - # Since the last call to calc_screen: - # screen and screeninfo may differ due to a completion menu being shown - # pos and cxy may differ due to edits, cursor movements, or completion menus + @property + def screeninfo(self) -> list[ScreenInfoRow]: + return self.layout.screeninfo - # Lines that are above both the old and new cursor position can't have changed, - # unless the terminal has been resized (which might cause reflowing) or we've - # entered or left paste mode (which changes prompts, causing reflowing). + def collect_keymap(self) -> Keymap: + return default_keymap + + def calc_screen(self) -> RenderedScreen: + """Translate the editable buffer into a base rendered screen.""" num_common_lines = 0 offset = 0 if self.last_refresh_cache.valid(self): - offset, num_common_lines = self.last_refresh_cache.get_cached_location(self) - - screen = self.last_refresh_cache.screen - del screen[num_common_lines:] - - screeninfo = self.last_refresh_cache.screeninfo - del screeninfo[num_common_lines:] - - last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets - del last_refresh_line_end_offsets[num_common_lines:] + if ( + self.invalidation.buffer_from_pos is None + and not ( + self.invalidation.full + or self.invalidation.prompt + or self.invalidation.layout + or self.invalidation.theme + ) + and (self.invalidation.message or self.invalidation.overlay) + ): + # Fast path: only overlays or messages changed. + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + reuse_full=True, + ) + assert not self.last_refresh_cache.line_end_offsets or ( + self.last_refresh_cache.line_end_offsets[-1] >= len(self.buffer) + ), "Buffer modified without invalidate_buffer() call" + else: + offset, num_common_lines = self.last_refresh_cache.get_cached_location( + self, + self._buffer_refresh_from_pos(), + ) + + base_render_lines = self.last_refresh_cache.render_lines[:num_common_lines] + layout_rows = self.last_refresh_cache.layout_rows[:num_common_lines] + last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets[:num_common_lines] + + source_lines = self._build_source_lines(offset, num_common_lines) + content_lines = self._build_content_lines( + source_lines, + prompt_from_cache=bool(offset and self.buffer[offset - 1] != "\n"), + ) + layout_result = self._layout_content(content_lines, offset) + base_render_lines.extend(self._render_wrapped_rows(layout_result.wrapped_rows)) + layout_rows.extend(layout_result.layout_map.rows) + last_refresh_line_end_offsets.extend(layout_result.line_end_offsets) - pos = self.pos - pos -= offset + self.layout = LayoutMap(tuple(layout_rows)) + self.cxy = self.pos2xy() + if not source_lines: + # reuse_full path: _build_source_lines didn't run, + # so lxy wasn't updated. Derive it from the buffer. + self.lxy = self._compute_lxy() + self.last_refresh_cache.update_cache( + self, + base_render_lines, + layout_rows, + last_refresh_line_end_offsets, + ) + return RenderedScreen(tuple(base_render_lines), self.cxy) - prompt_from_cache = (offset and self.buffer[offset - 1] != "\n") + def _buffer_refresh_from_pos(self) -> int: + """Return buffer position from which to rebuild content. - if self.can_colorize: - colors = list(gen_colors(self.get_unicode())) + Returns 0 (full rebuild) when no incremental position is known. + """ + buffer_from_pos = self.invalidation.buffer_rebuild_from_pos + if buffer_from_pos is not None: + return buffer_from_pos + return 0 + + def _compute_lxy(self) -> CursorXY: + """Derive logical cursor (col, lineno) from the buffer and pos.""" + text = "".join(self.buffer[:self.pos]) + lineno = text.count("\n") + if lineno: + col = self.pos - text.rindex("\n") - 1 else: - colors = None - trace("colors = {colors}", colors=colors) + col = self.pos + return col, lineno + + def _build_source_lines( + self, + offset: int, + first_lineno: int, + ) -> tuple[SourceLine, ...]: + if offset == len(self.buffer) and (offset > 0 or first_lineno > 0): + return () + + pos = self.pos - offset lines = "".join(self.buffer[offset:]).split("\n") cursor_found = False lines_beyond_cursor = 0 - for ln, line in enumerate(lines, num_common_lines): + source_lines: list[SourceLine] = [] + current_offset = offset + + for line_index, line in enumerate(lines): + lineno = first_lineno + line_index + has_newline = line_index < len(lines) - 1 line_len = len(line) + cursor_index: int | None = None if 0 <= pos <= line_len: - self.lxy = pos, ln + cursor_index = pos + self.lxy = pos, lineno cursor_found = True elif cursor_found: lines_beyond_cursor += 1 if lines_beyond_cursor > self.console.height: - # No need to keep formatting lines. - # The console can't show them. break + + source_lines.append( + SourceLine( + lineno=lineno, + text=line, + start_offset=current_offset, + has_newline=has_newline, + cursor_index=cursor_index, + ) + ) + pos -= line_len + 1 + current_offset += line_len + (1 if has_newline else 0) + + return tuple(source_lines) + + def _build_content_lines( + self, + source_lines: tuple[SourceLine, ...], + *, + prompt_from_cache: bool, + ) -> tuple[ContentLine, ...]: + if self.can_colorize: + colors = list(self.gen_colors(self.get_unicode())) + else: + colors = None + trace("colors = {colors}", colors=colors) + + content_lines: list[ContentLine] = [] + for source_line in source_lines: if prompt_from_cache: - # Only the first line's prompt can come from the cache prompt_from_cache = False prompt = "" else: - prompt = self.get_prompt(ln, line_len >= pos >= 0) - while "\n" in prompt: - pre_prompt, _, prompt = prompt.partition("\n") - last_refresh_line_end_offsets.append(offset) - screen.append(pre_prompt) - screeninfo.append((0, [])) - pos -= line_len + 1 - prompt, prompt_len = self.process_prompt(prompt) - chars, char_widths = disp_str(line, colors, offset) - wrapcount = (sum(char_widths) + prompt_len) // self.console.width - if wrapcount == 0 or not char_widths: - offset += line_len + 1 # Takes all of the line plus the newline - last_refresh_line_end_offsets.append(offset) - screen.append(prompt + "".join(chars)) - screeninfo.append((prompt_len, char_widths)) - else: - pre = prompt - prelen = prompt_len - for wrap in range(wrapcount + 1): - index_to_wrap_before = 0 - column = 0 - for char_width in char_widths: - if column + char_width + prelen >= self.console.width: - break - index_to_wrap_before += 1 - column += char_width - if len(chars) > index_to_wrap_before: - offset += index_to_wrap_before - post = "\\" - after = [1] - else: - offset += index_to_wrap_before + 1 # Takes the newline - post = "" - after = [] - last_refresh_line_end_offsets.append(offset) - render = pre + "".join(chars[:index_to_wrap_before]) + post - render_widths = char_widths[:index_to_wrap_before] + after - screen.append(render) - screeninfo.append((prelen, render_widths)) - chars = chars[index_to_wrap_before:] - char_widths = char_widths[index_to_wrap_before:] - pre = "" - prelen = 0 - self.screeninfo = screeninfo - self.cxy = self.pos2xy() - if self.msg: - for mline in self.msg.split("\n"): - screen.append(mline) - screeninfo.append((0, [])) - - self.last_refresh_cache.update_cache(self, screen, screeninfo) - return screen + prompt = self.get_prompt(source_line.lineno, source_line.cursor_on_line) + content_lines.append( + ContentLine( + source=source_line, + prompt=build_prompt_content(prompt), + body=build_body_fragments( + source_line.text, + colors, + source_line.start_offset, + ), + ) + ) + return tuple(content_lines) + + def _layout_content( + self, + content_lines: tuple[ContentLine, ...], + offset: int, + ) -> LayoutResult: + return layout_content_lines(content_lines, self.console.width, offset) + + def _render_wrapped_rows( + self, + wrapped_rows: tuple[WrappedRow, ...], + ) -> list[RenderLine]: + return [ + self._render_line( + row.prompt_text, + row.fragments, + row.suffix, + ) + for row in wrapped_rows + ] + + def _render_message_lines(self) -> tuple[RenderLine, ...]: + if not self.msg: + return () + width = self.console.width + render_lines: list[RenderLine] = [] + for message_line in self.msg.split("\n"): + # If self.msg is larger than console width, make it fit. + # TODO: try to split between words? + if not message_line: + render_lines.append(RenderLine.from_rendered_text("")) + continue + for offset in range(0, len(message_line), width): + render_lines.append( + RenderLine.from_rendered_text(message_line[offset : offset + width]) + ) + return tuple(render_lines) + + def get_screen_overlays(self) -> tuple[ScreenOverlay, ...]: + return () + + def compose_rendered_screen(self, base_screen: RenderedScreen) -> RenderedScreen: + overlays = list(self.get_screen_overlays()) + message_lines = self._render_message_lines() + if message_lines: + overlays.append(ScreenOverlay(len(base_screen.lines), message_lines)) + if not overlays: + return base_screen + return RenderedScreen(base_screen.lines, base_screen.cursor, tuple(overlays)) + + _prompt_cell_cache: dict[PromptCellCacheKey, tuple[RenderCell, ...]] = field( + init=False, default_factory=dict, repr=False + ) + + def _render_line( + self, + prefix: str, + fragments: tuple[ContentFragment, ...], + suffix: str = "", + ) -> RenderLine: + cells: list[RenderCell] = [] + if prefix: + cache_key = (prefix, self.can_colorize) + cached = self._prompt_cell_cache.get(cache_key) + if cached is None: + prompt_cells = RenderLine.from_rendered_text(prefix).cells + if self.can_colorize and prompt_cells and not ANSI_ESCAPE_SEQUENCE.search(prefix): + prompt_style = StyleRef.from_tag("prompt", THEME()["prompt"]) + prompt_cells = tuple( + RenderCell( + cell.text, + cell.width, + style=prompt_style if cell.text else cell.style, + controls=cell.controls, + ) + for cell in prompt_cells + ) + self._prompt_cell_cache[cache_key] = prompt_cells + cached = prompt_cells + cells.extend(cached) + cells.extend( + RenderCell(fragment.text, fragment.width, style=fragment.style) + for fragment in fragments + ) + if suffix: + cells.extend(RenderLine.from_rendered_text(suffix).cells) + return RenderLine.from_cells(cells) @staticmethod def process_prompt(prompt: str) -> tuple[str, int]: @@ -396,9 +656,8 @@ def process_prompt(prompt: str) -> tuple[str, int]: (\x01 and \x02) removed. The length ignores anything between those brackets as well as any ANSI escape sequences. """ - out_prompt = unbracket(prompt, including_content=False) - visible_prompt = unbracket(prompt, including_content=True) - return out_prompt, wlen(visible_prompt) + prompt_content = build_prompt_content(prompt) + return prompt_content.text, prompt_content.width def bow(self, p: int | None = None) -> int: """Return the 0-based index of the word break preceding p most @@ -460,10 +719,10 @@ def eol(self, p: int | None = None) -> int: def max_column(self, y: int) -> int: """Return the last x-offset for line y""" - return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + return self.layout.max_column(y) def max_row(self) -> int: - return len(self.screeninfo) - 1 + return self.layout.max_row() def get_arg(self, default: int = 1) -> int: """Return any prefix argument that the user has supplied, @@ -489,10 +748,6 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: prompt = self.ps3 else: prompt = self.ps1 - - if self.can_colorize: - t = THEME() - prompt = f"{t.prompt}{prompt}{t.reset}" return prompt def push_input_trans(self, itrans: input.KeymapTranslator) -> None: @@ -504,65 +759,48 @@ def pop_input_trans(self) -> None: def setpos_from_xy(self, x: int, y: int) -> None: """Set pos according to coordinates x, y""" - pos = 0 - i = 0 - while i < y: - prompt_len, char_widths = self.screeninfo[i] - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - pos += offset - 1 # -1 cause backslash is not in buffer - else: - pos += offset + 1 # +1 cause newline is in buffer - i += 1 - - j = 0 - cur_x = self.screeninfo[i][0] - while cur_x < x: - if self.screeninfo[i][1][j] == 0: - j += 1 # prevent potential future infinite loop - continue - cur_x += self.screeninfo[i][1][j] - j += 1 - pos += 1 - - self.pos = pos + self.pos = self.layout.xy_to_pos(x, y) - def pos2xy(self) -> tuple[int, int]: + def pos2xy(self) -> CursorXY: """Return the x, y coordinates of position 'pos'.""" + assert 0 <= self.pos <= len(self.buffer) + return self.layout.pos_to_xy(self.pos) + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + start = self.pos + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.invalidate_buffer(start) - prompt_len, y = 0, 0 - char_widths: list[int] = [] - pos = self.pos - assert 0 <= pos <= len(self.buffer) + def invalidate_cursor(self) -> None: + self.invalidation = self.invalidation.with_cursor() - # optimize for the common case: typing at the end of the buffer - if pos == len(self.buffer) and len(self.screeninfo) > 0: - y = len(self.screeninfo) - 1 - prompt_len, char_widths = self.screeninfo[y] - return prompt_len + sum(char_widths), y + def invalidate_buffer(self, from_pos: int) -> None: + self.invalidation = self.invalidation.with_buffer(from_pos) - for prompt_len, char_widths in self.screeninfo: - offset = len(char_widths) - in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width - if in_wrapped_line: - offset -= 1 # need to remove line-wrapping backslash + def invalidate_prompt(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_prompt() - if offset >= pos: - break + def invalidate_layout(self) -> None: + self.invalidation = self.invalidation.with_layout() - if not in_wrapped_line: - offset += 1 # there's a newline in buffer + def invalidate_theme(self) -> None: + self._prompt_cell_cache.clear() + self.invalidation = self.invalidation.with_theme() - pos -= offset - y += 1 - return prompt_len + sum(char_widths[:pos]), y + def invalidate_message(self) -> None: + self.invalidation = self.invalidation.with_message() - def insert(self, text: str | list[str]) -> None: - """Insert 'text' at the insertion point.""" - self.buffer[self.pos : self.pos] = list(text) - self.pos += len(text) - self.dirty = True + def invalidate_overlay(self) -> None: + self.invalidation = self.invalidation.with_overlay() + + def invalidate_full(self) -> None: + self.invalidation = self.invalidation.with_full() + + def clear_invalidation(self) -> None: + self.invalidation = RefreshInvalidation.empty() def update_cursor(self) -> None: """Move the cursor to reflect changes in self.pos""" @@ -574,7 +812,7 @@ def after_command(self, cmd: Command) -> None: """This function is called to allow post command cleanup.""" if getattr(cmd, "kills_digit_arg", True): if self.arg is not None: - self.dirty = True + self.invalidate_prompt() self.arg = None def prepare(self) -> None: @@ -587,9 +825,15 @@ def prepare(self) -> None: self.finished = False del self.buffer[:] self.pos = 0 - self.dirty = True + self.layout = LayoutMap.empty() + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + self.rendered_screen = RenderedScreen.empty() + self.invalidate_full() self.last_command = None - self.calc_screen() + base_screen = self.calc_screen() + self.rendered_screen = self.compose_rendered_screen(base_screen) + self.invalidation = RefreshInvalidation.empty() except BaseException: self.restore() raise @@ -598,7 +842,7 @@ def prepare(self) -> None: cmd = self.scheduled_commands.pop() self.do_cmd((cmd, [])) - def last_command_is(self, cls: type) -> bool: + def last_command_is(self, cls: CommandClass) -> bool: if not self.last_command: return False return issubclass(cls, self.last_command) @@ -619,27 +863,51 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + def finish(self) -> None: """Called when a command signals that we're finished.""" pass def error(self, msg: str = "none") -> None: self.msg = "! " + msg + " " - self.dirty = True + self.invalidate_message() self.console.beep() def update_screen(self) -> None: - if self.dirty: + if self.invalidation.is_cursor_only: + self.update_cursor() + self.clear_invalidation() + elif self.invalidation.needs_screen_refresh: self.refresh() def refresh(self) -> None: """Recalculate and refresh the screen.""" + self.console.height, self.console.width = self.console.getheightwidth() # this call sets up self.cxy, so call it first. - self.screen = self.calc_screen() - self.console.refresh(self.screen, self.cxy) - self.dirty = False + base_screen = self.calc_screen() + rendered_screen = self.compose_rendered_screen(base_screen) + self.rendered_screen = rendered_screen + trace( + "reader.refresh cursor={cursor} lines={lines} " + "dims=({width},{height}) invalidation={invalidation}", + cursor=self.cxy, + lines=len(rendered_screen.composed_lines), + width=self.console.width, + height=self.console.height, + invalidation=self.invalidation, + ) + self.console.refresh(rendered_screen) + self.clear_invalidation() - def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + def do_cmd(self, cmd: CommandInput) -> None: """`cmd` is a tuple of "event_name" and "event", which in the current implementation is always just the "buffer" which happens to be a list of single-character strings.""" @@ -656,13 +924,14 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: command.do() self.after_command(command) - - if self.dirty: - self.refresh() - else: - self.update_cursor() - - if not isinstance(cmd, commands.digit_arg): + if ( + not self.invalidation.needs_screen_refresh + and not self.invalidation.is_cursor_only + ): + self.invalidate_cursor() + self.update_screen() + + if command_type is not commands.digit_arg: self.last_command = command_type self.finished = bool(command.finish) @@ -695,7 +964,7 @@ def handle1(self, block: bool = True) -> bool: if self.msg: self.msg = "" - self.dirty = True + self.invalidate_message() while True: # We use the same timeout as in readline.c: 100ms @@ -712,9 +981,13 @@ def handle1(self, block: bool = True) -> bool: if event.evt == "key": self.input_trans.push(event) elif event.evt == "scroll": + self.invalidate_full() self.refresh() + return True elif event.evt == "resize": + self.invalidate_full() self.refresh() + return True else: translate = False diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 23b8fa6b9c7625e..e4370b0d1462eaf 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -37,9 +37,10 @@ from rlcompleter import Completer as RLCompleter from . import commands, historical_reader -from .completing_reader import CompletingReader +from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer +from .fancycompleter import Completer as FancyCompleter, colorize_matches Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -55,7 +56,7 @@ # types Command = commands.Command from collections.abc import Callable, Collection -from .types import Callback, Completer, KeySpec, CommandName +from .types import Callback, Completer, KeySpec, CommandName, CompletionAction TYPE_CHECKING = False @@ -103,6 +104,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) + colorize_completions: Callable[[list[str], list[Any]], list[str]] | None = None @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -134,7 +136,7 @@ def get_stem(self) -> str: p -= 1 return "".join(b[p + 1 : self.pos]) - def get_completions(self, stem: str) -> list[str]: + def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None]: module_completions = self.get_module_completions() if module_completions is not None: return module_completions @@ -144,7 +146,7 @@ def get_completions(self, stem: str) -> list[str]: while p > 0 and b[p - 1] != "\n": p -= 1 num_spaces = 4 - ((self.pos - p) % 4) - return [" " * num_spaces] + return [" " * num_spaces], None result = [] function = self.config.readline_completer if function is not None: @@ -162,14 +164,23 @@ def get_completions(self, stem: str) -> list[str]: break result.append(next) state += 1 - # emulate the behavior of the standard readline that sorts - # the completions before displaying them. - result.sort() - return result - - def get_module_completions(self) -> list[str] | None: - line = self.get_line() - return self.config.module_completer.get_completions(line) + # Emulate readline's sorting using the visible text rather than + # the raw ANSI escape sequences used for colorized matches. + result.sort(key=stripcolor) + return result, None + + def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: + line = stripcolor(self.get_line()) + colorize_completions = self.config.colorize_completions + result = self.config.module_completer.get_completions( + line, include_values=bool(colorize_completions) + ) + if result is None: + return None + names, values, action = result + if colorize_completions: + names = colorize_completions(names, values) + return names, action def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: @@ -276,7 +287,7 @@ class maybe_accept(commands.Command): def do(self) -> None: r: ReadlineAlikeReader r = self.reader # type: ignore[assignment] - r.dirty = True # this is needed to hide the completion menu, if visible + r.invalidate_overlay() # hide completion menu, if visible # if there are already several lines and the cursor # is not on the last one, always insert a new \n. @@ -336,7 +347,7 @@ def do(self) -> None: break r.pos -= repeat del b[r.pos : r.pos + repeat] - r.dirty = True + r.invalidate_buffer(r.pos) else: self.reader.error("can't backspace at start") @@ -412,8 +423,12 @@ def set_completer_delims(self, delimiters: Collection[str]) -> None: def get_completer_delims(self) -> str: return "".join(sorted(self.config.completer_delims)) - def _histline(self, line: str) -> str: + def _histline(self, line: str, *, sanitize_nuls: bool = False) -> str: line = line.rstrip("\n") + if "\0" in line: + if not sanitize_nuls: + raise ValueError("embedded null character") + line = line.replace("\0", "") return line def get_history_length(self) -> int: @@ -446,9 +461,12 @@ def read_history_file(self, filename: str = gethistoryfile()) -> None: if line.endswith("\r"): buffer.append(line+'\n') else: - line = self._histline(line) + line = self._histline(line, sanitize_nuls=True) if buffer: - line = self._histline("".join(buffer).replace("\r", "") + line) + line = self._histline( + "".join(buffer).replace("\r", "") + line, + sanitize_nuls=True, + ) del buffer[:] if line: history.append(line) @@ -608,8 +626,19 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + use_basic_completer = ( + not sys.flags.ignore_environment + and os.getenv("PYTHON_BASIC_COMPLETER") + ) + completer_cls = RLCompleter if use_basic_completer else FancyCompleter + completer = completer_cls(namespace) + _wrapper.config.readline_completer = completer.complete + if isinstance(completer, FancyCompleter) and completer.use_colors: + theme = completer.theme + def _colorize(names: list[str], values: list[object]) -> list[str]: + return colorize_matches(names, values, theme) + _wrapper.config.colorize_completions = _colorize _wrapper.config.module_completer = ModuleCompleter(namespace) - _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/render.py b/Lib/_pyrepl/render.py new file mode 100644 index 000000000000000..b835778e366c539 --- /dev/null +++ b/Lib/_pyrepl/render.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Literal, Protocol, Self + +from _colorize import ANSIColors + +from .types import CursorXY +from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, str_width + +type RenderStyle = StyleRef | str | None +type LineUpdateKind = Literal[ + "insert_char", + "replace_char", + "replace_span", + "delete_then_insert", + "rewrite_suffix", +] + + +class _ThemeSyntax(Protocol): + """Protocol for theme objects that map tag names to SGR escape strings.""" + def __getitem__(self, key: str, /) -> str: ... + + +@dataclass(frozen=True, slots=True) +class RenderCell: + """One terminal cell: a character, its column width, and SGR style. + + A screen row like ``>>> def`` is a sequence of cells:: + + > > > d e f + ╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯╰─╯ + """ + + text: str + width: int + style: StyleRef = field(default_factory=StyleRef) + controls: tuple[str, ...] = () + + @property + def terminal_text(self) -> str: + return render_cells((self,)) + + +def _theme_style(theme: _ThemeSyntax, tag: str) -> str: + return theme[tag] + + +def _style_escape(style: StyleRef) -> str: + if style.sgr: + return style.sgr + if style.tag is None: + return "" + return _theme_style(THEME(), style.tag) + + +def _update_terminal_state(state: str, escape: str) -> str: + if escape in {ANSIColors.RESET, "\x1b[m"}: + return "" + return state + escape + + +def _cells_from_rendered_text(text: str) -> tuple[RenderCell, ...]: + if not text: + return () + + cells: list[RenderCell] = [] + pending_controls: list[str] = [] + active_sgr = "" + index = 0 + + def append_plain_text(segment: str) -> None: + nonlocal pending_controls + if not segment: + return + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + pending_controls = [] + for char in segment: + cells.append( + RenderCell( + char, + str_width(char), + style=StyleRef.from_sgr(active_sgr), + ) + ) + + for match in ANSI_ESCAPE_SEQUENCE.finditer(text): + append_plain_text(text[index : match.start()]) + escape = match.group(0) + if escape.endswith("m"): + active_sgr = _update_terminal_state(active_sgr, escape) + else: + pending_controls.append(escape) + index = match.end() + + append_plain_text(text[index:]) + if pending_controls: + cells.append(RenderCell("", 0, controls=tuple(pending_controls))) + + return tuple(cells) + + +@dataclass(frozen=True, slots=True) +class RenderLine: + """One physical screen row as a tuple of :class:`RenderCell` objects. + + ``text`` is the pre-rendered terminal string (characters + SGR escapes); + ``width`` is the total visible column count. + """ + + cells: tuple[RenderCell, ...] + text: str + width: int + + @classmethod + def from_cells(cls, cells: Iterable[RenderCell]) -> Self: + cell_tuple = tuple(cells) + return cls( + cells=cell_tuple, + text=render_cells(cell_tuple), + width=sum(cell.width for cell in cell_tuple), + ) + + @classmethod + def from_parts( + cls, + parts: Sequence[str], + widths: Sequence[int], + styles: Sequence[RenderStyle] | None = None, + ) -> Self: + if styles is None: + return cls.from_cells( + RenderCell(text, width) + for text, width in zip(parts, widths) + ) + + cells: list[RenderCell] = [] + for text, width, style in zip(parts, widths, styles): + if isinstance(style, StyleRef): + cells.append(RenderCell(text, width, style=style)) + elif style is None: + cells.append(RenderCell(text, width)) + else: + cells.append(RenderCell(text, width, style=StyleRef.from_tag(style))) + return cls.from_cells(cells) + + @classmethod + def from_rendered_text(cls, text: str) -> Self: + return cls.from_cells(_cells_from_rendered_text(text)) + + +@dataclass(frozen=True, slots=True) +class ScreenOverlay: + """An overlay that replaces or inserts lines at a screen position. + + If *insert* is True, lines are spliced in (shifting content down); + if False (default), lines replace existing content at *y*. + + Overlays are used to display tab completion menus and status messages. + For example, a tab-completion menu inserted below the input:: + + >>> os.path.j ← line 0 (base content) + join ← ScreenOverlay(y=1, insert=True) + junction ← (pushes remaining lines down) + ... ← line 1 (shifted down by 2) + """ + y: int + lines: tuple[RenderLine, ...] + insert: bool = False + + +@dataclass(frozen=True, slots=True) +class RenderedScreen: + """The complete screen state: content lines, cursor, and overlays. + + ``lines`` holds the base content; ``composed_lines`` is the final + result after overlays (completion menus, messages) are applied:: + + lines: composed_lines: + ┌──────────────────┐ ┌──────────────────┐ + │>>> os.path.j │ │>>> os.path.j │ + │... │ ──► │ join │ ← overlay + └──────────────────┘ │... │ + └──────────────────┘ + """ + + lines: tuple[RenderLine, ...] + cursor: CursorXY + overlays: tuple[ScreenOverlay, ...] = () + composed_lines: tuple[RenderLine, ...] = field(init=False, default=()) + + def __post_init__(self) -> None: + object.__setattr__(self, "composed_lines", self._compose()) + + def _compose(self) -> tuple[RenderLine, ...]: + """Apply overlays in tuple order; inserts shift subsequent positions.""" + if not self.overlays: + return self.lines + + lines = list(self.lines) + y_offset = 0 + for overlay in self.overlays: + adjusted_y = overlay.y + y_offset + assert adjusted_y >= 0, ( + f"Overlay y={overlay.y} with offset={y_offset} is negative; " + "overlays must be sorted by ascending y" + ) + if overlay.insert: + # Splice overlay lines in, pushing existing content down. + lines[adjusted_y:adjusted_y] = overlay.lines + y_offset += len(overlay.lines) + else: + # Replace existing lines at the overlay position. + target_len = adjusted_y + len(overlay.lines) + if len(lines) < target_len: + lines.extend([EMPTY_RENDER_LINE] * (target_len - len(lines))) + for index, line in enumerate(overlay.lines): + lines[adjusted_y + index] = line + return tuple(lines) + + @classmethod + def empty(cls) -> Self: + return cls((), (0, 0), ()) + + @classmethod + def from_screen_lines( + cls, + screen: Sequence[str], + cursor: CursorXY, + ) -> Self: + return cls( + tuple(RenderLine.from_rendered_text(line) for line in screen), + cursor, + (), + ) + + def with_overlay( + self, + y: int, + lines: Iterable[RenderLine], + ) -> Self: + return type(self)( + self.lines, + self.cursor, + self.overlays + (ScreenOverlay(y, tuple(lines)),), + ) + + @property + def screen_lines(self) -> tuple[str, ...]: + return tuple(line.text for line in self.composed_lines) + + +@dataclass(frozen=True, slots=True) +class LineDiff: + """The changed region between an old and new version of one screen row. + + When the user types ``e`` so the row changes from + ``>>> nam`` to ``>>> name``:: + + >>> n a m old + >>> n a m e new + ╰─╯ + start_cell=7, new_cells=("m","e"), old_cells=("m",) + """ + + start_cell: int + start_x: int + old_cells: tuple[RenderCell, ...] + new_cells: tuple[RenderCell, ...] + old_width: int + new_width: int + + @property + def old_text(self) -> str: + return render_cells(self.old_cells) + + @property + def new_text(self) -> str: + return render_cells(self.new_cells) + + @property + def old_changed_width(self) -> int: + return sum(cell.width for cell in self.old_cells) + + @property + def new_changed_width(self) -> int: + return sum(cell.width for cell in self.new_cells) + + +EMPTY_RENDER_LINE = RenderLine(cells=(), text="", width=0) + + +@dataclass(frozen=True, slots=True) +class LineUpdate: + kind: LineUpdateKind + y: int + start_cell: int + start_x: int + """Screen x-coordinate where the update begins. Used for cursor positioning.""" + cells: tuple[RenderCell, ...] + char_width: int = 0 + clear_eol: bool = False + reset_to_margin: bool = False + """If True, the console must resync the cursor position after writing + (needed when cells contain non-SGR escape sequences that may move the cursor).""" + text: str = field(init=False, default="") + + def __post_init__(self) -> None: + object.__setattr__(self, "text", render_cells(self.cells)) + + +def _controls_require_cursor_resync(controls: Sequence[str]) -> bool: + # Anything beyond SGR means the cursor may no longer be where we left it. + return any(not control.endswith("m") for control in controls) + + +def requires_cursor_resync(cells: Sequence[RenderCell]) -> bool: + return any(_controls_require_cursor_resync(cell.controls) for cell in cells) + + +def render_cells( + cells: Sequence[RenderCell], + visual_style: str | None = None, +) -> str: + """Render a sequence of cells into a terminal string with SGR escapes. + + Tracks the active SGR state to emit resets only when the style + actually changes, minimizing output bytes. + + If *visual_style* is given (used by redraw visualization), it is appended + to every cell's style. + """ + rendered: list[str] = [] + active_escape = "" + for cell in cells: + if cell.controls: + rendered.extend(cell.controls) + if not cell.text: + continue + + target_escape = _style_escape(cell.style) + if visual_style is not None: + target_escape += visual_style + if target_escape != active_escape: + if active_escape: + rendered.append(ANSIColors.RESET) + if target_escape: + rendered.append(target_escape) + active_escape = target_escape + rendered.append(cell.text) + + if active_escape: + rendered.append(ANSIColors.RESET) + return "".join(rendered) + + +def diff_render_lines(old: RenderLine, new: RenderLine) -> LineDiff | None: + if old == new: + return None + + prefix = 0 + start_x = 0 + max_prefix = min(len(old.cells), len(new.cells)) + while prefix < max_prefix and old.cells[prefix] == new.cells[prefix]: + # Stop at any cell with non-SGR controls, since those might affect + # cursor position and must be re-emitted. + if old.cells[prefix].controls: + break + start_x += old.cells[prefix].width + prefix += 1 + + old_suffix = len(old.cells) + new_suffix = len(new.cells) + while old_suffix > prefix and new_suffix > prefix: + old_cell = old.cells[old_suffix - 1] + new_cell = new.cells[new_suffix - 1] + if old_cell.controls or new_cell.controls or old_cell != new_cell: + break + old_suffix -= 1 + new_suffix -= 1 + + # Extend diff range to include trailing zero-width combining characters, + # so we never render a combining char without its base character. + while old_suffix < len(old.cells) and old.cells[old_suffix].width == 0: + old_suffix += 1 + while new_suffix < len(new.cells) and new.cells[new_suffix].width == 0: + new_suffix += 1 + + return LineDiff( + start_cell=prefix, + start_x=start_x, + old_cells=old.cells[prefix:old_suffix], + new_cells=new.cells[prefix:new_suffix], + old_width=old.width, + new_width=new.width, + ) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index ff1bdab9fea0785..c169d0191bd8333 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -31,7 +31,6 @@ import sys import code import warnings -import errno from .readline import _get_reader, multiline_input, append_history_file @@ -125,7 +124,7 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False @@ -162,7 +161,7 @@ def maybe_run_command(statement: str) -> bool: if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) - r.dirty = True + r.invalidate_full() r.refresh() console.write("\nKeyboardInterrupt\n") console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index 943ee12f964b296..395867805196a5c 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -32,3 +32,9 @@ def trace(line: str, *k: object, **kw: object) -> None: line = line.format(*k, **kw) trace_file.write(line + "\n") trace_file.flush() + + +def trace_text(text: str, limit: int = 60) -> str: + if len(text) > limit: + text = text[:limit] + "..." + return repr(text) diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py index c5b7ebc1a406bdf..919763158eb123a 100644 --- a/Lib/_pyrepl/types.py +++ b/Lib/_pyrepl/types.py @@ -4,7 +4,13 @@ type SimpleContextManager = Iterator[None] type KeySpec = str # like r"\C-c" type CommandName = str # like "interrupt" -type EventTuple = tuple[CommandName, str] +type EventData = list[str] +type EventTuple = tuple[CommandName, EventData] +type CursorXY = tuple[int, int] +type Dimensions = tuple[int, int] +type ScreenInfoRow = tuple[int, list[int]] +type Keymap = tuple[tuple[KeySpec, CommandName], ...] type Completer = Callable[[str, int], str | None] type CharBuffer = list[str] type CharWidths = list[int] +type CompletionAction = tuple[str, Callable[[], str | None]] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 09247de748ee3bd..8edd1c36a7232d7 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -31,14 +31,27 @@ import time import types import platform +from collections.abc import Callable +from dataclasses import dataclass from fcntl import ioctl +from typing import TYPE_CHECKING, overload + +from _colorize import ANSIColors from . import terminfo from .console import Console, Event from .fancy_termios import tcgetattr, tcsetattr, TermState -from .trace import trace +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .unix_eventqueue import EventQueue -from .utils import wlen # declare posix optional to allow None assignment on other platforms posix: types.ModuleType | None @@ -47,14 +60,12 @@ except ImportError: posix = None -TYPE_CHECKING = False - # types if TYPE_CHECKING: - from typing import AbstractSet, IO, Literal, overload, cast -else: - overload = lambda func: None - cast = lambda typ, val: val + from typing import AbstractSet, IO, Literal + +type _MoveFunc = Callable[[int, int], None] +type _PendingWrite = tuple[str | bytes, bool] class InvalidTerminal(RuntimeError): @@ -140,7 +151,47 @@ def poll(self, timeout: float | None = None) -> list[int]: poll = MinimalPoll # type: ignore[assignment] +@dataclass(frozen=True, slots=True) +class UnixRefreshPlan: + """Instructions for updating the terminal after a screen change. + + After the user types ``e`` to complete ``name``:: + + Before: >>> def greet(nam|): + ▲ + LineUpdate here: insert_char "e" + + After: >>> def greet(name|): + ▲ + + Only the changed cells are sent to the terminal; unchanged rows + are skipped entirely. + """ + + grow_lines: int + """Number of blank lines to append at the bottom to accommodate new content.""" + use_tall_mode: bool + """Use absolute cursor addressing via ``cup`` instead of relative moves. + Activated when content exceeds one screen height.""" + offset: int + """Vertical scroll offset: the buffer row displayed at the top of the terminal window.""" + reverse_scroll: int + """Number of lines to scroll backwards (content moves down).""" + forward_scroll: int + """Number of lines to scroll forwards (content moves up).""" + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + """Row indices to erase (old content with no replacement).""" + rendered_screen: RenderedScreen + cursor: tuple[int, int] + + class UnixConsole(Console): + __buffer: list[_PendingWrite] + __gone_tall: bool + __move: _MoveFunc + __offset: int + def __init__( self, f_in: IO[bytes] | int = 0, @@ -219,7 +270,7 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.event_queue = EventQueue( self.input_fd, self.encoding, self.terminfo ) - self.cursor_visible = 1 + self.cursor_visible = True signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -239,33 +290,50 @@ def change_encoding(self, encoding: str) -> None: """ self.encoding = encoding - def refresh(self, screen, c_xy): + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ + c_xy = rendered_screen.cursor + trace( + "unix.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) + + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> UnixRefreshPlan: cx, cy = c_xy + height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) + + grow_lines = 0 if not self.__gone_tall: - while len(self.screen) < min(len(screen), self.height): - self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") - else: - while len(self.screen) < len(screen): - self.screen.append("") + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) + elif len(previous_lines) < line_count: + previous_lines.extend([EMPTY_RENDER_LINE] * (line_count - len(previous_lines))) - if len(screen) > self.height: - self.__gone_tall = 1 - self.__move = self.__move_tall - - px, py = self.posxy - old_offset = offset = self.__offset - height = self.height + use_tall_mode = self.__gone_tall or line_count > height # we make sure the cursor is on the screen, and that we're # using all of the screen if we can @@ -273,56 +341,115 @@ def refresh(self, screen, c_xy): offset = cy elif cy >= offset + height: offset = cy - height + 1 - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] - # use hardware scrolling if we have it. + reverse_scroll = 0 + forward_scroll = 0 if old_offset > offset and self._ri: + reverse_scroll = old_offset - offset + for _ in range(reverse_scroll): + if oldscr: + oldscr.pop(-1) + oldscr.insert(0, EMPTY_RENDER_LINE) + elif old_offset < offset and self._ind: + forward_scroll = offset - old_offset + for _ in range(forward_scroll): + if oldscr: + oldscr.pop(0) + oldscr.append(EMPTY_RENDER_LINE) + + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) + + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "unix.refresh plan grow={grow} tall={tall} offset={offset} " + "reverse_scroll={reverse_scroll} forward_scroll={forward_scroll} " + "updates={updates} clears={clears}", + grow=grow_lines, + tall=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return UnixRefreshPlan( + grow_lines=grow_lines, + use_tall_mode=use_tall_mode, + offset=offset, + reverse_scroll=reverse_scroll, + forward_scroll=forward_scroll, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) + + def __apply_refresh_plan(self, plan: UnixRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "unix.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) + + for _ in range(plan.grow_lines): + self.__hide_cursor() + if screen_line_count: + self.__move(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 + + if plan.use_tall_mode and not self.__gone_tall: + self.__gone_tall = True + self.__move = self.__move_tall + + old_offset = self.__offset + if plan.reverse_scroll: self.__hide_cursor() self.__write_code(self._cup, 0, 0) self.posxy = 0, old_offset - for i in range(old_offset - offset): + for _ in range(plan.reverse_scroll): self.__write_code(self._ri) - oldscr.pop(-1) - oldscr.insert(0, "") - elif old_offset < offset and self._ind: + elif plan.forward_scroll: self.__hide_cursor() self.__write_code(self._cup, self.height - 1, 0) self.posxy = 0, old_offset + self.height - 1 - for i in range(offset - old_offset): + for _ in range(plan.forward_scroll): self.__write_code(self._ind) - oldscr.pop(0) - oldscr.append("") - self.__offset = offset + self.__offset = plan.offset - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) - y = len(newscr) - while y < len(oldscr): + for y in plan.cleared_lines: self.__hide_cursor() self.__move(0, y) self.posxy = 0, y self.__write_code(self._el) - y += 1 self.__show_cursor() - - self.screen = screen.copy() self.move_cursor(cx, cy) self.flushoutput() + self.sync_rendered_screen(plan.rendered_screen, self.posxy) - def move_cursor(self, x, y): + def move_cursor(self, x: int, y: int) -> None: """ Move the cursor to the specified position on the screen. @@ -331,16 +458,25 @@ def move_cursor(self, x, y): - y (int): Y coordinate. """ if y < self.__offset or y >= self.__offset + self.height: - self.event_queue.insert(Event("scroll", None)) + trace( + "unix.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) + self.event_queue.insert(Event("scroll", "")) else: + trace("unix.move_cursor x={x} y={y}", x=x, y=y) self.__move(x, y) self.posxy = x, y self.flushoutput() - def prepare(self): + def prepare(self) -> None: """ Prepare the console for input/output operations. """ + trace("unix.prepare") self.__buffer = [] self.__svtermstate = tcgetattr(self.input_fd) @@ -352,21 +488,22 @@ def prepare(self): raw.iflag |= termios.BRKINT raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN) raw.lflag |= termios.ISIG - raw.cc[termios.VMIN] = 1 - raw.cc[termios.VTIME] = 0 + raw.cc[termios.VMIN] = b"\x01" + raw.cc[termios.VTIME] = b"\x00" self.__input_fd_set(raw) - # In macOS terminal we need to deactivate line wrap via ANSI escape code + # Apple Terminal will re-wrap lines for us unless we preempt the + # damage. if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") - self.screen = [] self.height, self.width = self.getheightwidth() self.posxy = 0, 0 - self.__gone_tall = 0 + self.__gone_tall = False self.__move = self.__move_short self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) self.__maybe_write_code(self._smkx) @@ -377,10 +514,12 @@ def prepare(self): self.__enable_bracketed_paste() - def restore(self): + def restore(self) -> None: """ Restore the console to the default state """ + trace("unix.restore") + self.__write(ANSIColors.RESET) self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() @@ -445,7 +584,7 @@ def wait(self, timeout: float | None = None) -> bool: or bool(self.pollob.poll(timeout)) ) - def set_cursor_vis(self, visible): + def set_cursor_vis(self, visible: bool) -> None: """ Set the visibility of the cursor. @@ -513,10 +652,12 @@ def finish(self): """ Finish console operations and flush the output buffer. """ - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write(ANSIColors.RESET) self.__write("\n\r") self.flushoutput() @@ -541,7 +682,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] trace("getpending({a})", a=amount) @@ -565,7 +706,7 @@ def getpending(self): while not self.event_queue.empty(): e2 = self.event_queue.get() e.data += e2.data - e.raw += e.raw + e.raw += e2.raw amount = 10000 raw = self.__read(amount) @@ -578,11 +719,12 @@ def clear(self): """ Clear the console screen. """ + trace("unix.clear") self.__write_code(self._clear) - self.__gone_tall = 1 + self.__gone_tall = True self.__move = self.__move_tall self.posxy = 0, 0 - self.screen = [] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) @property def input_hook(self): @@ -633,98 +775,178 @@ def __setup_movement(self): self.__move = self.__move_short - def __write_changed_line(self, y, oldline, newline, px_coord): - # this is frustrating; there's no reason to test (say) - # self.dch1 inside the loop -- but alternative ways of - # structuring this function are equally painful (I'm trying to - # avoid writing code generators these days...) - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 - - px_pos = 0 - j = 0 - for c in oldline: - if j >= px_coord: - break - j += wlen(c) - px_pos += 1 - - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" - ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 + @staticmethod + def __cell_index_from_x(line: RenderLine, x_coord: int) -> int: + width = 0 + index = 0 + while index < len(line.cells) and width < x_coord: + width += line.cells[index].width + index += 1 + return index + + def __plan_changed_line( + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + # NOTE: The shared replace_char / replace_span / rewrite_suffix logic + # is duplicated in WindowsConsole.__plan_changed_line. Keep changes to + # these common cases synchronised between the two files. Yes, this is + # duplicated on purpose; the two backends agree just enough to make a + # shared helper a trap. Unix-only cases (insert_char, delete_then_insert) + # rely on terminal capabilities (ich1/dch1) that are unavailable on + # Windows. + diff = diff_render_lines(oldline, newline) + if diff is None: + return None - # if we need to insert a single character right after the first detected change - if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + start_cell = diff.start_cell + start_x = diff.start_x + + if ( + self.ich1 + and not diff.old_cells + and (visible_new_cells := tuple( + cell for cell in diff.new_cells if cell.width + )) + and len(visible_new_cells) == 1 + and all(cell.width == 0 for cell in diff.new_cells[1:]) + and oldline.cells[start_cell:] == newline.cells[start_cell + 1 :] + ): + px_cell = self.__cell_index_from_x(oldline, px_coord) if ( y == self.posxy[1] - and x_coord > self.posxy[0] - and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + and start_x > self.posxy[0] + and oldline.cells[px_cell:start_cell] + == newline.cells[px_cell + 1 : start_cell + 1] ): - x_pos = px_pos - x_coord = px_coord - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y - - # if it's a single character change in the middle of the line - elif ( - x_coord < minlen - and oldline[x_pos + 1 :] == newline[x_pos + 1 :] - and wlen(oldline[x_pos]) == wlen(newline[x_pos]) + start_cell = px_cell + start_x = px_coord + planned_cells = diff.new_cells + changed_cell = visible_new_cells[0] + return LineUpdate( + kind="insert_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) - self.__write(newline[x_pos]) - self.posxy = x_coord + character_width, y - - # if this is the last character to fit in the line and we edit in the middle of the line - elif ( + planned_cells = diff.new_cells + changed_cell = planned_cells[0] + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if diff.old_changed_width == diff.new_changed_width: + planned_cells = diff.new_cells + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=diff.new_changed_width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + if ( self.dch1 and self.ich1 - and wlen(newline) == self.width - and x_coord < wlen(newline) - 2 - and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + and newline.width == self.width + and start_x < newline.width - 2 + and newline.cells[start_cell + 1 : -1] == oldline.cells[start_cell:-2] ): + planned_cells = (newline.cells[start_cell],) + changed_cell = planned_cells[0] + return LineUpdate( + kind="delete_then_insert", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=planned_cells, + char_width=changed_cell.width, + reset_to_margin=requires_cursor_resync(planned_cells), + ) + + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=requires_cursor_resync(suffix_cells), + ) + + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "unix.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + if update.kind == "insert_char": + self.__move(update.start_x, update.y) + self.__write_code(self.ich1) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind in {"replace_char", "replace_span"}: + self.__move(update.start_x, update.y) + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y + elif update.kind == "delete_then_insert": self.__hide_cursor() - self.__move(self.width - 2, y) - self.posxy = self.width - 2, y + self.__move(self.width - 2, update.y) + self.posxy = self.width - 2, update.y self.__write_code(self.dch1) - - character_width = wlen(newline[x_pos]) - self.__move(x_coord, y) + self.__move(update.start_x, update.y) self.__write_code(self.ich1) - self.__write(newline[x_pos]) - self.posxy = character_width + 1, y - + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y else: self.__hide_cursor() - self.__move(x_coord, y) - if wlen(oldline) > wlen(newline): + self.__move(update.start_x, update.y) + if update.clear_eol: self.__write_code(self._el) - self.__write(newline[x_pos:]) - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = update.start_x + update.char_width, update.y - if "\x1b" in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin: + # Non-SGR terminal controls can affect the cursor position. + self.move_cursor(0, update.y) def __write(self, text): - self.__buffer.append((text, 0)) + self.__buffer.append((text, False)) def __write_code(self, fmt, *args): - self.__buffer.append((terminfo.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), True)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -775,30 +997,38 @@ def __move_tall(self, x, y): self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() - self.event_queue.insert(Event("resize", None)) + self.event_queue.insert(Event("resize", "")) def __hide_cursor(self): if self.cursor_visible: self.__maybe_write_code(self._civis) - self.cursor_visible = 0 + self.cursor_visible = False def __show_cursor(self): if not self.cursor_visible: self.__maybe_write_code(self._cnorm) - self.cursor_visible = 1 + self.cursor_visible = True def repaint(self): + composed = self._rendered_screen.composed_lines + trace( + "unix.repaint gone_tall={gone_tall} screen_lines={lines} offset={offset}", + gone_tall=self.__gone_tall, + lines=len(composed), + offset=self.__offset, + ) if not self.__gone_tall: self.posxy = 0, self.posxy[1] self.__write("\r") - ns = len(self.screen) * ["\000" * self.width] - self.screen = ns + ns = len(composed) * ["\000" * self.width] else: self.posxy = 0, self.__offset self.__move(0, self.__offset) ns = self.height * ["\000" * self.width] - self.screen = ns + self.sync_rendered_screen( + RenderedScreen.from_screen_lines(ns, self.posxy), + self.posxy, + ) def __tputs(self, fmt, prog=delayprog): """A Python implementation of the curses tputs function; the @@ -808,7 +1038,7 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) while True: m = prog.search(fmt) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 06cddef851bb404..230dae35af665ab 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -9,6 +9,7 @@ import _colorize from collections import deque +from dataclasses import dataclass from io import StringIO from tokenize import TokenInfo as TI from typing import Iterable, Iterator, Match, NamedTuple, Self @@ -16,12 +17,13 @@ from .types import CharBuffer, CharWidths from .trace import trace + ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) -IDENTIFIERS_AFTER = {"def", "class"} -KEYWORD_CONSTANTS = {"True", "False", "None"} -BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} +IDENTIFIERS_AFTER = frozenset({"def", "class"}) +KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) +BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) def THEME(**kwargs): @@ -59,6 +61,21 @@ class ColorSpan(NamedTuple): tag: str +class StyledChar(NamedTuple): + text: str + width: int + tag: str | None = None + + +def _ascii_control_repr(c: str) -> str | None: + code = ord(c) + if code < 32: + return "^" + chr(code + 64) + if code == 127: + return "^?" + return None + + @functools.cache def str_width(c: str) -> int: if ord(c) < 128: @@ -226,8 +243,8 @@ def gen_colors_from_token_stream( yield ColorSpan(span, "builtin") -keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} -keyword_first_sets_case = {"False", "None", "True"} +keyword_first_sets_match = frozenset({"False", "None", "True", "await", "lambda", "not"}) +keyword_first_sets_case = frozenset({"False", "None", "True"}) def is_soft_keyword_used(*tokens: TI | None) -> bool: @@ -242,7 +259,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), TI(string="match"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...") ): return True case ( @@ -257,7 +274,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{") ): return True case ( @@ -276,10 +293,71 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: TI(T.NAME, string=s) ): return not keyword.iskeyword(s) + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":" | ";"), + TI(string="lazy"), + TI(string="import") | TI(string="from") + ): + return True case _: return False +def iter_display_chars( + buffer: str, + colors: list[ColorSpan] | None = None, + start_index: int = 0, +) -> Iterator[StyledChar]: + """Yield visible display characters with widths and semantic color tags. + + Note: ``colors`` is consumed in place as spans are processed -- callers + that split a buffer across multiple calls rely on this mutation to track + which spans have already been handled. + """ + + if not buffer: + return + + color_idx = 0 + if colors: + while color_idx < len(colors) and colors[color_idx].span.end < start_index: + color_idx += 1 + + active_tag = None + if colors and color_idx < len(colors) and colors[color_idx].span.start < start_index: + active_tag = colors[color_idx].tag + + for i, c in enumerate(buffer, start_index): + if colors and color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + if control := _ascii_control_repr(c): + text = control + width = len(control) + elif ord(c) < 128: + text = c + width = 1 + elif unicodedata.category(c).startswith("C"): + text = r"\u%04x" % ord(c) + width = len(text) + else: + text = c + width = str_width(c) + + yield StyledChar(text, width, active_tag) + + if colors and color_idx < len(colors) and colors[color_idx].span.end == i: + color_idx += 1 + active_tag = None + # Check if the next span starts at the same position + if color_idx < len(colors) and colors[color_idx].span.start == i: + active_tag = colors[color_idx].tag + + # Remove consumed spans so callers see the mutation + if color_idx > 0 and colors: + del colors[:color_idx] + + def disp_str( buffer: str, colors: list[ColorSpan] | None = None, @@ -315,53 +393,18 @@ def disp_str( (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) """ + styled_chars = list(iter_display_chars(buffer, colors, start_index)) chars: CharBuffer = [] char_widths: CharWidths = [] - - if not buffer: - return chars, char_widths - - while colors and colors[0].span.end < start_index: - # move past irrelevant spans - colors.pop(0) - theme = THEME(force_color=force_color) - pre_color = "" - post_color = "" - if colors and colors[0].span.start < start_index: - # looks like we're continuing a previous color (e.g. a multiline str) - pre_color = theme[colors[0].tag] - - for i, c in enumerate(buffer, start_index): - if colors and colors[0].span.start == i: # new color starts now - pre_color = theme[colors[0].tag] - - if c == "\x1a": # CTRL-Z on Windows - chars.append(c) - char_widths.append(2) - elif ord(c) < 128: - chars.append(c) - char_widths.append(1) - elif unicodedata.category(c).startswith("C"): - c = r"\u%04x" % ord(c) - chars.append(c) - char_widths.append(len(c)) - else: - chars.append(c) - char_widths.append(str_width(c)) - - if colors and colors[0].span.end == i: # current color ends now - post_color = theme.reset - colors.pop(0) - chars[-1] = pre_color + chars[-1] + post_color - pre_color = "" - post_color = "" - - if colors and colors[0].span.start < i and colors[0].span.end > i: - # even though the current color should be continued, reset it for now. - # the next call to `disp_str()` will revive it. - chars[-1] += theme.reset + for index, styled_char in enumerate(styled_chars): + previous_tag = styled_chars[index - 1].tag if index else None + next_tag = styled_chars[index + 1].tag if index + 1 < len(styled_chars) else None + prefix = theme[styled_char.tag] if styled_char.tag and styled_char.tag != previous_tag else "" + suffix = theme.reset if styled_char.tag and styled_char.tag != next_tag else "" + chars.append(prefix + styled_char.text + suffix) + char_widths.append(styled_char.width) return chars, char_widths @@ -379,13 +422,35 @@ def prev_next_window[T]( """ iterator = iter(iterable) - window = deque((None, next(iterator)), maxlen=3) + try: + first = next(iterator) + except StopIteration: + return + window = deque((None, first), maxlen=3) try: for x in iterator: window.append(x) yield tuple(window) - except Exception: - raise finally: window.append(None) yield tuple(window) + + +@dataclass(frozen=True, slots=True) +class StyleRef: + tag: str | None = None # From THEME().syntax, e.g. "keyword", "builtin" + sgr: str = "" + + @classmethod + def from_tag(cls, tag: str, sgr: str = "") -> Self: + return cls(tag=tag, sgr=sgr) + + @classmethod + def from_sgr(cls, sgr: str) -> Self: + if not sgr: + return cls() + return cls(sgr=sgr) + + @property + def is_plain(self) -> bool: + return self.tag is None and not self.sgr diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index f9f5988af0b9eff..3768a22ad16f7bb 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -25,6 +25,7 @@ import ctypes import types +from dataclasses import dataclass from ctypes.wintypes import ( _COORD, WORD, @@ -37,20 +38,29 @@ SHORT, ) from ctypes import Structure, POINTER, Union +from typing import TYPE_CHECKING + +from _colorize import ANSIColors + from .console import Event, Console -from .trace import trace -from .utils import wlen +from .render import ( + EMPTY_RENDER_LINE, + LineUpdate, + RenderLine, + RenderedScreen, + requires_cursor_resync, + diff_render_lines, + render_cells, +) +from .trace import trace, trace_text from .windows_eventqueue import EventQueue try: - from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined] + from ctypes import get_last_error, WinDLL, windll, WinError # type: ignore[attr-defined] except: # Keep MyPy happy off Windows from ctypes import CDLL as WinDLL, cdll as windll - def GetLastError() -> int: - return 42 - def get_last_error() -> int: return 42 @@ -66,8 +76,6 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: except ImportError: nt = None -TYPE_CHECKING = False - if TYPE_CHECKING: from typing import IO @@ -126,6 +134,17 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: class _error(Exception): pass + +@dataclass(frozen=True, slots=True) +class WindowsRefreshPlan: + grow_lines: int + offset: int + scroll_lines: int + line_updates: tuple[LineUpdate, ...] + cleared_lines: tuple[int, ...] + rendered_screen: RenderedScreen + cursor: tuple[int, int] + def _supports_vt(): try: return nt._supports_virtual_terminal() @@ -149,18 +168,19 @@ def __init__( # Save original console modes so we can recover on cleanup. original_input_mode = DWORD() - GetConsoleMode(InHandle, original_input_mode) + if not GetConsoleMode(InHandle, original_input_mode): + raise WinError(get_last_error()) trace(f'saved original input mode 0x{original_input_mode.value:x}') self.__original_input_mode = original_input_mode.value - SetConsoleMode( + if not SetConsoleMode( OutHandle, ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING, - ) + ): + raise WinError(get_last_error()) - self.screen: list[str] = [] self.width = 80 self.height = 25 self.__offset = 0 @@ -171,73 +191,124 @@ def __init__( # Console I/O is redirected, fallback... self.out = None - def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: + def refresh(self, rendered_screen: RenderedScreen) -> None: """ Refresh the console screen. Parameters: - - screen (list): List of strings representing the screen contents. - - c_xy (tuple): Cursor position (x, y) on the screen. + - rendered_screen: Structured rendered screen contents and cursor. """ - cx, cy = c_xy - - while len(self.screen) < min(len(screen), self.height): - self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") - self.posxy = 0, len(self.screen) - self.screen.append("") + c_xy = rendered_screen.cursor + trace( + "windows.refresh start cursor={cursor} lines={lines} prev_lines={prev_lines} " + "offset={offset} posxy={posxy}", + cursor=c_xy, + lines=len(rendered_screen.composed_lines), + prev_lines=len(self._rendered_screen.composed_lines), + offset=self.__offset, + posxy=self.posxy, + ) + plan = self.__plan_refresh(rendered_screen, c_xy) + self.__apply_refresh_plan(plan) - px, py = self.posxy - old_offset = offset = self.__offset + def __plan_refresh( + self, + rendered_screen: RenderedScreen, + c_xy: tuple[int, int], + ) -> WindowsRefreshPlan: + cx, cy = c_xy height = self.height + old_offset = offset = self.__offset + prev_composed = self._rendered_screen.composed_lines + previous_lines = list(prev_composed) + next_lines = list(rendered_screen.composed_lines) + line_count = len(next_lines) + + grow_lines = max( + min(line_count, height) - len(prev_composed), + 0, + ) + previous_lines.extend([EMPTY_RENDER_LINE] * grow_lines) - # we make sure the cursor is on the screen, and that we're - # using all of the screen if we can + scroll_lines = 0 if cy < offset: offset = cy elif cy >= offset + height: offset = cy - height + 1 scroll_lines = offset - old_offset + previous_lines.extend([EMPTY_RENDER_LINE] * scroll_lines) + elif offset > 0 and line_count < offset + height: + offset = max(line_count - height, 0) + next_lines.append(EMPTY_RENDER_LINE) + + oldscr = previous_lines[old_offset : old_offset + height] + newscr = next_lines[offset : offset + height] + + line_updates: list[LineUpdate] = [] + px, _ = self.posxy + for y, oldline, newline in zip(range(offset, offset + height), oldscr, newscr): + update = self.__plan_changed_line(y, oldline, newline, px) + if update is not None: + line_updates.append(update) + + cleared_lines = tuple(range(offset + len(newscr), offset + len(oldscr))) + console_rendered_screen = RenderedScreen(tuple(next_lines), c_xy) + trace( + "windows.refresh plan grow={grow} offset={offset} scroll_lines={scroll_lines} " + "updates={updates} clears={clears}", + grow=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + updates=len(line_updates), + clears=len(cleared_lines), + ) + return WindowsRefreshPlan( + grow_lines=grow_lines, + offset=offset, + scroll_lines=scroll_lines, + line_updates=tuple(line_updates), + cleared_lines=cleared_lines, + rendered_screen=console_rendered_screen, + cursor=(cx, cy), + ) - # Scrolling the buffer as the current input is greater than the visible - # portion of the window. We need to scroll the visible portion and the - # entire history - self._scroll(scroll_lines, self._getscrollbacksize()) - self.posxy = self.posxy[0], self.posxy[1] + scroll_lines - self.__offset += scroll_lines + def __apply_refresh_plan(self, plan: WindowsRefreshPlan) -> None: + cx, cy = plan.cursor + trace( + "windows.refresh apply cursor={cursor} updates={updates} clears={clears}", + cursor=plan.cursor, + updates=len(plan.line_updates), + clears=len(plan.cleared_lines), + ) + visual_style = self.begin_redraw_visualization() + screen_line_count = len(self._rendered_screen.composed_lines) - for i in range(scroll_lines): - self.screen.append("") - elif offset > 0 and len(screen) < offset + height: - offset = max(len(screen) - height, 0) - screen.append("") + for _ in range(plan.grow_lines): + self._hide_cursor() + if screen_line_count: + self._move_relative(0, screen_line_count - 1) + self.__write("\n") + self.posxy = 0, screen_line_count + screen_line_count += 1 - oldscr = self.screen[old_offset : old_offset + height] - newscr = screen[offset : offset + height] + if plan.scroll_lines: + self._scroll(plan.scroll_lines, self._getscrollbacksize()) + self.posxy = self.posxy[0], self.posxy[1] + plan.scroll_lines - self.__offset = offset + self.__offset = plan.offset self._hide_cursor() - for ( - y, - oldline, - newline, - ) in zip(range(offset, offset + height), oldscr, newscr): - if oldline != newline: - self.__write_changed_line(y, oldline, newline, px) - - y = len(newscr) - while y < len(oldscr): + for update in plan.line_updates: + self.__apply_line_update(update, visual_style) + + for y in plan.cleared_lines: self._move_relative(0, y) self.posxy = 0, y self._erase_to_end() - y += 1 self._show_cursor() - - self.screen = screen self.move_cursor(cx, cy) + self.sync_rendered_screen(plan.rendered_screen, self.posxy) @property def input_hook(self): @@ -246,42 +317,98 @@ def input_hook(self): if nt is not None and nt._is_inputhook_installed(): return nt._inputhook - def __write_changed_line( - self, y: int, oldline: str, newline: str, px_coord: int - ) -> None: - minlen = min(wlen(oldline), wlen(newline)) - x_pos = 0 - x_coord = 0 - - # reuse the oldline as much as possible, but stop as soon as we - # encounter an ESCAPE, because it might be the start of an escape - # sequence - while ( - x_coord < minlen - and oldline[x_pos] == newline[x_pos] - and newline[x_pos] != "\x1b" + def __plan_changed_line( # keep in sync with UnixConsole.__plan_changed_line + self, + y: int, + oldline: RenderLine, + newline: RenderLine, + px_coord: int, + ) -> LineUpdate | None: + diff = diff_render_lines(oldline, newline) + if diff is None: + return None + + start_cell = diff.start_cell + start_x = diff.start_x + if ( + len(diff.old_cells) == 1 + and len(diff.new_cells) == 1 + and diff.old_cells[0].width == diff.new_cells[0].width ): - x_coord += wlen(newline[x_pos]) - x_pos += 1 + changed_cell = diff.new_cells[0] + # Ctrl-Z (SUB) can reach here via RenderLine.from_rendered_text() + # for prompt/message lines, which bypasses iter_display_chars(). + # On Windows, raw \x1a causes console cursor anomalies, so we + # force a cursor resync when it appears. + return LineUpdate( + kind="replace_char", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=changed_cell.width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or "\x1a" in changed_cell.text + ), + ) + + if diff.old_changed_width == diff.new_changed_width: + return LineUpdate( + kind="replace_span", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=diff.new_cells, + char_width=diff.new_changed_width, + reset_to_margin=( + requires_cursor_resync(diff.new_cells) + or any("\x1a" in cell.text for cell in diff.new_cells) + ), + ) + + suffix_cells = newline.cells[start_cell:] + return LineUpdate( + kind="rewrite_suffix", + y=y, + start_cell=start_cell, + start_x=start_x, + cells=suffix_cells, + char_width=sum(cell.width for cell in suffix_cells), + clear_eol=oldline.width > newline.width, + reset_to_margin=( + requires_cursor_resync(suffix_cells) + or any("\x1a" in cell.text for cell in suffix_cells) + ), + ) - self._hide_cursor() - self._move_relative(x_coord, y) - if wlen(oldline) > wlen(newline): + def __apply_line_update( + self, + update: LineUpdate, + visual_style: str | None = None, + ) -> None: + text = render_cells(update.cells, visual_style) if visual_style else update.text + trace( + "windows.refresh update kind={kind} y={y} x={x} text={text} " + "clear_eol={clear_eol} reset_to_margin={reset}", + kind=update.kind, + y=update.y, + x=update.start_x, + text=trace_text(text), + clear_eol=update.clear_eol, + reset=update.reset_to_margin, + ) + original_y = self.posxy[1] + self._move_relative(update.start_x, update.y) + if update.clear_eol: self._erase_to_end() - self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.__write(text) + self.posxy = min(update.start_x + update.char_width, self.width - 1), update.y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if update.reset_to_margin or update.y != original_y: + # Non-SGR terminal controls or vertical movement require a cursor sync. + self.move_cursor(0, update.y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -300,7 +427,7 @@ def _scroll( if not ScrollConsoleScreenBuffer( OutHandle, scroll_rect, None, destination_origin, fill_info ): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def _hide_cursor(self): self.__write("\x1b[?25l") @@ -322,7 +449,7 @@ def _disable_bracketed_paste(self) -> None: def __write(self, text: str) -> None: if "\x1a" in text: - text = ''.join(["^Z" if x == '\x1a' else x for x in text]) + text = text.replace("\x1a", "^Z") if self.out is not None: self.out.write(text.encode(self.encoding, "replace")) @@ -334,29 +461,33 @@ def __write(self, text: str) -> None: def screen_xy(self) -> tuple[int, int]: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.dwCursorPosition.X, info.dwCursorPosition.Y def _erase_to_end(self) -> None: self.__write(ERASE_IN_LINE) def prepare(self) -> None: - trace("prepare") - self.screen = [] + trace("windows.prepare") self.height, self.width = self.getheightwidth() self.posxy = 0, 0 self.__offset = 0 + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) if self.__vt_support: - SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT) + if not SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT): + raise WinError(get_last_error()) self._enable_bracketed_paste() def restore(self) -> None: + trace("windows.restore") if self.__vt_support: + self.__write(ANSIColors.RESET) # Recover to original mode before running REPL self._disable_bracketed_paste() - SetConsoleMode(InHandle, self.__original_input_mode) + if not SetConsoleMode(InHandle, self.__original_input_mode): + raise WinError(get_last_error()) def _move_relative(self, x: int, y: int) -> None: """Moves relative to the current posxy""" @@ -377,8 +508,16 @@ def move_cursor(self, x: int, y: int) -> None: raise ValueError(f"Bad cursor position {x}, {y}") if y < self.__offset or y >= self.__offset + self.height: + trace( + "windows.move_cursor offscreen x={x} y={y} offset={offset} height={height}", + x=x, + y=y, + offset=self.__offset, + height=self.height, + ) self.event_queue.insert(Event("scroll", "")) else: + trace("windows.move_cursor x={x} y={y}", x=x, y=y) self._move_relative(x, y) self.posxy = x, y @@ -393,7 +532,7 @@ def getheightwidth(self) -> tuple[int, int]: and width of the terminal window in characters.""" info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return ( info.srWindow.Bottom - info.srWindow.Top + 1, info.srWindow.Right - info.srWindow.Left + 1, @@ -402,7 +541,7 @@ def getheightwidth(self) -> tuple[int, int]: def _getscrollbacksize(self) -> int: info = CONSOLE_SCREEN_BUFFER_INFO() if not GetConsoleScreenBufferInfo(OutHandle, info): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return info.srWindow.Bottom # type: ignore[no-any-return] @@ -410,7 +549,7 @@ def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec @@ -420,7 +559,7 @@ def _read_input_bulk( rec = (n * INPUT_RECORD)() read = DWORD() if not ReadConsoleInput(InHandle, rec, n, read): - raise WinError(GetLastError()) + raise WinError(get_last_error()) return rec, read.value @@ -499,17 +638,20 @@ def beep(self) -> None: def clear(self) -> None: """Wipe the screen""" + trace("windows.clear") self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.sync_rendered_screen(RenderedScreen.empty(), self.posxy) def finish(self) -> None: """Move the cursor to the end of the display and otherwise get ready for end. XXX could be merged with restore? Hmm.""" - y = len(self.screen) - 1 - while y >= 0 and not self.screen[y]: + rendered_lines = self._rendered_screen.composed_lines + y = len(rendered_lines) - 1 + while y >= 0 and not rendered_lines[y].text: y -= 1 self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write(ANSIColors.RESET) self.__write("\r\n") def flushoutput(self) -> None: @@ -522,7 +664,7 @@ def flushoutput(self) -> None: def forgetinput(self) -> None: """Forget all pending, but not yet processed input.""" if not FlushConsoleInputBuffer(InHandle): - raise WinError(GetLastError()) + raise WinError(get_last_error()) def getpending(self) -> Event: """Return the characters that have been typed but not yet @@ -549,11 +691,11 @@ def getpending(self) -> Event: # ignore SHIFT_PRESSED and special keys continue if ch == "\r": - ch += "\n" + ch = "\n" e.data += ch return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" if timeout is None: timeout = INFINITE @@ -566,7 +708,17 @@ def wait(self, timeout: float | None) -> bool: return False return True + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) + def repaint(self) -> None: + trace("windows.repaint unsupported") raise NotImplementedError("No repaint support") diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c66269a571967f8..5259a36123ea276 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data - self.__lines = None + self.__lines = [] self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] @@ -55,7 +55,6 @@ def __setup(self): if not data: data = self.__data self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) def __repr__(self): self.__setup() @@ -65,24 +64,22 @@ def __repr__(self): return "Type %s() to see the full %s text" % ((self.__name,)*2) def __call__(self): - self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: + try: + from _pyrepl.pager import get_pager + except ModuleNotFoundError: try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break + from pydoc import get_pager + except ModuleNotFoundError: + def get_pager(): + def _print(text, title=None): + print(text) + return _print + + self.__setup() + + pager = get_pager() + text = "\n".join(self.__lines) + pager(text, title=self.__name) class _Helper(object): diff --git a/Lib/_strptime.py b/Lib/_strptime.py index d011ddf8b181c3c..59ac96745aa15e2 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -7,7 +7,7 @@ FUNCTIONS: _getlang -- Figure out what language is being used for the locale - strptime -- Calculates the time struct represented by the passed-in string + _strptime -- Calculates the time struct represented by the passed-in string """ import os @@ -238,7 +238,7 @@ def __calc_date_time(self): current_format = current_format.replace(tz, "%Z") # Transform all non-ASCII digits to digits in range U+0660 to U+0669. if not current_format.isascii() and self.LC_alt_digits is None: - current_format = re_sub(r'\d(?\d\d|\d| \d)' % d @@ -418,6 +421,8 @@ def __init__(self, locale_time=None): mapping['W'] = mapping['U'].replace('U', 'W') base.__init__(mapping) + base.__setitem__('D', self.pattern('%m/%d/%y')) + base.__setitem__('F', self.pattern('%Y-%m-%d')) base.__setitem__('T', self.pattern('%H:%M:%S')) base.__setitem__('R', self.pattern('%H:%M')) base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) @@ -459,7 +464,8 @@ def pattern(self, format): format = re_sub(r'\s+', r'\\s+', format) format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR year_in_format = False - day_of_month_in_format = False + day_d_in_format = False + day_e_in_format = False def repl(m): directive = m.group()[1:] # exclude `%` symbol match directive: @@ -467,20 +473,30 @@ def repl(m): nonlocal year_in_format year_in_format = True case 'd': - nonlocal day_of_month_in_format - day_of_month_in_format = True + nonlocal day_d_in_format + day_d_in_format = True + case 'e': + nonlocal day_e_in_format + day_e_in_format = True return self[directive] format = re_sub(r'%[-_0^#]*[0-9]*([OE]?[:\\]?.?)', repl, format) - if day_of_month_in_format and not year_in_format: - import warnings - warnings.warn("""\ + if not year_in_format: + if day_d_in_format: + raise ValueError( + "Day of month directive '%d' may not be used without " + "a year directive. Parsing dates involving a day of " + "month without a year is ambiguous and fails to parse " + "leap day. Add a year to the input and format. " + "See https://github.com/python/cpython/issues/70647.") + if day_e_in_format: + import warnings + warnings.warn("""\ Parsing dates involving a day of month without a year specified is ambiguous -and fails to parse leap day. The default behavior will change in Python 3.15 -to either always raise an exception or to use a different default year (TBD). -To avoid trouble, add a specific year to the input & format. +and fails to parse leap day. '%e' without a year will become an error in Python 3.17. +To avoid trouble, add a specific year to the input and format. See https://github.com/python/cpython/issues/70647.""", - DeprecationWarning, - skip_file_prefixes=(os.path.dirname(__file__),)) + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) return format def compile(self, format): @@ -516,9 +532,10 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): - """Return a 2-tuple consisting of a time struct and an int containing - the number of microseconds based on the input string and the - format string.""" + """Return a 3-tuple consisting of a tuple with time components, + an int containing the number of microseconds, and an int + containing the microseconds part of the GMT offset, based on the + input string and the format string.""" for index, arg in enumerate([data_string, format]): if not isinstance(arg, str): diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index 0b9e5d3bbf6ef64..2af3885458b54f2 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -57,7 +57,7 @@ def thread_deleted(_, idt=idt): # as soon as the OS-level thread ends instead. local = wrlocal() if local is not None: - dct = local.dicts.pop(idt) + local.dicts.pop(idt) wrlocal = ref(self, local_deleted) wrthread = ref(thread, thread_deleted) thread.__dict__[key] = wrlocal diff --git a/Lib/abc.py b/Lib/abc.py index f8a4e11ce9c3b1e..08f708d1b892085 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -36,11 +36,15 @@ class C(ABC): def my_abstract_classmethod(cls, ...): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True def __init__(self, callable): + import warnings + warnings._deprecated('abc.abstractclassmethod', remove=(3, 21)) callable.__isabstractmethod__ = True super().__init__(callable) @@ -56,11 +60,15 @@ class C(ABC): def my_abstract_staticmethod(...): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True def __init__(self, callable): + import warnings + warnings._deprecated('abc.abstractstaticmethod', remove=(3, 21)) callable.__isabstractmethod__ = True super().__init__(callable) @@ -76,10 +84,23 @@ class C(ABC): def my_abstract_property(self): ... + .. deprecated-removed: 3.3 3.21 + """ __isabstractmethod__ = True + def __init__( + self, + fget=None, + fset=None, + fdel=None, + doc=None, + ): + import warnings + warnings._deprecated('abc.abstractproperty', remove=(3, 21)) + super().__init__(fget, fset, fdel, doc) + try: from _abc import (get_cache_token, _abc_init, _abc_register, diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 33907b1fc2a53a5..8204c762cce8a2b 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -47,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -94,6 +95,7 @@ def __init__( # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -113,7 +115,7 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: @@ -224,29 +226,6 @@ def evaluate( new_locals.transmogrify(self.__cell__) return result - def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): - import typing - import warnings - - if type_params is _sentinel: - typing._deprecation_warning_for_no_type_params_passed( - "typing.ForwardRef._evaluate" - ) - type_params = () - warnings._deprecated( - "ForwardRef._evaluate", - "{name} is a private API and is retained for compatibility, but will be removed" - " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.", - remove=(3, 16), - ) - return typing.evaluate_forward_ref( - self, - globals=globalns, - locals=localns, - type_params=type_params, - _recursive_guard=recursive_guard, - ) - @property def __forward_arg__(self): if self.__arg__ is not None: @@ -258,6 +237,24 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: @@ -279,7 +276,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -293,7 +296,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - tuple(sorted(self.__cell__.items())) if isinstance(self.__cell__, dict) else self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -312,7 +318,7 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" _Template = type(t"") @@ -348,6 +354,7 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): @@ -844,14 +851,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: return None, None - freevars = annotate.__code__.co_freevars new_closure = [] cell_dict = {} - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): cell_dict[name] = cell new_cell = None if allow_evaluation: @@ -915,7 +917,7 @@ def get_annotations( does not exist, the __annotate__ function is called. The FORWARDREF format uses __annotations__ if it exists and can be evaluated, and otherwise falls back to calling the __annotate__ function. - The SOURCE format tries __annotate__ first, and falls back to + The STRING format tries __annotate__ first, and falls back to using __annotations__, stringified using annotations_to_string(). This function handles several details for you: @@ -1033,13 +1035,26 @@ def get_annotations( obj_globals = obj_locals = unwrap = None if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} while True: if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue if functools := sys.modules.get("functools"): if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue break if hasattr(unwrap, "__globals__"): @@ -1101,7 +1116,7 @@ def _rewrite_star_unpack(arg): """If the given argument annotation expression is a star unpack e.g. `'*Ts'` rewrite it to a valid expression. """ - if arg.startswith("*"): + if arg.lstrip().startswith("*"): return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] else: return arg @@ -1146,3 +1161,14 @@ def _get_dunder_annotations(obj): if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/argparse.py b/Lib/argparse.py index 6b79747572f48f8..29e6ebb9634261a 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -89,8 +89,10 @@ import os as _os import re as _re import sys as _sys +from gettext import gettext as _ +from gettext import ngettext -from gettext import gettext as _, ngettext +lazy import _colorize SUPPRESS = '==SUPPRESS==' @@ -148,10 +150,25 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== +class _ColorlessTheme: + # A 'fake' theme for no colors + def __getattr__(self, name): + # _colorize's no_color themes are just all empty strings + # by directly using empty strings the import is avoided + if name.startswith("_"): + raise AttributeError(name) + return "" + +_colorless_theme = _ColorlessTheme() + class HelpFormatter(object): """Formatter for generating usage messages and argument help strings. @@ -166,7 +183,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=True, ): # default setting for width if width is None: @@ -174,7 +190,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -191,15 +206,35 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') - def _set_color(self, color): - from _colorize import can_colorize, decolor, get_theme + self._set_color(False) + + def _set_color(self, color, *, file=None): + # Set a new color setting and file, clear caches for theme and decolor + self._theme_color = color + self._theme_file = file + self._cached_theme = None + self._cached_decolor = None - if color and can_colorize(): - self._theme = get_theme(force_color=True).argparse - self._decolor = decolor + def _get_theme_and_decolor(self): + # If self._theme_color is false, this prevents _colorize from importing + if self._theme_color and _colorize.can_colorize(file=self._theme_file): + self._cached_theme = _colorize.get_theme(force_color=True).argparse + self._cached_decolor = _colorize.decolor else: - self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text + self._cached_theme = _colorless_theme + self._cached_decolor = _identity + + @property + def _theme(self): + if self._cached_theme is None: + self._get_theme_and_decolor() + return self._cached_theme + + @property + def _decolor(self): + if self._cached_decolor is None: + self._get_theme_and_decolor() + return self._cached_decolor # =============================== # Section and indentation methods @@ -336,27 +371,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -413,117 +438,141 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') + """Get usage parts with split index for optionals/positionals. - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: text = text % dict(prog=self._prog) text_width = max(self._width - self._current_indent, 11) indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' + text = self._fill_text(text, text_width, indent) + text = self._apply_text_markup(text) + return text + '\n\n' + + def _apply_text_markup(self, text): + """Apply color markup to text. + + Supported markup: + `...` or ``...`` - inline code (rendered with prog_extra color) + + When colors are disabled, backticks are preserved as-is. + """ + t = self._theme + if not t.reset: + return text + text = _re.sub( + r'(`{1,2})([^`]+)\1', + rf'{t.prog_extra}\2{t.reset}', + text, + ) + return text def _format_action(self, action): # determine the required width and the entry label @@ -664,7 +713,7 @@ def _format_args(self, action, default_metavar): def _expand_help(self, action): help_string = self._get_help_string(action) if '%' not in help_string: - return help_string + return self._apply_text_markup(help_string) params = dict(vars(action), prog=self._prog) for name in list(params): value = params[name] @@ -674,7 +723,43 @@ def _expand_help(self, action): params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) - return help_string % params + + t = self._theme + + result = help_string % params + + if not t.reset: + return result + + # Match format specifiers like: %s, %d, %(key)s, etc. + fmt_spec = r''' + % + (?: + % # %% escape + | + (?:\((?P[^)]*)\))? # key + [-#0\ +]* # flags + (?:\*|\d+)? # width + (?:\.(?:\*|\d+))? # precision + [hlL]? # length modifier + [diouxXeEfFgGcrsa] # conversion type + ) + ''' + + def colorize(match): + spec, key = match.group(0, 'key') + if spec == '%%': + return '%' + if key is not None: + # %(key)... - format and colorize + formatted = spec % {key: params[key]} + return f'{t.interpolated_value}{formatted}{t.reset}' + # bare %s etc. - format with full params dict, no colorization + return spec % params + + return self._apply_text_markup( + _re.sub(fmt_spec, colorize, help_string, flags=_re.VERBOSE) + ) def _iter_indented_subactions(self, action): try: @@ -744,11 +829,21 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + t = self._theme + default_str = _(" (default: %(default)s)") + prefix, suffix = default_str.split("%(default)s") + help += ( + f" {t.default}{prefix.lstrip()}{t.reset}" + f"%(default)s" + f"{t.default}{suffix}{t.reset}" + ) return help @@ -932,15 +1027,26 @@ def __init__(self, deprecated=False): _option_strings = [] + neg_option_strings = [] for option_string in option_strings: _option_strings.append(option_string) - if option_string.startswith('--'): - if option_string.startswith('--no-'): + if len(option_string) > 2 and option_string[0] == option_string[1]: + # two-dash long option: '--foo' -> '--no-foo' + if option_string.startswith('no-', 2): raise ValueError(f'invalid option name {option_string!r} ' f'for BooleanOptionalAction') - option_string = '--no-' + option_string[2:] + option_string = option_string[:2] + 'no-' + option_string[2:] _option_strings.append(option_string) + neg_option_strings.append(option_string) + elif len(option_string) > 2 and option_string[0] != option_string[1]: + # single-dash long option: '-foo' -> '-nofoo' + if option_string.startswith('no', 1): + raise ValueError(f'invalid option name {option_string!r} ' + f'for BooleanOptionalAction') + option_string = option_string[:1] + 'no' + option_string[1:] + _option_strings.append(option_string) + neg_option_strings.append(option_string) super().__init__( option_strings=_option_strings, @@ -950,11 +1056,12 @@ def __init__(self, required=required, help=help, deprecated=deprecated) + self.neg_option_strings = neg_option_strings def __call__(self, parser, namespace, values, option_string=None): if option_string in self.option_strings: - setattr(namespace, self.dest, not option_string.startswith('--no-')) + setattr(namespace, self.dest, option_string not in self.neg_option_strings) def format_usage(self): return ' | '.join(self.option_strings) @@ -1551,8 +1658,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1660,29 +1767,35 @@ def _get_positional_kwargs(self, dest, **kwargs): def _get_optional_kwargs(self, *args, **kwargs): # determine short and long option strings option_strings = [] - long_option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: + if option_string[0] not in self.prefix_chars: raise ValueError( f'invalid option string {option_string!r}: ' f'must start with a character {self.prefix_chars!r}') - - # strings starting with two prefix characters are long options option_strings.append(option_string) - if len(option_string) > 1 and option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' dest = kwargs.pop('dest', None) if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) + priority = 0 + for option_string in option_strings: + if len(option_string) <= 2: + # short option: '-x' -> 'x' + if priority < 1: + dest = option_string.lstrip(self.prefix_chars) + priority = 1 + elif option_string[1] not in self.prefix_chars: + # single-dash long option: '-foo' -> 'foo' + if priority < 2: + dest = option_string.lstrip(self.prefix_chars) + priority = 2 + else: + # two-dash long option: '--foo' -> 'foo' + dest = option_string.lstrip(self.prefix_chars) + break if not dest: - msg = f'dest= is required for options like {option_string!r}' + msg = f'dest= is required for options like {repr(option_strings)[1:-1]}' raise TypeError(msg) dest = dest.replace('-', '_') @@ -1740,8 +1853,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1896,15 +2009,16 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) @@ -1957,14 +2071,16 @@ def add_subparsers(self, **kwargs): self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix + # non-required optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: # Create formatter without color to avoid storing ANSI codes in prog formatter = self.formatter_class(prog=self.prog) formatter._set_color(False) positionals = self._get_positional_actions() + required_optionals = [action for action in self._get_optional_actions() + if action.required] groups = self._mutually_exclusive_groups - formatter.add_usage(None, positionals, groups, '') + formatter.add_usage(None, required_optionals + positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list @@ -2431,7 +2547,7 @@ def _parse_optional(self, arg_string): return None # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: + if arg_string[0] not in self.prefix_chars: return None # if the option string is present in the parser, return the action @@ -2540,7 +2656,7 @@ def _get_nargs_pattern(self, action): # allow any number of options or arguments elif nargs == REMAINDER: - nargs_pattern = '([AO]*)' if option else '(.*)' + nargs_pattern = '(.*)' # allow one argument followed by any number of options or arguments elif nargs == PARSER: @@ -2675,7 +2791,7 @@ def _check_value(self, action, value): if value not in choices: args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} + 'choices': ', '.join(repr(str(choice)) for choice in action.choices)} msg = _('invalid choice: %(value)r (choose from %(choices)s)') if self.suggest_on_error and isinstance(value, str): @@ -2693,14 +2809,16 @@ def _check_value(self, action, value): # Help-formatting methods # ======================= - def format_usage(self): - formatter = self._get_formatter() + def format_usage(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() - def format_help(self): - formatter = self._get_formatter() + def format_help(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, @@ -2722,11 +2840,22 @@ def format_help(self): # determine help from format above return formatter.format_help() - def _get_formatter(self): + def _get_formatter(self, file=None): formatter = self.formatter_class(prog=self.prog) - formatter._set_color(self.color) + formatter._set_color(self.color, file=file) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + # Validation never renders output, so force color off to avoid + # importing _colorize during add_argument. + if self._cached_formatter is None: + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) + self._cached_formatter = formatter + return self._cached_formatter + # ===================== # Help-printing methods # ===================== @@ -2734,12 +2863,26 @@ def _get_formatter(self): def print_usage(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_usage(), file) + formatter = self._get_formatter(file=file) + try: + usage_text = self.format_usage(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + usage_text = self.format_usage() + self._print_message(usage_text, file) def print_help(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_help(), file) + formatter = self._get_formatter(file=file) + try: + help_text = self.format_help(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + help_text = self.format_help() + self._print_message(help_text, file) def _print_message(self, message, file=None): if message: @@ -2750,12 +2893,11 @@ def _print_message(self, message, file=None): pass def _get_theme(self, file=None): - from _colorize import can_colorize, get_theme - - if self.color and can_colorize(file=file): - return get_theme(force_color=True).argparse + # If self.color is False, _colorize is not imported + if self.color and _colorize.can_colorize(file=file): + return _colorize.get_theme(force_color=True).argparse else: - return get_theme(force_no_color=True).argparse + return _colorless_theme # =============== # Exiting methods diff --git a/Lib/ast.py b/Lib/ast.py index d9743ba7ab40b12..f445b32040e5fb2 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -21,7 +21,7 @@ :license: Python License. """ from _ast import * - +lazy import warnings def parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None): @@ -117,21 +117,34 @@ def _convert_literal(node): def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -166,7 +179,9 @@ def _format(node, level=0): field_type = cls._field_types.get(name, object) if field_type is expr_context: if not keywords: - args_buffer.append(repr(value)) + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') continue if not keywords: args.extend(args_buffer) @@ -174,7 +189,7 @@ def _format(node, level=0): value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -187,14 +202,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -608,9 +630,11 @@ def __new__(cls, dims=(), **kwargs): def _dims_getter(self): """Deprecated. Use elts instead.""" + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) return self.elts def _dims_setter(self, value): + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) self.elts = value Tuple.dims = property(_dims_getter, _dims_setter) @@ -642,9 +666,9 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -657,11 +681,11 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -687,8 +711,28 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) +_deprecated = { + 'slice': globals().pop("slice"), + 'Index': globals().pop("Index"), + 'ExtSlice': globals().pop("ExtSlice"), + 'Suite': globals().pop("Suite"), + 'AugLoad': globals().pop("AugLoad"), + 'AugStore': globals().pop("AugStore"), + 'Param': globals().pop("Param") +} + +def __getattr__(attr): + try: + val = _deprecated[attr] + except KeyError: + raise AttributeError(f"module 'ast' has no attribute {attr!r}") from None + warnings._deprecated(f"ast.{attr}", remove=(3, 21)) + return val + if __name__ == '__main__': main() diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 32a5dbae03af216..2432e2dad74c23d 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -47,28 +47,3 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ - -def __getattr__(name: str): - import warnings - - match name: - case "AbstractEventLoopPolicy": - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return events._AbstractEventLoopPolicy - case "DefaultEventLoopPolicy": - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - if sys.platform == 'win32': - return windows_events._DefaultEventLoopPolicy - return unix_events._DefaultEventLoopPolicy - case "WindowsSelectorEventLoopPolicy": - if sys.platform == 'win32': - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return windows_events._WindowsSelectorEventLoopPolicy - # Else fall through to the AttributeError below. - case "WindowsProactorEventLoopPolicy": - if sys.platform == 'win32': - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return windows_events._WindowsProactorEventLoopPolicy - # Else fall through to the AttributeError below. - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index d078ebfa4cedbe4..7f0565d0b8ddc76 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -12,13 +12,16 @@ import types import warnings -from _colorize import get_theme -from _pyrepl.console import InteractiveColoredConsole +try: + from _colorize import get_theme + from _pyrepl.console import InteractiveColoredConsole as InteractiveConsole +except ModuleNotFoundError: + from code import InteractiveConsole from . import futures -class AsyncIOInteractiveConsole(InteractiveColoredConsole): +class AsyncIOInteractiveConsole(InteractiveConsole): def __init__(self, locals, loop): super().__init__(locals, filename="") @@ -86,22 +89,27 @@ def run(self): global return_code try: - banner = ( - f'asyncio REPL {sys.version} on {sys.platform}\n' - f'Use "await" directly instead of "asyncio.run()".\n' - f'Type "help", "copyright", "credits" or "license" ' - f'for more information.\n' - ) + if not sys.flags.quiet: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) - console.write(banner) + console.write(banner) - if startup_path := os.getenv("PYTHONSTARTUP"): + if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: @@ -151,24 +159,35 @@ def interrupt(self) -> None: parser = argparse.ArgumentParser( prog="python3 -m asyncio", description="Interactive asyncio shell and CLI tools", - color=True, ) subparsers = parser.add_subparsers(help="sub-commands", dest="command") ps = subparsers.add_parser( "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + ps.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) pstree.add_argument("pid", type=int, help="Process ID to inspect") + pstree.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + asyncio.tools.display_awaited_by_tasks_table(args.pid, retries=args.retries) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + asyncio.tools.display_awaited_by_tasks_tree(args.pid, retries=args.retries) sys.exit(0) case None: pass # continue to the interactive shell @@ -184,7 +203,10 @@ def interrupt(self) -> None: if os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: - from _pyrepl.main import CAN_USE_PYREPL + try: + from _pyrepl.main import CAN_USE_PYREPL + except ModuleNotFoundError: + CAN_USE_PYREPL = False return_code = 0 loop = asyncio.new_event_loop() @@ -240,4 +262,5 @@ def interrupt(self) -> None: break console.write('exiting asyncio REPL...\n') + loop.close() sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 8cbb71f708537fe..93bd7df993d8270 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -14,21 +14,24 @@ """ import collections +import contextvars import collections.abc import concurrent.futures import errno import heapq import itertools +import math import os import socket import stat import subprocess +import sys import threading import time import traceback -import sys import warnings import weakref +import inspect try: import ssl @@ -289,6 +292,7 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, self._ssl_shutdown_timeout = ssl_shutdown_timeout self._serving = False self._serving_forever_fut = None + self._context = contextvars.copy_context() def __repr__(self): return f'<{self.__class__.__name__} sockets={self.sockets!r}>' @@ -318,7 +322,7 @@ def _start_serving(self): self._loop._start_serving( self._protocol_factory, sock, self._ssl_context, self, self._backlog, self._ssl_handshake_timeout, - self._ssl_shutdown_timeout) + self._ssl_shutdown_timeout, context=self._context) def get_loop(self): return self._loop @@ -380,6 +384,7 @@ async def serve_forever(self): except exceptions.CancelledError: try: self.close() + self.close_clients() await self.wait_closed() finally: raise @@ -483,10 +488,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -507,7 +512,8 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - call_connection_made=True): + call_connection_made=True, + context=None): """Create SSL transport.""" raise NotImplementedError @@ -721,8 +727,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -835,7 +841,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -949,7 +955,7 @@ async def sock_sendfile(self, sock, file, offset=0, count=None, try: return await self._sock_sendfile_native(sock, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise return await self._sock_sendfile_fallback(sock, file, @@ -963,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1070,12 +1076,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1211,9 +1217,10 @@ async def _create_connection_transport( self, sock, protocol_factory, ssl, server_hostname, server_side=False, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() protocol = protocol_factory() waiter = self.create_future() @@ -1223,9 +1230,10 @@ async def _create_connection_transport( sock, protocol, sslcontext, waiter, server_side=server_side, server_hostname=server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: - transport = self._make_socket_transport(sock, protocol, waiter) + transport = self._make_socket_transport(sock, protocol, waiter, context=context) try: await waiter @@ -1270,15 +1278,14 @@ async def sendfile(self, transport, file, offset=0, count=None, try: return await self._sendfile_native(transport, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise if not fallback: - raise RuntimeError( + raise exceptions.SendfileNotAvailableError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1287,9 +1294,12 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) - blocksize = min(count, 16384) if count else 16384 + blocksize = ( + min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) + if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE + ) buf = bytearray(blocksize) total_sent = 0 proto = _SendfileFallbackProtocol(transp) @@ -1345,6 +1355,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) @@ -1532,11 +1553,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. @@ -2011,7 +2032,10 @@ def _run_once(self): event_list = None # Handle 'later' callbacks that are ready. - end_time = self.time() + self._clock_resolution + now = self.time() + # Ensure that `end_time` is strictly increasing + # when the clock resolution is too small. + end_time = now + max(self._clock_resolution, math.ulp(now)) while self._scheduled: handle = self._scheduled[0] if handle._when >= end_time: diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 321a4e5d5d18fb1..224b1883808a412 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -265,7 +265,7 @@ def _try_finish(self): # to avoid hanging forever in self._wait as otherwise _exit_waiters # would never be woken up, we wake them up here. for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): @@ -278,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..3a4246f6ccd4c24 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,7 +1,6 @@ -__all__ = 'iscoroutinefunction', 'iscoroutine' +__all__ = ('iscoroutine',) import collections.abc -import inspect import os import sys import types @@ -13,25 +12,6 @@ def _is_debug_mode(): bool(os.environ.get('PYTHONASYNCIODEBUG'))) -# A marker for iscoroutinefunction. -_is_coroutine = object() - - -def iscoroutinefunction(func): - import warnings - """Return True if func is a decorated coroutine function.""" - warnings._deprecated("asyncio.iscoroutinefunction", - f"{warnings._DEPRECATED_MSG}; " - "use inspect.iscoroutinefunction() instead", - remove=(3,16)) - return _iscoroutinefunction(func) - - -def _iscoroutinefunction(func): - return (inspect.iscoroutinefunction(func) or - getattr(func, '_is_coroutine', None) is _is_coroutine) - - # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. _COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a7fb55982abe9ca..807c70bc775aa2c 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -9,8 +9,6 @@ "AbstractServer", "Handle", "TimerHandle", - "get_event_loop_policy", - "set_event_loop_policy", "get_event_loop", "set_event_loop", "new_event_loop", @@ -26,7 +24,6 @@ import subprocess import sys import threading -import warnings from . import format_helpers @@ -374,8 +371,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -415,8 +412,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -479,8 +477,9 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -511,8 +510,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -552,7 +551,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" @@ -662,84 +662,11 @@ def set_debug(self, enabled): raise NotImplementedError -class _AbstractEventLoopPolicy: - """Abstract policy for accessing the event loop.""" +class _Local(threading.local): + _loop = None - def get_event_loop(self): - """Get the event loop for the current context. - Returns an event loop object implementing the AbstractEventLoop interface, - or raises an exception in case no event loop has been set for the - current context and the current policy does not specify to create one. - - It should never return None.""" - raise NotImplementedError - - def set_event_loop(self, loop): - """Set the event loop for the current context to loop.""" - raise NotImplementedError - - def new_event_loop(self): - """Create and return a new event loop object according to this - policy's rules. If there's need to set this loop as the event loop for - the current context, set_event_loop must be called explicitly.""" - raise NotImplementedError - -class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): - """Default policy implementation for accessing the event loop. - - In this policy, each thread has its own event loop. However, we - only automatically create an event loop by default for the main - thread; other threads by default have no event loop. - - Other policies may have different rules (e.g. a single global - event loop, or automatically creating an event loop per thread, or - using some other notion of context to which an event loop is - associated). - """ - - _loop_factory = None - - class _Local(threading.local): - _loop = None - - def __init__(self): - self._local = self._Local() - - def get_event_loop(self): - """Get the event loop for the current context. - - Returns an instance of EventLoop or raises an exception. - """ - if self._local._loop is None: - raise RuntimeError('There is no current event loop in thread %r.' - % threading.current_thread().name) - - return self._local._loop - - def set_event_loop(self, loop): - """Set the event loop.""" - if loop is not None and not isinstance(loop, AbstractEventLoop): - raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") - self._local._loop = loop - - def new_event_loop(self): - """Create a new event loop. - - You must call set_event_loop() to make this the current event - loop. - """ - return self._loop_factory() - - -# Event loop policy. The policy itself is always global, even if the -# policy's rules say that there is an event loop per thread (or other -# notion of context). The default policy is installed by the first -# call to get_event_loop_policy(). -_event_loop_policy = None - -# Lock for protecting the on-the-fly creation of the event loop policy. -_lock = threading.Lock() +_local = _Local() # A TLS for the running event loop, used by _get_running_loop. @@ -784,39 +711,19 @@ def _set_running_loop(loop): _running_loop.loop_pid = (loop, os.getpid()) -def _init_event_loop_policy(): - global _event_loop_policy - with _lock: - if _event_loop_policy is None: # pragma: no branch - if sys.platform == 'win32': - from .windows_events import _DefaultEventLoopPolicy - else: - from .unix_events import _DefaultEventLoopPolicy - _event_loop_policy = _DefaultEventLoopPolicy() - +def _get_event_loop(): + """Return the event loop set for the current thread. -def _get_event_loop_policy(): - """Get the current event loop policy.""" - if _event_loop_policy is None: - _init_event_loop_policy() - return _event_loop_policy - -def get_event_loop_policy(): - warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) - return _get_event_loop_policy() - -def _set_event_loop_policy(policy): - """Set the current event loop policy. - - If policy is None, the default policy is restored.""" - global _event_loop_policy - if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): - raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") - _event_loop_policy = policy + Raise a RuntimeError if no event loop has been set for the current + thread. This is the slow path of get_event_loop(); the running loop + is checked by the caller. + """ + loop = _local._loop + if loop is None: + raise RuntimeError('There is no current event loop in thread %r.' + % threading.current_thread().name) + return loop -def set_event_loop_policy(policy): - warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) - _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. @@ -825,23 +732,33 @@ def get_event_loop(): or similar API), this function will always return the running event loop. If there is no running event loop set, the function will return - the result of `get_event_loop_policy().get_event_loop()` call. + the loop set by ``set_event_loop()``, or raise a RuntimeError if + no loop has been set. """ # NOTE: this function is implemented in C (see _asynciomodule.c) current_loop = _get_running_loop() if current_loop is not None: return current_loop - return _get_event_loop_policy().get_event_loop() + return _get_event_loop() def set_event_loop(loop): - """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - _get_event_loop_policy().set_event_loop(loop) + """Set the event loop for the current thread to loop. + + If loop is None, the current event loop is unset. + """ + if loop is not None and not isinstance(loop, AbstractEventLoop): + raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") + _local._loop = loop def new_event_loop(): - """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return _get_event_loop_policy().new_event_loop() + """Create and return a new event loop object.""" + if sys.platform == 'win32': + from .windows_events import EventLoop + else: + from .unix_events import EventLoop + return EventLoop() # Alias pure-Python implementations for testing purposes. @@ -870,8 +787,8 @@ def new_event_loop(): if hasattr(os, 'fork'): def on_fork(): # Reset the loop and wakeupfd in the forked child process. - if _event_loop_policy is not None: - _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() + global _local + _local = _Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 6bd00a644789f10..11858a0274a69fd 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -79,6 +79,10 @@ def __init__(self, *, loop=None): loop object used by the future. If it's not provided, the future uses the default event loop. """ + if self._loop is not None: + raise RuntimeError(f"{self.__class__.__name__} object is already " + "initialized") + if loop is None: self._loop = events.get_event_loop() else: @@ -389,7 +393,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -398,7 +402,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index b5bfeb1630a1590..35f7fa62140bd49 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -112,13 +112,13 @@ def capture_call_graph( optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. - If the optional keyword-only 'limit' argument is provided, each call stack - in the resulting graph is truncated to include at most ``abs(limit)`` - entries. If 'limit' is positive, the entries left are the closest to - the invocation point. If 'limit' is negative, the topmost entries are - left. If 'limit' is omitted or None, all entries are present. - If 'limit' is 0, the call stack is not captured at all, only - "awaited by" information is present. + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. """ loop = events._get_running_loop() diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index fa3a94764b507ae..c59ea15111bef2b 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -145,10 +145,7 @@ def _wake_up_first(self): """Ensure that the first waiter will wake up.""" if not self._waiters: return - try: - fut = next(iter(self._waiters)) - except StopIteration: - return + fut = next(iter(self._waiters)) # .done() means that the waiter is already set to wake up. if not fut.done(): @@ -158,10 +155,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -353,9 +350,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +508,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index f404273c3ae5c1d..cf2902b4c76559e 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -642,7 +642,7 @@ def __init__(self, proactor): signal.set_wakeup_fd(self._csock.fileno()) def _make_socket_transport(self, sock, protocol, waiter=None, - extra=None, server=None): + extra=None, server=None, context=None): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) @@ -651,7 +651,7 @@ def _make_ssl_transport( *, server_side=False, server_hostname=None, extra=None, server=None, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, @@ -733,7 +733,7 @@ async def sock_accept(self, sock): async def _sock_sendfile_native(self, sock, file, offset, count): try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size @@ -756,8 +756,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() @@ -837,7 +836,7 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=None, - ssl_shutdown_timeout=None): + ssl_shutdown_timeout=None, context=None): def loop(f=None): try: diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 084fccaaff2ff7c..30004f2bc9bacd2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,11 +33,11 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ @@ -174,8 +174,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): if self._is_shutdown and self.empty(): @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,12 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -239,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index ba37e003a655c0a..774a317564df22c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -35,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index ff7e16df3c62737..4fa4dd5a885236e 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -53,7 +53,7 @@ def _test_selector_event(selector, fd, event): class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. - See events.EventLoop for API specification. + See events.AbstractEventLoop for API specification. """ def __init__(self, selector=None): @@ -67,10 +67,10 @@ def __init__(self, selector=None): self._transports = weakref.WeakValueDictionary() def _make_socket_transport(self, sock, protocol, waiter=None, *, - extra=None, server=None): + extra=None, server=None, context=None): self._ensure_fd_no_transport(sock) return _SelectorSocketTransport(self, sock, protocol, waiter, - extra, server) + extra, server, context=context) def _make_ssl_transport( self, rawsock, protocol, sslcontext, waiter=None, @@ -78,16 +78,17 @@ def _make_ssl_transport( extra=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, + context=None, ): self._ensure_fd_no_transport(rawsock) ssl_protocol = sslproto.SSLProtocol( self, protocol, sslcontext, waiter, server_side, server_hostname, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout + ssl_shutdown_timeout=ssl_shutdown_timeout, ) _SelectorSocketTransport(self, rawsock, ssl_protocol, - extra=extra, server=server) + extra=extra, server=server, context=context) return ssl_protocol._app_transport def _make_datagram_transport(self, sock, protocol, @@ -159,16 +160,16 @@ def _write_to_self(self): def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): self._add_reader(sock.fileno(), self._accept_connection, protocol_factory, sock, sslcontext, server, backlog, - ssl_handshake_timeout, ssl_shutdown_timeout) + ssl_handshake_timeout, ssl_shutdown_timeout, context) def _accept_connection( self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -204,21 +205,22 @@ def _accept_connection( self._start_serving, protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, - ssl_shutdown_timeout) + ssl_shutdown_timeout, context) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} + conn_context = context.copy() if context is not None else None accept = self._accept_connection2( protocol_factory, conn, extra, sslcontext, server, - ssl_handshake_timeout, ssl_shutdown_timeout) - self.create_task(accept) + ssl_handshake_timeout, ssl_shutdown_timeout, context=conn_context) + self.create_task(accept, context=conn_context) async def _accept_connection2( self, protocol_factory, conn, extra, sslcontext=None, server=None, ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT, - ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT): + ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT, context=None): protocol = None transport = None try: @@ -229,11 +231,12 @@ async def _accept_connection2( conn, protocol, sslcontext, waiter=waiter, server_side=True, extra=extra, server=server, ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout) + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, - server=server) + server=server, context=context) try: await waiter @@ -275,9 +278,9 @@ def _ensure_fd_no_transport(self, fd): f'File descriptor {fd!r} is used by transport ' f'{transport!r}') - def _add_reader(self, fd, callback, *args): + def _add_reader(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_READ, @@ -309,9 +312,9 @@ def _remove_reader(self, fd): else: return False - def _add_writer(self, fd, callback, *args): + def _add_writer(self, fd, callback, *args, context=None): self._check_closed() - handle = events.Handle(callback, args, self, None) + handle = events.Handle(callback, args, self, context=context) key = self._selector.get_map().get(fd) if key is None: self._selector.register(fd, selectors.EVENT_WRITE, @@ -530,11 +533,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -583,11 +587,12 @@ def _sock_sendall(self, fut, sock, view, pos): async def sock_sendto(self, sock, data, address): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -698,10 +703,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -770,7 +776,7 @@ class _SelectorTransport(transports._FlowControlMixin, # exception) _sock = None - def __init__(self, loop, sock, protocol, extra=None, server=None): + def __init__(self, loop, sock, protocol, extra=None, server=None, context=None): super().__init__(extra, loop) self._extra['socket'] = trsock.TransportSocket(sock) try: @@ -784,12 +790,13 @@ def __init__(self, loop, sock, protocol, extra=None, server=None): self._extra['peername'] = None self._sock = sock self._sock_fd = sock.fileno() - + self._context = context self._protocol_connected = False self.set_protocol(protocol) self._server = server self._buffer = collections.deque() + self._buffer_size = 0 self._conn_lost = 0 # Set when call to connection_lost scheduled. self._closing = False # Set when close() called. self._paused = False # Set when pause_reading() called @@ -866,7 +873,7 @@ def close(self): if not self._buffer: self._conn_lost += 1 self._loop._remove_writer(self._sock_fd) - self._loop.call_soon(self._call_connection_lost, None) + self._call_soon(self._call_connection_lost, None) def __del__(self, _warn=warnings.warn): if self._sock is not None: @@ -894,12 +901,13 @@ def _force_close(self, exc): return if self._buffer: self._buffer.clear() + self._buffer_size = 0 self._loop._remove_writer(self._sock_fd) if not self._closing: self._closing = True self._loop._remove_reader(self._sock_fd) self._conn_lost += 1 - self._loop.call_soon(self._call_connection_lost, exc) + self._call_soon(self._call_connection_lost, exc) def _call_connection_lost(self, exc): try: @@ -916,13 +924,18 @@ def _call_connection_lost(self, exc): self._server = None def get_write_buffer_size(self): - return sum(map(len, self._buffer)) + return self._buffer_size def _add_reader(self, fd, callback, *args): if not self.is_reading(): return - self._loop._add_reader(fd, callback, *args) + self._loop._add_reader(fd, callback, *args, context=self._context) + def _add_writer(self, fd, callback, *args): + self._loop._add_writer(fd, callback, *args, context=self._context) + + def _call_soon(self, callback, *args): + self._loop.call_soon(callback, *args, context=self._context) class _SelectorSocketTransport(_SelectorTransport): @@ -930,10 +943,9 @@ class _SelectorSocketTransport(_SelectorTransport): _sendfile_compatible = constants._SendfileMode.TRY_NATIVE def __init__(self, loop, sock, protocol, waiter=None, - extra=None, server=None): - + extra=None, server=None, context=None): self._read_ready_cb = None - super().__init__(loop, sock, protocol, extra, server) + super().__init__(loop, sock, protocol, extra, server, context) self._eof = False self._empty_waiter = None if _HAS_SENDMSG: @@ -945,14 +957,12 @@ def __init__(self, loop, sock, protocol, waiter=None, # decreases the latency (in some cases significantly.) base_events._set_nodelay(self._sock) - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def set_protocol(self, protocol): if isinstance(protocol, protocols.BufferedProtocol): @@ -1081,10 +1091,11 @@ def write(self, data): if not data: return # Not all was written; register write handler. - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) # Add it to the buffer. self._buffer.append(data) + self._buffer_size += len(data) self._maybe_pause_protocol() def _get_sendmsg_buffer(self): @@ -1104,6 +1115,7 @@ def _write_sendmsg(self): except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1119,6 +1131,7 @@ def _write_sendmsg(self): self._sock.shutdown(socket.SHUT_WR) def _adjust_leftover_buffer(self, nbytes: int) -> None: + self._buffer_size -= nbytes buffer = self._buffer while nbytes: b = buffer.popleft() @@ -1139,13 +1152,16 @@ def _write_send(self): if n != len(buffer): # Not all data was written self._buffer.appendleft(buffer[n:]) + self._buffer_size -= n except (BlockingIOError, InterruptedError): - pass + self._buffer.appendleft(buffer) + return except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: self._loop._remove_writer(self._sock_fd) self._buffer.clear() + self._buffer_size = 0 self._fatal_error(exc, 'Fatal write error on socket transport') if self._empty_waiter is not None: self._empty_waiter.set_exception(exc) @@ -1181,11 +1197,13 @@ def writelines(self, list_of_data): self._conn_lost += 1 return - self._buffer.extend([memoryview(data) for data in list_of_data]) + for data in list_of_data: + self._buffer.append(memoryview(data)) + self._buffer_size += len(data) self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) + self._add_writer(self._sock_fd, self._write_ready) self._maybe_pause_protocol() def can_write_eof(self): @@ -1218,7 +1236,6 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): - _buffer_factory = collections.deque _header_size = 8 def __init__(self, loop, sock, protocol, address=None, @@ -1226,14 +1243,12 @@ def __init__(self, loop, sock, protocol, address=None, super().__init__(loop, sock, protocol, extra) self._address = address self._buffer_size = 0 - self._loop.call_soon(self._protocol.connection_made, self) + self._call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called - self._loop.call_soon(self._add_reader, - self._sock_fd, self._read_ready) + self._call_soon(self._add_reader, self._sock_fd, self._read_ready) if waiter is not None: # only wake up the waiter when connection_made() has been called - self._loop.call_soon(futures._set_result_unless_cancelled, - waiter, None) + self._call_soon(futures._set_result_unless_cancelled, waiter, None) def get_write_buffer_size(self): return self._buffer_size @@ -1280,7 +1295,7 @@ def sendto(self, data, addr=None): self._sock.sendto(data, addr) return except (BlockingIOError, InterruptedError): - self._loop._add_writer(self._sock_fd, self._sendto_ready) + self._add_writer(self._sock_fd, self._sendto_ready) except OSError as exc: self._protocol.error_received(exc) return diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 59e22f523a85c5f..a28c11e928f806a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -539,17 +539,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. @@ -667,8 +667,7 @@ async def readuntil(self, separator=b'\n'): # adds data which makes separator be found. That's why we check for # EOF *after* inspecting the buffer. if self._eof: - chunk = bytes(self._buffer) - self._buffer.clear() + chunk = self._buffer.take_bytes() raise exceptions.IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. @@ -678,10 +677,9 @@ async def readuntil(self, separator=b'\n'): raise exceptions.LimitOverrunError( 'Separator is found, but chunk is longer than limit', match_start) - chunk = self._buffer[:match_end] - del self._buffer[:match_end] + chunk = self._buffer.take_bytes(match_end) self._maybe_resume_transport() - return bytes(chunk) + return chunk async def read(self, n=-1): """Read up to `n` bytes from the stream. @@ -716,20 +714,16 @@ async def read(self, n=-1): # collect everything in self._buffer, but that would # deadlock if the subprocess sends more than self.limit # bytes. So just call self.read(self._limit) until EOF. - blocks = [] - while True: - block = await self.read(self._limit) - if not block: - break - blocks.append(block) - return b''.join(blocks) + joined = bytearray() + while block := await self.read(self._limit): + joined += block + return joined.take_bytes() if not self._buffer and not self._eof: await self._wait_for_data('read') # This will work right even if buffer is less than n bytes - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(min(len(self._buffer), n)) self._maybe_resume_transport() return data @@ -760,18 +754,12 @@ async def readexactly(self, n): while len(self._buffer) < n: if self._eof: - incomplete = bytes(self._buffer) - self._buffer.clear() + incomplete = self._buffer.take_bytes() raise exceptions.IncompleteReadError(incomplete, n) await self._wait_for_data('readexactly') - if len(self._buffer) == n: - data = bytes(self._buffer) - self._buffer.clear() - else: - data = bytes(memoryview(self._buffer)[:n]) - del self._buffer[:n] + data = self._buffer.take_bytes(n) self._maybe_resume_transport() return data diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a68b2..e1ec025791a52e6 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -178,6 +181,9 @@ async def _aexit(self, et, exc): finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -278,3 +284,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause + the task group context manager to exit *without* + `asyncio.CancelledError` being raised. + + If `cancel()` is called before entering the task group, the group + will be cancelled upon entry. This is useful for patterns where + one piece of code passes an unused TaskGroup instance to another in + order to have the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fbd5c39a7c56ac4..9d20930dc300c67 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -524,6 +524,7 @@ def _on_completion(f): timeout_handle.cancel() for f in fs: f.remove_done_callback(_on_completion) + futures.future_discard_from_awaited_by(f, cur_task) done, pending = set(), set() for f in fs: @@ -624,10 +625,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -643,26 +645,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -1030,21 +1033,22 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de166..5001351b0976ecd 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 09342dc7c1310b0..65ddc285abd971d 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -25,7 +25,8 @@ class _State(enum.Enum): class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ def __init__(self, when: float | None) -> None: diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index f39e11fdd513b4d..2ac1738d15c6c72 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,6 +1,6 @@ """Tools to analyze tasks running in asyncio programs.""" -from collections import defaultdict, namedtuple +from collections import defaultdict from itertools import count from enum import Enum import sys @@ -27,10 +27,10 @@ def __init__( # ─── indexing helpers ─────────────────────────────────────────── def _format_stack_entry(elem: str|FrameInfo) -> str: if not isinstance(elem, str): - if elem.lineno == 0 and elem.filename == "": + if elem.location.lineno == 0 and elem.filename == "": return f"{elem.funcname}" else: - return f"{elem.funcname} {elem.filename}:{elem.lineno}" + return f"{elem.funcname} {elem.filename}:{elem.location.lineno}" return elem @@ -231,27 +231,38 @@ def exit_with_permission_help_text(): print( "Error: The specified process cannot be attached to due to insufficient permissions.\n" "See the Python documentation for details on required privileges and troubleshooting:\n" - "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + "https://docs.python.org/3/howto/remote_debugging.html#permission-requirements\n", + file=sys.stderr, ) sys.exit(1) -def _get_awaited_by_tasks(pid: int) -> list: - try: - return get_all_awaited_by(pid) - except RuntimeError as e: - while e.__context__ is not None: - e = e.__context__ - print(f"Error retrieving tasks: {e}") - sys.exit(1) - except PermissionError as e: - exit_with_permission_help_text() +_TRANSIENT_ERRORS = (RuntimeError, OSError, UnicodeDecodeError, MemoryError) + + +def _get_awaited_by_tasks(pid: int, retries: int = 3) -> list: + for attempt in range(retries + 1): + try: + return get_all_awaited_by(pid) + except PermissionError: + exit_with_permission_help_text() + except ProcessLookupError: + print(f"Error: process {pid} not found.", file=sys.stderr) + sys.exit(1) + except _TRANSIENT_ERRORS as e: + if attempt < retries: + continue + if isinstance(e, RuntimeError): + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}", file=sys.stderr) + sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +def display_awaited_by_tasks_table(pid: int, retries: int = 3) -> None: """Build and print a table of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) table = build_task_table(tasks) # Print the table in a simple tabular format print( @@ -262,10 +273,10 @@ def display_awaited_by_tasks_table(pid: int) -> None: print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") -def display_awaited_by_tasks_tree(pid: int) -> None: +def display_awaited_by_tasks_tree(pid: int, retries: int = 3) -> None: """Build and print a tree of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) try: result = build_async_tree(tasks) except CycleFoundException as e: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 1c1458127db5ac1..93dd8994e1bcd51 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -12,6 +12,7 @@ import sys import threading import warnings +import inspect from . import base_events from . import base_subprocess @@ -54,7 +55,8 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): @@ -94,7 +96,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -359,7 +361,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): "os.sendfile() is not available") try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size @@ -384,12 +386,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -423,20 +425,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -447,9 +449,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -834,7 +836,8 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + if (stdin == subprocess.PIPE + and (sys.platform.startswith('aix') or sys.platform == 'cygwin')): # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the @@ -886,8 +889,8 @@ def _do_wait(self, pid, pidfd, callback, args): pid) else: returncode = waitstatus_to_exitcode(status) - - os.close(pidfd) + finally: + os.close(pidfd) callback(pid, returncode, *args) class _ThreadedChildWatcher: @@ -962,11 +965,5 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - """UNIX event loop policy""" - _loop_factory = _UnixSelectorEventLoop - - SelectorEventLoop = _UnixSelectorEventLoop -_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..8241c10b32d105c 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -16,7 +16,6 @@ import time import weakref -from . import events from . import base_subprocess from . import futures from . import exceptions @@ -29,8 +28,7 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', - '_WindowsProactorEventLoopPolicy', 'EventLoop', + 'EventLoop', ) @@ -610,6 +608,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, @@ -890,14 +891,4 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop - -class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - _loop_factory = SelectorEventLoop - - -class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - _loop_factory = ProactorEventLoop - - -_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e291c3..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, @@ -104,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index f95132a42740517..fa562f74a810345 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,10 +1,9 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere -import struct import binascii @@ -26,6 +25,8 @@ ] +_NOT_SPECIFIED = ['NOT SPECIFIED'] + bytes_types = (bytes, bytearray) # Types acceptable as binary data def _bytes_from_decode_data(s): @@ -45,44 +46,90 @@ def _bytes_from_decode_data(s): # Base64 encoding/decoding uses binascii -def b64encode(s, altchars=None): +def b64encode(s, altchars=None, *, padded=True, wrapcol=0): """Encode the bytes-like object s using Base64 and return a bytes object. Optional altchars should be a byte string of length 2 which specifies an alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. + + If padded is false, omit padding in the output. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: - assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b'+/', altchars)) - return encoded + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False, + alphabet=alphabet) + return binascii.b2a_base64(s, padded=padded, wrapcol=wrapcol, newline=False) -def b64decode(s, altchars=None, validate=False): +def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 which specifies the alternative alphabet used instead of the '+' and '/' characters. + If padded is false, padding in input is not required. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded. - If validate is False (the default), characters that are neither in the - normal base-64 alphabet nor the alternative alphabet are discarded prior - to the padding check. If validate is True, these non-alphabet characters - in the input result in a binascii.Error. + If ignorechars is specified, it should be a byte string containing + characters to ignore from the input. The default value of validate is + True if ignorechars is specified, False otherwise. + + If validate is false, characters that are neither in the normal base-64 + alphabet nor the alternative alphabet are discarded prior to the + padding check. If validate is true, these non-alphabet characters in + the input result in a binascii.Error if they are not in ignorechars. For more information about the strict base64 check, see: https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64 """ s = _bytes_from_decode_data(s) + if validate is _NOT_SPECIFIED: + validate = ignorechars is not _NOT_SPECIFIED + badchar = None if altchars is not None: altchars = _bytes_from_decode_data(altchars) - assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b'+/')) - return binascii.a2b_base64(s, strict_mode=validate) + if len(altchars) != 2: + raise ValueError(f'invalid altchars: {altchars!r}') + if ignorechars is _NOT_SPECIFIED: + for b in b'+/': + if b not in altchars and b in s: + badchar = b + break + s = s.translate(bytes.maketrans(altchars, b'+/')) + else: + alphabet = binascii.BASE64_ALPHABET[:-2] + altchars + return binascii.a2b_base64(s, strict_mode=validate, + alphabet=alphabet, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if ignorechars is _NOT_SPECIFIED: + ignorechars = b'' + result = binascii.a2b_base64(s, strict_mode=validate, + padded=padded, ignorechars=ignorechars, + canonical=canonical) + if badchar is not None: + import warnings + if validate: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=True ' + f'will be an error in future Python versions', + DeprecationWarning, stacklevel=2) + else: + warnings.warn(f'invalid character {chr(badchar)!a} in Base64 data ' + f'with altchars={altchars!r} and validate=False ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result def standard_b64encode(s): @@ -103,19 +150,21 @@ def standard_b64decode(s): return b64decode(s) -_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') -def urlsafe_b64encode(s): +def urlsafe_b64encode(s, *, padded=True): """Encode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object to encode. The result is returned as a bytes object. The alphabet uses '-' instead of '+' and '_' instead of '/'. + + If padded is false, omit padding in the output. """ - return b64encode(s).translate(_urlsafe_encode_translation) + return binascii.b2a_base64(s, padded=padded, newline=False, + alphabet=binascii.URLSAFE_BASE64_ALPHABET) -def urlsafe_b64decode(s): +def urlsafe_b64decode(s, *, padded=False): """Decode bytes using the URL- and filesystem-safe Base64 alphabet. Argument s is a bytes-like object or ASCII string to decode. The result @@ -124,30 +173,53 @@ def urlsafe_b64decode(s): alphabet, and are not a plus '+' or slash '/', are discarded prior to the padding check. + If padded is false, padding in input is not required. + The alphabet uses '-' instead of '+' and '_' instead of '/'. """ s = _bytes_from_decode_data(s) + badchar = None + for b in b'+/': + if b in s: + badchar = b + break s = s.translate(_urlsafe_decode_translation) - return b64decode(s) + result = binascii.a2b_base64(s, strict_mode=False, padded=padded) + if badchar is not None: + import warnings + warnings.warn(f'invalid character {chr(badchar)!a} in URL-safe Base64 data ' + f'will be discarded in future Python versions', + FutureWarning, stacklevel=2) + return result # Base32 encoding/decoding must be done in Python _B32_ENCODE_DOCSTRING = ''' Encode the bytes-like objects using {encoding} and return a bytes object. + +If padded is false, omit padding in the output. + +If wrapcol is non-zero, insert a newline (b'\\n') character after at most +every wrapcol characters. ''' _B32_DECODE_DOCSTRING = ''' Decode the {encoding} encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + +If padded is false, padding in input is not required. + +ignorechars should be a byte string containing characters to ignore +from the input. {extra_args} The result is returned as a bytes object. A binascii.Error is raised if the input is incorrectly padded or if there are non-alphabet characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -155,374 +227,183 @@ def urlsafe_b64decode(s): the letter O). For security purposes the default is None, so that 0 and 1 are not allowed in the input. ''' -_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' -_b32hexalphabet = b'0123456789ABCDEFGHIJKLMNOPQRSTUV' -_b32tab2 = {} -_b32rev = {} - -def _b32encode(alphabet, s): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32tab2: - b32tab = [bytes((i,)) for i in alphabet] - _b32tab2[alphabet] = [a + b for a in b32tab for b in b32tab] - b32tab = None - - if not isinstance(s, bytes_types): - s = memoryview(s).tobytes() - leftover = len(s) % 5 - # Pad the last quantum with zero bits if necessary - if leftover: - s = s + b'\0' * (5 - leftover) # Don't use += ! - encoded = bytearray() - from_bytes = int.from_bytes - b32tab2 = _b32tab2[alphabet] - for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5]) # big endian - encoded += (b32tab2[c >> 30] + # bits 1 - 10 - b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 - b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 - b32tab2[c & 0x3ff] # bits 31 - 40 - ) - # Adjust for any leftover partial quanta - if leftover == 1: - encoded[-6:] = b'======' - elif leftover == 2: - encoded[-4:] = b'====' - elif leftover == 3: - encoded[-3:] = b'===' - elif leftover == 4: - encoded[-1:] = b'=' - return bytes(encoded) - -def _b32decode(alphabet, s, casefold=False, map01=None): - # Delay the initialization of the table to not waste memory - # if the function is never called - if alphabet not in _b32rev: - _b32rev[alphabet] = {v: k for k, v in enumerate(alphabet)} + +def b32encode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) +b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') + +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) - if len(s) % 8: - raise binascii.Error('Incorrect padding') # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be # either L (el) or I (eye). if map01 is not None: map01 = _bytes_from_decode_data(map01) - assert len(map01) == 1, repr(map01) s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - # Strip off pad characters from the right. We need to count the pad - # characters because this will tell us how many null bytes to remove from - # the end of the decoded string. - l = len(s) - s = s.rstrip(b'=') - padchars = l - len(s) - # Now decode the full quanta - decoded = bytearray() - b32rev = _b32rev[alphabet] - for i in range(0, len(s), 8): - quanta = s[i: i + 8] - acc = 0 - try: - for c in quanta: - acc = (acc << 5) + b32rev[c] - except KeyError: - raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5) # big endian - # Process the last, partial quanta - if l % 8 or padchars not in {0, 1, 3, 4, 6}: - raise binascii.Error('Incorrect padding') - if padchars and decoded: - acc <<= 5 * padchars - last = acc.to_bytes(5) # big endian - leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 - decoded[-5:] = last[:leftover] - return bytes(decoded) - - -def b32encode(s): - return _b32encode(_b32alphabet, s) -b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') - -def b32decode(s, casefold=False, map01=None): - return _b32decode(_b32alphabet, s, casefold, map01) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) -def b32hexencode(s): - return _b32encode(_b32hexalphabet, s) +def b32hexencode(s, *, padded=True, wrapcol=0): + return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol, + alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): + s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping - return _b32decode(_b32hexalphabet, s, casefold) + if casefold: + s = s.upper() + return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. -def b16encode(s): +def b16encode(s, *, wrapcol=0): """Encode the bytes-like object s using Base16 and return a bytes object. + + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. """ - return binascii.hexlify(s).upper() + if not wrapcol: + return binascii.hexlify(s).upper() + if wrapcol < 0: + raise ValueError('Negative wrapcol') + if wrapcol < 2: + wrapcol = 2 + return binascii.hexlify(s, bytes_per_sep=-(wrapcol//2), sep=b'\n').upper() -def b16decode(s, casefold=False): +def b16decode(s, casefold=False, *, ignorechars=b''): """Decode the Base16 encoded bytes-like object or ASCII string s. Optional casefold is a flag specifying whether a lowercase alphabet is acceptable as input. For security purposes, the default is False. + ignorechars should be a byte string containing characters to ignore + from the input. + The result is returned as a bytes object. A binascii.Error is raised if s is incorrectly padded or if there are non-alphabet characters present in the input. """ - s = _bytes_from_decode_data(s) - if casefold: - s = s.upper() - if s.translate(None, delete=b'0123456789ABCDEF'): - raise binascii.Error('Non-base16 digit found') - return binascii.unhexlify(s) + if not casefold: + s = _bytes_from_decode_data(s) + if not isinstance(ignorechars, bytes): + ignorechars = bytes(memoryview(ignorechars)) + for b in b'abcdef': + if b in s and b not in ignorechars: + raise binascii.Error('Non-base16 digit found') + s = s.translate(None, delete=b'abcdef') + return binascii.unhexlify(s, ignorechars=ignorechars) # # Ascii85 encoding/decoding # - -_a85chars = None -_a85chars2 = None -_A85START = b"<~" -_A85END = b"~>" - -def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): - # Helper function for a85encode and b85encode - if not isinstance(b, bytes_types): - b = memoryview(b).tobytes() - - padding = (-len(b)) % 4 - if padding: - b = b + b'\0' * padding - words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) - - chunks = [b'z' if foldnuls and not word else - b'y' if foldspaces and word == 0x20202020 else - (chars2[word // 614125] + - chars2[word // 85 % 7225] + - chars[word % 85]) - for word in words] - - if padding and not pad: - if chunks[-1] == b'z': - chunks[-1] = chars[0] * 5 - chunks[-1] = chunks[-1][:-padding] - - return b''.join(chunks) - def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): """Encode bytes-like object b using Ascii85 and return a bytes object. foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. - wrapcol controls whether the output should have newline (b'\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long, excluding the trailing newline. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. - """ - global _a85chars, _a85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _a85chars2 is None: - _a85chars = [bytes((i,)) for i in range(33, 118)] - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - - result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) - - if adobe: - result = _A85START + result - if wrapcol: - wrapcol = max(2 if adobe else 1, wrapcol) - chunks = [result[i: i + wrapcol] - for i in range(0, len(result), wrapcol)] - if adobe: - if len(chunks[-1]) + 2 > wrapcol: - chunks.append(b'') - result = b'\n'.join(chunks) - if adobe: - result += _A85END + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - return result + """ + return binascii.b2a_ascii85(b, foldspaces=foldspaces, + adobe=adobe, wrapcol=wrapcol, pad=pad) -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - b = _bytes_from_decode_data(b) - if adobe: - if not b.endswith(_A85END): - raise ValueError( - "Ascii85 encoded byte sequences must end " - "with {!r}".format(_A85END) - ) - if b.startswith(_A85START): - b = b[2:-2] # Strip off start/end markers - else: - b = b[:-2] - # - # We have to go through this stepwise, so as to ignore spaces and handle - # special short sequences - # - packI = struct.Struct('!I').pack - decoded = [] - decoded_append = decoded.append - curr = [] - curr_append = curr.append - curr_clear = curr.clear - for x in b + b'u' * 4: - if b'!'[0] <= x <= b'u'[0]: - curr_append(x) - if len(curr) == 5: - acc = 0 - for x in curr: - acc = 85 * acc + (x - 33) - try: - decoded_append(packI(acc)) - except struct.error: - raise ValueError('Ascii85 overflow') from None - curr_clear() - elif x == b'z'[0]: - if curr: - raise ValueError('z inside Ascii85 5-tuple') - decoded_append(b'\0\0\0\0') - elif foldspaces and x == b'y'[0]: - if curr: - raise ValueError('y inside Ascii85 5-tuple') - decoded_append(b'\x20\x20\x20\x20') - elif x in ignorechars: - # Skip whitespace - continue - else: - raise ValueError('Non-Ascii85 digit found: %c' % x) - - result = b''.join(decoded) - padding = 4 - len(curr) - if padding: - # Throw away the extra padding - result = result[:-padding] - return result - -# The following code is originally taken (with permission) from Mercurial + return binascii.a2b_ascii85(b, foldspaces=foldspaces, + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) -_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" - b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") -_b85chars = None -_b85chars2 = None -_b85dec = None - -def b85encode(b, pad=False): +def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ - global _b85chars, _b85chars2 - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85chars2 is None: - _b85chars = [bytes((i,)) for i in _b85alphabet] - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - return _85encode(b, _b85chars, _b85chars2, pad) - -def b85decode(b): + return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) + +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - global _b85dec - # Delay the initialization of tables to not waste memory - # if the function is never called - if _b85dec is None: - # we don't assign to _b85dec directly to avoid issues when - # multiple threads call this function simultaneously - b85dec_tmp = [None] * 256 - for i, c in enumerate(_b85alphabet): - b85dec_tmp[c] = i - _b85dec = b85dec_tmp - - b = _bytes_from_decode_data(b) - padding = (-len(b)) % 5 - b = b + b'~' * padding - out = [] - packI = struct.Struct('!I').pack - for i in range(0, len(b), 5): - chunk = b[i:i + 5] - acc = 0 - try: - for c in chunk: - acc = acc * 85 + _b85dec[c] - except TypeError: - for j, c in enumerate(chunk): - if _b85dec[c] is None: - raise ValueError('bad base85 character at position %d' - % (i + j)) from None - raise - try: - out.append(packI(acc)) - except struct.error: - raise ValueError('base85 overflow in hunk starting at byte %d' - % i) from None - - result = b''.join(out) - if padding: - result = result[:-padding] - return result + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) + +def z85encode(s, pad=False, *, wrapcol=0): + """Encode bytes-like object b in z85 format and return a bytes object. -_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz' - b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#') -# Translating b85 valid but z85 invalid chars to b'\x00' is required -# to prevent them from being decoded as b85 valid chars. -_z85_b85_decode_diff = b';_`|~' -_z85_decode_translation = bytes.maketrans( - _z85alphabet + _z85_b85_decode_diff, - _b85alphabet + b'\x00' * len(_z85_b85_decode_diff) -) -_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet) - -def z85encode(s): - """Encode bytes-like object b in z85 format and return a bytes object.""" - return b85encode(s).translate(_z85_encode_translation) - -def z85decode(s): + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. + + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. + """ + return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, + alphabet=binascii.Z85_ALPHABET) + +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - s = _bytes_from_decode_data(s) - s = s.translate(_z85_decode_translation) - try: - return b85decode(s) - except ValueError as e: - raise ValueError(e.args[0].replace('base85', 'z85')) from None + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it @@ -566,11 +447,10 @@ def encodebytes(s): """Encode a bytestring into a bytes object containing multiple lines of base-64 data.""" _input_type_check(s) - pieces = [] - for i in range(0, len(s), MAXBINSIZE): - chunk = s[i : i + MAXBINSIZE] - pieces.append(binascii.b2a_base64(chunk)) - return b"".join(pieces) + result = binascii.b2a_base64(s, wrapcol=MAXLINESIZE) + if result == b'\n': + return b'' + return result def decodebytes(s): diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235ac8e1..50cf2b3f5b3e453 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 4af82f2cb8c8485..cc6255f61ae2114 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -9,6 +9,5 @@ __all__ = ["run", "runctx", "Profile"] if __name__ == "__main__": - import sys from profiling.tracing.__main__ import main main() diff --git a/Lib/calendar.py b/Lib/calendar.py index d80c3fd9524776f..92fe6b7723fe268 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -686,28 +686,61 @@ def __init__(self, highlight_day=None, *args, **kwargs): super().__init__(*args, **kwargs) self.highlight_day = highlight_day - def formatweek(self, theweek, width, *, highlight_day=None): + def _get_theme(self): + from _colorize import get_theme + + return get_theme(tty_file=sys.stdout) + + def formatday(self, day, weekday, width, *, highlight_day=None): """ - Returns a single week in a string (no newline). + Returns a formatted day. """ - if highlight_day: - from _colorize import get_colors - - ansi = get_colors() - highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" - reset = ansi.RESET + if day == 0: + s = '' else: - highlight = reset = "" + s = f'{day:2}' + s = s.center(width) + if day == highlight_day: + theme = self._get_theme().calendar + s = f"{theme.highlight}{s}{theme.reset}" + return s + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ return ' '.join( - ( - f"{highlight}{self.formatday(d, wd, width)}{reset}" - if d == highlight_day - else self.formatday(d, wd, width) - ) + self.formatday(d, wd, width, highlight_day=highlight_day) for (d, wd) in theweek ) + def formatweekheader(self, width): + """ + Return a header for a week. + """ + header = super().formatweekheader(width) + theme = self._get_theme().calendar + return f"{theme.weekday}{header}{theme.reset}" + + def formatmonthname(self, theyear, themonth, width, withyear=True): + """ + Return a formatted month name. + """ + name = super().formatmonthname(theyear, themonth, width, withyear) + theme = self._get_theme().calendar + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + color = theme.highlight + name_only = name.strip() + colored_name = f"{color}{name_only}{theme.reset}" + return name.replace(name_only, colored_name, 1) + else: + color = theme.header + return f"{color}{name}{theme.reset}" + def formatmonth(self, theyear, themonth, w=0, l=0): """ Return a month's calendar string (multi-line). @@ -742,7 +775,9 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): colwidth = (w + 1) * 7 - 1 v = [] a = v.append - a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + theme = self._get_theme().calendar + year = repr(theyear).center(colwidth*m+c*(m-1)).rstrip() + a(f"{theme.header}{year}{theme.reset}") a('\n'*l) header = self.formatweekheader(w) for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): @@ -843,28 +878,30 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( "-w", "--width", type=int, default=2, - help="width of date column (default 2)" + help="width of date column" ) textgroup.add_argument( "-l", "--lines", type=int, default=1, - help="number of lines for each week (default 1)" + help="number of lines for each week" ) textgroup.add_argument( "-s", "--spacing", type=int, default=6, - help="spacing between months (default 6)" + help="spacing between months" ) textgroup.add_argument( "-m", "--months", type=int, default=3, - help="months per row (default 3)" + help="months per row" ) htmlgroup.add_argument( "-c", "--css", @@ -879,18 +916,18 @@ def main(args=None): parser.add_argument( "-e", "--encoding", default=None, - help="encoding to use for output (default utf-8)" + help="encoding to use for output" ) parser.add_argument( "-t", "--type", default="text", choices=("text", "html"), - help="output type (text or html)" + help="output type (`text` or `html`)" ) parser.add_argument( "-f", "--first-weekday", type=int, default=0, - help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + help="weekday (0 is Monday, 6 is Sunday) to start each week" ) parser.add_argument( "year", diff --git a/Lib/code.py b/Lib/code.py index f7e275d8801b7c3..df1d7199e339341 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -385,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a50..af6ab031157e790 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -93,7 +93,7 @@ class CodecInfo(tuple): def __new__(cls, encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None, - *, _is_text_encoding=None): + *, _is_text_encoding=None, _expat_decoding_table=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode @@ -104,6 +104,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None, self.streamreader = streamreader if _is_text_encoding is not None: self._is_text_encoding = _is_text_encoding + if _expat_decoding_table is not None: + self._expat_decoding_table = _expat_decoding_table return self def __repr__(self): diff --git a/Lib/codeop.py b/Lib/codeop.py index 8cac00442d99e3a..40e88423119bc4f 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags): try: compiler(source + "\n", filename, symbol, flags=flags) return None - except _IncompleteInputError as e: + except _IncompleteInputError: return None - except SyntaxError as e: + except SyntaxError: pass # fallthrough diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 25ac4d1d524bc2c..20f1e728733fec6 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -32,6 +32,8 @@ _sys.modules['collections.abc'] = _collections_abc abc = _collections_abc +lazy from copy import copy as _copy +lazy from heapq import nlargest as _nlargest from itertools import chain as _chain from itertools import repeat as _repeat from itertools import starmap as _starmap @@ -59,8 +61,6 @@ except ImportError: pass -heapq = None # Lazily imported - ################################################################################ ### OrderedDict @@ -328,14 +328,14 @@ def __ior__(self, other): return self def __or__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(self) new.update(other) return new def __ror__(self, other): - if not isinstance(other, dict): + if not isinstance(other, (dict, frozendict)): return NotImplemented new = self.__class__(other) new.update(self) @@ -634,12 +634,7 @@ def most_common(self, n=None): if n is None: return sorted(self.items(), key=_itemgetter(1), reverse=True) - # Lazy import to speedup Python startup time - global heapq - if heapq is None: - import heapq - - return heapq.nlargest(n, self.items(), key=_itemgetter(1)) + return _nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): '''Iterator over elements repeating each as many times as its count. @@ -1221,14 +1216,14 @@ def __repr__(self): def __or__(self, other): if isinstance(other, UserDict): return self.__class__(self.data | other.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(self.data | other) return NotImplemented def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) - if isinstance(other, dict): + if isinstance(other, (dict, frozendict)): return self.__class__(other | self.data) return NotImplemented @@ -1249,16 +1244,29 @@ def __copy__(self): def copy(self): if self.__class__ is UserDict: return UserDict(self.data.copy()) - import copy data = self.data try: self.data = {} - c = copy.copy(self) + c = _copy(self) finally: self.data = data c.update(self) return c + + # This method has a default implementation in MutableMapping, but dict's + # equivalent is last-in, first-out instead of first-in, first-out. + def popitem(self): + """Remove and return a (key, value) pair as a 2-tuple. + + Removes pairs in the same order as the wrapped mapping's popitem() + method. For dict objects (the default), that order is last-in, + first-out (LIFO). + Raises KeyError if the UserDict is empty. + """ + return self.data.popitem() + + @classmethod def fromkeys(cls, iterable, value=None): d = cls() @@ -1542,6 +1550,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1610,6 +1620,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/compileall.py b/Lib/compileall.py index 67fe370451e1efb..812a496611e043f 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -165,6 +165,14 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) + # Without a cache_tag, we can only create legacy .pyc files. None of our + # callers seem to expect this, so the best we can do is fail without raising + if not legacy and sys.implementation.cache_tag is None: + if not quiet: + print("No cache tag is available to generate .pyc path for", + repr(fullname)) + return False + dfile = None if ddir is not None: @@ -223,7 +231,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, cfile = importlib.util.cache_from_source(fullname) opt_cfiles[opt_level] = cfile - head, tail = name[:-3], name[-3:] + tail = name[-3:] if tail == '.py': if not force: try: @@ -318,7 +326,6 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.', - color=True, ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', @@ -330,10 +337,10 @@ def main(): parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' + help='output only error messages; `-qq` will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') + help='use legacy (pre-PEP 3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' @@ -359,28 +366,28 @@ def main(): 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) + '`FILE` to the list considered for compilation; ' + 'if `"-"`, names are read from `stdin`')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) + 'to the equivalent of `-l` `sys.path`')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), - help=('set .pyc invalidation mode; defaults to ' - '"checked-hash" if the SOURCE_DATE_EPOCH ' + help=('set `.pyc` invalidation mode; defaults to ' + '`"checked-hash"` if the `SOURCE_DATE_EPOCH` ' 'environment variable is set, and ' - '"timestamp" otherwise.')) + '`"timestamp"` otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is `-1` which uses the optimization level ' + 'of the Python interpreter itself (see `-O`).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', - help='Ignore symlinks pointing outsite of the DIR') + help='Ignore symlinks pointing outsite of the `DIR`') parser.add_argument('--hardlink-dupes', action='store_true', dest='hardlink_dupes', help='Hardlink duplicated pyc files') diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 84b25914b0aa93a..5326cf9b19cf883 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -61,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -103,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -140,8 +142,8 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ @@ -152,8 +154,8 @@ def compress(data, level=None, options=None, zstd_dict=None): def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index d709f5efc658fa4..8d3358152e9a889 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -36,9 +36,9 @@ def __init__(self, file, /, mode='r', *, *file* can be either an file-like object, or a file name to open. - *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for - creating exclusively, or 'a' for appending. These can equivalently be - given as 'rb', 'wb', 'xb' and 'ab' respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -296,26 +296,27 @@ def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', - 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. - - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. + + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index d6ac4b3e0b675f6..5d53a5ac50931d3 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,6 +17,9 @@ wait, as_completed) +lazy from .process import ProcessPoolExecutor +lazy from .thread import ThreadPoolExecutor + __all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', @@ -40,26 +43,9 @@ _interpreters = None if _interpreters: + lazy from .interpreter import InterpreterPoolExecutor # noqa: F401 __all__.append('InterpreterPoolExecutor') def __dir__(): return __all__ + ['__author__', '__doc__'] - - -def __getattr__(name): - global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor - - if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor - return ProcessPoolExecutor - - if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor - return ThreadPoolExecutor - - if _interpreters and name == 'InterpreterPoolExecutor': - from .interpreter import InterpreterPoolExecutor - return InterpreterPoolExecutor - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index f506ce68aea5b29..4e71331f9a0a594 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -194,15 +194,15 @@ def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: - fs: The sequence of Futures (possibly created by different Executors) to - iterate over. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. + fs: The sequence of Futures (possibly created by different + Executors) to iterate over. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. Returns: - An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + An iterator that yields the given Futures as they complete + (finished or cancelled). If any given Futures are duplicated, + they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -258,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): """Wait for the futures in the given sequence to complete. Args: - fs: The sequence of Futures (possibly created by different Executors) to - wait upon. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - return_when: Indicates when this function should return. The options - are: + fs: The sequence of Futures (possibly created by different + Executors) to wait upon. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + return_when: Indicates when this function should return. + The options are: FIRST_COMPLETED - Return when any future finishes or is cancelled. FIRST_EXCEPTION - Return when any future finishes by raising an - exception. If no future raises an exception + exception. If no future raises an exception then it is equivalent to ALL_COMPLETED. - ALL_COMPLETED - Return when all futures finish or are cancelled. + ALL_COMPLETED - Return when all futures finish or are + cancelled. Returns: A named 2-tuple of sets. The first set, named 'done', contains the @@ -404,11 +405,12 @@ def add_done_callback(self, fn): Args: fn: A callable that will be called with this future as its only - argument when the future completes or is cancelled. The callable - will always be called by a thread in the same process in which - it was added. If the future has already completed or been - cancelled then the callable will be called immediately. These - callables are called in the order that they were added. + argument when the future completes or is cancelled. The + callable will always be called by a thread in the same + process in which it was added. If the future has already + completed or been cancelled then the callable will be + called immediately. These callables are called in the + order that they were added. """ with self._condition: if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: @@ -423,17 +425,19 @@ def result(self, timeout=None): """Return the result of the call that the future represents. Args: - timeout: The number of seconds to wait for the result if the future - isn't done. If None, then there is no limit on the wait time. + timeout: The number of seconds to wait for the result if the + future isn't done. If None, then there is no limit on the + wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. - Exception: If the call raised then that exception will be raised. + TimeoutError: If the future didn't finish executing before the + given timeout. + Exception: If the call raised then that exception will be + raised. """ try: with self._condition: @@ -459,17 +463,17 @@ def exception(self, timeout=None): Args: timeout: The number of seconds to wait for the exception if the - future isn't done. If None, then there is no limit on the wait - time. + future isn't done. If None, then there is no limit on the + wait time. Returns: - The exception raised by the call that the future represents or None - if the call completed without raising. + The exception raised by the call that the future represents or + None if the call completed without raising. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. + TimeoutError: If the future didn't finish executing before the + given timeout. """ with self._condition: @@ -494,22 +498,23 @@ def set_running_or_notify_cancel(self): Should only be used by Executor implementations and unit tests. If the future has been cancelled (cancel() was called and returned - True) then any threads waiting on the future completing (though calls - to as_completed() or wait()) are notified and False is returned. + True) then any threads waiting on the future completing (though + calls to as_completed() or wait()) are notified and False is + returned. If the future was not cancelled then it is put in the running state (future calls to running() will return True) and True is returned. This method should be called by Executor implementations before - executing the work associated with this future. If this method returns - False then the work should not be executed. + executing the work associated with this future. If this method + returns False then the work should not be executed. Returns: False if the Future was cancelled, True otherwise. Raises: - RuntimeError: if this method was already called or if set_result() - or set_exception() was called. + RuntimeError: if this method was already called or if + set_result() or set_exception() was called. """ with self._condition: if self._state == CANCELLED: @@ -593,8 +598,9 @@ class Executor(object): def submit(self, fn, /, *args, **kwargs): """Submits a callable to be executed with the given arguments. - Schedules the callable to be executed as fn(*args, **kwargs) and returns - a Future instance representing the execution of the callable. + Schedules the callable to be executed as fn(*args, **kwargs) and + returns a Future instance representing the execution of the + callable. Returns: A Future representing the given call. @@ -607,25 +613,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: The size of the chunks the iterable will be broken + into before being passed to a child process. This argument + is only used by ProcessPoolExecutor; it is ignored by ThreadPoolExecutor. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if buffersize is not None and not isinstance(buffersize, int): @@ -679,8 +685,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): Args: wait: If True then shutdown will not return until all running - futures have finished executing and the resources used by the - executor have been reclaimed. + futures have finished executing and the resources used by + the executor have been reclaimed. cancel_futures: If True then shutdown will cancel all pending futures. Futures that are completed or running will not be cancelled. diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 85c1da2c7228943..fd3d45144b49a71 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -110,8 +110,8 @@ def __init__(self, max_workers=None, thread_name_prefix='', """Initializes a new InterpreterPoolExecutor instance. Args: - max_workers: The maximum number of interpreters that can be used to - execute the given calls. + max_workers: The maximum number of interpreters that can be used + to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. initializer: A callable or script used to initialize each worker interpreter. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a42afa68efcb148..8f200fc1c82613f 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -360,7 +360,7 @@ def run(self): if executor := self.executor_reference(): if process_exited: with self.shutdown_lock: - executor._adjust_process_count() + executor._replace_dead_worker() else: executor._idle_worker_semaphore.release() del executor @@ -469,11 +469,9 @@ def _terminate_broken(self, cause): executor._shutdown_thread = True executor = None - # All pending tasks are to be marked failed with the following - # BrokenProcessPool error - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + # All pending tasks are to be marked failed with a + # BrokenProcessPool error, as separate instances to avoid sharing + # a traceback (gh-101267). cause_str = None if cause is not None: cause_str = ''.join(cause) @@ -489,11 +487,15 @@ def _terminate_broken(self, cause): f"with exit code {p.exitcode}") if errors: cause_str = "\n".join(errors) - if cause_str: - bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") + cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): + bpe = BrokenProcessPool("A process in the process pool was " + "terminated abruptly while the future was " + "running or pending.") + if cause_tb is not None: + bpe.__cause__ = _RemoteTraceback(cause_tb) try: work_item.future.set_exception(bpe) except _base.InvalidStateError: @@ -656,19 +658,21 @@ def __init__(self, max_workers=None, mp_context=None, Args: max_workers: The maximum number of processes that can be used to - execute the given calls. If None or not given then as many - worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers created - using the multiprocessing.get_context('start method') API. This - object should provide SimpleQueue, Queue and Process. + execute the given calls. If None or not given then as many + worker processes will be created as the machine has + processors. + mp_context: A multiprocessing context to launch the workers + created using the multiprocessing.get_context('start method') + API. This object should provide SimpleQueue, Queue and + Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. - max_tasks_per_child: The maximum number of tasks a worker process - can complete before it will exit and be replaced with a fresh - worker process. The default of None means worker process will - live as long as the executor. Requires a non-'fork' mp_context - start method. When given, we default to using 'spawn' if no - mp_context is supplied. + max_tasks_per_child: The maximum number of tasks a worker + process can complete before it will exit and be replaced + with a fresh worker process. The default of None means + worker process will live as long as the executor. Requires + a non-'fork' mp_context start method. When given, we + default to using 'spawn' if no mp_context is supplied. """ _check_system_limits() @@ -768,6 +772,30 @@ def _start_executor_manager_thread(self): _threads_wakeups[self._executor_manager_thread] = \ self._executor_manager_thread_wakeup + def _replace_dead_worker(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + + # A replacement is pointless when shutting down with nothing left + # to run. Both attributes are read under _shutdown_lock, which + # shutdown() holds while setting _shutdown_thread. + assert self._shutdown_lock.locked() + if self._shutdown_thread and not self._pending_work_items: + return + + # gh-115634: A worker exited after reaching max_tasks_per_child and + # has been removed from self._processes. Do not consult + # _idle_worker_semaphore here: it counts task completions, not idle + # workers, so it can hold a stale token released by the now-dead + # worker. Trusting such a token would leave the pool a worker short, + # deadlocking once all workers reach their task limit. Spawning is + # safe from this (manager) thread despite gh-90622 because + # max_tasks_per_child is rejected for the "fork" start method. + if len(self._processes) < self._max_workers: + self._spawn_process() + def _adjust_process_count(self): # gh-132969: avoid error when state is reset and executor is still running, # which will happen when shutdown(wait=False) is called. @@ -780,12 +808,12 @@ def _adjust_process_count(self): process_count = len(self._processes) if process_count < self._max_workers: - # Assertion disabled as this codepath is also used to replace a - # worker that unexpectedly dies, even when using the 'fork' start - # method. That means there is still a potential deadlock bug. If a - # 'fork' mp_context worker dies, we'll be forking a new one when - # we know a thread is running (self._executor_manager_thread). - #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622' + # gh-90622: spawning a child via fork while another thread is + # running can deadlock in the child. submit() only calls this + # method when using a non-fork start method. + assert (self._safe_to_dynamically_spawn_children + or not self._executor_manager_thread), ( + 'https://github.com/python/cpython/issues/90622') self._spawn_process() def _launch_processes(self): @@ -838,24 +866,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: If greater than one, the iterables will be chopped + into chunks of size chunksize and submitted to the process + pool. If set to one, the items in the list will be sent + one at a time. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if chunksize < 1: diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index b5cc0b8944940dc..5f3ee0934de59d6 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -185,7 +185,8 @@ def put(self, obj, block=True, timeout=None, *, underlying data is actually shared. Furthermore, some types can be sent through a queue more efficiently than others. This group includes various immutable types like int, str, bytes, and - tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later @@ -223,7 +224,7 @@ def put(self, obj, block=True, timeout=None, *, while True: try: _queues.put(self._id, obj, unboundop) - except QueueFull as exc: + except QueueFull: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -258,7 +259,7 @@ def get(self, block=True, timeout=None, *, while True: try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -277,7 +278,7 @@ def get_nowait(self): """ try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) diff --git a/Lib/configparser.py b/Lib/configparser.py index 18af1eadaad1110..3c452afe8ade485 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -613,7 +616,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P